Bug 1419957 - Remove old style backfill/add new job features (#2982)

Since everything but standard retrigger/cancellation is now handled
client-side by tcactions, making the backend parts that provided pulse
messages to pulse_actions redundant (since it was decommissioned in
bug 1379172).

schemas/push-action-message.json

@@ -1,41 +0,0 @@
-{
-  "id":           "https://treeherder.mozilla.org/schemas/v1/resultset-action-message.json#",
-  "$schema":      "http://json-schema.org/draft-04/schema#",
-  "title":        "Notification of triggering jobs in a resultset",
-  "description":  "Event is dispatched when user/service issues a (trigger_missing_jobs, trigger_all_talos_jobs) action on a resultset",
-  "type":         "object",
-  "properties": {
-    "version": {
-      "title":                "Message-format version",
-      "enum":                 [1]
-    },
-    "project": {
-      "title":                "Project Name",
-      "description":          "Identifier for treeherder project, like `try` or `mozilla-central`.",
-      "type":                 "string"
-    },
-    "resultset_id": {
-      "title":                "Resultset ID",
-      "description":          "Project unique identifier for a resultset",
-      "type":                 "string"
-    },
-    "times": {
-      "title":                "Times",
-      "description":          "Number of times to execute the command for a resultset.",
-      "type":                 "number"
-    },
-    "action": {
-      "title":                "Action",
-      "description":          "Type of action issued on task",
-      "enum":                 ["cancel_all", "trigger_missing_jobs", "trigger_all_talos_jobs"],
-      "type":                 "string"
-    },
-    "requester": {
-      "title":                "Requester",
-      "description":          "The requester of the action (usually an email)",
-      "type":                 "string"
-    }
-  },
-  "additionalProperties":     true,
-  "required": ["version", "resultset_id", "project", "action", "requester"]
-}

schemas/push-runnable-job-action-message.json

@@ -1,45 +0,0 @@
-{
-  "id":           "https://treeherder.mozilla.org/schemas/v1/resultset-runnable-job-action-message.json#",
-  "$schema":      "http://json-schema.org/draft-04/schema#",
-  "title":        "Notification of triggering runnable jobs on a resultset",
-  "description":  "Event is dispatched when user/service asks for new runnable jobs on a resultset",
-  "type":         "object",
-  "properties": {
-    "version": {
-      "title":                "Message-format version",
-      "enum":                 [1]
-    },
-    "project": {
-      "title":                "Project Name",
-      "description":          "Identifier for treeherder project, like `try` or `mozilla-central`.",
-      "type":                 "string"
-    },
-    "resultset_id": {
-      "title":                "Resultset ID",
-      "description":          "Project unique identifier for a resultset",
-      "type":                 "string"
-    },
-    "requester": {
-      "title":                "Requester",
-      "description":          "The requester of the action (usually an email)",
-      "type":                 "string"
-    },
-    "requested_jobs": {
-      "title":                "Requested Jobs",
-      "description":          "The buildernames and TaskLabels that should be added to a push",
-      "type":                 "array"
-    },
-    "decision_task_id": {
-      "title":                "decision_task_id",
-      "description":          "The Gecko Decision Task's Task ID for this resultset",
-      "type":                 "string"
-    },
-    "timestamp": {
-      "title":                "timestamp",
-      "description":          "The UTC timestamp",
-      "type":                 "string"
-    }
-  },
-  "additionalProperties":     false,
-  "required": ["version", "resultset_id", "project", "requester", "requested_jobs", "decision_task_id"]
-}

treeherder/model/exchanges.py

@@ -10,42 +10,6 @@ class TreeherderPublisher(PulsePublisher):
     """
     exchange_prefix = "v1/"
 
-    push_action = Exchange(
-        exchange="resultset-actions",
-        title="Actions issued by push",
-        description="""
-            There are actions which can be done to a push
-            (eg: trigger_missing_jobs), they are published on this exchange
-        """,
-        routing_keys=[
-            Key(
-                name='project',
-                summary="Project (or branch) that this push belongs to"
-            ),
-            Key(
-                name="action",
-                summary="Type of action issued (i.e. trigger_missing_jobs)"
-            ),
-        ],
-        schema="https://treeherder.mozilla.org/schemas/v1/resultset-action-message.json#"
-    )
-
-    push_runnable_job_action = Exchange(
-        exchange="resultset-runnable-job-actions",
-        title="Runnable job actions issued by push",
-        description="""
-            This action is published when a user asks for new runnable jobs (chosen
-            by name) on a push.
-        """,
-        routing_keys=[
-            Key(
-                name='project',
-                summary="Project (or branch) that this push belongs to"
-            ),
-        ],
-        schema="https://treeherder.mozilla.org/schemas/v1/resultset-runnable-job-action-message.json#"
-    )
-
     job_action = Exchange(
         exchange="job-actions",
         title="Actions issued by jobs",

treeherder/model/tasks.py

@@ -1,5 +1,4 @@
 import os
-import time
 
 import newrelic.agent
 from celery import task
@@ -80,52 +79,3 @@ def publish_job_action(project, action, job_id, requester):
         job_id=job.id,
         requester=requester
     )
-
-
-@task(name='publish-push-action')
-def publish_push_action(project, action, push_id, requester, times=1):
-    newrelic.agent.add_custom_parameter("project", project)
-    newrelic.agent.add_custom_parameter("action", action)
-    newrelic.agent.add_custom_parameter("push_id", str(push_id))
-    newrelic.agent.add_custom_parameter("requester", requester)
-
-    publisher = pulse_connection.get_publisher()
-    if not publisher:
-        return
-
-    publisher.push_action(
-        version=1,
-        project=project,
-        action=action,
-        requester=requester,
-        resultset_id=push_id,
-        push_id=push_id,
-        times=times
-    )
-
-
-@task(name='publish-resultset-runnable-job-action')
-def publish_resultset_runnable_job_action(project, resultset_id, requester, requested_jobs, decision_task_id):
-    publish_push_runnable_job_action(project, resultset_id, requester, requested_jobs, decision_task_id)
-
-
-@task(name='publish-push-runnable-job-action')
-def publish_push_runnable_job_action(project, push_id, requester, requested_jobs, decision_task_id):
-    newrelic.agent.add_custom_parameter("project", project)
-    newrelic.agent.add_custom_parameter("push_id", str(push_id))
-    newrelic.agent.add_custom_parameter("requester", requester)
-
-    publisher = pulse_connection.get_publisher()
-    if not publisher:
-        return
-
-    timestamp = str(time.time())
-    publisher.push_runnable_job_action(
-        version=1,
-        project=project,
-        requester=requester,
-        push_id=push_id,
-        requested_jobs=requested_jobs,
-        decision_task_id=decision_task_id,
-        timestamp=timestamp
-    )

treeherder/webapp/api/jobs.py

@@ -411,20 +411,6 @@ class JobsViewSet(viewsets.ViewSet):
                             status=HTTP_404_NOT_FOUND)
         return Response({"message": "All jobs successfully retriggered."})
 
-    @detail_route(methods=['post'], permission_classes=[IsAuthenticated])
-    def backfill(self, request, project, pk=None):
-        """
-        Issue a "backfill" to the underlying build_system_type by scheduling a
-        pulse message.
-        """
-        try:
-            job = Job.objects.get(repository__name=project,
-                                  id=pk)
-            self._job_action_event(job, 'backfill', request.user.email)
-            return Response({"message": "backfilled job '{0}'".format(job.guid)})
-        except ObjectDoesNotExist:
-            return Response("No job with id: {0}".format(pk), status=HTTP_404_NOT_FOUND)
-
     @detail_route(methods=['get'])
     def failure_lines(self, request, project, pk=None):
         """

treeherder/webapp/api/push.py

@@ -2,7 +2,6 @@ import datetime
 
 from rest_framework import viewsets
 from rest_framework.decorators import detail_route
-from rest_framework.exceptions import ParseError
 from rest_framework.permissions import IsAuthenticated
 from rest_framework.response import Response
 from rest_framework.status import (HTTP_400_BAD_REQUEST,
@@ -14,9 +13,7 @@ from treeherder.model.models import (Commit,
                                      Job,
                                      Push,
                                      Repository)
-from treeherder.model.tasks import (publish_job_action,
+from treeherder.model.tasks import publish_job_action
-                                    publish_push_action,
-                                    publish_push_runnable_job_action)
 from treeherder.webapp.api import permissions
 from treeherder.webapp.api.utils import (to_datetime,
                                          to_timestamp)
@@ -197,13 +194,6 @@ class PushViewSet(viewsets.ViewSet):
         if not pk:  # pragma nocover
             return Response({"message": "push id required"}, status=HTTP_400_BAD_REQUEST)
 
-        # Sending 'cancel_all' action to pulse. Right now there is no listener
-        # for this, so we cannot remove 'cancel' action for each job below.
-        publish_push_action.apply_async(
-            args=[project, 'cancel_all', pk, request.user.email],
-            routing_key='publish_to_pulse'
-        )
-
         # Notify the build systems which created these jobs...
         for job in Job.objects.filter(push_id=pk).exclude(state='completed'):
             publish_job_action.apply_async(
@@ -221,68 +211,6 @@ class PushViewSet(viewsets.ViewSet):
 
         return Response({"message": "pending and running jobs canceled for push '{0}'".format(pk)})
 
-    @detail_route(methods=['post'], permission_classes=[IsAuthenticated])
-    def trigger_missing_jobs(self, request, project, pk=None):
-        """
-        Trigger jobs that are missing in a push.
-        """
-        if not pk:
-            return Response({"message": "push id required"}, status=HTTP_400_BAD_REQUEST)
-
-        publish_push_action.apply_async(
-            args=[project, "trigger_missing_jobs", pk, request.user.email],
-            routing_key='publish_to_pulse'
-        )
-
-        return Response({"message": "Missing jobs triggered for push '{0}'".format(pk)})
-
-    @detail_route(methods=['post'], permission_classes=[IsAuthenticated])
-    def trigger_all_talos_jobs(self, request, project, pk=None):
-        """
-        Trigger all the talos jobs in a push.
-        """
-        if not pk:
-            return Response({"message": "push id required"}, status=HTTP_400_BAD_REQUEST)
-
-        times = int(request.query_params.get('times', None))
-        if not times:
-            raise ParseError(detail="The 'times' parameter is mandatory for this endpoint")
-
-        publish_push_action.apply_async(
-            args=[project, "trigger_all_talos_jobs", pk, request.user.email,
-                  times],
-            routing_key='publish_to_pulse'
-        )
-
-        return Response({"message": "Talos jobs triggered for push '{0}'".format(pk)})
-
-    @detail_route(methods=['post'], permission_classes=[IsAuthenticated])
-    def trigger_runnable_jobs(self, request, project, pk=None):
-        """
-        Add new jobs to a push.
-        """
-        if not pk:
-            return Response({"message": "push id required"},
-                            status=HTTP_400_BAD_REQUEST)
-
-        # Making sure a push with this id exists
-        if not Push.objects.filter(id=pk).exists():
-            return Response({"message": "No push with id: {0}".format(pk)},
-                            status=HTTP_404_NOT_FOUND)
-
-        requested_jobs = request.data.get('requested_jobs', [])
-        decision_task_id = request.data.get('decision_task_id', [])
-        if not requested_jobs:
-            Response({"message": "The list of requested_jobs cannot be empty"},
-                     status=HTTP_400_BAD_REQUEST)
-
-        publish_push_runnable_job_action.apply_async(
-            args=[project, pk, request.user.email, requested_jobs, decision_task_id],
-            routing_key='publish_to_pulse'
-        )
-
-        return Response({"message": "New jobs added for push '{0}'".format(pk)})
-
     @detail_route()
     def status(self, request, project, pk=None):
         """

ui/js/controllers/jobs.js

@@ -216,8 +216,6 @@ treeherderApp.controller('ResultSetCtrl', [
                 $scope.resultset.id
             ).then(function (decisionTaskID) {
                 ThResultSetModel.triggerMissingJobs(
-                    $scope.resultset.id,
-                    $scope.repoName,
                     decisionTaskID
                 ).then(function (msg) {
                     thNotify.send(msg, "success");
@@ -245,8 +243,6 @@ treeherderApp.controller('ResultSetCtrl', [
                 $scope.resultset.id
             ).then(function (decisionTaskID) {
                 ThResultSetModel.triggerAllTalosJobs(
-                    $scope.resultset.id,
-                    $scope.repoName,
                     times,
                     decisionTaskID
                 ).then(function (msg) {
@@ -270,8 +266,8 @@ treeherderApp.controller('ResultSetCtrl', [
             if ($scope.user.loggedin) {
                 var buildernames = ThResultSetStore.getSelectedRunnableJobs($rootScope.repoName, $scope.resultset.id);
                 ThResultSetStore.getGeckoDecisionTaskId($rootScope.repoName, $scope.resultset.id).then(function (decisionTaskID) {
-                    ThResultSetModel.triggerNewJobs($scope.repoName, $scope.resultset.id, buildernames, decisionTaskID).then(function (results) {
+                    ThResultSetModel.triggerNewJobs(buildernames, decisionTaskID).then(function (result) {
-                        thNotify.send(results[1], "success");
+                        thNotify.send(result, "success");
                         ThResultSetStore.deleteRunnableJobs($scope.repoName, $scope.resultset);
                     }, function (e) {
                         thNotify.send(ThTaskclusterErrors.format(e), 'danger', { sticky: true });

ui/js/models/job.js

@@ -115,14 +115,6 @@ treeherder.factory('ThJobModel', [
                 });
         };
 
-        ThJobModel.backfill = function (repoName, pk, config) {
-            config = config || {};
-            var timeout = config.timeout || null;
-
-            return $http.post(ThJobModel.get_uri(repoName)+pk+"/backfill/",
-                              { timeout: timeout });
-        };
-
         ThJobModel.cancel = function (repoName, jobIds, config) {
             config = config || {};
             var timeout = config.timeout || null;

ui/js/models/resultset.js

@@ -191,12 +191,8 @@ treeherder.factory('ThResultSetModel', ['$rootScope', '$http', '$location',
                 return $http.post(thUrl.getProjectUrl("/resultset/", repoName) + uri);
             },
 
-            triggerMissingJobs: function (resultset_id, repoName, decisionTaskId) {
+            triggerMissingJobs: function (decisionTaskId) {
-                var uri = resultset_id + '/trigger_missing_jobs/';
-                return $http.post(thUrl.getProjectUrl("/resultset/", repoName) + uri).then(function () {
                 return tcactions.load(decisionTaskId).then((results) => {
-                        // After we trigger the buildbot jobs, we can go ahead and trigger tc
-                        // jobs directly.
                     const tc = thTaskcluster.client();
                     const actionTaskId = tc.slugid();
 
@@ -217,15 +213,10 @@ treeherder.factory('ThResultSetModel', ['$rootScope', '$http', '$location',
                         }
                     }
                 });
-                });
             },
 
-            triggerAllTalosJobs: function (resultset_id, repoName, times, decisionTaskId) {
+            triggerAllTalosJobs: function (times, decisionTaskId) {
-                let uri = resultset_id + '/trigger_all_talos_jobs/?times=' + times;
-                return $http.post(thUrl.getProjectUrl("/resultset/", repoName) + uri).then(function () {
                 return tcactions.load(decisionTaskId).then((results) => {
-                        // After we trigger the buildbot jobs, we can go ahead and trigger tc
-                        // jobs directly.
                     const tc = thTaskcluster.client();
                     const actionTaskId = tc.slugid();
 
@@ -264,10 +255,9 @@ treeherder.factory('ThResultSetModel', ['$rootScope', '$http', '$location',
                         });
                     });
                 });
-                });
             },
 
-            triggerNewJobs: function (repoName, resultset_id, buildernames, decisionTaskId) {
+            triggerNewJobs: function (buildernames, decisionTaskId) {
                 let tc = thTaskcluster.client();
                 let queue = new tc.Queue();
                 let url = queue.buildUrl(
@@ -283,37 +273,18 @@ treeherder.factory('ThResultSetModel', ['$rootScope', '$http', '$location',
                     let allLabels = _.keys(graph);
 
                     let tclabels = [];
-                    let bbnames = [];
 
                     buildernames.forEach(function (name) {
-                        // The following has 3 cases that it accounts for
+                        // The following has 2 cases that it accounts for
-                        // 1. The name is a buildbot buildername not scheduled through bbb, in which case we pass it on
+                        // 1. The name is a taskcluster task label, in which case we pass it on
-                        // 2. The name is a taskcluster task label, in which case we pass it on
+                        // 2. The name is a buildbot buildername _scheduled_ through bbb, in which case we
-                        // 3. The name is a buildbot buildername _scheduled_ through bbb, in which case we
                         //    translate it to the taskcluster label that triggers it.
                         name = builderToTask[name] || name;
                         if (_.includes(allLabels, name)) {
                             tclabels.push(name);
-                        } else {
-                            bbnames.push(name);
                         }
                     });
 
-                    return $q.all([
-                        $q.resolve().then(function () {
-                            if (bbnames.length === 0) {
-                                return;
-                            }
-                            let bbdata = {
-                                requested_jobs: bbnames,
-                                decision_task_id: decisionTaskId
-                            };
-                            return $http.post(
-                                thUrl.getProjectUrl("/resultset/", repoName) + resultset_id + '/trigger_runnable_jobs/',
-                                bbdata
-                            );
-                        }),
-                        $q.resolve().then(function () {
                     if (tclabels.length === 0) {
                         return;
                     }
@@ -352,8 +323,6 @@ treeherder.factory('ThResultSetModel', ['$rootScope', '$http', '$location',
                             return queue.createTask(actionTaskId, task).then(() => `Request sent to trigger new jobs via actions.yml (${actionTaskId})`);
                         });
                     });
-                        }),
-                    ]);
                 });
             },
         };

ui/js/services/taskcluster.js

@@ -14,7 +14,7 @@ treeherder.factory('thTaskcluster', ['$rootScope', 'localStorageService',
             refreshTimestamps: function (task) {
                 // Take a taskcluster task and make all of the timestamps
                 // new again. This is pretty much lifted verbatim from
-                // mozilla_ci_tools which is used by pulse_actions.
+                // mozilla_ci_tools which was used by pulse_actions.
                 // We need to do this because action tasks are created with
                 // timestamps for expires/created/deadline that are based
                 // on the time of the original decision task creation. We must

ui/plugins/controller.js

@@ -327,12 +327,11 @@ treeherder.controller('PluginCtrl', [
 
                 var job_id_list = _.map(jobs, 'id');
                 // The logic here is somewhat complicated because we need to support
-                // two use cases the first is the case where we notify a system
+                // two use cases the first is the case where we notify a system other
-                // other then buildbot that a retrigger has been requested. The
+                // then buildbot that a retrigger has been requested (eg mozilla-taskcluster).
-                // second is when we have the buildapi id and need to send a request
+                // The second is when we have the buildapi id and need to send a request
                 // to the self serve api (which does not listen over pulse!).
                 ThJobModel.retrigger($scope.repoName, job_id_list).then(function () {
-                    // XXX: Remove this after 1134929 is resolved.
                     return ThJobDetailModel.getJobDetails({
                         title: "buildbot_request_id",
                         repository: $scope.repoName,
@@ -431,18 +430,7 @@ treeherder.controller('PluginCtrl', [
                         });
                     });
             } else {
-                ThJobModel.backfill(
+                thNotify.send('Unable to backfill this job type!', 'danger', { sticky: true });
-                    $scope.repoName,
-                    $scope.job.id
-                ).then(function () {
-                    thNotify.send("Request sent to backfill jobs", 'success');
-                }, function (e) {
-                    // Generic error eg. the user doesn't have LDAP access
-                    thNotify.send(
-                        ThModelErrors.format(e, "Unable to send backfill"),
-                        'danger'
-                    );
-                });
             }
         };