mozilla
/
treeherder
зеркало из https://github.com/mozilla/treeherder.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

Bug 1419957 - Remove old style backfill/add new job features (#2982) 

Since everything but standard retrigger/cancellation is now handled
client-side by tcactions, making the backend parts that provided pulse
messages to pulse_actions redundant (since it was decommissioned in
bug 1379172).
This commit is contained in:
Ed Morley 2017-12-05 22:42:06 +00:00 коммит произвёл GitHub
Родитель ffc7fe6c79
Коммит 958cc079a7
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: 4AEE18F83AFDEB23
11 изменённых файлов: 100 добавлений и 413 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

41
schemas/push-action-message.json
Просмотреть файл

@ -1,41 +0,0 @@
{
"id": "https://treeherder.mozilla.org/schemas/v1/resultset-action-message.json#",
"$schema": "http://json-schema.org/draft-04/schema#",
"title": "Notification of triggering jobs in a resultset",
"description": "Event is dispatched when user/service issues a (trigger_missing_jobs, trigger_all_talos_jobs) action on a resultset",
"type": "object",
"properties": {
"version": {
"title": "Message-format version",
"enum": [1]
},
"project": {
"title": "Project Name",
"description": "Identifier for treeherder project, like `try` or `mozilla-central`.",
"type": "string"
},
"resultset_id": {
"title": "Resultset ID",
"description": "Project unique identifier for a resultset",
"type": "string"
},
"times": {
"title": "Times",
"description": "Number of times to execute the command for a resultset.",
"type": "number"
},
"action": {
"title": "Action",
"description": "Type of action issued on task",
"enum": ["cancel_all", "trigger_missing_jobs", "trigger_all_talos_jobs"],
"type": "string"
},
"requester": {
"title": "Requester",
"description": "The requester of the action (usually an email)",
"type": "string"
}
},
"additionalProperties": true,
"required": ["version", "resultset_id", "project", "action", "requester"]
}

45
schemas/push-runnable-job-action-message.json
Просмотреть файл

@ -1,45 +0,0 @@
{
"id": "https://treeherder.mozilla.org/schemas/v1/resultset-runnable-job-action-message.json#",
"$schema": "http://json-schema.org/draft-04/schema#",
"title": "Notification of triggering runnable jobs on a resultset",
"description": "Event is dispatched when user/service asks for new runnable jobs on a resultset",
"type": "object",
"properties": {
"version": {
"title": "Message-format version",
"enum": [1]
},
"project": {
"title": "Project Name",
"description": "Identifier for treeherder project, like `try` or `mozilla-central`.",
"type": "string"
},
"resultset_id": {
"title": "Resultset ID",
"description": "Project unique identifier for a resultset",
"type": "string"
},
"requester": {
"title": "Requester",
"description": "The requester of the action (usually an email)",
"type": "string"
},
"requested_jobs": {
"title": "Requested Jobs",
"description": "The buildernames and TaskLabels that should be added to a push",
"type": "array"
},
"decision_task_id": {
"title": "decision_task_id",
"description": "The Gecko Decision Task's Task ID for this resultset",
"type": "string"
},
"timestamp": {
"title": "timestamp",
"description": "The UTC timestamp",
"type": "string"
}
},
"additionalProperties": false,
"required": ["version", "resultset_id", "project", "requester", "requested_jobs", "decision_task_id"]
}

36
treeherder/model/exchanges.py
Просмотреть файл

@ -10,42 +10,6 @@ class TreeherderPublisher(PulsePublisher):
"""
exchange_prefix = "v1/"
push_action = Exchange(
exchange="resultset-actions",
title="Actions issued by push",
description="""
There are actions which can be done to a push
(eg: trigger_missing_jobs), they are published on this exchange
""",
routing_keys=[
Key(
name='project',
summary="Project (or branch) that this push belongs to"
),
Key(
name="action",
summary="Type of action issued (i.e. trigger_missing_jobs)"
),
],
schema="https://treeherder.mozilla.org/schemas/v1/resultset-action-message.json#"
)
push_runnable_job_action = Exchange(
exchange="resultset-runnable-job-actions",
title="Runnable job actions issued by push",
description="""
This action is published when a user asks for new runnable jobs (chosen
by name) on a push.
""",
routing_keys=[
Key(
name='project',
summary="Project (or branch) that this push belongs to"
),
],
schema="https://treeherder.mozilla.org/schemas/v1/resultset-runnable-job-action-message.json#"
)
job_action = Exchange(
exchange="job-actions",
title="Actions issued by jobs",

50
treeherder/model/tasks.py
Просмотреть файл

@ -1,5 +1,4 @@
import os
import time
import newrelic.agent
from celery import task
@ -80,52 +79,3 @@ def publish_job_action(project, action, job_id, requester):
job_id=job.id,
requester=requester
)
@task(name='publish-push-action')
def publish_push_action(project, action, push_id, requester, times=1):
newrelic.agent.add_custom_parameter("project", project)
newrelic.agent.add_custom_parameter("action", action)
newrelic.agent.add_custom_parameter("push_id", str(push_id))
newrelic.agent.add_custom_parameter("requester", requester)
publisher = pulse_connection.get_publisher()
if not publisher:
return
publisher.push_action(
version=1,
project=project,
action=action,
requester=requester,
resultset_id=push_id,
push_id=push_id,
times=times
)
@task(name='publish-resultset-runnable-job-action')
def publish_resultset_runnable_job_action(project, resultset_id, requester, requested_jobs, decision_task_id):
publish_push_runnable_job_action(project, resultset_id, requester, requested_jobs, decision_task_id)
@task(name='publish-push-runnable-job-action')
def publish_push_runnable_job_action(project, push_id, requester, requested_jobs, decision_task_id):
newrelic.agent.add_custom_parameter("project", project)
newrelic.agent.add_custom_parameter("push_id", str(push_id))
newrelic.agent.add_custom_parameter("requester", requester)
publisher = pulse_connection.get_publisher()
if not publisher:
return
timestamp = str(time.time())
publisher.push_runnable_job_action(
version=1,
project=project,
requester=requester,
push_id=push_id,
requested_jobs=requested_jobs,
decision_task_id=decision_task_id,
timestamp=timestamp
)

14
treeherder/webapp/api/jobs.py
Просмотреть файл

@ -411,20 +411,6 @@ class JobsViewSet(viewsets.ViewSet):
status=HTTP_404_NOT_FOUND)
return Response({"message": "All jobs successfully retriggered."})
@detail_route(methods=['post'], permission_classes=[IsAuthenticated])
def backfill(self, request, project, pk=None):
"""
Issue a "backfill" to the underlying build_system_type by scheduling a
pulse message.
"""
try:
job = Job.objects.get(repository__name=project,
id=pk)
self._job_action_event(job, 'backfill', request.user.email)
return Response({"message": "backfilled job '{0}'".format(job.guid)})
except ObjectDoesNotExist:
return Response("No job with id: {0}".format(pk), status=HTTP_404_NOT_FOUND)
@detail_route(methods=['get'])
def failure_lines(self, request, project, pk=None):
"""

74
treeherder/webapp/api/push.py
Просмотреть файл

@ -2,7 +2,6 @@ import datetime
from rest_framework import viewsets
from rest_framework.decorators import detail_route
from rest_framework.exceptions import ParseError
from rest_framework.permissions import IsAuthenticated
from rest_framework.response import Response
from rest_framework.status import (HTTP_400_BAD_REQUEST,
@ -14,9 +13,7 @@ from treeherder.model.models import (Commit,
Job,
Push,
Repository)
from treeherder.model.tasks import (publish_job_action,
publish_push_action,
publish_push_runnable_job_action)
from treeherder.model.tasks import publish_job_action
from treeherder.webapp.api import permissions
from treeherder.webapp.api.utils import (to_datetime,
to_timestamp)
@ -197,13 +194,6 @@ class PushViewSet(viewsets.ViewSet):
if not pk: # pragma nocover
return Response({"message": "push id required"}, status=HTTP_400_BAD_REQUEST)
# Sending 'cancel_all' action to pulse. Right now there is no listener
# for this, so we cannot remove 'cancel' action for each job below.
publish_push_action.apply_async(
args=[project, 'cancel_all', pk, request.user.email],
routing_key='publish_to_pulse'
)
# Notify the build systems which created these jobs...
for job in Job.objects.filter(push_id=pk).exclude(state='completed'):
publish_job_action.apply_async(
@ -221,68 +211,6 @@ class PushViewSet(viewsets.ViewSet):
return Response({"message": "pending and running jobs canceled for push '{0}'".format(pk)})
@detail_route(methods=['post'], permission_classes=[IsAuthenticated])
def trigger_missing_jobs(self, request, project, pk=None):
"""
Trigger jobs that are missing in a push.
"""
if not pk:
return Response({"message": "push id required"}, status=HTTP_400_BAD_REQUEST)
publish_push_action.apply_async(
args=[project, "trigger_missing_jobs", pk, request.user.email],
routing_key='publish_to_pulse'
)
return Response({"message": "Missing jobs triggered for push '{0}'".format(pk)})
@detail_route(methods=['post'], permission_classes=[IsAuthenticated])
def trigger_all_talos_jobs(self, request, project, pk=None):
"""
Trigger all the talos jobs in a push.
"""
if not pk:
return Response({"message": "push id required"}, status=HTTP_400_BAD_REQUEST)
times = int(request.query_params.get('times', None))
if not times:
raise ParseError(detail="The 'times' parameter is mandatory for this endpoint")
publish_push_action.apply_async(
args=[project, "trigger_all_talos_jobs", pk, request.user.email,
times],
routing_key='publish_to_pulse'
)
return Response({"message": "Talos jobs triggered for push '{0}'".format(pk)})
@detail_route(methods=['post'], permission_classes=[IsAuthenticated])
def trigger_runnable_jobs(self, request, project, pk=None):
"""
Add new jobs to a push.
"""
if not pk:
return Response({"message": "push id required"},
status=HTTP_400_BAD_REQUEST)
# Making sure a push with this id exists
if not Push.objects.filter(id=pk).exists():
return Response({"message": "No push with id: {0}".format(pk)},
status=HTTP_404_NOT_FOUND)
requested_jobs = request.data.get('requested_jobs', [])
decision_task_id = request.data.get('decision_task_id', [])
if not requested_jobs:
Response({"message": "The list of requested_jobs cannot be empty"},
status=HTTP_400_BAD_REQUEST)
publish_push_runnable_job_action.apply_async(
args=[project, pk, request.user.email, requested_jobs, decision_task_id],
routing_key='publish_to_pulse'
)
return Response({"message": "New jobs added for push '{0}'".format(pk)})
@detail_route()
def status(self, request, project, pk=None):
"""

8
ui/js/controllers/jobs.js
Просмотреть файл

@ -216,8 +216,6 @@ treeherderApp.controller('ResultSetCtrl', [
$scope.resultset.id
).then(function (decisionTaskID) {
ThResultSetModel.triggerMissingJobs(
$scope.resultset.id,
$scope.repoName,
decisionTaskID
).then(function (msg) {
thNotify.send(msg, "success");
@ -245,8 +243,6 @@ treeherderApp.controller('ResultSetCtrl', [
$scope.resultset.id
).then(function (decisionTaskID) {
ThResultSetModel.triggerAllTalosJobs(
$scope.resultset.id,
$scope.repoName,
times,
decisionTaskID
).then(function (msg) {
@ -270,8 +266,8 @@ treeherderApp.controller('ResultSetCtrl', [
if ($scope.user.loggedin) {
var buildernames = ThResultSetStore.getSelectedRunnableJobs($rootScope.repoName, $scope.resultset.id);
ThResultSetStore.getGeckoDecisionTaskId($rootScope.repoName, $scope.resultset.id).then(function (decisionTaskID) {
ThResultSetModel.triggerNewJobs($scope.repoName, $scope.resultset.id, buildernames, decisionTaskID).then(function (results) {
thNotify.send(results[1], "success");
ThResultSetModel.triggerNewJobs(buildernames, decisionTaskID).then(function (result) {
thNotify.send(result, "success");
ThResultSetStore.deleteRunnableJobs($scope.repoName, $scope.resultset);
}, function (e) {
thNotify.send(ThTaskclusterErrors.format(e), 'danger', { sticky: true });

8
ui/js/models/job.js
Просмотреть файл

@ -115,14 +115,6 @@ treeherder.factory('ThJobModel', [
});
};
ThJobModel.backfill = function (repoName, pk, config) {
config = config || {};
var timeout = config.timeout || null;
return $http.post(ThJobModel.get_uri(repoName)+pk+"/backfill/",
{ timeout: timeout });
};
ThJobModel.cancel = function (repoName, jobIds, config) {
config = config || {};
var timeout = config.timeout || null;

43
ui/js/models/resultset.js
Просмотреть файл

@ -191,12 +191,8 @@ treeherder.factory('ThResultSetModel', ['$rootScope', '$http', '$location',
return $http.post(thUrl.getProjectUrl("/resultset/", repoName) + uri);
},
triggerMissingJobs: function (resultset_id, repoName, decisionTaskId) {
var uri = resultset_id + '/trigger_missing_jobs/';
return $http.post(thUrl.getProjectUrl("/resultset/", repoName) + uri).then(function () {
triggerMissingJobs: function (decisionTaskId) {
return tcactions.load(decisionTaskId).then((results) => {
// After we trigger the buildbot jobs, we can go ahead and trigger tc
// jobs directly.
const tc = thTaskcluster.client();
const actionTaskId = tc.slugid();
@ -217,15 +213,10 @@ treeherder.factory('ThResultSetModel', ['$rootScope', '$http', '$location',
}
}
});
});
},
triggerAllTalosJobs: function (resultset_id, repoName, times, decisionTaskId) {
let uri = resultset_id + '/trigger_all_talos_jobs/?times=' + times;
return $http.post(thUrl.getProjectUrl("/resultset/", repoName) + uri).then(function () {
triggerAllTalosJobs: function (times, decisionTaskId) {
return tcactions.load(decisionTaskId).then((results) => {
// After we trigger the buildbot jobs, we can go ahead and trigger tc
// jobs directly.
const tc = thTaskcluster.client();
const actionTaskId = tc.slugid();
@ -264,10 +255,9 @@ treeherder.factory('ThResultSetModel', ['$rootScope', '$http', '$location',
});
});
});
});
},
triggerNewJobs: function (repoName, resultset_id, buildernames, decisionTaskId) {
triggerNewJobs: function (buildernames, decisionTaskId) {
let tc = thTaskcluster.client();
let queue = new tc.Queue();
let url = queue.buildUrl(
@ -283,37 +273,18 @@ treeherder.factory('ThResultSetModel', ['$rootScope', '$http', '$location',
let allLabels = _.keys(graph);
let tclabels = [];
let bbnames = [];
buildernames.forEach(function (name) {
// The following has 3 cases that it accounts for
// 1. The name is a buildbot buildername not scheduled through bbb, in which case we pass it on
// 2. The name is a taskcluster task label, in which case we pass it on
// 3. The name is a buildbot buildername _scheduled_ through bbb, in which case we
// The following has 2 cases that it accounts for
// 1. The name is a taskcluster task label, in which case we pass it on
// 2. The name is a buildbot buildername _scheduled_ through bbb, in which case we
// translate it to the taskcluster label that triggers it.
name = builderToTask[name] || name;
if (_.includes(allLabels, name)) {
tclabels.push(name);
} else {
bbnames.push(name);
}
});
return $q.all([
$q.resolve().then(function () {
if (bbnames.length === 0) {
return;
}
let bbdata = {
requested_jobs: bbnames,
decision_task_id: decisionTaskId
};
return $http.post(
thUrl.getProjectUrl("/resultset/", repoName) + resultset_id + '/trigger_runnable_jobs/',
bbdata
);
}),
$q.resolve().then(function () {
if (tclabels.length === 0) {
return;
}
@ -352,8 +323,6 @@ treeherder.factory('ThResultSetModel', ['$rootScope', '$http', '$location',
return queue.createTask(actionTaskId, task).then(() => `Request sent to trigger new jobs via actions.yml (${actionTaskId})`);
});
});
}),
]);
});
},
};

2
ui/js/services/taskcluster.js
Просмотреть файл

@ -14,7 +14,7 @@ treeherder.factory('thTaskcluster', ['$rootScope', 'localStorageService',
refreshTimestamps: function (task) {
// Take a taskcluster task and make all of the timestamps
// new again. This is pretty much lifted verbatim from
// mozilla_ci_tools which is used by pulse_actions.
// mozilla_ci_tools which was used by pulse_actions.
// We need to do this because action tasks are created with
// timestamps for expires/created/deadline that are based
// on the time of the original decision task creation. We must

20
ui/plugins/controller.js
Просмотреть файл

@ -327,12 +327,11 @@ treeherder.controller('PluginCtrl', [
var job_id_list = _.map(jobs, 'id');
// The logic here is somewhat complicated because we need to support
// two use cases the first is the case where we notify a system
// other then buildbot that a retrigger has been requested. The
// second is when we have the buildapi id and need to send a request
// two use cases the first is the case where we notify a system other
// then buildbot that a retrigger has been requested (eg mozilla-taskcluster).
// The second is when we have the buildapi id and need to send a request
// to the self serve api (which does not listen over pulse!).
ThJobModel.retrigger($scope.repoName, job_id_list).then(function () {
// XXX: Remove this after 1134929 is resolved.
return ThJobDetailModel.getJobDetails({
title: "buildbot_request_id",
repository: $scope.repoName,
@ -431,18 +430,7 @@ treeherder.controller('PluginCtrl', [
});
});
} else {
ThJobModel.backfill(
$scope.repoName,
$scope.job.id
).then(function () {
thNotify.send("Request sent to backfill jobs", 'success');
}, function (e) {
// Generic error eg. the user doesn't have LDAP access
thNotify.send(
ThModelErrors.format(e, "Unable to send backfill"),
'danger'
);
});
thNotify.send('Unable to backfill this job type!', 'danger', { sticky: true });
}
};