Bug 1419957 - Remove old style backfill/add new job features (#2982)

Since everything but standard retrigger/cancellation is now handled
client-side by tcactions, making the backend parts that provided pulse
messages to pulse_actions redundant (since it was decommissioned in
bug 1379172).
This commit is contained in:
Ed Morley 2017-12-05 22:42:06 +00:00 коммит произвёл GitHub
Родитель ffc7fe6c79
Коммит 958cc079a7
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: 4AEE18F83AFDEB23
11 изменённых файлов: 100 добавлений и 413 удалений

Просмотреть файл

@ -1,41 +0,0 @@
{
"id": "https://treeherder.mozilla.org/schemas/v1/resultset-action-message.json#",
"$schema": "http://json-schema.org/draft-04/schema#",
"title": "Notification of triggering jobs in a resultset",
"description": "Event is dispatched when user/service issues a (trigger_missing_jobs, trigger_all_talos_jobs) action on a resultset",
"type": "object",
"properties": {
"version": {
"title": "Message-format version",
"enum": [1]
},
"project": {
"title": "Project Name",
"description": "Identifier for treeherder project, like `try` or `mozilla-central`.",
"type": "string"
},
"resultset_id": {
"title": "Resultset ID",
"description": "Project unique identifier for a resultset",
"type": "string"
},
"times": {
"title": "Times",
"description": "Number of times to execute the command for a resultset.",
"type": "number"
},
"action": {
"title": "Action",
"description": "Type of action issued on task",
"enum": ["cancel_all", "trigger_missing_jobs", "trigger_all_talos_jobs"],
"type": "string"
},
"requester": {
"title": "Requester",
"description": "The requester of the action (usually an email)",
"type": "string"
}
},
"additionalProperties": true,
"required": ["version", "resultset_id", "project", "action", "requester"]
}

Просмотреть файл

@ -1,45 +0,0 @@
{
"id": "https://treeherder.mozilla.org/schemas/v1/resultset-runnable-job-action-message.json#",
"$schema": "http://json-schema.org/draft-04/schema#",
"title": "Notification of triggering runnable jobs on a resultset",
"description": "Event is dispatched when user/service asks for new runnable jobs on a resultset",
"type": "object",
"properties": {
"version": {
"title": "Message-format version",
"enum": [1]
},
"project": {
"title": "Project Name",
"description": "Identifier for treeherder project, like `try` or `mozilla-central`.",
"type": "string"
},
"resultset_id": {
"title": "Resultset ID",
"description": "Project unique identifier for a resultset",
"type": "string"
},
"requester": {
"title": "Requester",
"description": "The requester of the action (usually an email)",
"type": "string"
},
"requested_jobs": {
"title": "Requested Jobs",
"description": "The buildernames and TaskLabels that should be added to a push",
"type": "array"
},
"decision_task_id": {
"title": "decision_task_id",
"description": "The Gecko Decision Task's Task ID for this resultset",
"type": "string"
},
"timestamp": {
"title": "timestamp",
"description": "The UTC timestamp",
"type": "string"
}
},
"additionalProperties": false,
"required": ["version", "resultset_id", "project", "requester", "requested_jobs", "decision_task_id"]
}

Просмотреть файл

@ -10,42 +10,6 @@ class TreeherderPublisher(PulsePublisher):
"""
exchange_prefix = "v1/"
push_action = Exchange(
exchange="resultset-actions",
title="Actions issued by push",
description="""
There are actions which can be done to a push
(eg: trigger_missing_jobs), they are published on this exchange
""",
routing_keys=[
Key(
name='project',
summary="Project (or branch) that this push belongs to"
),
Key(
name="action",
summary="Type of action issued (i.e. trigger_missing_jobs)"
),
],
schema="https://treeherder.mozilla.org/schemas/v1/resultset-action-message.json#"
)
push_runnable_job_action = Exchange(
exchange="resultset-runnable-job-actions",
title="Runnable job actions issued by push",
description="""
This action is published when a user asks for new runnable jobs (chosen
by name) on a push.
""",
routing_keys=[
Key(
name='project',
summary="Project (or branch) that this push belongs to"
),
],
schema="https://treeherder.mozilla.org/schemas/v1/resultset-runnable-job-action-message.json#"
)
job_action = Exchange(
exchange="job-actions",
title="Actions issued by jobs",

Просмотреть файл

@ -1,5 +1,4 @@
import os
import time
import newrelic.agent
from celery import task
@ -80,52 +79,3 @@ def publish_job_action(project, action, job_id, requester):
job_id=job.id,
requester=requester
)
@task(name='publish-push-action')
def publish_push_action(project, action, push_id, requester, times=1):
newrelic.agent.add_custom_parameter("project", project)
newrelic.agent.add_custom_parameter("action", action)
newrelic.agent.add_custom_parameter("push_id", str(push_id))
newrelic.agent.add_custom_parameter("requester", requester)
publisher = pulse_connection.get_publisher()
if not publisher:
return
publisher.push_action(
version=1,
project=project,
action=action,
requester=requester,
resultset_id=push_id,
push_id=push_id,
times=times
)
@task(name='publish-resultset-runnable-job-action')
def publish_resultset_runnable_job_action(project, resultset_id, requester, requested_jobs, decision_task_id):
publish_push_runnable_job_action(project, resultset_id, requester, requested_jobs, decision_task_id)
@task(name='publish-push-runnable-job-action')
def publish_push_runnable_job_action(project, push_id, requester, requested_jobs, decision_task_id):
newrelic.agent.add_custom_parameter("project", project)
newrelic.agent.add_custom_parameter("push_id", str(push_id))
newrelic.agent.add_custom_parameter("requester", requester)
publisher = pulse_connection.get_publisher()
if not publisher:
return
timestamp = str(time.time())
publisher.push_runnable_job_action(
version=1,
project=project,
requester=requester,
push_id=push_id,
requested_jobs=requested_jobs,
decision_task_id=decision_task_id,
timestamp=timestamp
)

Просмотреть файл

@ -411,20 +411,6 @@ class JobsViewSet(viewsets.ViewSet):
status=HTTP_404_NOT_FOUND)
return Response({"message": "All jobs successfully retriggered."})
@detail_route(methods=['post'], permission_classes=[IsAuthenticated])
def backfill(self, request, project, pk=None):
"""
Issue a "backfill" to the underlying build_system_type by scheduling a
pulse message.
"""
try:
job = Job.objects.get(repository__name=project,
id=pk)
self._job_action_event(job, 'backfill', request.user.email)
return Response({"message": "backfilled job '{0}'".format(job.guid)})
except ObjectDoesNotExist:
return Response("No job with id: {0}".format(pk), status=HTTP_404_NOT_FOUND)
@detail_route(methods=['get'])
def failure_lines(self, request, project, pk=None):
"""

Просмотреть файл

@ -2,7 +2,6 @@ import datetime
from rest_framework import viewsets
from rest_framework.decorators import detail_route
from rest_framework.exceptions import ParseError
from rest_framework.permissions import IsAuthenticated
from rest_framework.response import Response
from rest_framework.status import (HTTP_400_BAD_REQUEST,
@ -14,9 +13,7 @@ from treeherder.model.models import (Commit,
Job,
Push,
Repository)
from treeherder.model.tasks import (publish_job_action,
publish_push_action,
publish_push_runnable_job_action)
from treeherder.model.tasks import publish_job_action
from treeherder.webapp.api import permissions
from treeherder.webapp.api.utils import (to_datetime,
to_timestamp)
@ -197,13 +194,6 @@ class PushViewSet(viewsets.ViewSet):
if not pk: # pragma nocover
return Response({"message": "push id required"}, status=HTTP_400_BAD_REQUEST)
# Sending 'cancel_all' action to pulse. Right now there is no listener
# for this, so we cannot remove 'cancel' action for each job below.
publish_push_action.apply_async(
args=[project, 'cancel_all', pk, request.user.email],
routing_key='publish_to_pulse'
)
# Notify the build systems which created these jobs...
for job in Job.objects.filter(push_id=pk).exclude(state='completed'):
publish_job_action.apply_async(
@ -221,68 +211,6 @@ class PushViewSet(viewsets.ViewSet):
return Response({"message": "pending and running jobs canceled for push '{0}'".format(pk)})
@detail_route(methods=['post'], permission_classes=[IsAuthenticated])
def trigger_missing_jobs(self, request, project, pk=None):
"""
Trigger jobs that are missing in a push.
"""
if not pk:
return Response({"message": "push id required"}, status=HTTP_400_BAD_REQUEST)
publish_push_action.apply_async(
args=[project, "trigger_missing_jobs", pk, request.user.email],
routing_key='publish_to_pulse'
)
return Response({"message": "Missing jobs triggered for push '{0}'".format(pk)})
@detail_route(methods=['post'], permission_classes=[IsAuthenticated])
def trigger_all_talos_jobs(self, request, project, pk=None):
"""
Trigger all the talos jobs in a push.
"""
if not pk:
return Response({"message": "push id required"}, status=HTTP_400_BAD_REQUEST)
times = int(request.query_params.get('times', None))
if not times:
raise ParseError(detail="The 'times' parameter is mandatory for this endpoint")
publish_push_action.apply_async(
args=[project, "trigger_all_talos_jobs", pk, request.user.email,
times],
routing_key='publish_to_pulse'
)
return Response({"message": "Talos jobs triggered for push '{0}'".format(pk)})
@detail_route(methods=['post'], permission_classes=[IsAuthenticated])
def trigger_runnable_jobs(self, request, project, pk=None):
"""
Add new jobs to a push.
"""
if not pk:
return Response({"message": "push id required"},
status=HTTP_400_BAD_REQUEST)
# Making sure a push with this id exists
if not Push.objects.filter(id=pk).exists():
return Response({"message": "No push with id: {0}".format(pk)},
status=HTTP_404_NOT_FOUND)
requested_jobs = request.data.get('requested_jobs', [])
decision_task_id = request.data.get('decision_task_id', [])
if not requested_jobs:
Response({"message": "The list of requested_jobs cannot be empty"},
status=HTTP_400_BAD_REQUEST)
publish_push_runnable_job_action.apply_async(
args=[project, pk, request.user.email, requested_jobs, decision_task_id],
routing_key='publish_to_pulse'
)
return Response({"message": "New jobs added for push '{0}'".format(pk)})
@detail_route()
def status(self, request, project, pk=None):
"""

Просмотреть файл

@ -216,8 +216,6 @@ treeherderApp.controller('ResultSetCtrl', [
$scope.resultset.id
).then(function (decisionTaskID) {
ThResultSetModel.triggerMissingJobs(
$scope.resultset.id,
$scope.repoName,
decisionTaskID
).then(function (msg) {
thNotify.send(msg, "success");
@ -245,8 +243,6 @@ treeherderApp.controller('ResultSetCtrl', [
$scope.resultset.id
).then(function (decisionTaskID) {
ThResultSetModel.triggerAllTalosJobs(
$scope.resultset.id,
$scope.repoName,
times,
decisionTaskID
).then(function (msg) {
@ -270,8 +266,8 @@ treeherderApp.controller('ResultSetCtrl', [
if ($scope.user.loggedin) {
var buildernames = ThResultSetStore.getSelectedRunnableJobs($rootScope.repoName, $scope.resultset.id);
ThResultSetStore.getGeckoDecisionTaskId($rootScope.repoName, $scope.resultset.id).then(function (decisionTaskID) {
ThResultSetModel.triggerNewJobs($scope.repoName, $scope.resultset.id, buildernames, decisionTaskID).then(function (results) {
thNotify.send(results[1], "success");
ThResultSetModel.triggerNewJobs(buildernames, decisionTaskID).then(function (result) {
thNotify.send(result, "success");
ThResultSetStore.deleteRunnableJobs($scope.repoName, $scope.resultset);
}, function (e) {
thNotify.send(ThTaskclusterErrors.format(e), 'danger', { sticky: true });

Просмотреть файл

@ -115,14 +115,6 @@ treeherder.factory('ThJobModel', [
});
};
ThJobModel.backfill = function (repoName, pk, config) {
config = config || {};
var timeout = config.timeout || null;
return $http.post(ThJobModel.get_uri(repoName)+pk+"/backfill/",
{ timeout: timeout });
};
ThJobModel.cancel = function (repoName, jobIds, config) {
config = config || {};
var timeout = config.timeout || null;

Просмотреть файл

@ -191,83 +191,73 @@ treeherder.factory('ThResultSetModel', ['$rootScope', '$http', '$location',
return $http.post(thUrl.getProjectUrl("/resultset/", repoName) + uri);
},
triggerMissingJobs: function (resultset_id, repoName, decisionTaskId) {
var uri = resultset_id + '/trigger_missing_jobs/';
return $http.post(thUrl.getProjectUrl("/resultset/", repoName) + uri).then(function () {
return tcactions.load(decisionTaskId).then((results) => {
// After we trigger the buildbot jobs, we can go ahead and trigger tc
// jobs directly.
const tc = thTaskcluster.client();
const actionTaskId = tc.slugid();
triggerMissingJobs: function (decisionTaskId) {
return tcactions.load(decisionTaskId).then((results) => {
const tc = thTaskcluster.client();
const actionTaskId = tc.slugid();
// In this case we have actions.json tasks
if (results) {
const missingtask = _.find(results.actions, { name: 'run-missing-tests' });
// We'll fall back to actions.yaml if this isn't true
if (missingtask) {
return tcactions.submit({
action: missingtask,
actionTaskId,
decisionTaskId,
taskId: null,
task: null,
input: {},
staticActionVariables: results.staticActionVariables,
}).then(() => `Request sent to trigger missing jobs via actions.json (${actionTaskId})`);
}
// In this case we have actions.json tasks
if (results) {
const missingtask = _.find(results.actions, { name: 'run-missing-tests' });
// We'll fall back to actions.yaml if this isn't true
if (missingtask) {
return tcactions.submit({
action: missingtask,
actionTaskId,
decisionTaskId,
taskId: null,
task: null,
input: {},
staticActionVariables: results.staticActionVariables,
}).then(() => `Request sent to trigger missing jobs via actions.json (${actionTaskId})`);
}
});
}
});
},
triggerAllTalosJobs: function (resultset_id, repoName, times, decisionTaskId) {
let uri = resultset_id + '/trigger_all_talos_jobs/?times=' + times;
return $http.post(thUrl.getProjectUrl("/resultset/", repoName) + uri).then(function () {
return tcactions.load(decisionTaskId).then((results) => {
// After we trigger the buildbot jobs, we can go ahead and trigger tc
// jobs directly.
const tc = thTaskcluster.client();
const actionTaskId = tc.slugid();
triggerAllTalosJobs: function (times, decisionTaskId) {
return tcactions.load(decisionTaskId).then((results) => {
const tc = thTaskcluster.client();
const actionTaskId = tc.slugid();
// In this case we have actions.json tasks
if (results) {
const talostask = _.find(results.actions, { name: 'run-all-talos' });
// We'll fall back to actions.yaml if this isn't true
if (talostask) {
return tcactions.submit({
action: talostask,
actionTaskId,
decisionTaskId,
taskId: null,
task: null,
input: { times },
staticActionVariables: results.staticActionVariables,
}).then(function () {
return `Request sent to trigger all talos jobs ${times} time(s) via actions.json (${actionTaskId})`;
});
}
// In this case we have actions.json tasks
if (results) {
const talostask = _.find(results.actions, { name: 'run-all-talos' });
// We'll fall back to actions.yaml if this isn't true
if (talostask) {
return tcactions.submit({
action: talostask,
actionTaskId,
decisionTaskId,
taskId: null,
task: null,
input: { times },
staticActionVariables: results.staticActionVariables,
}).then(function () {
return `Request sent to trigger all talos jobs ${times} time(s) via actions.json (${actionTaskId})`;
});
}
}
// Otherwise we'll figure things out with actions.yml
const queue = new tc.Queue();
const url = queue.buildUrl(queue.getLatestArtifact, decisionTaskId, 'public/action.yml');
return $http.get(url).then(function (resp) {
let action = resp.data;
let template = $interpolate(action);
action = template({
action: 'add-talos',
action_args: '--decision-task-id=' + decisionTaskId + ' --times=' + times,
});
let task = thTaskcluster.refreshTimestamps(jsyaml.safeLoad(action));
return queue.createTask(actionTaskId, task).then(function () {
return `Request sent to trigger all talos jobs ${times} time(s) via actions.yml (${actionTaskId})`;
});
// Otherwise we'll figure things out with actions.yml
const queue = new tc.Queue();
const url = queue.buildUrl(queue.getLatestArtifact, decisionTaskId, 'public/action.yml');
return $http.get(url).then(function (resp) {
let action = resp.data;
let template = $interpolate(action);
action = template({
action: 'add-talos',
action_args: '--decision-task-id=' + decisionTaskId + ' --times=' + times,
});
let task = thTaskcluster.refreshTimestamps(jsyaml.safeLoad(action));
return queue.createTask(actionTaskId, task).then(function () {
return `Request sent to trigger all talos jobs ${times} time(s) via actions.yml (${actionTaskId})`;
});
});
});
},
triggerNewJobs: function (repoName, resultset_id, buildernames, decisionTaskId) {
triggerNewJobs: function (buildernames, decisionTaskId) {
let tc = thTaskcluster.client();
let queue = new tc.Queue();
let url = queue.buildUrl(
@ -283,77 +273,56 @@ treeherder.factory('ThResultSetModel', ['$rootScope', '$http', '$location',
let allLabels = _.keys(graph);
let tclabels = [];
let bbnames = [];
buildernames.forEach(function (name) {
// The following has 3 cases that it accounts for
// 1. The name is a buildbot buildername not scheduled through bbb, in which case we pass it on
// 2. The name is a taskcluster task label, in which case we pass it on
// 3. The name is a buildbot buildername _scheduled_ through bbb, in which case we
// The following has 2 cases that it accounts for
// 1. The name is a taskcluster task label, in which case we pass it on
// 2. The name is a buildbot buildername _scheduled_ through bbb, in which case we
// translate it to the taskcluster label that triggers it.
name = builderToTask[name] || name;
if (_.includes(allLabels, name)) {
tclabels.push(name);
} else {
bbnames.push(name);
}
});
return $q.all([
$q.resolve().then(function () {
if (bbnames.length === 0) {
return;
}
let bbdata = {
requested_jobs: bbnames,
decision_task_id: decisionTaskId
};
return $http.post(
thUrl.getProjectUrl("/resultset/", repoName) + resultset_id + '/trigger_runnable_jobs/',
bbdata
);
}),
$q.resolve().then(function () {
if (tclabels.length === 0) {
return;
if (tclabels.length === 0) {
return;
}
return tcactions.load(decisionTaskId).then((results) => {
const actionTaskId = tc.slugid();
// In this case we have actions.json tasks
if (results) {
const addjobstask = _.find(results.actions, { name: 'add-new-jobs' });
// We'll fall back to actions.yaml if this isn't true
if (addjobstask) {
return tcactions.submit({
action: addjobstask,
actionTaskId,
decisionTaskId,
taskId: null,
task: null,
input: { tasks: tclabels },
staticActionVariables: results.staticActionVariables,
}).then(() => `Request sent to trigger new jobs via actions.json (${actionTaskId})`);
}
}
return tcactions.load(decisionTaskId).then((results) => {
const actionTaskId = tc.slugid();
// In this case we have actions.json tasks
if (results) {
const addjobstask = _.find(results.actions, { name: 'add-new-jobs' });
// We'll fall back to actions.yaml if this isn't true
if (addjobstask) {
return tcactions.submit({
action: addjobstask,
actionTaskId,
decisionTaskId,
taskId: null,
task: null,
input: { tasks: tclabels },
staticActionVariables: results.staticActionVariables,
}).then(() => `Request sent to trigger new jobs via actions.json (${actionTaskId})`);
}
}
// Otherwise we'll figure things out with actions.yml
let url = queue.buildUrl(queue.getLatestArtifact, decisionTaskId, 'public/action.yml');
return $http.get(url).then(function (resp) {
let action = resp.data;
let template = $interpolate(action);
let taskLabels = tclabels.join(',');
action = template({
action: 'add-tasks',
action_args: `--decision-id=${decisionTaskId} --task-labels=${taskLabels}`,
});
let task = thTaskcluster.refreshTimestamps(jsyaml.safeLoad(action));
return queue.createTask(actionTaskId, task).then(() => `Request sent to trigger new jobs via actions.yml (${actionTaskId})`);
});
// Otherwise we'll figure things out with actions.yml
let url = queue.buildUrl(queue.getLatestArtifact, decisionTaskId, 'public/action.yml');
return $http.get(url).then(function (resp) {
let action = resp.data;
let template = $interpolate(action);
let taskLabels = tclabels.join(',');
action = template({
action: 'add-tasks',
action_args: `--decision-id=${decisionTaskId} --task-labels=${taskLabels}`,
});
}),
]);
let task = thTaskcluster.refreshTimestamps(jsyaml.safeLoad(action));
return queue.createTask(actionTaskId, task).then(() => `Request sent to trigger new jobs via actions.yml (${actionTaskId})`);
});
});
});
},
};

Просмотреть файл

@ -14,7 +14,7 @@ treeherder.factory('thTaskcluster', ['$rootScope', 'localStorageService',
refreshTimestamps: function (task) {
// Take a taskcluster task and make all of the timestamps
// new again. This is pretty much lifted verbatim from
// mozilla_ci_tools which is used by pulse_actions.
// mozilla_ci_tools which was used by pulse_actions.
// We need to do this because action tasks are created with
// timestamps for expires/created/deadline that are based
// on the time of the original decision task creation. We must

Просмотреть файл

@ -327,12 +327,11 @@ treeherder.controller('PluginCtrl', [
var job_id_list = _.map(jobs, 'id');
// The logic here is somewhat complicated because we need to support
// two use cases the first is the case where we notify a system
// other then buildbot that a retrigger has been requested. The
// second is when we have the buildapi id and need to send a request
// two use cases the first is the case where we notify a system other
// then buildbot that a retrigger has been requested (eg mozilla-taskcluster).
// The second is when we have the buildapi id and need to send a request
// to the self serve api (which does not listen over pulse!).
ThJobModel.retrigger($scope.repoName, job_id_list).then(function () {
// XXX: Remove this after 1134929 is resolved.
return ThJobDetailModel.getJobDetails({
title: "buildbot_request_id",
repository: $scope.repoName,
@ -431,18 +430,7 @@ treeherder.controller('PluginCtrl', [
});
});
} else {
ThJobModel.backfill(
$scope.repoName,
$scope.job.id
).then(function () {
thNotify.send("Request sent to backfill jobs", 'success');
}, function (e) {
// Generic error eg. the user doesn't have LDAP access
thNotify.send(
ThModelErrors.format(e, "Unable to send backfill"),
'danger'
);
});
thNotify.send('Unable to backfill this job type!', 'danger', { sticky: true });
}
};