зеркало из https://github.com/mozilla/treeherder.git
merged
This commit is contained in:
Jonathan Eads
Коммит
ce17bbfab4
|
|
@ -146,11 +146,11 @@ def test_objectstore_update_content(jm, sample_data):
|
|||
"""
|
||||
Test updating an object of the objectstore.
|
||||
"""
|
||||
original_obj = sample_data.job_data[0]
|
||||
original_obj = sample_data.job_data[100]
|
||||
jm.store_job_data([original_obj])
|
||||
|
||||
obj_updated = original_obj.copy()
|
||||
obj_updated["job"]["state"] = "new_state"
|
||||
obj_updated["job"]["state"] = "pending"
|
||||
|
||||
jm.store_job_data([obj_updated])
|
||||
|
||||
|
|
@ -165,4 +165,4 @@ def test_objectstore_update_content(jm, sample_data):
|
|||
stored_blob = json.loads(stored_objs[0]["json_blob"])
|
||||
|
||||
# check that the blob was updated
|
||||
assert stored_blob["job"]["state"] == "new_state"
|
||||
assert stored_blob["job"]["state"] == "pending"
|
||||
|
|
|
|||
|
|
@ -2,109 +2,6 @@ import json
|
|||
|
||||
from django.core.urlresolvers import reverse
|
||||
|
||||
from tests import test_utils
|
||||
from thclient import TreeherderJobCollection
|
||||
|
||||
|
||||
def test_update_state_success(webapp, eleven_jobs_processed, jm):
|
||||
"""
|
||||
test setting the state of a job via webtest.
|
||||
extected result are:
|
||||
- return code 200
|
||||
- return message successful
|
||||
- job status updated
|
||||
"""
|
||||
|
||||
job = jm.get_job_list(0, 1)[0]
|
||||
job_id = job["id"]
|
||||
new_state = "coalesced"
|
||||
|
||||
# use the backdoor to set the state of the job to something we can
|
||||
# change. because we can't change it once it's ``completed``
|
||||
jm.get_jobs_dhub().execute(
|
||||
proc='jobs_test.updates.set_state_any',
|
||||
placeholders=["running", job_id],
|
||||
)
|
||||
|
||||
uri = reverse("jobs-update-state", kwargs={
|
||||
"project": jm.project,
|
||||
"pk": job_id
|
||||
})
|
||||
|
||||
data = {
|
||||
"project": jm.project,
|
||||
"pk": job_id,
|
||||
"state": new_state
|
||||
}
|
||||
|
||||
resp = test_utils.post_job_data(jm.project, uri, data)
|
||||
|
||||
assert resp.status_int == 200
|
||||
assert resp.json["message"] == "state updated to '{0}'".format(new_state)
|
||||
assert jm.get_job(job_id)[0]["state"] == new_state
|
||||
|
||||
|
||||
def test_update_state_invalid_state(webapp, eleven_jobs_processed, jm):
|
||||
"""
|
||||
test setting the state of a job via webtest with invalid state.
|
||||
extected result are:
|
||||
- return code 400
|
||||
"""
|
||||
|
||||
job = jm.get_job_list(0, 1)[0]
|
||||
job_id = job["id"]
|
||||
old_state = job["state"]
|
||||
new_state = "chokey"
|
||||
|
||||
uri = reverse("jobs-update-state", kwargs={
|
||||
"project": jm.project,
|
||||
"pk": job_id
|
||||
})
|
||||
|
||||
data = {
|
||||
"project": jm.project,
|
||||
"pk": job_id,
|
||||
"state": new_state
|
||||
}
|
||||
|
||||
resp = test_utils.post_job_data(
|
||||
jm.project, uri, data, status=404, expect_errors=True)
|
||||
|
||||
assert resp.json["message"] == ("'{0}' is not a valid state. Must be "
|
||||
"one of: {1}".format(
|
||||
new_state,
|
||||
", ".join(jm.STATES)
|
||||
))
|
||||
assert jm.get_job(job_id)[0]["state"] == old_state
|
||||
|
||||
|
||||
def test_update_state_invalid_job_id(webapp, eleven_jobs_processed, jm):
|
||||
"""
|
||||
test setting the state of a job via webtest with invalid job_id.
|
||||
extected result are:
|
||||
- return code 404
|
||||
"""
|
||||
|
||||
job_id = -32767
|
||||
new_state = "pending"
|
||||
|
||||
url = reverse("jobs-update-state", kwargs={
|
||||
"project": jm.project,
|
||||
"pk": job_id
|
||||
})
|
||||
|
||||
data = {
|
||||
"project": jm.project,
|
||||
"pk": job_id,
|
||||
"state": new_state
|
||||
}
|
||||
|
||||
expect_errors = '{"message": "\'chokey\' is not a valid state. Must be one of: pending, running, completed, coalesced"}'
|
||||
|
||||
test_utils.post_job_data(
|
||||
jm.project, url, data, status=404, expect_errors=True)
|
||||
|
||||
|
||||
def test_job_list(webapp, eleven_jobs_processed, jm):
|
||||
"""
|
||||
test retrieving a list of ten json blobs from the jobs-list
|
||||
|
|
|
|||
|
|
@ -29,9 +29,10 @@ def test_resultset_list(webapp, eleven_jobs_processed, jm):
|
|||
u'revision_hash',
|
||||
u'revision',
|
||||
u'revision_list',
|
||||
u'job_count',
|
||||
u'platforms',
|
||||
u'result_types'
|
||||
u'revision_count',
|
||||
u'revisions_uri',
|
||||
u'job_counts',
|
||||
u'platforms'
|
||||
])
|
||||
for rs in rs_list:
|
||||
assert set(rs.keys()) == exp_keys
|
||||
|
|
|
|||
|
|
@ -10,7 +10,6 @@ RESULT_DICT = {
|
|||
6: "usercancel"
|
||||
}
|
||||
|
||||
|
||||
####
|
||||
# The following variables were taken from util.py
|
||||
#
|
||||
|
|
|
|||
|
|
@ -50,10 +50,10 @@ class EventsPublisher(object):
|
|||
|
||||
class JobStatusPublisher(EventsPublisher):
|
||||
|
||||
def publish(self, job_id, result_set_id, branch, status):
|
||||
def publish(self, job_id, resultset, branch, status):
|
||||
message = {
|
||||
"id": job_id,
|
||||
"result_set_id": result_set_id,
|
||||
"resultset": resultset,
|
||||
"event": "job",
|
||||
"branch": branch,
|
||||
"status": status
|
||||
|
|
|
|||
|
|
@ -34,6 +34,12 @@ def parse_log(project, job_id, result_set_id, check_errors=False):
|
|||
failure_publisher = JobFailurePublisher(settings.BROKER_URL)
|
||||
|
||||
try:
|
||||
# return the resultset with the job id to identify if the UI wants
|
||||
# to fetch the whole thing.
|
||||
resultset = jm.get_result_set_by_id(result_set_id=result_set_id)[0]
|
||||
del(resultset["active_status"])
|
||||
del(resultset["revision_hash"])
|
||||
|
||||
log_references = jm.get_log_references(job_id)
|
||||
|
||||
# we may have many log references per job
|
||||
|
|
@ -79,7 +85,7 @@ def parse_log(project, job_id, result_set_id, check_errors=False):
|
|||
|
||||
# store the artifacts generated
|
||||
jm.store_job_artifact(artifact_list)
|
||||
status_publisher.publish(job_id, result_set_id, project, 'processed')
|
||||
status_publisher.publish(job_id, resultset, project, 'processed')
|
||||
if check_errors:
|
||||
failure_publisher.publish(job_id, project)
|
||||
|
||||
|
|
|
|||
|
|
@ -26,7 +26,18 @@ class JobsModel(TreeherderModelBase):
|
|||
CT_JOBS = "jobs"
|
||||
CT_OBJECTSTORE = "objectstore"
|
||||
CONTENT_TYPES = [CT_JOBS, CT_OBJECTSTORE]
|
||||
STATES = ["pending", "running", "completed", "coalesced"]
|
||||
RESULTS = [
|
||||
"busted",
|
||||
"exception",
|
||||
"testfailed",
|
||||
"unknown",
|
||||
"usercancel",
|
||||
"retry",
|
||||
"success",
|
||||
]
|
||||
INCOMPLETE_STATES = ["running", "pending"]
|
||||
STATES = INCOMPLETE_STATES + ["completed", "coalesced"]
|
||||
|
||||
# list of searchable columns, i.e. those who have an index
|
||||
# it would be nice to get this directly from the db and cache it
|
||||
INDEXED_COLUMNS = {
|
||||
|
|
@ -103,12 +114,14 @@ class JobsModel(TreeherderModelBase):
|
|||
)
|
||||
return data
|
||||
|
||||
def get_job_list(self, offset, limit, conditions=None):
|
||||
def get_job_list(self, offset, limit, full=True, conditions=None):
|
||||
"""
|
||||
Retrieve a list of jobs.
|
||||
Mainly used by the restful api to list the jobs
|
||||
|
||||
joblist: a list of job ids to limit which jobs are returned.
|
||||
full: whether to return all reference data or just what is
|
||||
needed for the UI.
|
||||
"""
|
||||
|
||||
placeholders = []
|
||||
|
|
@ -131,8 +144,10 @@ class JobsModel(TreeherderModelBase):
|
|||
|
||||
repl = [self.refdata_model.get_db_name(), replace_str]
|
||||
|
||||
proc = "jobs.selects.get_job_list"
|
||||
|
||||
if full:
|
||||
proc = "jobs.selects.get_job_list_full"
|
||||
else:
|
||||
proc = "jobs.selects.get_job_list"
|
||||
data = self.get_jobs_dhub().execute(
|
||||
proc=proc,
|
||||
replace=repl,
|
||||
|
|
@ -254,10 +269,11 @@ class JobsModel(TreeherderModelBase):
|
|||
|
||||
return result_set_id_lookup
|
||||
|
||||
def get_result_set_list(self, offset, limit, conditions=None):
|
||||
def get_result_set_list(self, offset, limit, full=True, conditions=None):
|
||||
"""
|
||||
Retrieve a list of ``result_sets`` (also known as ``pushes``)
|
||||
with associated revisions. No jobs
|
||||
If ``full`` is set to ``True`` then return revisions, too.
|
||||
No jobs
|
||||
|
||||
Mainly used by the restful api to list the pushes in the UI
|
||||
"""
|
||||
|
|
@ -301,20 +317,22 @@ class JobsModel(TreeherderModelBase):
|
|||
return_list = []
|
||||
for result in result_set_ids:
|
||||
|
||||
detail = aggregate_details[ result['id'] ][0]
|
||||
|
||||
return_list.append(
|
||||
{
|
||||
"id":result['id'],
|
||||
"revision_hash":result['revision_hash'],
|
||||
"push_timestamp":result['push_timestamp'],
|
||||
"repository_id":detail['repository_id'],
|
||||
"revision":detail['revision'],
|
||||
"author":detail['author'],
|
||||
"comments":detail['comments'],
|
||||
"revision_list":aggregate_details[ result['id'] ]
|
||||
}
|
||||
)
|
||||
detail = aggregate_details[result['id']][0]
|
||||
list_item = {
|
||||
"id": result['id'],
|
||||
"revision_hash": result['revision_hash'],
|
||||
"push_timestamp": result['push_timestamp'],
|
||||
"repository_id": detail['repository_id'],
|
||||
"revision": detail['revision'],
|
||||
"author": detail['author'],
|
||||
"revision_count": len(aggregate_details[result['id']])
|
||||
}
|
||||
if full:
|
||||
list_item.update({
|
||||
"comments": detail['comments'],
|
||||
"revision_list": aggregate_details[result['id']]
|
||||
})
|
||||
return_list.append(list_item)
|
||||
|
||||
return return_list
|
||||
|
||||
|
|
@ -338,6 +356,20 @@ class JobsModel(TreeherderModelBase):
|
|||
return lookups
|
||||
|
||||
|
||||
def get_resultset_revisions_list(self, result_set_id):
|
||||
"""
|
||||
Return the revisions for the given resultset
|
||||
"""
|
||||
|
||||
proc = "jobs.selects.get_result_set_details"
|
||||
lookups = self.get_jobs_dhub().execute(
|
||||
proc=proc,
|
||||
debug_show=self.DEBUG,
|
||||
replace=[result_set_id],
|
||||
)
|
||||
return lookups
|
||||
|
||||
|
||||
def get_result_set_details(self, result_set_ids):
|
||||
"""
|
||||
Retrieve all revisions associated with a set of ``result_set``
|
||||
|
|
@ -387,7 +419,7 @@ class JobsModel(TreeherderModelBase):
|
|||
|
||||
return aggregate_details
|
||||
|
||||
def get_result_set_job_list(self, result_set_ids, **kwargs):
|
||||
def get_result_set_job_list(self, result_set_ids, full=True, **kwargs):
|
||||
"""
|
||||
Retrieve a list of ``jobs`` and results for a result_set.
|
||||
|
||||
|
|
@ -409,7 +441,10 @@ class JobsModel(TreeherderModelBase):
|
|||
if "job_type_name" in kwargs:
|
||||
repl.append(" AND jt.`name` = '{0}'".format(kwargs["job_type_name"]))
|
||||
|
||||
proc = "jobs.selects.get_result_set_job_list"
|
||||
if full:
|
||||
proc = "jobs.selects.get_result_set_job_list_full"
|
||||
else:
|
||||
proc = "jobs.selects.get_result_set_job_list"
|
||||
data = self.get_jobs_dhub().execute(
|
||||
proc=proc,
|
||||
placeholders=result_set_ids,
|
||||
|
|
|
|||
|
|
@ -208,6 +208,30 @@
|
|||
"host":"read_host"
|
||||
},
|
||||
"get_job_list":{
|
||||
"sql":"SELECT
|
||||
j.id,
|
||||
j.`option_collection_hash`,
|
||||
mp.`platform` as platform,
|
||||
jt.`name` as job_type_name,
|
||||
jt.`symbol` as job_type_symbol,
|
||||
jg.`name` as job_group_name,
|
||||
jg.`symbol` as job_group_symbol,
|
||||
j.`result_set_id`,
|
||||
j.`result`,
|
||||
j.`state`
|
||||
FROM `job` as j
|
||||
LEFT JOIN `REP0`.`machine_platform` as mp
|
||||
ON j.`machine_platform_id` = mp.id
|
||||
LEFT JOIN `REP0`.`job_type` as jt
|
||||
ON j.`job_type_id` = jt.id
|
||||
LEFT JOIN `REP0`.`job_group` as jg
|
||||
ON jt.`job_group_id` = jg.id
|
||||
WHERE 1
|
||||
REP1",
|
||||
|
||||
"host":"read_host"
|
||||
},
|
||||
"get_job_list_full":{
|
||||
"sql":"SELECT
|
||||
j.id,
|
||||
j.`job_guid`,
|
||||
|
|
@ -370,23 +394,40 @@
|
|||
"host": "read_host"
|
||||
},
|
||||
"get_result_set_by_id":{
|
||||
"sql":"SELECT
|
||||
`rs`.`id`,
|
||||
`rs`.`revision_hash`,
|
||||
`rs`.`push_timestamp`,
|
||||
`rev`.`revision`,
|
||||
`rev`.`author`,
|
||||
`rev`.`repository_id`,
|
||||
`rev`.`comments`
|
||||
FROM result_set as rs
|
||||
LEFT JOIN revision_map as rm
|
||||
ON rs.id = rm.result_set_id
|
||||
LEFT JOIN revision as rev
|
||||
ON rm.revision_id = rev.id
|
||||
WHERE `rs`.`id` = ?",
|
||||
"sql":"SELECT *
|
||||
FROM result_set
|
||||
WHERE id = ?",
|
||||
"host": "read_host"
|
||||
},
|
||||
"get_result_set_job_list":{
|
||||
"sql":"SELECT
|
||||
j.`id`,
|
||||
j.`option_collection_hash`,
|
||||
mp.`platform` as platform,
|
||||
jt.`name` as job_type_name,
|
||||
jt.`symbol` as job_type_symbol,
|
||||
jg.`name` as job_group_name,
|
||||
jg.`symbol` as job_group_symbol,
|
||||
j.`result_set_id`,
|
||||
j.`id` as job_id,
|
||||
j.`result`,
|
||||
j.`state`,
|
||||
j.`reason`
|
||||
FROM `job` as j
|
||||
LEFT JOIN `REP0`.`machine` as m
|
||||
ON j.`machine_id` = m.id
|
||||
LEFT JOIN `REP0`.`machine_platform` as mp
|
||||
ON j.`machine_platform_id` = mp.id
|
||||
LEFT JOIN `REP0`.`job_type` as jt
|
||||
ON j.`job_type_id` = jt.id
|
||||
LEFT JOIN `REP0`.`job_group` as jg
|
||||
ON jt.`job_group_id` = jg.id
|
||||
WHERE j.`result_set_id` IN (REP1)
|
||||
REP2
|
||||
",
|
||||
"host": "read_host"
|
||||
},
|
||||
"get_result_set_job_list_full":{
|
||||
"sql":"SELECT
|
||||
j.`job_guid`,
|
||||
j.`job_coalesced_to_guid`,
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ import simplejson as json
|
|||
import itertools
|
||||
import oauth2 as oauth
|
||||
import urllib
|
||||
from collections import defaultdict
|
||||
|
||||
from django.db import models
|
||||
from django.conf import settings
|
||||
|
|
@ -288,13 +289,12 @@ class JobsViewSet(viewsets.ViewSet):
|
|||
job["artifacts"].append(art)
|
||||
|
||||
option_collections = jm.refdata_model.get_all_option_collections()
|
||||
option_collections[job["option_collection_hash"]]['opt']
|
||||
job["platform_opt"] = option_collections[job["option_collection_hash"]]['opt']
|
||||
|
||||
return Response(job)
|
||||
else:
|
||||
return Response("No job with id: {0}".format(pk), 404)
|
||||
|
||||
|
||||
@with_jobs
|
||||
def list(self, request, project, jm):
|
||||
"""
|
||||
|
|
@ -305,7 +305,8 @@ class JobsViewSet(viewsets.ViewSet):
|
|||
|
||||
limit_condition = filters.pop("limit", set([("=", "0,10")])).pop()
|
||||
offset, limit = limit_condition[1].split(",")
|
||||
objs = jm.get_job_list(offset, limit, filters)
|
||||
full = request.QUERY_PARAMS.get('full', 'true').lower() == 'true'
|
||||
objs = jm.get_job_list(offset, limit, full, filters)
|
||||
|
||||
if objs:
|
||||
option_collections = jm.refdata_model.get_all_option_collections()
|
||||
|
|
@ -376,34 +377,43 @@ class ResultSetViewSet(viewsets.ViewSet):
|
|||
|
||||
limit_condition = filters.pop("limit", set([("=", "0,10")])).pop()
|
||||
offset, limit = limit_condition[1].split(",")
|
||||
full = request.QUERY_PARAMS.get('full', "true").lower() == "true"
|
||||
|
||||
objs = jm.get_result_set_list(
|
||||
offset,
|
||||
limit,
|
||||
full,
|
||||
filters
|
||||
)
|
||||
return Response(self.get_resultsets_with_jobs(jm, objs, {}))
|
||||
return Response(self.get_resultsets_with_jobs(jm, objs, full, {}))
|
||||
|
||||
@with_jobs
|
||||
def retrieve(self, request, project, jm, pk=None):
|
||||
"""
|
||||
GET method implementation for detail view of ``resultset``
|
||||
"""
|
||||
filters = ["job_type_name"]
|
||||
filter_kwargs = dict(
|
||||
(k, v) for k, v in request.QUERY_PARAMS.iteritems()
|
||||
if k in filters
|
||||
)
|
||||
filters = UrlQueryFilter({"id": pk}).parse()
|
||||
|
||||
rs = jm.get_result_set_by_id(pk)
|
||||
if rs:
|
||||
resultsets = self.get_resultsets_with_jobs(jm, [rs[0]], filter_kwargs)
|
||||
return Response(resultsets[0])
|
||||
full = request.QUERY_PARAMS.get('full', "true").lower() == "true"
|
||||
|
||||
objs = jm.get_result_set_list(0, 1, full, filters)
|
||||
if objs:
|
||||
rs = self.get_resultsets_with_jobs(jm, objs, full, {})
|
||||
return Response(rs[0])
|
||||
else:
|
||||
return Response("No resultset with id: {0}".format(pk), 404)
|
||||
|
||||
@link()
|
||||
@with_jobs
|
||||
def revisions(self, request, project, jm, pk=None):
|
||||
"""
|
||||
GET method for revisions of a resultset
|
||||
"""
|
||||
objs = jm.get_resultset_revisions_list(pk)
|
||||
return Response(objs)
|
||||
|
||||
@staticmethod
|
||||
def get_resultsets_with_jobs(jm, rs_list, filter_kwargs):
|
||||
def get_resultsets_with_jobs(jm, rs_list, full, filter_kwargs):
|
||||
"""Convert db result of resultsets in a list to JSON"""
|
||||
|
||||
# Fetch the job results all at once, then parse them out in memory.
|
||||
|
|
@ -411,9 +421,13 @@ class ResultSetViewSet(viewsets.ViewSet):
|
|||
rs_map = {}
|
||||
for rs in rs_list:
|
||||
rs_map[rs["id"]] = rs
|
||||
# all rs should have the revisions_uri, so add it here
|
||||
rs["revisions_uri"] = reverse("resultset-revisions",
|
||||
kwargs={"project": jm.project, "pk": rs["id"]})
|
||||
|
||||
jobs_ungrouped = jm.get_result_set_job_list(
|
||||
rs_map.keys(),
|
||||
full,
|
||||
**filter_kwargs
|
||||
)
|
||||
|
||||
|
|
@ -443,8 +457,8 @@ class ResultSetViewSet(viewsets.ViewSet):
|
|||
# of resultsets to be returned.
|
||||
del(rs_map[rs_id])
|
||||
|
||||
result_types = []
|
||||
job_count = 0
|
||||
job_counts = dict.fromkeys(
|
||||
jm.RESULTS + jm.INCOMPLETE_STATES + ["total"], 0)
|
||||
|
||||
#itertools needs the elements to be sorted by the grouper
|
||||
by_platform = sorted(list(resultset_group), key=platform_grouper)
|
||||
|
|
@ -477,15 +491,16 @@ class ResultSetViewSet(viewsets.ViewSet):
|
|||
job["id"] = job["job_id"]
|
||||
del(job["job_id"])
|
||||
del(job["result_set_id"])
|
||||
del(job["option_collection_hash"])
|
||||
|
||||
job["resource_uri"] = reverse("jobs-detail",
|
||||
kwargs={"project": jm.project, "pk": job["id"]})
|
||||
|
||||
if job["state"] == "completed":
|
||||
result_types.append(job["result"])
|
||||
job_counts[job["result"]] += 1
|
||||
else:
|
||||
result_types.append(job["state"])
|
||||
job_count += 1
|
||||
job_counts[job["state"]] += 1
|
||||
job_counts["total"] += 1
|
||||
|
||||
platforms.append({
|
||||
"name": platform_name,
|
||||
|
|
@ -497,8 +512,7 @@ class ResultSetViewSet(viewsets.ViewSet):
|
|||
#can be used to determine the resultset's severity
|
||||
resultset.update({
|
||||
"platforms": platforms,
|
||||
"result_types": list(set(result_types)),
|
||||
"job_count": job_count,
|
||||
"job_counts": job_counts,
|
||||
})
|
||||
|
||||
# the resultsets left in the map have no jobs, so fill in the fields
|
||||
|
|
@ -506,11 +520,14 @@ class ResultSetViewSet(viewsets.ViewSet):
|
|||
for rs in rs_map.values():
|
||||
rs.update({
|
||||
"platforms": [],
|
||||
"result_types": [],
|
||||
"job_count": 0,
|
||||
"job_counts": dict.fromkeys(
|
||||
jm.RESULTS + jm.INCOMPLETE_STATES + ["total"], 0),
|
||||
})
|
||||
resultsets.append(rs)
|
||||
return sorted(resultsets, key=lambda x: x["push_timestamp"], reverse=True)
|
||||
return sorted(
|
||||
resultsets,
|
||||
key=lambda x: x["push_timestamp"],
|
||||
reverse=True)
|
||||
|
||||
|
||||
@with_jobs
|
||||
|
|
@ -520,7 +537,7 @@ class ResultSetViewSet(viewsets.ViewSet):
|
|||
POST method implementation
|
||||
"""
|
||||
try:
|
||||
jm.store_result_set_data( request.DATA )
|
||||
jm.store_result_set_data(request.DATA)
|
||||
except DatasetNotFoundError as e:
|
||||
return Response({"message": str(e)}, status=404)
|
||||
except Exception as e: # pragma nocover
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче