зеркало из https://github.com/mozilla/treeherder.git
transform tinderbox lines to structured artifacts
This commit is contained in:
mdoglio
Родитель
0e35319284
Коммит
5c22521311
|
|
@ -1,7 +1,17 @@
|
|||
{
|
||||
"tinderbox_printlines": [
|
||||
"mozharness_revlink: http://hg.mozilla.org/build/mozharness/rev/c43ba6cb3db3",
|
||||
"crashtest<br/>810/0/22"
|
||||
{
|
||||
"url": "http://hg.mozilla.org/build/mozharness/rev/c43ba6cb3db3",
|
||||
"content_type": "link",
|
||||
"value": "http://hg.mozilla.org/build/mozharness/rev/c43ba6cb3db3",
|
||||
"title": "mozharness_revlink"
|
||||
},
|
||||
{
|
||||
"url": null,
|
||||
"content_type": "html",
|
||||
"value": "810/0/22",
|
||||
"title": "crashtest"
|
||||
}
|
||||
],
|
||||
"logurl": "file:///home/vagrant/treeherder-service/tests/sample_data/logs/mozilla-central_fedora-b2g_test-crashtest-1-bm54-tests1-linux-build50.txt.gz"
|
||||
}
|
||||
|
|
@ -1,7 +1,7 @@
|
|||
{
|
||||
"tinderbox_printlines": [
|
||||
"mozharness_revlink: http://hg.mozilla.org/build/mozharness/rev/c43ba6cb3db3",
|
||||
"mochitest-plain2<br/>206414/0/18501"
|
||||
{"url": "http://hg.mozilla.org/build/mozharness/rev/c43ba6cb3db3", "content_type": "link", "value": "http://hg.mozilla.org/build/mozharness/rev/c43ba6cb3db3", "title": "mozharness_revlink"},
|
||||
{"url": null, "content_type": "html", "value": "206414/0/18501", "title": "mochitest-plain2"}
|
||||
],
|
||||
"logurl": "file:///home/vagrant/treeherder-service/tests/sample_data/logs/mozilla-central_mountainlion-debug_test-mochitest-2-bm80-tests1-macosx-build93.txt.gz"
|
||||
}
|
||||
|
|
@ -1,7 +1,7 @@
|
|||
{
|
||||
"tinderbox_printlines": [
|
||||
"mozharness_revlink: http://hg.mozilla.org/build/mozharness/rev/c43ba6cb3db3",
|
||||
"mochitest-plain2<br/>206415/0/18500"
|
||||
{"url": "http://hg.mozilla.org/build/mozharness/rev/c43ba6cb3db3", "content_type": "link", "value": "http://hg.mozilla.org/build/mozharness/rev/c43ba6cb3db3", "title": "mozharness_revlink"},
|
||||
{"url": null, "content_type": "html", "value": "206415/0/18500", "title": "mochitest-plain2"}
|
||||
],
|
||||
"logurl": "file:///home/vagrant/treeherder-service/tests/sample_data/logs/mozilla-central_mountainlion_test-mochitest-2-bm77-tests1-macosx-build141.txt.gz"
|
||||
}
|
||||
|
|
@ -1,7 +1,7 @@
|
|||
{
|
||||
"tinderbox_printlines": [
|
||||
"mozharness_revlink: http://hg.mozilla.org/build/mozharness/rev/97228ea173cb",
|
||||
"mochitest-plain2<br/><em class=\"testfail\">T-FAIL</em>"
|
||||
{"url": "http://hg.mozilla.org/build/mozharness/rev/97228ea173cb", "content_type": "link", "value": "http://hg.mozilla.org/build/mozharness/rev/97228ea173cb", "title": "mozharness_revlink"},
|
||||
{"url": null, "content_type": "html", "value": "<em class=\"testfail\">T-FAIL</em>", "title": "mochitest-plain2"}
|
||||
],
|
||||
"logurl": "file:///home/vagrant/treeherder-service/tests/sample_data/logs/mozilla-central_mountainlion_test-mochitest-2-bm80-tests1-macosx-build138.txt.gz"
|
||||
}
|
||||
|
|
@ -1,6 +1,6 @@
|
|||
{
|
||||
"tinderbox_printlines": [
|
||||
"mochitest-browser-chrome<br/>31776/<em class=\"testfail\">2</em>/40"
|
||||
{"url": null, "content_type": "html", "value": "31776/<em class=\"testfail\">2</em>/40", "title": "mochitest-browser-chrome"}
|
||||
],
|
||||
"logurl": "file:///home/vagrant/treeherder-service/tests/sample_data/logs/mozilla-esr17_xp_test_pgo-mochitest-browser-chrome-bm74-tests1-windows-build12.txt.gz"
|
||||
}
|
||||
|
|
@ -1,9 +1,9 @@
|
|||
{
|
||||
"tinderbox_printlines": [
|
||||
"mozharness_revlink: http://hg.mozilla.org/build/mozharness/rev/c43ba6cb3db3",
|
||||
"mochitest-chrome<br/><em class=\"testfail\">T-FAIL</em> <em class=\"testfail\">CRASH</em>",
|
||||
"mochitest-a11y<br/><em class=\"testfail\">T-FAIL</em> <em class=\"testfail\">CRASH</em>",
|
||||
"mochitest-plugins<br/><em class=\"testfail\">T-FAIL</em> <em class=\"testfail\">CRASH</em>"
|
||||
{"url": "http://hg.mozilla.org/build/mozharness/rev/c43ba6cb3db3", "content_type": "link", "value": "http://hg.mozilla.org/build/mozharness/rev/c43ba6cb3db3", "title": "mozharness_revlink"},
|
||||
{"url": null, "content_type": "html", "value": "<em class=\"testfail\">T-FAIL</em> <em class=\"testfail\">CRASH</em>", "title": "mochitest-chrome"},
|
||||
{"url": null, "content_type": "html", "value": "<em class=\"testfail\">T-FAIL</em> <em class=\"testfail\">CRASH</em>", "title": "mochitest-a11y"},
|
||||
{"url": null, "content_type": "html", "value": "<em class=\"testfail\">T-FAIL</em> <em class=\"testfail\">CRASH</em>", "title": "mochitest-plugins"}
|
||||
],
|
||||
"logurl": "file:///home/vagrant/treeherder-service/tests/sample_data/logs/mozilla-inbound_ubuntu64_vm-debug_test-mochitest-other-bm53-tests1-linux-build122.txt.gz"
|
||||
}
|
||||
|
|
@ -1,5 +1,6 @@
|
|||
import re
|
||||
import datetime
|
||||
import json
|
||||
|
||||
|
||||
class ParserBase(object):
|
||||
|
|
@ -182,6 +183,7 @@ class StepParser(ParserBase):
|
|||
|
||||
RE_TINDERBOXPRINT = re.compile('.*?TinderboxPrint: (.*)$')
|
||||
|
||||
RE_UPLOADED_TO = re.compile("<a href='(http://[A-Za-z/\.0-9\-_]+)'>([A-Za-z/\.0-9\-_]+)</a>")
|
||||
|
||||
class TinderboxPrintParser(ParserBase):
|
||||
|
||||
|
|
@ -194,7 +196,51 @@ class TinderboxPrintParser(ParserBase):
|
|||
if "TinderboxPrint: " in line:
|
||||
match = RE_TINDERBOXPRINT.match(line)
|
||||
if match:
|
||||
self.artifact.append(match.group(1))
|
||||
artifact = {}
|
||||
line = match.group(1)
|
||||
if "<a href='http://graphs.mozilla.org" in line:
|
||||
return
|
||||
# by default use the whole line
|
||||
title = line
|
||||
value = ""
|
||||
content_type = "text"
|
||||
url = None
|
||||
splitters = (": ", "<br/>")
|
||||
|
||||
splitters_used = [s for s in splitters if s in line]
|
||||
|
||||
if splitters_used:
|
||||
title, value = line.split(splitters_used[0], 1)
|
||||
|
||||
# if it's a json string, return it as is.
|
||||
try:
|
||||
artifact = json.loads(value)
|
||||
self.artifact.append(artifact)
|
||||
except ValueError:
|
||||
# if it's not a json string, let's parse it
|
||||
if "link" in title:
|
||||
content_type = "link"
|
||||
url = value
|
||||
if "uploaded" in value:
|
||||
uploaded_to_chunks = RE_UPLOADED_TO.match(title)
|
||||
if uploaded_to_chunks:
|
||||
title = "artifact uploaded"
|
||||
value = uploaded_to_chunks.group(2)
|
||||
url = uploaded_to_chunks.group(1)
|
||||
content_type = "link"
|
||||
|
||||
if (splitters_used[0] == "<br/>" or
|
||||
("<" in splitters_used[0]) or
|
||||
("<" in value)):
|
||||
content_type = "html"
|
||||
|
||||
artifact["title"] = title
|
||||
artifact["value"] = value
|
||||
artifact["content_type"] = content_type
|
||||
artifact["url"] = url
|
||||
|
||||
self.artifact.append(artifact)
|
||||
|
||||
|
||||
|
||||
RE_INFO = re.compile((
|
||||
|
|
|
|||
|
|
@ -1616,6 +1616,8 @@ class JobsModel(TreeherderModelBase):
|
|||
|
||||
result_sets = []
|
||||
|
||||
time_now = int(time.time())
|
||||
|
||||
if log_placeholders:
|
||||
for index, log_ref in enumerate(log_placeholders):
|
||||
job_guid = log_ref[0]
|
||||
|
|
@ -1626,7 +1628,7 @@ class JobsModel(TreeherderModelBase):
|
|||
|
||||
# Replace job_guid with id
|
||||
log_placeholders[index][0] = job_id
|
||||
|
||||
log_placeholders[index].append(time_now)
|
||||
task = dict()
|
||||
task['job_guid'] = job_guid
|
||||
task['log_url'] = log_ref[2]
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
from optparse import make_option
|
||||
from django.utils.six.moves import input
|
||||
|
||||
from django.conf import settings
|
||||
|
||||
|
|
@ -21,12 +22,27 @@ class Command(BaseCommand):
|
|||
dest='readonly_host',
|
||||
default=settings.TREEHERDER_DATABASE_HOST,
|
||||
help='Readonly host to associate the datasource to'),
|
||||
make_option('--reset',
|
||||
action='store_true',
|
||||
dest='reset',
|
||||
default=False,
|
||||
help='Reset the datasources if they already exists'),
|
||||
)
|
||||
|
||||
def handle(self, *args, **options):
|
||||
if options["reset"]:
|
||||
confirm = input("""You have requested an init of the datasources.
|
||||
This will IRREVERSIBLY DESTROY all data in the jobs and objectstore databases.
|
||||
Are you sure you want to do this?
|
||||
|
||||
Type 'yes' to continue, or 'no' to cancel: """)
|
||||
if confirm == "yes":
|
||||
for ds in Datasource.objects.all():
|
||||
ds.delete()
|
||||
|
||||
projects = Repository.objects.all().values_list('name', flat=True)
|
||||
for project in projects:
|
||||
for contenttype in ("jobs","objectstore"):
|
||||
for contenttype in ("jobs", "objectstore"):
|
||||
Datasource.objects.get_or_create(
|
||||
contenttype=contenttype,
|
||||
dataset=1,
|
||||
|
|
|
|||
|
|
@ -151,9 +151,10 @@
|
|||
"sql":"INSERT INTO `job_log_url` (
|
||||
`job_id`,
|
||||
`name`,
|
||||
`url`
|
||||
`url`,
|
||||
`parse_timestamp`
|
||||
)
|
||||
VALUES (?,?,?)",
|
||||
VALUES (?,?,?,?)",
|
||||
|
||||
"host":"master_host"
|
||||
},
|
||||
|
|
|
|||
|
|
@ -256,17 +256,22 @@ DROP TABLE IF EXISTS `job_log_url`;
|
|||
* job_id - References job.id
|
||||
* name - Name of log file
|
||||
* url - URL to log file
|
||||
* parse_status - the status of the log parsing
|
||||
**************************/
|
||||
CREATE TABLE `job_log_url` (
|
||||
`id` bigint(20) unsigned NOT NULL AUTO_INCREMENT,
|
||||
`job_id` bigint(20) unsigned NOT NULL,
|
||||
`name` varchar(50) COLLATE utf8_bin NOT NULL,
|
||||
`url` varchar(255) COLLATE utf8_bin NOT NULL,
|
||||
`parse_status` enum('pending', 'parsed', 'failed') COLLATE utf8_bin DEFAULT 'pending',
|
||||
`parse_timestamp` int(10) NOT NULL,
|
||||
`active_status` enum('active','onhold','deleted') COLLATE utf8_bin DEFAULT 'active',
|
||||
PRIMARY KEY (`id`),
|
||||
KEY `idx_job_id` (`job_id`),
|
||||
KEY `idx_name` (`name`),
|
||||
KEY `idx_parse_timestamp` (`parse_timestamp`),
|
||||
KEY `idx_active_status` (`active_status`),
|
||||
KEY `idx_parse_status` (`parse_status`),
|
||||
CONSTRAINT `fk_job_log_url` FOREIGN KEY (`job_id`) REFERENCES `job` (`id`)
|
||||
) ENGINE={engine} DEFAULT CHARSET=utf8 COLLATE=utf8_bin;
|
||||
/*!40101 SET character_set_client = @saved_cs_client */;
|
||||
|
|
|
|||
|
|
@ -0,0 +1,14 @@
|
|||
from rest_framework import viewsets
|
||||
from rest_framework.response import Response
|
||||
from rest_framework.authentication import SessionAuthentication
|
||||
from treeherder.webapp.api.permissions import IsStaffOrReadOnly
|
||||
|
||||
from treeherder.model.derived import JobsModel
|
||||
|
||||
|
||||
class LogUrlViewset(viewsets.ViewSetw):
|
||||
authentication_classes = (SessionAuthentication,)
|
||||
permission_classes = (IsStaffOrReadOnly,)
|
||||
|
||||
def
|
||||
|
||||
Загрузка…
Ссылка в новой задаче