From 5c22521311757b91cc3d2be90054210cfe2a2e1e Mon Sep 17 00:00:00 2001 From: mdoglio Date: Wed, 11 Jun 2014 15:02:56 +0200 Subject: [PATCH] transform tinderbox lines to structured artifacts --- ...bm54-tests1-linux-build50.jobartifact.json | 14 +++++- ...m80-tests1-macosx-build93.jobartifact.json | 4 +- ...77-tests1-macosx-build141.jobartifact.json | 4 +- ...80-tests1-macosx-build138.jobartifact.json | 4 +- ...74-tests1-windows-build12.jobartifact.json | 2 +- ...m53-tests1-linux-build122.jobartifact.json | 8 ++-- treeherder/log_parser/parsers.pyx | 48 ++++++++++++++++++- treeherder/model/derived/jobs.py | 4 +- .../management/commands/init_datasources.py | 18 ++++++- treeherder/model/sql/jobs.json | 5 +- .../template_schema/project_jobs_1.sql.tmpl | 5 ++ treeherder/webapp/api/log_url.py | 14 ++++++ 12 files changed, 112 insertions(+), 18 deletions(-) create mode 100644 treeherder/webapp/api/log_url.py diff --git a/tests/sample_data/logs/mozilla-central_fedora-b2g_test-crashtest-1-bm54-tests1-linux-build50.jobartifact.json b/tests/sample_data/logs/mozilla-central_fedora-b2g_test-crashtest-1-bm54-tests1-linux-build50.jobartifact.json index 0cefb24dd..a853b2c43 100644 --- a/tests/sample_data/logs/mozilla-central_fedora-b2g_test-crashtest-1-bm54-tests1-linux-build50.jobartifact.json +++ b/tests/sample_data/logs/mozilla-central_fedora-b2g_test-crashtest-1-bm54-tests1-linux-build50.jobartifact.json @@ -1,7 +1,17 @@ { "tinderbox_printlines": [ - "mozharness_revlink: http://hg.mozilla.org/build/mozharness/rev/c43ba6cb3db3", - "crashtest
810/0/22" + { + "url": "http://hg.mozilla.org/build/mozharness/rev/c43ba6cb3db3", + "content_type": "link", + "value": "http://hg.mozilla.org/build/mozharness/rev/c43ba6cb3db3", + "title": "mozharness_revlink" + }, + { + "url": null, + "content_type": "html", + "value": "810/0/22", + "title": "crashtest" + } ], "logurl": "file:///home/vagrant/treeherder-service/tests/sample_data/logs/mozilla-central_fedora-b2g_test-crashtest-1-bm54-tests1-linux-build50.txt.gz" } \ No newline at end of file diff --git a/tests/sample_data/logs/mozilla-central_mountainlion-debug_test-mochitest-2-bm80-tests1-macosx-build93.jobartifact.json b/tests/sample_data/logs/mozilla-central_mountainlion-debug_test-mochitest-2-bm80-tests1-macosx-build93.jobartifact.json index d56bc82c6..463b1968c 100644 --- a/tests/sample_data/logs/mozilla-central_mountainlion-debug_test-mochitest-2-bm80-tests1-macosx-build93.jobartifact.json +++ b/tests/sample_data/logs/mozilla-central_mountainlion-debug_test-mochitest-2-bm80-tests1-macosx-build93.jobartifact.json @@ -1,7 +1,7 @@ { "tinderbox_printlines": [ - "mozharness_revlink: http://hg.mozilla.org/build/mozharness/rev/c43ba6cb3db3", - "mochitest-plain2
206414/0/18501" + {"url": "http://hg.mozilla.org/build/mozharness/rev/c43ba6cb3db3", "content_type": "link", "value": "http://hg.mozilla.org/build/mozharness/rev/c43ba6cb3db3", "title": "mozharness_revlink"}, + {"url": null, "content_type": "html", "value": "206414/0/18501", "title": "mochitest-plain2"} ], "logurl": "file:///home/vagrant/treeherder-service/tests/sample_data/logs/mozilla-central_mountainlion-debug_test-mochitest-2-bm80-tests1-macosx-build93.txt.gz" } \ No newline at end of file diff --git a/tests/sample_data/logs/mozilla-central_mountainlion_test-mochitest-2-bm77-tests1-macosx-build141.jobartifact.json b/tests/sample_data/logs/mozilla-central_mountainlion_test-mochitest-2-bm77-tests1-macosx-build141.jobartifact.json index dcabd0558..13337035d 100644 --- a/tests/sample_data/logs/mozilla-central_mountainlion_test-mochitest-2-bm77-tests1-macosx-build141.jobartifact.json +++ b/tests/sample_data/logs/mozilla-central_mountainlion_test-mochitest-2-bm77-tests1-macosx-build141.jobartifact.json @@ -1,7 +1,7 @@ { "tinderbox_printlines": [ - "mozharness_revlink: http://hg.mozilla.org/build/mozharness/rev/c43ba6cb3db3", - "mochitest-plain2
206415/0/18500" + {"url": "http://hg.mozilla.org/build/mozharness/rev/c43ba6cb3db3", "content_type": "link", "value": "http://hg.mozilla.org/build/mozharness/rev/c43ba6cb3db3", "title": "mozharness_revlink"}, + {"url": null, "content_type": "html", "value": "206415/0/18500", "title": "mochitest-plain2"} ], "logurl": "file:///home/vagrant/treeherder-service/tests/sample_data/logs/mozilla-central_mountainlion_test-mochitest-2-bm77-tests1-macosx-build141.txt.gz" } \ No newline at end of file diff --git a/tests/sample_data/logs/mozilla-central_mountainlion_test-mochitest-2-bm80-tests1-macosx-build138.jobartifact.json b/tests/sample_data/logs/mozilla-central_mountainlion_test-mochitest-2-bm80-tests1-macosx-build138.jobartifact.json index 5bbe568c2..1ec7dfde0 100644 --- a/tests/sample_data/logs/mozilla-central_mountainlion_test-mochitest-2-bm80-tests1-macosx-build138.jobartifact.json +++ b/tests/sample_data/logs/mozilla-central_mountainlion_test-mochitest-2-bm80-tests1-macosx-build138.jobartifact.json @@ -1,7 +1,7 @@ { "tinderbox_printlines": [ - "mozharness_revlink: http://hg.mozilla.org/build/mozharness/rev/97228ea173cb", - "mochitest-plain2
T-FAIL" + {"url": "http://hg.mozilla.org/build/mozharness/rev/97228ea173cb", "content_type": "link", "value": "http://hg.mozilla.org/build/mozharness/rev/97228ea173cb", "title": "mozharness_revlink"}, + {"url": null, "content_type": "html", "value": "T-FAIL", "title": "mochitest-plain2"} ], "logurl": "file:///home/vagrant/treeherder-service/tests/sample_data/logs/mozilla-central_mountainlion_test-mochitest-2-bm80-tests1-macosx-build138.txt.gz" } \ No newline at end of file diff --git a/tests/sample_data/logs/mozilla-esr17_xp_test_pgo-mochitest-browser-chrome-bm74-tests1-windows-build12.jobartifact.json b/tests/sample_data/logs/mozilla-esr17_xp_test_pgo-mochitest-browser-chrome-bm74-tests1-windows-build12.jobartifact.json index 426dc371d..a519b0c2e 100644 --- a/tests/sample_data/logs/mozilla-esr17_xp_test_pgo-mochitest-browser-chrome-bm74-tests1-windows-build12.jobartifact.json +++ b/tests/sample_data/logs/mozilla-esr17_xp_test_pgo-mochitest-browser-chrome-bm74-tests1-windows-build12.jobartifact.json @@ -1,6 +1,6 @@ { "tinderbox_printlines": [ - "mochitest-browser-chrome
31776/2/40" + {"url": null, "content_type": "html", "value": "31776/2/40", "title": "mochitest-browser-chrome"} ], "logurl": "file:///home/vagrant/treeherder-service/tests/sample_data/logs/mozilla-esr17_xp_test_pgo-mochitest-browser-chrome-bm74-tests1-windows-build12.txt.gz" } \ No newline at end of file diff --git a/tests/sample_data/logs/mozilla-inbound_ubuntu64_vm-debug_test-mochitest-other-bm53-tests1-linux-build122.jobartifact.json b/tests/sample_data/logs/mozilla-inbound_ubuntu64_vm-debug_test-mochitest-other-bm53-tests1-linux-build122.jobartifact.json index 4802688a7..8d1b909e4 100644 --- a/tests/sample_data/logs/mozilla-inbound_ubuntu64_vm-debug_test-mochitest-other-bm53-tests1-linux-build122.jobartifact.json +++ b/tests/sample_data/logs/mozilla-inbound_ubuntu64_vm-debug_test-mochitest-other-bm53-tests1-linux-build122.jobartifact.json @@ -1,9 +1,9 @@ { "tinderbox_printlines": [ - "mozharness_revlink: http://hg.mozilla.org/build/mozharness/rev/c43ba6cb3db3", - "mochitest-chrome
T-FAIL CRASH", - "mochitest-a11y
T-FAIL CRASH", - "mochitest-plugins
T-FAIL CRASH" + {"url": "http://hg.mozilla.org/build/mozharness/rev/c43ba6cb3db3", "content_type": "link", "value": "http://hg.mozilla.org/build/mozharness/rev/c43ba6cb3db3", "title": "mozharness_revlink"}, + {"url": null, "content_type": "html", "value": "T-FAIL CRASH", "title": "mochitest-chrome"}, + {"url": null, "content_type": "html", "value": "T-FAIL CRASH", "title": "mochitest-a11y"}, + {"url": null, "content_type": "html", "value": "T-FAIL CRASH", "title": "mochitest-plugins"} ], "logurl": "file:///home/vagrant/treeherder-service/tests/sample_data/logs/mozilla-inbound_ubuntu64_vm-debug_test-mochitest-other-bm53-tests1-linux-build122.txt.gz" } \ No newline at end of file diff --git a/treeherder/log_parser/parsers.pyx b/treeherder/log_parser/parsers.pyx index c9bfe9df0..569171692 100644 --- a/treeherder/log_parser/parsers.pyx +++ b/treeherder/log_parser/parsers.pyx @@ -1,5 +1,6 @@ import re import datetime +import json class ParserBase(object): @@ -182,6 +183,7 @@ class StepParser(ParserBase): RE_TINDERBOXPRINT = re.compile('.*?TinderboxPrint: (.*)$') +RE_UPLOADED_TO = re.compile("([A-Za-z/\.0-9\-_]+)") class TinderboxPrintParser(ParserBase): @@ -194,7 +196,51 @@ class TinderboxPrintParser(ParserBase): if "TinderboxPrint: " in line: match = RE_TINDERBOXPRINT.match(line) if match: - self.artifact.append(match.group(1)) + artifact = {} + line = match.group(1) + if "" or + ("<" in splitters_used[0]) or + ("<" in value)): + content_type = "html" + + artifact["title"] = title + artifact["value"] = value + artifact["content_type"] = content_type + artifact["url"] = url + + self.artifact.append(artifact) + RE_INFO = re.compile(( diff --git a/treeherder/model/derived/jobs.py b/treeherder/model/derived/jobs.py index 748478f52..656a2d642 100644 --- a/treeherder/model/derived/jobs.py +++ b/treeherder/model/derived/jobs.py @@ -1616,6 +1616,8 @@ class JobsModel(TreeherderModelBase): result_sets = [] + time_now = int(time.time()) + if log_placeholders: for index, log_ref in enumerate(log_placeholders): job_guid = log_ref[0] @@ -1626,7 +1628,7 @@ class JobsModel(TreeherderModelBase): # Replace job_guid with id log_placeholders[index][0] = job_id - + log_placeholders[index].append(time_now) task = dict() task['job_guid'] = job_guid task['log_url'] = log_ref[2] diff --git a/treeherder/model/management/commands/init_datasources.py b/treeherder/model/management/commands/init_datasources.py index f1539b4b2..5cc88149b 100644 --- a/treeherder/model/management/commands/init_datasources.py +++ b/treeherder/model/management/commands/init_datasources.py @@ -1,4 +1,5 @@ from optparse import make_option +from django.utils.six.moves import input from django.conf import settings @@ -21,12 +22,27 @@ class Command(BaseCommand): dest='readonly_host', default=settings.TREEHERDER_DATABASE_HOST, help='Readonly host to associate the datasource to'), + make_option('--reset', + action='store_true', + dest='reset', + default=False, + help='Reset the datasources if they already exists'), ) def handle(self, *args, **options): + if options["reset"]: + confirm = input("""You have requested an init of the datasources. +This will IRREVERSIBLY DESTROY all data in the jobs and objectstore databases. +Are you sure you want to do this? + +Type 'yes' to continue, or 'no' to cancel: """) + if confirm == "yes": + for ds in Datasource.objects.all(): + ds.delete() + projects = Repository.objects.all().values_list('name', flat=True) for project in projects: - for contenttype in ("jobs","objectstore"): + for contenttype in ("jobs", "objectstore"): Datasource.objects.get_or_create( contenttype=contenttype, dataset=1, diff --git a/treeherder/model/sql/jobs.json b/treeherder/model/sql/jobs.json index eb1fe7d4b..b5d02e844 100644 --- a/treeherder/model/sql/jobs.json +++ b/treeherder/model/sql/jobs.json @@ -151,9 +151,10 @@ "sql":"INSERT INTO `job_log_url` ( `job_id`, `name`, - `url` + `url`, + `parse_timestamp` ) - VALUES (?,?,?)", + VALUES (?,?,?,?)", "host":"master_host" }, diff --git a/treeherder/model/sql/template_schema/project_jobs_1.sql.tmpl b/treeherder/model/sql/template_schema/project_jobs_1.sql.tmpl index 20d625365..1de805555 100644 --- a/treeherder/model/sql/template_schema/project_jobs_1.sql.tmpl +++ b/treeherder/model/sql/template_schema/project_jobs_1.sql.tmpl @@ -256,17 +256,22 @@ DROP TABLE IF EXISTS `job_log_url`; * job_id - References job.id * name - Name of log file * url - URL to log file + * parse_status - the status of the log parsing **************************/ CREATE TABLE `job_log_url` ( `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT, `job_id` bigint(20) unsigned NOT NULL, `name` varchar(50) COLLATE utf8_bin NOT NULL, `url` varchar(255) COLLATE utf8_bin NOT NULL, + `parse_status` enum('pending', 'parsed', 'failed') COLLATE utf8_bin DEFAULT 'pending', + `parse_timestamp` int(10) NOT NULL, `active_status` enum('active','onhold','deleted') COLLATE utf8_bin DEFAULT 'active', PRIMARY KEY (`id`), KEY `idx_job_id` (`job_id`), KEY `idx_name` (`name`), + KEY `idx_parse_timestamp` (`parse_timestamp`), KEY `idx_active_status` (`active_status`), + KEY `idx_parse_status` (`parse_status`), CONSTRAINT `fk_job_log_url` FOREIGN KEY (`job_id`) REFERENCES `job` (`id`) ) ENGINE={engine} DEFAULT CHARSET=utf8 COLLATE=utf8_bin; /*!40101 SET character_set_client = @saved_cs_client */; diff --git a/treeherder/webapp/api/log_url.py b/treeherder/webapp/api/log_url.py new file mode 100644 index 000000000..391f182f4 --- /dev/null +++ b/treeherder/webapp/api/log_url.py @@ -0,0 +1,14 @@ +from rest_framework import viewsets +from rest_framework.response import Response +from rest_framework.authentication import SessionAuthentication +from treeherder.webapp.api.permissions import IsStaffOrReadOnly + +from treeherder.model.derived import JobsModel + + +class LogUrlViewset(viewsets.ViewSetw): + authentication_classes = (SessionAuthentication,) + permission_classes = (IsStaffOrReadOnly,) + + def +