Get to submit to treeherder
This commit is contained in:
Hannes Verschore
Родитель
060dd2a2d5
Коммит
661bf80cf4
|
|
@ -0,0 +1,13 @@
|
|||
<?php
|
||||
|
||||
// Add field 'treeherder' to the awfy_run database.
|
||||
// And mark all previous runs supposedly submitted to treeherder
|
||||
|
||||
$migrate = function() {
|
||||
mysql_query("ALTER TABLE `awfy_run` ADD `treeherder` BOOLEAN NOT NULL;") or die(mysql_error());
|
||||
mysql_query("UPDATE awfy_run SET `treeherder` = 1 WHERE status != 0;") or die(mysql_error());
|
||||
};
|
||||
|
||||
$rollback = function() {
|
||||
mysql_query("ALTER TABLE `awfy_run` DROP `treeherder`;") or die(mysql_error());
|
||||
};
|
||||
|
|
@ -1,73 +0,0 @@
|
|||
# vim: set ts=4 sw=4 tw=99 et:
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
try:
|
||||
import MySQLdb as mdb
|
||||
except:
|
||||
import mysqldb as mdb
|
||||
try:
|
||||
import ConfigParser
|
||||
except:
|
||||
import configparser as ConfigParser
|
||||
|
||||
db = None
|
||||
version = None
|
||||
path = None
|
||||
|
||||
|
||||
queries = 0
|
||||
class DB:
|
||||
def __init__(self, host, user, pw, name):
|
||||
self.host = host
|
||||
self.user = user
|
||||
self.pw = pw
|
||||
self.name = name
|
||||
self.connect()
|
||||
def connect(self):
|
||||
if self.host[0] == '/':
|
||||
self.db = mdb.connect(unix_socket=self.host, user=self.user, passwd=self.pw,
|
||||
db=self.name, use_unicode=True)
|
||||
else:
|
||||
self.db = mdb.connect(self.host, self.user, self.pw, self.name, use_unicode=True)
|
||||
def cursor(self):
|
||||
return DBCursor(self.db.cursor())
|
||||
def commit(self):
|
||||
return self.db.commit()
|
||||
class DBCursor:
|
||||
def __init__(self, cursor):
|
||||
self.cursor = cursor
|
||||
def execute(self, sql, data=None):
|
||||
global queries
|
||||
queries+=1
|
||||
exe = self.cursor.execute(sql, data);
|
||||
self.description = self.cursor.description
|
||||
self.lastrowid = self.cursor.lastrowid
|
||||
self.rowcount = self.cursor.rowcount
|
||||
return exe
|
||||
def fetchone(self):
|
||||
return self.cursor.fetchone();
|
||||
def fetchall(self):
|
||||
return self.cursor.fetchall();
|
||||
|
||||
def Startup():
|
||||
global db, version, path
|
||||
config = ConfigParser.RawConfigParser()
|
||||
config.read("/etc/awfy-server.config")
|
||||
|
||||
host = config.get('mysql', 'host')
|
||||
user = config.get('mysql', 'user')
|
||||
pw = config.get('mysql', 'pass')
|
||||
name = config.get('mysql', 'db_name')
|
||||
|
||||
db = DB(host, user, pw, name)
|
||||
c = db.cursor()
|
||||
c.execute("SELECT `value` FROM awfy_config WHERE `key` = 'version'")
|
||||
row = c.fetchone()
|
||||
version = int(row[0])
|
||||
|
||||
path = config.get('general', 'data_folder')
|
||||
|
||||
Startup()
|
||||
|
||||
|
|
@ -3,11 +3,14 @@
|
|||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
import awfy
|
||||
from optparse import OptionParser
|
||||
import sys
|
||||
import time
|
||||
|
||||
sys.path.append("../server")
|
||||
import awfy
|
||||
import tables
|
||||
from optparse import OptionParser
|
||||
|
||||
|
||||
parser = OptionParser(usage="usage: %prog [options]")
|
||||
parser.add_option("-n", "--non-existing", dest="nonexistonly", action="store_true", default=False,
|
||||
|
|
|
|||
|
|
@ -3,6 +3,9 @@
|
|||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
import sys
|
||||
|
||||
sys.path.append("../server")
|
||||
import awfy
|
||||
import tables
|
||||
|
||||
|
|
|
|||
|
|
@ -3,12 +3,13 @@
|
|||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
import awfy
|
||||
from optparse import OptionParser
|
||||
import sys
|
||||
import time
|
||||
import tables
|
||||
|
||||
from optparse import OptionParser
|
||||
sys.path.append("../server")
|
||||
import awfy
|
||||
import tables
|
||||
|
||||
parser = OptionParser(usage="usage: %prog [options]")
|
||||
parser.add_option( "--dry-run", dest="dryrun", action="store_true", default=False,
|
||||
|
|
|
|||
|
|
@ -3,6 +3,9 @@
|
|||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
import sys
|
||||
|
||||
sys.path.append("../server")
|
||||
import awfy
|
||||
|
||||
c = awfy.db.cursor()
|
||||
|
|
|
|||
|
|
@ -7,4 +7,9 @@ db_name = ???
|
|||
[general]
|
||||
data_folder = /home/awfy
|
||||
machine_timeout = 480 ; 8 hours (480 minutes)
|
||||
slack_webhook = ???
|
||||
|
||||
[treeherder]
|
||||
host = ???
|
||||
user = ???
|
||||
secret = ???
|
||||
|
|
|
|||
|
|
@ -15,6 +15,9 @@ except:
|
|||
db = None
|
||||
version = None
|
||||
path = None
|
||||
th_host = None
|
||||
th_user = None
|
||||
th_secret = None
|
||||
|
||||
|
||||
queries = 0
|
||||
|
|
@ -52,7 +55,7 @@ class DBCursor:
|
|||
return self.cursor.fetchall();
|
||||
|
||||
def Startup():
|
||||
global db, version, path
|
||||
global db, version, path, th_host, th_user, th_secret
|
||||
config = ConfigParser.RawConfigParser()
|
||||
config.read("/etc/awfy-server.config")
|
||||
|
||||
|
|
@ -69,5 +72,9 @@ def Startup():
|
|||
|
||||
path = config.get('general', 'data_folder')
|
||||
|
||||
Startup()
|
||||
if config.has_section('treeherder'):
|
||||
th_host = config.get('treeherder', 'host')
|
||||
th_user = config.get('treeherder', 'user')
|
||||
th_secret = config.get('treeherder', 'secret')
|
||||
|
||||
Startup()
|
||||
|
|
|
|||
|
|
@ -28,6 +28,13 @@ def camelcase(string):
|
|||
class DBTable(object):
|
||||
globalcache = {}
|
||||
|
||||
@classmethod
|
||||
def FromId(class_, id):
|
||||
obj = class_(row[0])
|
||||
if obj.exists():
|
||||
return obj
|
||||
return None
|
||||
|
||||
def __init__(self, id):
|
||||
self.id = int(id)
|
||||
self.initialized = False
|
||||
|
|
@ -142,6 +149,19 @@ class Run(DBTable):
|
|||
def table():
|
||||
return "awfy_run"
|
||||
|
||||
@staticmethod
|
||||
def fromSortOrder(machine_id, sort_order):
|
||||
c = awfy.db.cursor()
|
||||
c.execute("SELECT id \
|
||||
FROM awfy_run \
|
||||
WHERE machine = %s AND \
|
||||
sort_order = %s", (machine_id, sort_order))
|
||||
rows = c.fetchall()
|
||||
if len(rows) == 0:
|
||||
return None
|
||||
assert len(rows) == 1
|
||||
return Run(rows[0][0])
|
||||
|
||||
def initialize(self):
|
||||
if self.initialized:
|
||||
return
|
||||
|
|
@ -348,6 +368,19 @@ class Build(DBTable):
|
|||
def table():
|
||||
return "awfy_build"
|
||||
|
||||
@staticmethod
|
||||
def fromRunAndMode(run_id, mode_id):
|
||||
c = awfy.db.cursor()
|
||||
c.execute("SELECT id \
|
||||
FROM awfy_build \
|
||||
WHERE run_id = %s AND \
|
||||
mode_id = %s", (run_id, mode_id))
|
||||
rows = c.fetchall()
|
||||
if len(rows) == 0:
|
||||
return None
|
||||
assert len(rows) == 1
|
||||
return Build(rows[0][0])
|
||||
|
||||
def getScores(self):
|
||||
scores = []
|
||||
c = awfy.db.cursor()
|
||||
|
|
@ -591,6 +624,29 @@ class Score(RegressionTools):
|
|||
def table():
|
||||
return "awfy_score"
|
||||
|
||||
@staticmethod
|
||||
def fromBuildAndSuite(build_id, suite_version_id):
|
||||
c = awfy.db.cursor()
|
||||
c.execute("SELECT id \
|
||||
FROM awfy_score \
|
||||
WHERE build_id = %s AND \
|
||||
suite_version_id = %s", (build_id, suite_version_id))
|
||||
rows = c.fetchall()
|
||||
if len(rows) == 0:
|
||||
return None
|
||||
assert len(rows) == 1
|
||||
return Score(rows[0][0])
|
||||
|
||||
def getBreakdowns(self):
|
||||
c = awfy.db.cursor()
|
||||
c.execute("SELECT awfy_breakdown.id \
|
||||
FROM awfy_breakdown \
|
||||
WHERE score_id = %s", (self.id,))
|
||||
breakdowns = []
|
||||
for row in c.fetchall():
|
||||
breakdowns.append(Breakdown(row[0]))
|
||||
return breakdowns
|
||||
|
||||
def sane(self):
|
||||
if self.get("suite_version_id") == -1:
|
||||
return False
|
||||
|
|
@ -0,0 +1,12 @@
|
|||
[{
|
||||
"machine": 28,
|
||||
"modes": [{
|
||||
"mode": "jmim",
|
||||
"repo": "mozilla-inbound",
|
||||
"platform": ["linux", "linux32", "x86"],
|
||||
"tier": 2,
|
||||
"job_symbol": "shell",
|
||||
"job_name": "shell",
|
||||
"enabled": true
|
||||
}]
|
||||
}]
|
||||
|
|
@ -0,0 +1,55 @@
|
|||
{
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"machine": {
|
||||
"description": "The id of the machine in the db",
|
||||
"type": "integer"
|
||||
},
|
||||
"modes": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"required": ["mode","repo","tier","job_symbol","job_name","platform","enabled"],
|
||||
"properties": {
|
||||
"enabled": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"mode": {
|
||||
"description": "the mode name in the db",
|
||||
"type": "string"
|
||||
},
|
||||
"repo": {
|
||||
"description": "The repo this dataset is testing. E.g. mozilla-inbound",
|
||||
"type": "string"
|
||||
},
|
||||
"tier": {
|
||||
"description": "On a scale of 1-3 how important the data is",
|
||||
"type": "integer",
|
||||
"minimum": 1,
|
||||
"maximum": 3
|
||||
},
|
||||
"job_symbol": {
|
||||
"description": "The symbol used to report on treeherder",
|
||||
"type": "string"
|
||||
},
|
||||
"job_name": {
|
||||
"description": "The name used to report on treeherder",
|
||||
"type": "string"
|
||||
},
|
||||
"platform": {
|
||||
"description": "An array explaining the platform. [OS, OS+bits, CPU]",
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "string"
|
||||
},
|
||||
"minItems": 3,
|
||||
"maxItems": 3
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,226 @@
|
|||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import requests
|
||||
import socket
|
||||
import time
|
||||
from urlparse import urljoin, urlparse
|
||||
import uuid
|
||||
|
||||
try:
|
||||
from thclient import TreeherderClient, TreeherderJob, TreeherderJobCollection
|
||||
except:
|
||||
print "run 'sudo pip install treeherder-client' to install the needed libraries"
|
||||
exit()
|
||||
|
||||
RESULTSET_FRAGMENT = 'api/project/{repository}/resultset/?revision={revision}'
|
||||
JOB_FRAGMENT = '/#/jobs?repo={repository}&revision={revision}'
|
||||
BUILD_STATES = ['running', 'completed']
|
||||
|
||||
logging.basicConfig(format='%(asctime)s %(levelname)s | %(message)s', datefmt='%H:%M:%S')
|
||||
logger = logging.getLogger('mozmill-ci')
|
||||
logger.setLevel(logging.INFO)
|
||||
|
||||
class Submission(object):
|
||||
"""Class for submitting reports to Treeherder."""
|
||||
|
||||
def __init__(self, repository, revision, settings,
|
||||
treeherder_url=None, treeherder_client_id=None, treeherder_secret=None):
|
||||
"""Creates new instance of the submission class.
|
||||
|
||||
:param repository: Name of the repository the build has been built from.
|
||||
:param revision: Changeset of the repository the build has been built from.
|
||||
:param settings: Settings for the Treeherder job as retrieved from the config file.
|
||||
:param treeherder_url: URL of the Treeherder instance.
|
||||
:param treeherder_client_id: The client ID necessary for the Hawk authentication.
|
||||
:param treeherder_secret: The secret key necessary for the Hawk authentication.
|
||||
|
||||
"""
|
||||
self.repository = repository
|
||||
self.revision = revision
|
||||
self.settings = settings
|
||||
|
||||
self._job_details = []
|
||||
|
||||
self.url = treeherder_url
|
||||
self.client_id = treeherder_client_id
|
||||
self.secret = treeherder_secret
|
||||
|
||||
if not self.client_id or not self.secret and self.url != "mock":
|
||||
raise ValueError('The client_id and secret for Treeherder must be set.')
|
||||
|
||||
def _get_treeherder_platform(self):
|
||||
"""Returns the Treeherder equivalent platform identifier of the current platform."""
|
||||
|
||||
#Todo
|
||||
return ('linux', 'linux64', 'x86_64')
|
||||
|
||||
def create_job(self, data=None, **kwargs):
|
||||
"""Creates a new instance of a Treeherder job for submission.
|
||||
|
||||
:param data: Job data to use for initilization, e.g. from a previous submission, optional
|
||||
:param kwargs: Dictionary of necessary values to build the job details. The
|
||||
properties correlate to the placeholders in config.py.
|
||||
|
||||
"""
|
||||
data = data or {}
|
||||
|
||||
job = TreeherderJob(data=data)
|
||||
|
||||
# If no data is available we have to set all properties
|
||||
if not data:
|
||||
job.add_job_guid(str(uuid.uuid4()))
|
||||
job.add_tier(self.settings['treeherder']['tier'])
|
||||
|
||||
job.add_product_name('firefox')
|
||||
|
||||
job.add_project(self.repository)
|
||||
job.add_revision_hash(self.retrieve_revision_hash())
|
||||
|
||||
# Add platform and build information
|
||||
job.add_machine(socket.getfqdn())
|
||||
platform = self._get_treeherder_platform()
|
||||
job.add_machine_info(*platform)
|
||||
job.add_build_info(*platform)
|
||||
|
||||
# TODO debug or others?
|
||||
job.add_option_collection({'opt': True})
|
||||
|
||||
# TODO: Add e10s group once we run those tests
|
||||
job.add_group_name(self.settings['treeherder']['group_name'].format(**kwargs))
|
||||
job.add_group_symbol(self.settings['treeherder']['group_symbol'].format(**kwargs))
|
||||
|
||||
# Bug 1174973 - for now we need unique job names even in different groups
|
||||
job.add_job_name(self.settings['treeherder']['job_name'].format(**kwargs))
|
||||
job.add_job_symbol(self.settings['treeherder']['job_symbol'].format(**kwargs))
|
||||
|
||||
job.add_start_timestamp(int(time.time()))
|
||||
|
||||
# Bug 1175559 - Workaround for HTTP Error
|
||||
job.add_end_timestamp(0)
|
||||
|
||||
return job
|
||||
|
||||
def retrieve_revision_hash(self):
|
||||
"""Retrieves the unique hash for the current revision."""
|
||||
if not self.url:
|
||||
raise ValueError('URL for Treeherder is missing.')
|
||||
|
||||
lookup_url = urljoin(self.url,
|
||||
RESULTSET_FRAGMENT.format(repository=self.repository,
|
||||
revision=self.revision))
|
||||
|
||||
if self.url == "mock":
|
||||
logger.info('Pretend to get revision hash from: {}'.format(lookup_url))
|
||||
return None
|
||||
|
||||
logger.info('Getting revision hash from: {}'.format(lookup_url))
|
||||
|
||||
response = requests.get(lookup_url)
|
||||
response.raise_for_status()
|
||||
|
||||
if not response.json():
|
||||
raise ValueError('Unable to determine revision hash for {}. '
|
||||
'Perhaps it has not been ingested by '
|
||||
'Treeherder?'.format(self.revision))
|
||||
|
||||
return response.json()['results'][0]['revision_hash']
|
||||
|
||||
def submit(self, job):
|
||||
"""Submit the job to treeherder.
|
||||
|
||||
:param job: Treeherder job instance to use for submission.
|
||||
|
||||
"""
|
||||
job.add_submit_timestamp(int(time.time()))
|
||||
|
||||
# We can only submit job info once, so it has to be done in completed
|
||||
if self._job_details:
|
||||
job.add_artifact('Job Info', 'json', {'job_details': self._job_details})
|
||||
|
||||
job_collection = TreeherderJobCollection()
|
||||
job_collection.add(job)
|
||||
|
||||
logger.info('Sending results to Treeherder: {}'.format(job_collection.to_json()))
|
||||
if self.url == 'mock':
|
||||
logger.info('Pretending to submit job')
|
||||
return
|
||||
|
||||
url = urlparse(self.url)
|
||||
client = TreeherderClient(protocol=url.scheme, host=url.hostname,
|
||||
client_id=self.client_id, secret=self.secret)
|
||||
client.post_collection(self.repository, job_collection)
|
||||
|
||||
logger.info('Results are available to view at: {}'.format(
|
||||
urljoin(self.url,
|
||||
JOB_FRAGMENT.format(repository=self.repository,
|
||||
revision=self.revision))))
|
||||
|
||||
def submit_running_job(self, job):
|
||||
"""Submit job as state running.
|
||||
|
||||
:param job: Treeherder job instance to use for submission.
|
||||
|
||||
"""
|
||||
job.add_state('running')
|
||||
self.submit(job)
|
||||
|
||||
def submit_completed_job(self, job, perfdata, state="success"):
|
||||
"""Submit job as state completed.
|
||||
|
||||
:param job: Treeherder job instance to use for submission.
|
||||
:param state: success, testfailed, busted, skipped, exception, retry, usercancel
|
||||
:param uploaded_logs: List of uploaded logs to reference in the job.
|
||||
|
||||
"""
|
||||
job.add_state('completed')
|
||||
job.add_result(state)
|
||||
|
||||
jsondata = json.dumps({'performance_data': perfdata})
|
||||
job.add_artifact('performance_data', 'json', jsondata)
|
||||
|
||||
job.add_end_timestamp(int(time.time()))
|
||||
|
||||
self.submit(job)
|
||||
|
||||
|
||||
def upload_log_files(guid, logs,
|
||||
bucket_name=None, access_key_id=None, access_secret_key=None):
|
||||
"""Upload all specified logs to Amazon S3.
|
||||
|
||||
:param guid: Unique ID which is used as subfolder name for all log files.
|
||||
:param logs: List of log files to upload.
|
||||
:param bucket_name: Name of the S3 bucket.
|
||||
:param access_key_id: Client ID used for authentication.
|
||||
:param access_secret_key: Secret key for authentication.
|
||||
|
||||
"""
|
||||
# If no AWS credentials are given we don't upload anything.
|
||||
if not bucket_name:
|
||||
logger.info('No AWS Bucket name specified - skipping upload of artifacts.')
|
||||
return {}
|
||||
|
||||
s3_bucket = S3Bucket(bucket_name, access_key_id=access_key_id,
|
||||
access_secret_key=access_secret_key)
|
||||
|
||||
uploaded_logs = {}
|
||||
|
||||
for log in logs:
|
||||
try:
|
||||
if os.path.isfile(logs[log]):
|
||||
remote_path = '{dir}/{filename}'.format(dir=str(guid),
|
||||
filename=os.path.basename(log))
|
||||
url = s3_bucket.upload(logs[log], remote_path)
|
||||
|
||||
uploaded_logs.update({log: {'path': logs[log], 'url': url}})
|
||||
logger.info('Uploaded {path} to {url}'.format(path=logs[log], url=url))
|
||||
|
||||
except Exception:
|
||||
logger.exception('Failure uploading "{path}" to S3'.format(path=logs[log]))
|
||||
|
||||
return uploaded_logs
|
||||
|
||||
|
|
@ -0,0 +1,163 @@
|
|||
#!/usr/bin/env python
|
||||
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
from submission import Submission
|
||||
import json
|
||||
import sys
|
||||
|
||||
sys.path.append("../server")
|
||||
import awfy
|
||||
import tables
|
||||
|
||||
def first(gen):
|
||||
return list(gen)[0]
|
||||
|
||||
class Submitter(object):
|
||||
def __init__(self):
|
||||
pass
|
||||
|
||||
"""
|
||||
Submit the given data
|
||||
"""
|
||||
def submit(self, revision, data, mode_info):
|
||||
repo = mode_info["repo"]
|
||||
settings = {
|
||||
"treeherder": {
|
||||
'group_symbol': 'AWFY',
|
||||
'group_name': 'AWFY',
|
||||
'job_name': mode_info["job_name"],
|
||||
'job_symbol': mode_info["job_symbol"],
|
||||
"tier": mode_info["tier"]
|
||||
}
|
||||
}
|
||||
|
||||
th = Submission(repo, revision,
|
||||
treeherder_url = awfy.th_host,
|
||||
treeherder_client_id = awfy.th_user,
|
||||
treeherder_secret = awfy.th_secret,
|
||||
settings = settings)
|
||||
|
||||
job = th.create_job(None)
|
||||
th.submit_completed_job(job, data)
|
||||
|
||||
"""
|
||||
Takes all scores/subscores from a build and submit the data to treeherder.
|
||||
"""
|
||||
def submitBuild(self, build):
|
||||
revision = build.get("cset")
|
||||
machine_id = build.get("run").get("machine_id")
|
||||
mode_symbol = build.get("mode").get("mode")
|
||||
perfdata = []
|
||||
|
||||
mode_info = config.mode_info(machine_id, mode_symbol)
|
||||
if not mode_info:
|
||||
print "Couldn't submit", revision, "with mode", mode_symbol
|
||||
print "No data found in the config file about how to submit it"
|
||||
return
|
||||
|
||||
scores = build.getScores()
|
||||
for score in scores:
|
||||
if not score.get("suite_version") or not score.get("suite_version").exists():
|
||||
continue
|
||||
suite_version = score.get("suite_version")
|
||||
perfdata.append({
|
||||
"name": suite_version.get("name"),
|
||||
"score": score.get("score"),
|
||||
"lowerIsBetter": suite_version.get("suite").get("better_direction") == -1,
|
||||
"subscores": {}
|
||||
})
|
||||
for breakdown in score.getBreakdowns():
|
||||
if not breakdown.get("suite_test") or not breakdown.get("suite_test").exists():
|
||||
continue
|
||||
suite_test = breakdown.get("suite_test")
|
||||
perfdata[-1]["subscores"][suite_test.get("name")] = breakdown.get("score")
|
||||
|
||||
data = self.transform(perfdata)
|
||||
|
||||
self.submit(revision, data, mode_info)
|
||||
|
||||
|
||||
"""
|
||||
Takes all builds from a run and submit the enabled ones to treeherder.
|
||||
"""
|
||||
def submitRun(self, run):
|
||||
# Annonate run that it was forwared to treeherder.
|
||||
run.update({"treeherder": 1})
|
||||
awfy.db.commit()
|
||||
|
||||
# Send the data.
|
||||
modes = config.modes(run.get("machine_id"))
|
||||
for mode in modes:
|
||||
mode = first(tables.Mode.where({"mode": mode}))
|
||||
build = tables.Build.fromRunAndMode(run.id, mode.id)
|
||||
self.submitBuild(build)
|
||||
|
||||
"""
|
||||
transforms the intermediate representation of benchmark results pulled from the DB
|
||||
into the canonical format needed by treeherder/perfherder
|
||||
"""
|
||||
def transform(self, tests):
|
||||
data = {
|
||||
"framework": {
|
||||
"name": "awfy"
|
||||
},
|
||||
"suites": []
|
||||
}
|
||||
for test in tests:
|
||||
testdata = {
|
||||
"name": test["name"],
|
||||
"value": float(test["score"]),
|
||||
"subtests": [],
|
||||
"lowerIsBetter": bool(test["lowerIsBetter"])
|
||||
}
|
||||
if "subscores" not in test:
|
||||
test["subscores"] = []
|
||||
for subtest in test["subscores"]:
|
||||
subtestdata = {
|
||||
"lowerIsBetter": bool(test["lowerIsBetter"]),
|
||||
"name": subtest,
|
||||
"value": float(test["subscores"][subtest])
|
||||
}
|
||||
testdata["subtests"].append(subtestdata)
|
||||
data["suites"].append(testdata)
|
||||
return data
|
||||
|
||||
class Config(object):
|
||||
def __init__(self, filename):
|
||||
fp = open(filename)
|
||||
self.data = json.load(fp)
|
||||
fp.close()
|
||||
|
||||
def modes(self, machine_id):
|
||||
for machine_data in self.data:
|
||||
if str(machine_data["machine"]) != str(machine_id):
|
||||
continue
|
||||
return [mode["mode"] for mode in machine_data["modes"] if mode["enabled"]]
|
||||
return []
|
||||
def mode_info(self, machine_id, mode_symbol):
|
||||
for machine_data in self.data:
|
||||
if str(machine_data["machine"]) != str(machine_id):
|
||||
continue
|
||||
for mode in machine_data["modes"]:
|
||||
if mode["mode"] != mode_symbol:
|
||||
continue
|
||||
return mode
|
||||
return None
|
||||
def validate(self, schema):
|
||||
from jsonschema import validate
|
||||
fp = open(schema)
|
||||
schema = json.load(fp)
|
||||
fp.close()
|
||||
validate(self.data, schema)
|
||||
|
||||
if __name__ == '__main__':
|
||||
config = Config("config.json")
|
||||
config.validate("config.schema")
|
||||
|
||||
submitter = Submitter()
|
||||
for run in tables.Run.where({"status": 1, "treeherder": 0}):
|
||||
submitter.submitRun(run)
|
||||
|
||||
Загрузка…
Ссылка в новой задаче