зеркало из https://github.com/mozilla/gecko-dev.git
Bug 771554 - Removing autolog and having tps write-out results to JSON so that coversheet can handle them, r=jgriffin, DONTBUILD(NPOTB)
This commit is contained in:
Sam Garrett
Родитель
d83c5899ef
Коммит
3bdf0ebcd3
|
|
@ -5,11 +5,11 @@
|
|||
import sys
|
||||
from setuptools import setup, find_packages
|
||||
|
||||
version = '0.3'
|
||||
version = '0.4'
|
||||
|
||||
deps = ['mozinfo >= 0.3.3', 'mozprofile >= 0.4',
|
||||
'mozprocess >= 0.4', 'mozrunner >= 5.8', 'mozinstall >= 1.4',
|
||||
'mozautolog >= 0.2.4', 'mozautoeslib >= 0.1.1', 'httplib2 >= 0.7.3']
|
||||
'httplib2 >= 0.7.3']
|
||||
|
||||
# we only support python 2.6+ right now
|
||||
assert sys.version_info[0] == 2
|
||||
|
|
|
|||
|
|
@ -8,7 +8,6 @@ import optparse
|
|||
import os
|
||||
import sys
|
||||
import time
|
||||
import traceback
|
||||
|
||||
from threading import RLock
|
||||
|
||||
|
|
@ -16,19 +15,10 @@ from tps import TPSFirefoxRunner, TPSTestRunner
|
|||
|
||||
def main():
|
||||
parser = optparse.OptionParser()
|
||||
parser.add_option("--email-results",
|
||||
action = "store_true", dest = "emailresults",
|
||||
default = False,
|
||||
help = "email the test results to the recipients defined "
|
||||
"in the config file")
|
||||
parser.add_option("--mobile",
|
||||
action = "store_true", dest = "mobile",
|
||||
default = False,
|
||||
help = "run with mobile settings")
|
||||
parser.add_option("--autolog",
|
||||
action = "store_true", dest = "autolog",
|
||||
default = False,
|
||||
help = "post results to Autolog")
|
||||
parser.add_option("--testfile",
|
||||
action = "store", type = "string", dest = "testfile",
|
||||
default = '../../services/sync/tests/tps/test_sync.js',
|
||||
|
|
@ -38,6 +28,10 @@ def main():
|
|||
action = "store", type = "string", dest = "logfile",
|
||||
default = 'tps.log',
|
||||
help = "path to the log file [default: %default]")
|
||||
parser.add_option("--resultfile",
|
||||
action = "store", type = "string", dest = "resultfile",
|
||||
default = 'tps_result.json',
|
||||
help = "path to the result file [default: %default]")
|
||||
parser.add_option("--binary",
|
||||
action = "store", type = "string", dest = "binary",
|
||||
default = None,
|
||||
|
|
@ -90,43 +84,13 @@ def main():
|
|||
extensionDir = "%s:/%s" % (m.group(0)[1:2], extensionDir[3:])
|
||||
extensionDir = extensionDir.replace("/", "\\")
|
||||
|
||||
if options.binary is None:
|
||||
while True:
|
||||
try:
|
||||
# If no binary is specified, start the pulse build monitor, and wait
|
||||
# until we receive build notifications before running tests.
|
||||
monitor = TPSPulseMonitor(extensionDir,
|
||||
config=config,
|
||||
autolog=options.autolog,
|
||||
emailresults=options.emailresults,
|
||||
testfile=options.testfile,
|
||||
logfile=options.logfile,
|
||||
rlock=rlock)
|
||||
print "waiting for pulse build notifications"
|
||||
|
||||
if options.pulsefile:
|
||||
# For testing purposes, inject a pulse message directly into
|
||||
# the monitor.
|
||||
builddata = json.loads(open(options.pulsefile, 'r').read())
|
||||
monitor.onBuildComplete(builddata)
|
||||
|
||||
monitor.listen()
|
||||
except KeyboardInterrupt:
|
||||
sys.exit()
|
||||
except:
|
||||
traceback.print_exc()
|
||||
print 'sleeping 5 minutes'
|
||||
time.sleep(300)
|
||||
|
||||
TPS = TPSTestRunner(extensionDir,
|
||||
emailresults=options.emailresults,
|
||||
testfile=options.testfile,
|
||||
logfile=options.logfile,
|
||||
binary=options.binary,
|
||||
config=config,
|
||||
rlock=rlock,
|
||||
mobile=options.mobile,
|
||||
autolog=options.autolog,
|
||||
ignore_unused_engines=options.ignore_unused_engines)
|
||||
TPS.run_tests()
|
||||
|
||||
|
|
|
|||
|
|
@ -1,158 +0,0 @@
|
|||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
import datetime
|
||||
|
||||
def GenerateEmailBody(data, numpassed, numfailed, serverUrl, buildUrl):
|
||||
|
||||
now = datetime.datetime.now()
|
||||
builddate = datetime.datetime.strptime(data['productversion']['buildid'],
|
||||
'%Y%m%d%H%M%S')
|
||||
tree = data['productversion']['repository']
|
||||
|
||||
row = """
|
||||
<tr>
|
||||
<td><a href="http://hg.mozilla.org/services/services-central/file/default/services/sync/tests/tps/{name}">{name}</a></td>
|
||||
<td>{state}</td>
|
||||
<td>{message}</td>
|
||||
</tr>
|
||||
"""
|
||||
|
||||
rowWithLog = """
|
||||
<tr>
|
||||
<td><a href="http://hg.mozilla.org/services/services-central/file/default/services/sync/tests/tps/{name}">{name}</a></td>
|
||||
<td>{state}</td>
|
||||
<td>{message} [<a href="{logurl}">view log</a>]</td>
|
||||
</tr>
|
||||
"""
|
||||
|
||||
rows = ""
|
||||
for test in data['tests']:
|
||||
if test.get('logurl'):
|
||||
rows += rowWithLog.format(name=test['name'],
|
||||
state=test['state'],
|
||||
message=test['message'] if test['message'] else 'None',
|
||||
logurl=test['logurl'])
|
||||
else:
|
||||
rows += row.format(name=test['name'],
|
||||
state=test['state'],
|
||||
message=test['message'] if test['message'] else 'None')
|
||||
|
||||
firefox_version = data['productversion']['version']
|
||||
if buildUrl is not None:
|
||||
firefox_version = "<a href='%s'>%s</a>" % (buildUrl, firefox_version)
|
||||
body = """
|
||||
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
|
||||
<head>
|
||||
<title>TPS</title>
|
||||
<style type="text/css">
|
||||
#headertable {{ border: solid 1px black; margin-bottom: 2em; border-collapse: collapse; font-size: 0.8em; }}
|
||||
#headertable th {{ border: solid 1px black; background-color: lightgray; padding: 4px; }}
|
||||
#headertable td {{ border: solid 1px black; padding: 4px; }}
|
||||
.light {{ color: gray; }}
|
||||
.pass, a.pass:link, a.pass:visited {{ color: green; font-weight: bold; }}
|
||||
.fail, a.fail:link, a.fail:visited {{ color: red; font-weight: bold; }}
|
||||
.rightgray {{ text-align: right; background-color: lightgray; }}
|
||||
#summarytable {{ border: solid 1px black; margin-bottom: 2em; border-collapse: collapse; font-size: 0.8em; }}
|
||||
#summarytable th {{ border: solid 1px black; background-color: lightgray; padding: 4px; }}
|
||||
#summarytable td {{ border: solid 1px black; padding: 4px; }}
|
||||
</style>
|
||||
</head>
|
||||
|
||||
<body>
|
||||
<div id="content">
|
||||
|
||||
<h2>TPS Testrun Details</h2>
|
||||
|
||||
<table id="headertable">
|
||||
|
||||
<tr>
|
||||
<td class="rightgray">Testrun Date</td>
|
||||
<td>{date}</td>
|
||||
|
||||
</tr>
|
||||
<tr>
|
||||
<td class="rightgray">Firefox Version</td>
|
||||
<td>{firefox_version}</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td class="rightgray">Firefox Build Date</td>
|
||||
<td>{firefox_date}</td>
|
||||
</tr>
|
||||
|
||||
<tr>
|
||||
<td class="rightgray">Firefox Sync Version / Type</td>
|
||||
<td>{sync_version} / {sync_type}
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td class="rightgray">Firefox Sync Changeset</td>
|
||||
<td>
|
||||
|
||||
<a href="{repository}/rev/{changeset}">
|
||||
|
||||
{changeset}</a> / {sync_tree}
|
||||
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td class="rightgray">Sync Server</td>
|
||||
<td>{server}</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td class="rightgray">OS</td>
|
||||
<td>{os}</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td class="rightgray">Passed Tests</td>
|
||||
|
||||
<td>
|
||||
<span class="{passclass}">{numpassed}</span>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td class="rightgray">Failed Tests</td>
|
||||
<td>
|
||||
|
||||
<span class="{failclass}">{numfailed}</span>
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
<table id="summarytable">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Testcase</th>
|
||||
<th>Result</th>
|
||||
<th>Message</th>
|
||||
</tr>
|
||||
</thead>
|
||||
|
||||
{rows}
|
||||
|
||||
</table>
|
||||
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
|
||||
""".format(date=now.ctime(),
|
||||
firefox_version=firefox_version,
|
||||
firefox_date=builddate.ctime(),
|
||||
sync_version=data['addonversion']['version'],
|
||||
sync_type=data['synctype'],
|
||||
sync_tree=tree[tree.rfind("/") + 1:],
|
||||
repository=data['productversion']['repository'],
|
||||
changeset=data['productversion']['changeset'],
|
||||
os=data['os'],
|
||||
rows=rows,
|
||||
numpassed=numpassed,
|
||||
numfailed=numfailed,
|
||||
passclass="pass" if numpassed > 0 else "light",
|
||||
failclass="fail" if numfailed > 0 else "light",
|
||||
server=serverUrl if serverUrl != "" else "default"
|
||||
)
|
||||
|
||||
return body
|
||||
|
|
@ -1,50 +0,0 @@
|
|||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
import smtplib
|
||||
from email.mime.multipart import MIMEMultipart
|
||||
from email.mime.text import MIMEText
|
||||
|
||||
def SendEmail(From=None, To=None, Subject='No Subject',
|
||||
TextData=None, HtmlData=None,
|
||||
Server='mail.mozilla.com', Port=465,
|
||||
Username=None, Password=None):
|
||||
"""Sends an e-mail.
|
||||
|
||||
From is an e-mail address, To is a list of e-mail adresses.
|
||||
|
||||
TextData and HtmlData are both strings. You can specify one or both.
|
||||
If you specify both, the e-mail will be sent as a MIME multipart
|
||||
alternative; i.e., the recipient will see the HTML content if his
|
||||
viewer supports it, otherwise he'll see the text content.
|
||||
"""
|
||||
|
||||
if From is None or To is None:
|
||||
raise Exception("Both From and To must be specified")
|
||||
if TextData is None and HtmlData is None:
|
||||
raise Exception("Must specify either TextData or HtmlData")
|
||||
|
||||
server = smtplib.SMTP_SSL(Server, Port)
|
||||
|
||||
if Username is not None and Password is not None:
|
||||
server.login(Username, Password)
|
||||
|
||||
if HtmlData is None:
|
||||
msg = MIMEText(TextData)
|
||||
elif TextData is None:
|
||||
msg = MIMEMultipart()
|
||||
msg.preamble = Subject
|
||||
msg.attach(MIMEText(HtmlData, 'html'))
|
||||
else:
|
||||
msg = MIMEMultipart('alternative')
|
||||
msg.attach(MIMEText(TextData, 'plain'))
|
||||
msg.attach(MIMEText(HtmlData, 'html'))
|
||||
|
||||
msg['Subject'] = Subject
|
||||
msg['From'] = From
|
||||
msg['To'] = ', '.join(To)
|
||||
|
||||
server.sendmail(From, To, msg.as_string())
|
||||
|
||||
server.quit()
|
||||
|
|
@ -46,7 +46,6 @@ class TempFile(object):
|
|||
|
||||
__del__ = cleanup
|
||||
|
||||
|
||||
class TPSTestRunner(object):
|
||||
|
||||
default_env = { 'MOZ_CRASHREPORTER_DISABLE': '1',
|
||||
|
|
@ -86,14 +85,15 @@ class TPSTestRunner(object):
|
|||
ffDateRe = re.compile(
|
||||
r"Firefox builddate: (?P<ffdate>.*)\n")
|
||||
|
||||
def __init__(self, extensionDir, emailresults=False, testfile="sync.test",
|
||||
def __init__(self, extensionDir,
|
||||
testfile="sync.test",
|
||||
binary=None, config=None, rlock=None, mobile=False,
|
||||
autolog=False, logfile="tps.log",
|
||||
logfile="tps.log", resultfile="tps_result.json",
|
||||
ignore_unused_engines=False):
|
||||
self.extensions = []
|
||||
self.emailresults = emailresults
|
||||
self.testfile = testfile
|
||||
self.logfile = os.path.abspath(logfile)
|
||||
self.resultfile = resultfile
|
||||
self.binary = binary
|
||||
self.ignore_unused_engines = ignore_unused_engines
|
||||
self.config = config if config else {}
|
||||
|
|
@ -105,7 +105,6 @@ class TPSTestRunner(object):
|
|||
self.nightly = False
|
||||
self.rlock = rlock
|
||||
self.mobile = mobile
|
||||
self.autolog = autolog
|
||||
self.tpsxpi = None
|
||||
self.firefoxRunner = None
|
||||
self.extensionDir = extensionDir
|
||||
|
|
@ -132,6 +131,25 @@ class TPSTestRunner(object):
|
|||
if printToConsole:
|
||||
print msg
|
||||
|
||||
def writeToResultFile(self, postdata, body=None,
|
||||
sendTo='crossweave@mozilla.com'):
|
||||
"""Writes results to test file"""
|
||||
f = open(self.resultfile, 'a')
|
||||
if body is not None:
|
||||
postdata['body'] = body
|
||||
if self.numpassed is not None:
|
||||
postdata['numpassed'] = self.numpassed
|
||||
if self.numfailed is not None:
|
||||
postdata['numfailed'] = self.numfailed
|
||||
if self.firefoxRunner and self.firefoxRunner.url:
|
||||
postdata['firefoxrunnerurl'] = self.firefoxRunner.url
|
||||
|
||||
postdata['sendTo'] = sendTo
|
||||
results = {}
|
||||
results['results'] = postdata
|
||||
f.write(json.dumps(results, indent=2))
|
||||
f.close()
|
||||
|
||||
def _zip_add_file(self, zip, file, rootDir):
|
||||
zip.write(os.path.join(rootDir, file), file)
|
||||
|
||||
|
|
@ -314,26 +332,19 @@ class TPSTestRunner(object):
|
|||
traceback.print_exc()
|
||||
self.numpassed = 0
|
||||
self.numfailed = 1
|
||||
if self.emailresults:
|
||||
try:
|
||||
self.sendEmail('<pre>%s</pre>' % traceback.format_exc(),
|
||||
sendTo='crossweave@mozilla.com')
|
||||
except:
|
||||
traceback.print_exc()
|
||||
else:
|
||||
raise
|
||||
|
||||
try:
|
||||
self.writeToResultFile(self.postdata,
|
||||
'<pre>%s</pre>' % traceback.format_exc())
|
||||
except:
|
||||
traceback.print_exc()
|
||||
else:
|
||||
try:
|
||||
if self.autolog:
|
||||
self.postToAutolog()
|
||||
if self.emailresults:
|
||||
self.sendEmail()
|
||||
self.writeToResultFile(self.postdata)
|
||||
except:
|
||||
traceback.print_exc()
|
||||
try:
|
||||
self.sendEmail('<pre>%s</pre>' % traceback.format_exc(),
|
||||
sendTo='crossweave@mozilla.com')
|
||||
self.writeToResultFile(self.postdata,
|
||||
'<pre>%s</pre>' % traceback.format_exc())
|
||||
except:
|
||||
traceback.print_exc()
|
||||
|
||||
|
|
@ -409,100 +420,3 @@ class TPSTestRunner(object):
|
|||
'addonversion': self.addonversion,
|
||||
'synctype': self.synctype,
|
||||
}
|
||||
|
||||
def sendEmail(self, body=None, sendTo=None):
|
||||
# send the result e-mail
|
||||
if self.config.get('email') and self.config['email'].get('username') \
|
||||
and self.config['email'].get('password'):
|
||||
|
||||
from tps.sendemail import SendEmail
|
||||
from tps.emailtemplate import GenerateEmailBody
|
||||
|
||||
if body is None:
|
||||
buildUrl = None
|
||||
if self.firefoxRunner and self.firefoxRunner.url:
|
||||
buildUrl = self.firefoxRunner.url
|
||||
body = GenerateEmailBody(self.postdata,
|
||||
self.numpassed,
|
||||
self.numfailed,
|
||||
self.config['account']['serverURL'],
|
||||
buildUrl)
|
||||
|
||||
subj = "TPS Report: "
|
||||
if self.numfailed == 0 and self.numpassed > 0:
|
||||
subj += "YEEEAAAHHH"
|
||||
else:
|
||||
subj += "PC LOAD LETTER"
|
||||
|
||||
changeset = self.postdata['productversion']['changeset'] if \
|
||||
self.postdata and self.postdata.get('productversion') and \
|
||||
self.postdata['productversion'].get('changeset') \
|
||||
else 'unknown'
|
||||
subj +=", changeset " + changeset + "; " + str(self.numfailed) + \
|
||||
" failed, " + str(self.numpassed) + " passed"
|
||||
|
||||
To = [sendTo] if sendTo else None
|
||||
if not To:
|
||||
if self.numfailed > 0 or self.numpassed == 0:
|
||||
To = self.config['email'].get('notificationlist')
|
||||
else:
|
||||
To = self.config['email'].get('passednotificationlist')
|
||||
|
||||
if To:
|
||||
SendEmail(From=self.config['email']['username'],
|
||||
To=To,
|
||||
Subject=subj,
|
||||
HtmlData=body,
|
||||
Username=self.config['email']['username'],
|
||||
Password=self.config['email']['password'])
|
||||
|
||||
def postToAutolog(self):
|
||||
from mozautolog import RESTfulAutologTestGroup as AutologTestGroup
|
||||
|
||||
group = AutologTestGroup(
|
||||
harness='crossweave',
|
||||
testgroup='crossweave-%s' % self.synctype,
|
||||
server=self.config.get('es'),
|
||||
restserver=self.config.get('restserver'),
|
||||
machine=socket.gethostname(),
|
||||
platform=self.config.get('platform', None),
|
||||
os=self.config.get('os', None),
|
||||
)
|
||||
tree = self.postdata['productversion']['repository']
|
||||
group.set_primary_product(
|
||||
tree=tree[tree.rfind("/")+1:],
|
||||
version=self.postdata['productversion']['version'],
|
||||
buildid=self.postdata['productversion']['buildid'],
|
||||
buildtype='opt',
|
||||
revision=self.postdata['productversion']['changeset'],
|
||||
)
|
||||
group.add_test_suite(
|
||||
passed=self.numpassed,
|
||||
failed=self.numfailed,
|
||||
todo=0,
|
||||
)
|
||||
for test in self.results:
|
||||
if test['state'] != "TEST-PASS":
|
||||
errorlog = self.errorlogs.get(test['name'])
|
||||
errorlog_filename = errorlog.filename if errorlog else None
|
||||
group.add_test_failure(
|
||||
test = test['name'],
|
||||
status = test['state'],
|
||||
text = test['message'],
|
||||
logfile = errorlog_filename
|
||||
)
|
||||
try:
|
||||
group.submit()
|
||||
except:
|
||||
self.sendEmail('<pre>%s</pre>' % traceback.format_exc(),
|
||||
sendTo='crossweave@mozilla.com')
|
||||
return
|
||||
|
||||
# Iterate through all testfailure objects, and update the postdata
|
||||
# dict with the testfailure logurl's, if any.
|
||||
for tf in group.testsuites[-1].testfailures:
|
||||
result = [x for x in self.results if x.get('name') == tf.test]
|
||||
if not result:
|
||||
continue
|
||||
result[0]['logurl'] = tf.logurl
|
||||
|
||||
|
|
|
|||
|
|
@ -8,32 +8,27 @@ from testrunner import TPSTestRunner
|
|||
|
||||
class TPSTestThread(Thread):
|
||||
|
||||
def __init__(self, extensionDir, builddata=None, emailresults=False,
|
||||
testfile=None, logfile=None, rlock=None, config=None,
|
||||
autolog=False):
|
||||
def __init__(self, extensionDir, builddata=None,
|
||||
testfile=None, logfile=None, rlock=None, config=None):
|
||||
assert(builddata)
|
||||
assert(config)
|
||||
self.extensionDir = extensionDir
|
||||
self.builddata = builddata
|
||||
self.emailresults = emailresults
|
||||
self.testfile = testfile
|
||||
self.logfile = logfile
|
||||
self.rlock = rlock
|
||||
self.config = config
|
||||
self.autolog = autolog
|
||||
Thread.__init__(self)
|
||||
|
||||
def run(self):
|
||||
# run the tests in normal mode ...
|
||||
TPS = TPSTestRunner(self.extensionDir,
|
||||
emailresults=self.emailresults,
|
||||
testfile=self.testfile,
|
||||
logfile=self.logfile,
|
||||
binary=self.builddata['buildurl'],
|
||||
config=self.config,
|
||||
rlock=self.rlock,
|
||||
mobile=False,
|
||||
autolog=self.autolog)
|
||||
mobile=False)
|
||||
TPS.run_tests()
|
||||
|
||||
# Get the binary used by this TPS instance, and use it in subsequent
|
||||
|
|
@ -42,14 +37,12 @@ class TPSTestThread(Thread):
|
|||
|
||||
# ... and then again in mobile mode
|
||||
TPS_mobile = TPSTestRunner(self.extensionDir,
|
||||
emailresults=self.emailresults,
|
||||
testfile=self.testfile,
|
||||
logfile=self.logfile,
|
||||
binary=binary,
|
||||
config=self.config,
|
||||
rlock=self.rlock,
|
||||
mobile=True,
|
||||
autolog=self.autolog)
|
||||
mobile=True)
|
||||
TPS_mobile.run_tests()
|
||||
|
||||
# ... and again via the staging server, if credentials are present
|
||||
|
|
@ -62,12 +55,10 @@ class TPSTestThread(Thread):
|
|||
stageconfig = self.config.copy()
|
||||
stageconfig['account'] = stageaccount.copy()
|
||||
TPS_stage = TPSTestRunner(self.extensionDir,
|
||||
emailresults=self.emailresults,
|
||||
testfile=self.testfile,
|
||||
logfile=self.logfile,
|
||||
binary=binary,
|
||||
config=stageconfig,
|
||||
rlock=self.rlock,
|
||||
mobile=False,
|
||||
autolog=self.autolog)
|
||||
mobile=False)#, autolog=self.autolog)
|
||||
TPS_stage.run_tests()
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче