diff --git a/tools/buildbot-configs/testing/talos/tryperfmaster/configs/sample.config b/tools/buildbot-configs/testing/talos/tryperfmaster/configs/sample.config new file mode 100644 index 00000000000..32c90e52bd2 --- /dev/null +++ b/tools/buildbot-configs/testing/talos/tryperfmaster/configs/sample.config @@ -0,0 +1,115 @@ +# Sample Talos configuration file + +# The title of the report +title: firefox_testing + +#csv_file: 'out.csv' +results_server: 'graphs-stage.mozilla.org' +results_link: '/bulk.cgi' + +# Path to Firefox to test +firefox: firefox/firefox + +branch: testbranch + +buildid: testbuildid + +profile_path: base_profile + +init_url: getInfo.html + +# Preferences to set in the test (use "preferences : {}" for no prefs) +preferences : + browser.shell.checkDefaultBrowser : false + browser.warnOnQuit : false + browser.link.open_newwindow : 2 + dom.allow_scripts_to_close_windows : true + dom.disable_open_during_load: false + dom.max_script_run_time : 0 + browser.dom.window.dump.enabled: true + network.proxy.type : 1 + network.proxy.http : localhost + network.proxy.http_port : 80 + network.proxy.share_proxy_settings : true + dom.disable_window_flip : true + dom.disable_window_move_resize : true + security.enable_java : false + extensions.checkCompatibility : false + extensions.update.notifyUser: false + +# Extensions to install in test (use "extensions: {}" for none) +# Need quotes around guid because of curly braces +# extensions : +# "{12345678-1234-1234-1234-abcd12345678}" : c:\path\to\unzipped\xpi +# foo@sample.com : c:\path\to\other\unzipped\xpi +extensions : {} + +#any directories whose contents need to be installed in the browser before running the tests +# this assumes that the directories themselves already exist in the firefox path +dirs: + chrome : page_load_test/chrome + components : page_load_test/components + +# Environment variables to set during test (use env: {} for none) +env : + NO_EM_RESTART : 1 +# Tests to run +# url : (REQUIRED) url to load into the given firefox browser +# url_mod : (OPTIONAL) a bit of code to be evaled and added to the given url during each cycle of the test +# resolution: (REQUIRED) how long (in seconds) to pause between counter sampling +# cycles : (REQUIRED) how many times to run the test +# counters : (REQUIRED) types of system activity to monitor during test run, an be empty +# For possible values of counters argument on Windows, see +# http://technet2.microsoft.com/WindowsServer/en/Library/86b5d116-6fb3-427b-af8c-9077162125fe1033.mspx?mfr=true +# Possible values on Linux and Mac: +# counters : ['Private Bytes', 'RSS'] +# Standard windows values: +# counters : ['Working Set', 'Private Bytes', '% Processor Time'] + +# to set up a new test it must have the correct configuration options and drop information in a standard format +# the format is seen in the regular expressions in ttest.py +# to see how the data passed from the browser is processed see send_to_graph an send_to_csv in run_tests.py +tests : + ts : + url : startup_test/startup_test.html?begin= + url_mod : str(int(time.time()*1000)) + resolution : 1 + cycles : 20 + win_counters : [] + unix_counters : [] + tp: + url : '-tp page_load_test/manifest.txt -tpchrome -tpformat tinderbox -tpcycles 10 -tptimeout 120000' + resolution : 1 + cycles : 1 + win_counters : ['Working Set', 'Private Bytes', '% Processor Time'] + unix_counters : ['RSS', 'Private Bytes'] +# tp_js: +# url : '"http://localhost/page_load_test/framecycler.html?quit=1&cycles=5"' +# resolution : 1 +# cycles : 1 +# win_counters : ['Working Set', 'Private Bytes', '% Processor Time'] +# unix_counters: ['RSS', 'Private Bytes'] +# tdhtml: +# url: '-tp page_load_test/dhtml/dhtml.manifest -tpchrome -tpformat tinderbox -tpcycles 5' +# resolution : 1 +# cycles : 1 +# win_counters : [] +# unix_counters : [] +# tgfx: +# url: '-tp page_load_test/gfx/gfx.manifest -tpchrome -tpformat tinderbox -tpcycles 5 -tprender' +# resolution : 1 +# cycles : 1 +# win_counters : [] +# unix_counters : [] +# tsvg: +# url: '-tp page_load_test/svg/svg.manifest -tpchrome -tpformat tinderbox -tpcycles 5' +# resolution : 1 +# cycles : 1 +# win_counters : [] +# unix_counters : [] +# twinopen: +# url: startup_test/twinopen/winopen.xul?phase1=20 +# resolution: 1 +# cycles: 1 +# win_counters: [] +# unix_counters: [] diff --git a/tools/buildbot-configs/testing/talos/tryperfmaster/master.cfg b/tools/buildbot-configs/testing/talos/tryperfmaster/master.cfg new file mode 100644 index 00000000000..1fbbe8a6475 --- /dev/null +++ b/tools/buildbot-configs/testing/talos/tryperfmaster/master.cfg @@ -0,0 +1,361 @@ +# This is a sample buildmaster config file. It must be installed as +# 'master.cfg' in your buildmaster's base directory (although the filename +# can be changed with the --basedir option to 'mktap buildbot master'). + +# It has one job: define a dictionary named BuildmasterConfig. This +# dictionary has a variety of keys to control different aspects of the +# buildmaster. They are documented in docs/config.xhtml . + +import os.path +# from buildbot.changes.freshcvs import FreshCVSSource +from buildbot.process import factory +from buildbot.scheduler import Scheduler, Periodic +from buildbot.status import html +from buildbot import locks +from buildbot.steps.transfer import FileDownload +from buildbot.steps.shell import ShellCommand + +# from auth import authlist, debugPassword + +import perfrunner +reload(perfrunner) +from perfrunner import * + +### +### Tinderbox builder names and build directories +### +WIN32_TRUNK_BUILDER="Try server win32 builder" +LINUX_TRUNK_BUILDER="Try server linux builder" +MAC_TRUNK_BUILDER="Try server mac builder" +TRUNK_BUILDDIR="https://build.mozilla.org/tryserver-builds/" + +CVSROOT=":pserver:anonymous@cvs-mirror.mozilla.org:/cvsroot" + + +# This is the dictionary that the buildmaster pays attention to. We also use +# a shorter alias to save typing. +c = BuildmasterConfig = {} + +## +## Misc Config +## + +c['debugPassword'] = "mozilla" +#c['manhole'] = Manhole(9999, "admin", "password") +c['projectName'] = "Talos" +c['projectURL'] = "http://quality.mozilla.org/en/projects/automation/talos" +c['buildbotURL'] = "http://qm-rhel02.mozilla.org:2007" +c['slavePortnum'] = 9985 + +## +## Slaves +## + +c['bots'] = [("qm-pxp-try01", "w1nd3rs"), + ("qm-ptiger-try01", "mac1nt0sh"), + ("qm-pubuntu-try01", "l1nux")] + +## +## Status +## + +c['status'] = [] +c['status'].append(html.Waterfall(http_port=2007, + css="/build/tryperfmaster/waterfall.css")) + +from buildbot.status.tinderbox import TinderboxMailNotifier +from buildbot.status.mail import MailNotifier + +c['status'].append(MailNotifier( + fromaddr="talos@qa.mozilla", + sendToInterestedUsers=False, + extraRecipients=["anodelman@mozilla.com", + "rcampbell@mozilla.com"], + mode="failing", + builders=None, # send for all builders + relayhost="smtp.mozilla.org")) + +# this is the system installed TinderboxMailNotifier +c['status'].append(TinderboxMailNotifier( + fromaddr="rcampbell@mozilla.com", + tree="MozillaTry", # move to MozillaTry + extraRecipients=["tinderbox-daemon@tinderbox.mozilla.org",], + relayhost="smtp.mozilla.org", + builders=["WINNT 5.1 talos try trunk", + "MacOSX Darwin 8.8.1 talos try trunk", + "Linux talos try trunk"], + logCompression="bzip2")) + + + +## +## Sources +## + +from tinderboxpoller import TinderboxPoller + +c['sources'] = [] + +c['sources'].append(TinderboxPoller( + tinderboxURL = "http://tinderbox.mozilla.org", + tree = "MozillaTry", + branch = "win32", + machine = WIN32_TRUNK_BUILDER, + pollInterval = 10 * 60) +) + +c['sources'].append(TinderboxPoller( + tinderboxURL = "http://tinderbox.mozilla.org", + tree = "MozillaTry", + branch = "linux", + machine = LINUX_TRUNK_BUILDER, + pollInterval = 10 * 60) +) + +c['sources'].append(TinderboxPoller( + tinderboxURL = "http://tinderbox.mozilla.org", + tree = "MozillaTry", + branch = "mac", + machine = MAC_TRUNK_BUILDER, + pollInterval = 10 * 60) +) + +## +## Schedulers +## + +c['schedulers'] = [] + +c['schedulers'].append(Scheduler(name="WinXP try perfrun scheduler", + branch="win32", + treeStableTimer=5*60, + builderNames=["WINNT 5.1 talos try trunk"])) + +c['schedulers'].append(Scheduler(name="Mac OS X try perfrun scheduler", + branch="mac", + treeStableTimer=5*60, + builderNames=["MacOSX Darwin 8.8.1 talos try trunk"])) + +c['schedulers'].append(Scheduler(name="Linux try perfrun scheduler", + branch="linux", + treeStableTimer=5*60, + builderNames=["Linux talos try trunk"])) + +# the 'builders' list defines the Builders. Each one is configured with a +# dictionary, using the following keys: +# name (required): the name used to describe this bilder +# slavename (required): which slave to use, must appear in c['bots'] +# builddir (required): which subdirectory to run the builder in +# factory (required): a BuildFactory to define how the build is run +# periodicBuildTime (optional): if set, force a build every N seconds + +win32_trunk_steps = factory.BuildFactory() +win32_trunk_steps.addStep(MozillaChangePusher) +win32_trunk_steps.addStep(ShellCommand, + workdir=".", + description="Cleanup", + command=["rm", "-rf", "*.zip", "talos/", "firefox/"], + env=MozillaEnvironments['vc8perf']) +win32_trunk_steps.addStep(ShellCommand, + command=["cvs", "-d", CVSROOT, "co", "-d", "talos", + "mozilla/testing/performance/talos"], + workdir=".", + description="checking out talos", + haltOnFailure=True, + flunkOnFailure=True, + env=MozillaEnvironments['vc8perf']) +win32_trunk_steps.addStep(FileDownload, + mastersrc="scripts/generate-tpcomponent.py", + slavedest="generate-tpcomponent.py", + workdir="talos/page_load_test") +win32_trunk_steps.addStep(FileDownload, + mastersrc="configs/sample.config", + slavedest="sample.config", + workdir="talos/") +win32_trunk_steps.addStep(ShellCommand, + command=["python", "generate-tpcomponent.py"], + workdir="talos/page_load_test", + description="setting up pageloader", + haltOnFailure=True, + flunkOnFailure=True, + env=MozillaEnvironments['vc8perf']) +win32_trunk_steps.addStep(MozillaTryServerWgetLatest, + workdir=".", + branch="1.9", + url=TRUNK_BUILDDIR, + filenameSearchString="win32.zip", + env=MozillaEnvironments['vc8perf']) +win32_trunk_steps.addStep(MozillaInstallZip, + workdir=".", + branch="1.9", + haltOnFailure=True, + env=MozillaEnvironments['vc8perf']) +# TODO: remove this when msys bug is fixed +win32_trunk_steps.addStep(ShellCommand, + workdir="firefox/", + description="chmod files (see msys bug)", + command=["chmod", "-v", "-R", "a+x", "."], + env=MozillaEnvironments['vc8perf']) +win32_trunk_steps.addStep(MozillaUpdateConfig, + workdir="talos/", + branch="1.9", + haltOnFailure=True, + executablePath=r"C:\talos-slave\win32-trunk-mini\firefox\firefox.exe", + configPath=".", + env=MozillaEnvironments['vc8perf']) +win32_trunk_steps.addStep(MozillaRunPerfTests, + workdir="talos/", + branch="1.9", + timeout=14400, + haltOnFailure=True, + command=['python', 'run_tests.py', '--noisy'], + env=MozillaEnvironments['vc8perf']) + +linux_trunk_steps = factory.BuildFactory() +linux_trunk_steps.addStep(MozillaChangePusher) +linux_trunk_steps.addStep(ShellCommand, + workdir=".", + description="Cleanup", + command=["rm", "-rf", "*.bz2", "talos/", "firefox/"], + env=MozillaEnvironments['linux']) +linux_trunk_steps.addStep(ShellCommand, + command=["cvs", "-d", CVSROOT, "co", "-d", "talos", + "mozilla/testing/performance/talos"], + workdir=".", + description="checking out talos", + haltOnFailure=True, + flunkOnFailure=True, + env=MozillaEnvironments['linux']) +linux_trunk_steps.addStep(FileDownload, + mastersrc="scripts/generate-tpcomponent.py", + slavedest="generate-tpcomponent.py", + workdir="talos/page_load_test") +linux_trunk_steps.addStep(FileDownload, + mastersrc="configs/sample.config", + slavedest="sample.config", + workdir="talos/") +linux_trunk_steps.addStep(ShellCommand, + command=["python", "generate-tpcomponent.py"], + workdir="talos/page_load_test", + description="setting up pageloader", + haltOnFailure=True, + flunkOnFailure=True, + env=MozillaEnvironments['linux']) +linux_trunk_steps.addStep(MozillaTryServerWgetLatest, + workdir=".", + branch="1.9", + url=TRUNK_BUILDDIR, + filenameSearchString="try-linux.tar.bz2", + env=MozillaEnvironments['linux']) +linux_trunk_steps.addStep(MozillaInstallTarBz2, + workdir=".", + branch="LINUX-PL", + haltOnFailure=True, + env=MozillaEnvironments['linux']) +linux_trunk_steps.addStep(MozillaUpdateConfig, + workdir="talos/", + branch="1.9", + haltOnFailure=True, + # TODO: when talos supports relative paths on win32 + # make this a relative path + executablePath="../firefox/firefox", + configPath=".", + env=MozillaEnvironments['linux']) +linux_trunk_steps.addStep(MozillaRunPerfTests, + workdir="talos/", + branch="LINUX-PL", + timeout=21600, + haltOnFailure=True, + command=['python', 'run_tests.py', '--noisy'], + env=MozillaEnvironments['linux']) + + +mac_trunk_steps = factory.BuildFactory() +mac_trunk_steps.addStep(MozillaChangePusher) +mac_trunk_steps.addStep(ShellCommand, + workdir=".", + description="Cleanup", + command="rm -vrf *", + env=MozillaEnvironments['mac']) +mac_trunk_steps.addStep(ShellCommand, + command=["cvs", "-d", CVSROOT, "co", "-d", "talos", + "mozilla/testing/performance/talos"], + workdir=".", + description="checking out talos", + haltOnFailure=True, + flunkOnFailure=True, + env=MozillaEnvironments['mac']) +mac_trunk_steps.addStep(FileDownload, + mastersrc="scripts/generate-tpcomponent.py", + slavedest="generate-tpcomponent.py", + workdir="talos/page_load_test") +mac_trunk_steps.addStep(FileDownload, + mastersrc="scripts/installdmg.sh", + slavedest="installdmg.sh", + workdir=".") +mac_trunk_steps.addStep(FileDownload, + mastersrc="configs/sample.config", + slavedest="sample.config", + workdir="talos/") +mac_trunk_steps.addStep(ShellCommand, + command=["python", "generate-tpcomponent.py"], + workdir="talos/page_load_test", + description="setting up pageloader", + haltOnFailure=True, + flunkOnFailure=True, + env=MozillaEnvironments['mac']) +mac_trunk_steps.addStep(MozillaTryServerWgetLatest, + workdir=".", + branch="1.9", + url=TRUNK_BUILDDIR, + filenameSearchString="try-mac.dmg", + env=MozillaEnvironments['mac']) +mac_trunk_steps.addStep(MozillaInstallDmg, + workdir=".", + branch="MAC-PL", + haltOnFailure=True, + env=MozillaEnvironments['mac']) +mac_trunk_steps.addStep(MozillaUpdateConfig, + workdir="talos/", + branch="1.9", + haltOnFailure=True, + executablePath="../Minefield.app/Contents/MacOS/firefox", + configPath=".", + env=MozillaEnvironments['mac']) +mac_trunk_steps.addStep(MozillaRunPerfTests, + workdir="talos/", + branch="MAC-PL", + timeout=21600, + haltOnFailure=True, + command=['python', 'run_tests.py', '--noisy'], + env=MozillaEnvironments['mac']) + +win32_trunk_builder = { + 'name': "WINNT 5.1 talos try trunk", + 'slavenames': ['qm-pxp-try01'], + 'builddir': "win32-trunk-mini", + 'factory': win32_trunk_steps, + 'category': "Firefox Trunk" +} + +linux_trunk_builder = { + 'name': "Linux talos try trunk", + 'slavenames': ['qm-pubuntu-try01'], + 'builddir': "linux-try-trunk", + 'factory': linux_trunk_steps, + 'category': "Firefox Trunk" +} + +mac_trunk_builder = { + 'name': "MacOSX Darwin 8.8.1 talos try trunk", + 'slavenames': ['qm-ptiger-try01'], + 'builddir': "mac-trunk", + 'factory': mac_trunk_steps, + 'category': "Firefox Trunk" +} + +c['builders'] = [] + +c['builders'].append(win32_trunk_builder) +c['builders'].append(linux_trunk_builder) +c['builders'].append(mac_trunk_builder) diff --git a/tools/buildbot-configs/testing/talos/tryperfmaster/perfrunner.py b/tools/buildbot-configs/testing/talos/tryperfmaster/perfrunner.py new file mode 100644 index 00000000000..bd0be021aec --- /dev/null +++ b/tools/buildbot-configs/testing/talos/tryperfmaster/perfrunner.py @@ -0,0 +1,518 @@ +# -*- Python -*- + +from buildbot.process.buildstep import BuildStep +from buildbot.buildset import BuildSet +from buildbot.sourcestamp import SourceStamp +from buildbot.steps.shell import ShellCommand +from buildbot.status.builder import SUCCESS, WARNINGS, FAILURE, SKIPPED, EXCEPTION + +import re, urllib, sys, os +from time import mktime, strptime, strftime, localtime +from datetime import datetime +from os import path +import copy + +MozillaEnvironments = { } + +# platform SDK location. we can build both from one generic template. +# modified from vc8 environment +MozillaEnvironments['vc8perf'] = { + "MOZ_CRASHREPORTER_NO_REPORT": '1', + "MOZ_NO_REMOTE": '1', + "NO_EM_RESTART": '1', + "XPCOM_DEBUG_BREAK": 'warn', + "CYGWINBASE": 'C:\\cygwin', + "PATH": 'C:\\Python24;' + \ + 'C:\\Python24\\Scripts;' + \ + 'C:\\cygwin\\bin;' + \ + 'C:\\WINDOWS\\System32;' + \ + 'C:\\program files\\gnuwin32\\bin;' + \ + 'C:\\WINDOWS;' +} + +MozillaEnvironments['linux'] = { + "MOZ_CRASHREPORTER_NO_REPORT": '1', + "MOZ_NO_REMOTE": '1', + "NO_EM_RESTART": '1', + "XPCOM_DEBUG_BREAK": 'warn', + "DISPLAY": ":0", +} + +MozillaEnvironments['mac'] = { + "MOZ_NO_REMOTE": '1', + "NO_EM_RESTART": '1', + "XPCOM_DEBUG_BREAK": 'warn', + "MOZ_CRASHREPORTER_NO_REPORT": '1', + # for extracting dmg's + "PAGER": '/bin/cat', +} + +class ApacheDirectory: + sortByDateString = "?C=M;O=A" + lineParsingRegexp = r' (.*?)[\s]+(\d\d-\w\w\w-\d\d\d\d \d\d:\d\d)\s+(.+?)\s' + + def __init__(self, url): + self.page = [] + self.url = url + self.lineParserRE = re.compile(self.lineParsingRegexp) + + def _retrievePageAt(self, urlString): + content = [] + try: + opener = urllib.URLopener() + page = opener.open(urlString + self.sortByDateString) + content = page.readlines() + opener.close() + except: + print "unable to retrieve page at: " + self.url + return content + + def _buildPage(self, lines): + for line in lines: + if line.startswith(" 1: + builderName = self.step_status.build.builder.name + remainingChanges = changes[1:] # everything but the first + # get rid of the rest of the changes in the Build and BuildStatus + changes = changes[:1] # only the first one + self.step_status.build.changes = changes + bs = BuildSet([builderName], SourceStamp(changes=remainingChanges)) + # submit the buildset back to the BuildMaster + self.build.builder.botmaster.parent.submitBuildSet(bs) + self.finished(SUCCESS) + return + + self.finished(SKIPPED) + return SKIPPED + + +class MozillaWgetLatest(ShellCommand): + """Download built Firefox client from nightly staging directory.""" + haltOnFailure = True + + def __init__(self, **kwargs): + assert kwargs['url'] != "" + assert kwargs['filenameSearchString'] != "" + self.url = kwargs['url'] + self.filenameSearchString = kwargs['filenameSearchString'] + self.branch = "HEAD" + self.fileURL = "" + if 'branch' in kwargs: + self.branch = kwargs['branch'] + if not 'command' in kwargs: + kwargs['command'] = ["wget"] + ShellCommand.__init__(self, **kwargs) + + def getFilename(self): + return self.filename + + def describe(self, done=False): + return ["Wget Download"] + + def start(self): + urlGetter = LatestFileURL(self.url, self.filenameSearchString) + (self.fileURL, self.filename) = urlGetter.getLatestFilename() + if self.branch: + self.setProperty("fileURL", self.fileURL) + self.setProperty("filename", self.filename) + self.setCommand(["wget", "-nv", "-N", "--no-check-certificate", self.fileURL]) + ShellCommand.start(self) + + def evaluateCommand(self, cmd): + superResult = ShellCommand.evaluateCommand(self, cmd) + if SUCCESS != superResult: + return FAILURE + if None != re.search('ERROR', cmd.logs['stdio'].getText()): + return FAILURE + return SUCCESS + + +class MozillaTryServerWgetLatest(MozillaWgetLatest): + def evaluateCommand(self, cmd): + who, rest = self.getProperty("filename").split('-', 1) + identifier = rest.split("-firefox-try")[0] + msg = 'TinderboxPrint: %s\n' % who + msg += 'TinderboxPrint: %s\n' % identifier + self.addCompleteLog("header", msg) + + return MozillaWgetLatest.evaluateCommand(self, cmd) + + +class MozillaInstallZip(ShellCommand): + """Install given file, unzipping to executablePath""" + + def __init__(self, **kwargs): + self.filename = "" + self.branch = "" + if 'branch' in kwargs: + self.branch = kwargs['branch'] + if 'filename' in kwargs: + self.filename = kwargs['filename'] + if not 'command' in kwargs: + kwargs['command'] = ["unzip", "-o"] + ShellCommand.__init__(self, **kwargs) + + def describe(self, done=False): + return ["Install zip"] + + def start(self): + # removed the mkdir because this happens on the master, not the slave + if not self.filename: + if self.branch: + self.filename = self.getProperty("filename") + else: + return FAILURE + if self.filename: + self.command.append(self.filename) + ShellCommand.start(self) + + def evaluateCommand(self, cmd): + superResult = ShellCommand.evaluateCommand(self, cmd) + if SUCCESS != superResult: + return FAILURE + if None != re.search('ERROR', cmd.logs['stdio'].getText()): + return FAILURE + if None != re.search('Usage:', cmd.logs['stdio'].getText()): + return FAILURE + return SUCCESS + + +class MozillaUpdateConfig(ShellCommand): + """Configure YAML file for run_tests.py""" + + def __init__(self, **kwargs): + self.title = "default" + self.branch = "" + self.currentDate = "" + if 'build' in kwargs: + self.title = kwargs['build'].slavename + self.changes = kwargs['build'].source.changes + self.buildid = strftime("%Y%m%d%H%M", localtime(self.changes[0].when)) + if 'branch' in kwargs: + self.branch = kwargs['branch'] + assert 'configPath' in kwargs + assert 'executablePath' in kwargs + self.configPath = kwargs['configPath'] + self.exePath = kwargs['executablePath'] + if not 'command' in kwargs: + kwargs['command'] = ["python", "PerfConfigurator.py", "-v", + "-e", self.exePath, "-c", self.configPath, + "-t", self.title, "-b", self.branch, + "-i", self.buildid] + ShellCommand.__init__(self, **kwargs) + + def describe(self, done=False): + return ["Update config"] + + def evaluateCommand(self, cmd): + superResult = ShellCommand.evaluateCommand(self, cmd) + if SUCCESS != superResult: + return FAILURE + stdioText = cmd.logs['stdio'].getText() + if None != re.search('ERROR', stdioText): + return FAILURE + if None != re.search('USAGE:', stdioText): + return FAILURE + configFileMatch = re.search('outputName\s*=\s*(\w*?.yml)', stdioText) + if not configFileMatch: + return FAILURE + else: + self.setProperty("configFile", configFileMatch.group(1)) + return SUCCESS + + +class MozillaRunPerfTests(ShellCommand): + """Run the performance tests""" + + def __init__(self, **kwargs): + if 'branch' in kwargs: + self.branch = kwargs['branch'] + if not 'command' in kwargs: + kwargs['command'] = ["python", "run_tests.py"] + ShellCommand.__init__(self, **kwargs) + + def describe(self, done=False): + return ["Run performance tests"] + + def createSummary(self, log): + summary = [] + for line in log.readlines(): + if "RETURN:" in line: + summary.append(line.replace("RETURN:", "TinderboxPrint:")) + self.addCompleteLog('summary', "\n".join(summary)) + + def start(self): + """docstring for start""" + self.command = copy.copy(self.command) + self.command.append(self.getProperty("configFile")) + ShellCommand.start(self) + + def evaluateCommand(self, cmd): + superResult = ShellCommand.evaluateCommand(self, cmd) + stdioText = cmd.logs['stdio'].getText() + if SUCCESS != superResult: + return FAILURE + if None != re.search('ERROR', stdioText): + return FAILURE + if None != re.search('USAGE:', stdioText): + return FAILURE + return SUCCESS + +class MozillaInstallTarBz2(ShellCommand): + """Install given file, unzipping to executablePath""" + + def __init__(self, **kwargs): + self.filename = "" + self.branch = "" + if 'branch' in kwargs: + self.branch = kwargs['branch'] + if 'filename' in kwargs: + self.filename = kwargs['filename'] + if not 'command' in kwargs: + kwargs['command'] = ["tar", "-jvxf"] + ShellCommand.__init__(self, **kwargs) + + def describe(self, done=False): + return ["Install tar.gz"] + + def start(self): + if not self.filename: + if self.branch: + self.filename = self.getProperty("filename") + else: + return FAILURE + if self.filename: + self.command.append(self.filename) + ShellCommand.start(self) + + def evaluateCommand(self, cmd): + superResult = ShellCommand.evaluateCommand(self, cmd) + if SUCCESS != superResult: + return FAILURE + return SUCCESS + +class MozillaInstallTarGz(ShellCommand): + """Install given file, unzipping to executablePath""" + + def __init__(self, **kwargs): + self.filename = "" + self.branch = "" + if 'branch' in kwargs: + self.branch = kwargs['branch'] + if 'filename' in kwargs: + self.filename = kwargs['filename'] + if not 'command' in kwargs: + kwargs['command'] = ["tar", "-zvxf"] + ShellCommand.__init__(self, **kwargs) + + def describe(self, done=False): + return ["Install tar.gz"] + + def start(self): + if not self.filename: + if self.branch: + self.filename = self.getProperty("filename") + else: + return FAILURE + if self.filename: + self.command.append(self.filename) + ShellCommand.start(self) + + def evaluateCommand(self, cmd): + superResult = ShellCommand.evaluateCommand(self, cmd) + if SUCCESS != superResult: + return FAILURE + return SUCCESS + +class MozillaWgetFromChange(ShellCommand): + """Download built Firefox client from current change's filenames.""" + haltOnFailure = True + + def __init__(self, **kwargs): + self.branch = "HEAD" + self.fileURL = "" + self.filename = "" + self.filenameSearchString = "en-US.win32.zip" + if 'filenameSearchString' in kwargs: + self.filenameSearchString = kwargs['filenameSearchString'] + if 'url' in kwargs: + self.url = kwargs['url'] + else: + self.url = kwargs['build'].source.changes[0].files[0] + if 'branch' in kwargs: + self.branch = kwargs['branch'] + if not 'command' in kwargs: + kwargs['command'] = ["wget"] + ShellCommand.__init__(self, **kwargs) + + def getFilename(self): + return self.filename + + def describe(self, done=False): + return ["Wget Download"] + + def start(self): + urlGetter = LatestFileURL(self.url, self.filenameSearchString) + self.filename = urlGetter.getLatestFilename() + self.fileURL = self.url + self.filename + if self.branch: + self.setProperty("fileURL", self.fileURL) + self.setProperty("filename", self.filename) + self.setCommand(["wget", "-nv", "-N", self.fileURL]) + ShellCommand.start(self) + + def evaluateCommand(self, cmd): + superResult = ShellCommand.evaluateCommand(self, cmd) + if SUCCESS != superResult: + return FAILURE + if None != re.search('ERROR', cmd.logs['stdio'].getText()): + return FAILURE + return SUCCESS + +class MozillaUpdateConfigFromChange(ShellCommand): + """Configure YAML file for run_tests.py""" + + def __init__(self, **kwargs): + self.title = "default" + self.branch = "" + self.currentDate = "" + if 'build' in kwargs: + self.title = kwargs['build'].slavename + self.changes = kwargs['build'].source.changes + self.buildid = self.changes[0].comments.split(',')[0] + if 'branch' in kwargs: + self.branch = kwargs['branch'] + assert 'configPath' in kwargs + assert 'executablePath' in kwargs + self.configPath = kwargs['configPath'] + self.exePath = kwargs['executablePath'] + if not 'command' in kwargs: + kwargs['command'] = ["python", "PerfConfigurator.py", "-v", "-e", + self.exePath, "-c", self.configPath, + "-t", self.title, "-b", self.branch, "-d", + "-i", self.buildid] + ShellCommand.__init__(self, **kwargs) + + def describe(self, done=False): + return ["Update config"] + + def evaluateCommand(self, cmd): + superResult = ShellCommand.evaluateCommand(self, cmd) + if SUCCESS != superResult: + return FAILURE + stdioText = cmd.logs['stdio'].getText() + if None != re.search('ERROR', stdioText): + return FAILURE + if None != re.search('USAGE:', stdioText): + return FAILURE + configFileMatch = re.search('outputName\s*=\s*(\w*?.yml)', stdioText) + if not configFileMatch: + return FAILURE + else: + self.setProperty("configFile", configFileMatch.group(1)) + return SUCCESS + +class MozillaInstallDmg(ShellCommand): + """Install given file, copying to workdir""" + + def __init__(self, **kwargs): + self.filename = "" + self.branch = "" + if 'branch' in kwargs: + self.branch = kwargs['branch'] + if 'filename' in kwargs: + self.filename = kwargs['filename'] + if not 'command' in kwargs: + kwargs['command'] = ["bash", "installdmg.sh", "$FILENAME"] + ShellCommand.__init__(self, **kwargs) + + def describe(self, done=False): + return ["Install dmg"] + + def start(self): + if not self.filename: + if self.branch: + self.filename = self.getProperty("filename") + else: + return FAILURE + + for i in range(len(self.command)): + if self.command[i] == "$FILENAME": + self.command[i] = self.filename + ShellCommand.start(self) + + def evaluateCommand(self, cmd): + superResult = ShellCommand.evaluateCommand(self, cmd) + if SUCCESS != superResult: + return FAILURE + return SUCCESS + + +def main(argv=None): + if argv is None: + argv = sys.argv + # tester = LatestFileURL('https://build.mozilla.org/tryserver-builds/', "en-US.win32.zip") + tester = LatestFileURL('https://build.mozilla.org/tryserver-builds/', 'win32.zip') + tester.testrun() + return 0 + +if __name__ == '__main__': + main() diff --git a/tools/buildbot-configs/testing/talos/tryperfmaster/scripts/generate-tpcomponent.py b/tools/buildbot-configs/testing/talos/tryperfmaster/scripts/generate-tpcomponent.py new file mode 100644 index 00000000000..06c7c0555c4 --- /dev/null +++ b/tools/buildbot-configs/testing/talos/tryperfmaster/scripts/generate-tpcomponent.py @@ -0,0 +1,83 @@ +# Generates the proper chrome/ and component/ directories that can be dropped +# into a profile to enable the pageloader +# Original Author: Alice Nodelman (anodelman@mozilla.com) +# Modified by: Ben Hearsum (bhearsum@mozilla.com) + +import os +import sys +import zipfile +import shutil +import tempfile + +# create the temp directory +tmp_dir = tempfile.mkdtemp() +pageloader_dir = os.path.join(tmp_dir, "pageloader") +# where the chrome/ and component/ directories will be put +working_dir = "." +chrome_dir = os.path.join(working_dir, 'chrome') +components_dir = os.path.join(working_dir, 'components') +# where the pageloader will be checked out from +cvsroot = ":pserver:anonymous@cvs-mirror.mozilla.org:/cvsroot" +module = "mozilla/layout/tools/pageloader" + +def removedir(rdir): + if os.path.isdir(rdir): + for root, dirs, files in os.walk(rdir, topdown=False): + for name in files: + os.remove(os.path.join(root, name)) + for name in dirs: + os.rmdir(os.path.join(root, name)) + os.rmdir(rdir) + +def zipdir(zip, zdir): + if os.path.isdir(zdir): + for root, dirs, files in os.walk(zdir, topdown=False): + for name in files: + zip.write(os.path.join(root, name), os.path.join(os.path.basename(zdir), name)) + else: + zip.write(zdir) + + +# bail if the directories already exist +if os.path.exists(chrome_dir): + print "chrome/ directory exists, bailing out" + sys.exit(1) +if os.path.exists(components_dir): + print "components/ directory exists, bailing out" + sys.exit(1) + +oldcwd = os.getcwd() +os.chdir(tmp_dir) +# exit if cvs throws an error +if os.system("cvs -d%s co -d pageloader %s" % (cvsroot, module)): + print "could not retrieve pageloader, bailing out" + sys.exit(1) +os.chdir(oldcwd) + +#create the directory structure in the working_dir +os.mkdir(chrome_dir) +os.mkdir(os.path.join(chrome_dir, 'content')) +os.mkdir(components_dir) + +#create the pageloader.manifest file +f = open(os.path.join(chrome_dir, 'pageloader.manifest'), 'w') +f.write('content pageloader jar:pageloader.jar!/content/\n') +f.close() + +shutil.copy(os.path.join(pageloader_dir, 'pageloader.xul'), os.path.join(chrome_dir, 'content', 'pageloader.xul')) +shutil.copy(os.path.join(pageloader_dir, 'quit.js'), os.path.join(chrome_dir, 'content', 'quit.js')) +shutil.copy(os.path.join(pageloader_dir, 'pageloader.js'), os.path.join(chrome_dir, 'content', 'pageloader.js')) +shutil.copy(os.path.join(pageloader_dir, 'report.js'), os.path.join(chrome_dir, 'content', 'report.js')) + +# create pageloader.jar +jar = zipfile.ZipFile(os.path.join(chrome_dir, 'pageloader.jar'), 'w') +zipdir(jar, os.path.join(chrome_dir, 'content')) +jar.close() + +removedir(os.path.join(chrome_dir, 'content')) + +shutil.copy(os.path.join(pageloader_dir, 'tp-cmdline.js'), os.path.join(components_dir, 'tp-cmdline.js')) + +#get rid of the temporary directory +removedir(tmp_dir) + diff --git a/tools/buildbot-configs/testing/talos/tryperfmaster/scripts/installdmg.sh b/tools/buildbot-configs/testing/talos/tryperfmaster/scripts/installdmg.sh new file mode 100644 index 00000000000..830fd52490b --- /dev/null +++ b/tools/buildbot-configs/testing/talos/tryperfmaster/scripts/installdmg.sh @@ -0,0 +1,6 @@ +#!/bin/sh + +mkdir -p mnt +echo "y" | PAGER="/bin/cat" hdiutil attach -verbose -noautoopen -mountpoint ./mnt $1 +rsync -a ./mnt/* . +hdiutil detach mnt diff --git a/tools/buildbot-configs/testing/talos/tryperfmaster/tinderboxpoller.py b/tools/buildbot-configs/testing/talos/tryperfmaster/tinderboxpoller.py new file mode 100644 index 00000000000..96fe7ff2e6a --- /dev/null +++ b/tools/buildbot-configs/testing/talos/tryperfmaster/tinderboxpoller.py @@ -0,0 +1,216 @@ +import time + +from urllib2 import urlopen +from twisted.python import log, failure +from twisted.internet import defer, reactor +from twisted.internet.task import LoopingCall + +from buildbot.changes import base, changes + +class InvalidResultError(Exception): + def __init__(self, value="InvalidResultError"): + self.value = value + def __str__(self): + return repr(self.value) + +class EmptyResult(Exception): + pass + +class NoMoreBuildNodes(Exception): + pass + +class NoMoreFileNodes(Exception): + pass + +class TinderboxResult: + """I hold a list of dictionaries representing build nodes + items = hostname, status and date of change""" + + nodes = [] + + def __init__(self, nodes): + self.nodes = nodes + + def __eq__(self, other): + if len(self.nodes) != len(other.nodes): + return False + for i in range(len(self.nodes)): + if self.nodes[i] != other.nodes[i]: + return False + + return True + + def nodeForHostname(self, nameString): + """returnt the node for a nameString""" + for node in self.nodes: + if nameString in node['hostname']: + return node + + return None + + +class TinderboxParser: + """I parse the pipe-delimited result from a Tinderbox quickparse query.""" + + def __init__(self, tinderboxQuery): + nodes = [] + f = urlopen(tinderboxQuery.geturl()) + s = f.read() + f.close() + lines = s.split('\n') + for line in lines: + if line == "": continue + elements = line.split('|') + if elements[0] == 'State': continue + items = {'hostname': elements[2], 'status': elements[3], 'date': elements[4]} + nodes.append(items) + self.tinderboxResult = TinderboxResult(nodes) + + def getData(self): + return self.tinderboxResult + + +class TinderboxPoller(base.ChangeSource): + """This source will poll a tinderbox server for changes and submit + them to the change master.""" + + compare_attrs = ["tinderboxURL", "pollInterval", "tree", "branch"] + + parent = None # filled in when we're added + loop = None + volatile = ['loop'] + working = False + + def __init__(self, tinderboxURL, branch, tree="Firefox", machine="", pollInterval=30): + """ + @type tinderboxURL: string + @param tinderboxURL: The base URL of the Tinderbox server + (ie. http://tinderbox.mozilla.org) + @type tree: string + @param tree: The tree to look for changes in. + For example, Firefox trunk is 'Firefox' + @type branch: string + @param branch: The branch to look for changes in. This must + match the 'branch' option for the Scheduler. + @type machine: string + @param machine: A machine name to search for. Changes will + only register for machines that match the + substring "machine" + @type pollInterval: int + @param pollInterval: The time (in seconds) between queries for + changes + """ + + self.tinderboxURL = tinderboxURL + self.tree = tree + self.branch = branch + self.machine = machine + self.pollInterval = pollInterval + self.previousChange = '' + self.lastPoll = time.time() + self.lastChange = time.time() + + def startService(self): + self.loop = LoopingCall(self.poll) + base.ChangeSource.startService(self) + + reactor.callLater(0, self.loop.start, self.pollInterval) + + def stopService(self): + self.loop.stop() + return base.ChangeSource.stopService(self) + + def describe(self): + str = "" + str += "Getting changes from the Tinderbox service running at %s " \ + % self.tinderboxURL + str += "
Using tree: %s, branch %s, hostname %s" % (self.tree, self.branch, self.machine) + return str + + def poll(self): + if self.working: + log.msg("Not polling Tinderbox because last poll is still working") + else: + self.working = True + d = self._get_changes() + d.addCallback(self._process_changes) + d.addBoth(self._finished) + return + + def _finished(self, res): + assert self.working + self.working = False + + # check for failure + if isinstance(res, failure.Failure): + log.msg("Tinderbox poll failed: %s" % res) + return res + + def _make_url(self): + # build the tinderbox URL + url = self.tinderboxURL + url += "/" + self.tree + url += "/" + "quickparse.txt" + + return url + + def _get_changes(self): + url = self._make_url() + log.msg("Polling Tinderbox tree at %s" % url) + + self.lastPoll = time.time() + # get the page, in pipe-delimited format + return defer.maybeDeferred(urlopen, url) + + def _process_changes(self, query): + try: + tp = TinderboxParser(query) + result = tp.getData() + except InvalidResultError, e: + log.msg("Could not process Tinderbox query: " + e.value) + return + except EmptyResult: + return + + # check machine substring in result set + if self.machine: + node = result.nodeForHostname(self.machine) + if node: + result = TinderboxResult([node]) + else: + return + + # see if there are any new changes + if self.previousChange: + if (self.previousChange == result.nodes): + return + oldResults = result.nodes + result.nodes = [] + for node in oldResults: + if node not in self.previousChange: + result.nodes.append(node) + self.previousChange = oldResults + else: + self.previousChange = result.nodes + return + + allBuildDates = [] + for buildNode in result.nodes: + buildDate = int(buildNode['date']) + if self.lastChange > buildDate: + # change too old + continue + allBuildDates.append(buildDate) + c = changes.Change(who = buildNode['hostname'], + files = ['TODO: filename goes here'], + comments = buildNode['status'], + branch = self.branch, + when = buildDate) + self.parent.addChange(c) + + # do not allow repeats - count the last change as the largest + # build start time that has been seen + if allBuildDates: + self.lastChange = max(allBuildDates) + + diff --git a/tools/buildbot-configs/testing/talos/tryperfmaster/waterfall.css b/tools/buildbot-configs/testing/talos/tryperfmaster/waterfall.css new file mode 100644 index 00000000000..fde5a6f71c9 --- /dev/null +++ b/tools/buildbot-configs/testing/talos/tryperfmaster/waterfall.css @@ -0,0 +1,64 @@ +body { + background: url("http://www.mozilla.org/images/mozilla-banner.gif") + no-repeat top left; + font-family: Verdana, Courier, sans-sarif; + font-size: 80%; + padding-top: 80px; +} + +a:link, a:visited { + color: #000; +} + +td.Event, td.BuildStep, td.Activity, td.Change, td.Time, td.Builder, td.LastBuild { + border: 1px solid #777; +} + +td.Event, td.Change, td.Time, td.Builder { + background-color: #F3F3F3; +} + +td.green { + background-color: #72ff75; +} + +.table { + /* border-spacing: 2px; */ + border-spacing: 0px; + width: 1%; + border-collapse: seperate; +} + +td { + padding: 0.5ex 1ex; + text-align: center; + border: 1px solid #DDD; +} + +/* Activity states */ +.idle { + background-color: #F3F3F3; +} + +/* LastBuild, BuildStep states */ +.success { + background-color: #72ff75; +} +.failure { + background-color: red; +} +.warnings { + background-color: orange; +} +.exception { + background-color: #c000c0; +} +.start,.running { + background-color: yellow; +} + +/* color trunk/branch builder links differently */ + +.Builder a[href*="trunk"] { + color: #05e; +} \ No newline at end of file