зеркало из https://github.com/mozilla/pjs.git
bug 398192: set-up talos slaves to test try server builds - import in-use configs. r=robcee, patch=me
This commit is contained in:
bhearsum%mozilla.com
Родитель
36c21a120c
Коммит
33862e0af1
|
|
@ -0,0 +1,115 @@
|
|||
# Sample Talos configuration file
|
||||
|
||||
# The title of the report
|
||||
title: firefox_testing
|
||||
|
||||
#csv_file: 'out.csv'
|
||||
results_server: 'graphs-stage.mozilla.org'
|
||||
results_link: '/bulk.cgi'
|
||||
|
||||
# Path to Firefox to test
|
||||
firefox: firefox/firefox
|
||||
|
||||
branch: testbranch
|
||||
|
||||
buildid: testbuildid
|
||||
|
||||
profile_path: base_profile
|
||||
|
||||
init_url: getInfo.html
|
||||
|
||||
# Preferences to set in the test (use "preferences : {}" for no prefs)
|
||||
preferences :
|
||||
browser.shell.checkDefaultBrowser : false
|
||||
browser.warnOnQuit : false
|
||||
browser.link.open_newwindow : 2
|
||||
dom.allow_scripts_to_close_windows : true
|
||||
dom.disable_open_during_load: false
|
||||
dom.max_script_run_time : 0
|
||||
browser.dom.window.dump.enabled: true
|
||||
network.proxy.type : 1
|
||||
network.proxy.http : localhost
|
||||
network.proxy.http_port : 80
|
||||
network.proxy.share_proxy_settings : true
|
||||
dom.disable_window_flip : true
|
||||
dom.disable_window_move_resize : true
|
||||
security.enable_java : false
|
||||
extensions.checkCompatibility : false
|
||||
extensions.update.notifyUser: false
|
||||
|
||||
# Extensions to install in test (use "extensions: {}" for none)
|
||||
# Need quotes around guid because of curly braces
|
||||
# extensions :
|
||||
# "{12345678-1234-1234-1234-abcd12345678}" : c:\path\to\unzipped\xpi
|
||||
# foo@sample.com : c:\path\to\other\unzipped\xpi
|
||||
extensions : {}
|
||||
|
||||
#any directories whose contents need to be installed in the browser before running the tests
|
||||
# this assumes that the directories themselves already exist in the firefox path
|
||||
dirs:
|
||||
chrome : page_load_test/chrome
|
||||
components : page_load_test/components
|
||||
|
||||
# Environment variables to set during test (use env: {} for none)
|
||||
env :
|
||||
NO_EM_RESTART : 1
|
||||
# Tests to run
|
||||
# url : (REQUIRED) url to load into the given firefox browser
|
||||
# url_mod : (OPTIONAL) a bit of code to be evaled and added to the given url during each cycle of the test
|
||||
# resolution: (REQUIRED) how long (in seconds) to pause between counter sampling
|
||||
# cycles : (REQUIRED) how many times to run the test
|
||||
# counters : (REQUIRED) types of system activity to monitor during test run, an be empty
|
||||
# For possible values of counters argument on Windows, see
|
||||
# http://technet2.microsoft.com/WindowsServer/en/Library/86b5d116-6fb3-427b-af8c-9077162125fe1033.mspx?mfr=true
|
||||
# Possible values on Linux and Mac:
|
||||
# counters : ['Private Bytes', 'RSS']
|
||||
# Standard windows values:
|
||||
# counters : ['Working Set', 'Private Bytes', '% Processor Time']
|
||||
|
||||
# to set up a new test it must have the correct configuration options and drop information in a standard format
|
||||
# the format is seen in the regular expressions in ttest.py
|
||||
# to see how the data passed from the browser is processed see send_to_graph an send_to_csv in run_tests.py
|
||||
tests :
|
||||
ts :
|
||||
url : startup_test/startup_test.html?begin=
|
||||
url_mod : str(int(time.time()*1000))
|
||||
resolution : 1
|
||||
cycles : 20
|
||||
win_counters : []
|
||||
unix_counters : []
|
||||
tp:
|
||||
url : '-tp page_load_test/manifest.txt -tpchrome -tpformat tinderbox -tpcycles 10 -tptimeout 120000'
|
||||
resolution : 1
|
||||
cycles : 1
|
||||
win_counters : ['Working Set', 'Private Bytes', '% Processor Time']
|
||||
unix_counters : ['RSS', 'Private Bytes']
|
||||
# tp_js:
|
||||
# url : '"http://localhost/page_load_test/framecycler.html?quit=1&cycles=5"'
|
||||
# resolution : 1
|
||||
# cycles : 1
|
||||
# win_counters : ['Working Set', 'Private Bytes', '% Processor Time']
|
||||
# unix_counters: ['RSS', 'Private Bytes']
|
||||
# tdhtml:
|
||||
# url: '-tp page_load_test/dhtml/dhtml.manifest -tpchrome -tpformat tinderbox -tpcycles 5'
|
||||
# resolution : 1
|
||||
# cycles : 1
|
||||
# win_counters : []
|
||||
# unix_counters : []
|
||||
# tgfx:
|
||||
# url: '-tp page_load_test/gfx/gfx.manifest -tpchrome -tpformat tinderbox -tpcycles 5 -tprender'
|
||||
# resolution : 1
|
||||
# cycles : 1
|
||||
# win_counters : []
|
||||
# unix_counters : []
|
||||
# tsvg:
|
||||
# url: '-tp page_load_test/svg/svg.manifest -tpchrome -tpformat tinderbox -tpcycles 5'
|
||||
# resolution : 1
|
||||
# cycles : 1
|
||||
# win_counters : []
|
||||
# unix_counters : []
|
||||
# twinopen:
|
||||
# url: startup_test/twinopen/winopen.xul?phase1=20
|
||||
# resolution: 1
|
||||
# cycles: 1
|
||||
# win_counters: []
|
||||
# unix_counters: []
|
||||
|
|
@ -0,0 +1,361 @@
|
|||
# This is a sample buildmaster config file. It must be installed as
|
||||
# 'master.cfg' in your buildmaster's base directory (although the filename
|
||||
# can be changed with the --basedir option to 'mktap buildbot master').
|
||||
|
||||
# It has one job: define a dictionary named BuildmasterConfig. This
|
||||
# dictionary has a variety of keys to control different aspects of the
|
||||
# buildmaster. They are documented in docs/config.xhtml .
|
||||
|
||||
import os.path
|
||||
# from buildbot.changes.freshcvs import FreshCVSSource
|
||||
from buildbot.process import factory
|
||||
from buildbot.scheduler import Scheduler, Periodic
|
||||
from buildbot.status import html
|
||||
from buildbot import locks
|
||||
from buildbot.steps.transfer import FileDownload
|
||||
from buildbot.steps.shell import ShellCommand
|
||||
|
||||
# from auth import authlist, debugPassword
|
||||
|
||||
import perfrunner
|
||||
reload(perfrunner)
|
||||
from perfrunner import *
|
||||
|
||||
###
|
||||
### Tinderbox builder names and build directories
|
||||
###
|
||||
WIN32_TRUNK_BUILDER="Try server win32 builder"
|
||||
LINUX_TRUNK_BUILDER="Try server linux builder"
|
||||
MAC_TRUNK_BUILDER="Try server mac builder"
|
||||
TRUNK_BUILDDIR="https://build.mozilla.org/tryserver-builds/"
|
||||
|
||||
CVSROOT=":pserver:anonymous@cvs-mirror.mozilla.org:/cvsroot"
|
||||
|
||||
|
||||
# This is the dictionary that the buildmaster pays attention to. We also use
|
||||
# a shorter alias to save typing.
|
||||
c = BuildmasterConfig = {}
|
||||
|
||||
##
|
||||
## Misc Config
|
||||
##
|
||||
|
||||
c['debugPassword'] = "mozilla"
|
||||
#c['manhole'] = Manhole(9999, "admin", "password")
|
||||
c['projectName'] = "Talos"
|
||||
c['projectURL'] = "http://quality.mozilla.org/en/projects/automation/talos"
|
||||
c['buildbotURL'] = "http://qm-rhel02.mozilla.org:2007"
|
||||
c['slavePortnum'] = 9985
|
||||
|
||||
##
|
||||
## Slaves
|
||||
##
|
||||
|
||||
c['bots'] = [("qm-pxp-try01", "w1nd3rs"),
|
||||
("qm-ptiger-try01", "mac1nt0sh"),
|
||||
("qm-pubuntu-try01", "l1nux")]
|
||||
|
||||
##
|
||||
## Status
|
||||
##
|
||||
|
||||
c['status'] = []
|
||||
c['status'].append(html.Waterfall(http_port=2007,
|
||||
css="/build/tryperfmaster/waterfall.css"))
|
||||
|
||||
from buildbot.status.tinderbox import TinderboxMailNotifier
|
||||
from buildbot.status.mail import MailNotifier
|
||||
|
||||
c['status'].append(MailNotifier(
|
||||
fromaddr="talos@qa.mozilla",
|
||||
sendToInterestedUsers=False,
|
||||
extraRecipients=["anodelman@mozilla.com",
|
||||
"rcampbell@mozilla.com"],
|
||||
mode="failing",
|
||||
builders=None, # send for all builders
|
||||
relayhost="smtp.mozilla.org"))
|
||||
|
||||
# this is the system installed TinderboxMailNotifier
|
||||
c['status'].append(TinderboxMailNotifier(
|
||||
fromaddr="rcampbell@mozilla.com",
|
||||
tree="MozillaTry", # move to MozillaTry
|
||||
extraRecipients=["tinderbox-daemon@tinderbox.mozilla.org",],
|
||||
relayhost="smtp.mozilla.org",
|
||||
builders=["WINNT 5.1 talos try trunk",
|
||||
"MacOSX Darwin 8.8.1 talos try trunk",
|
||||
"Linux talos try trunk"],
|
||||
logCompression="bzip2"))
|
||||
|
||||
|
||||
|
||||
##
|
||||
## Sources
|
||||
##
|
||||
|
||||
from tinderboxpoller import TinderboxPoller
|
||||
|
||||
c['sources'] = []
|
||||
|
||||
c['sources'].append(TinderboxPoller(
|
||||
tinderboxURL = "http://tinderbox.mozilla.org",
|
||||
tree = "MozillaTry",
|
||||
branch = "win32",
|
||||
machine = WIN32_TRUNK_BUILDER,
|
||||
pollInterval = 10 * 60)
|
||||
)
|
||||
|
||||
c['sources'].append(TinderboxPoller(
|
||||
tinderboxURL = "http://tinderbox.mozilla.org",
|
||||
tree = "MozillaTry",
|
||||
branch = "linux",
|
||||
machine = LINUX_TRUNK_BUILDER,
|
||||
pollInterval = 10 * 60)
|
||||
)
|
||||
|
||||
c['sources'].append(TinderboxPoller(
|
||||
tinderboxURL = "http://tinderbox.mozilla.org",
|
||||
tree = "MozillaTry",
|
||||
branch = "mac",
|
||||
machine = MAC_TRUNK_BUILDER,
|
||||
pollInterval = 10 * 60)
|
||||
)
|
||||
|
||||
##
|
||||
## Schedulers
|
||||
##
|
||||
|
||||
c['schedulers'] = []
|
||||
|
||||
c['schedulers'].append(Scheduler(name="WinXP try perfrun scheduler",
|
||||
branch="win32",
|
||||
treeStableTimer=5*60,
|
||||
builderNames=["WINNT 5.1 talos try trunk"]))
|
||||
|
||||
c['schedulers'].append(Scheduler(name="Mac OS X try perfrun scheduler",
|
||||
branch="mac",
|
||||
treeStableTimer=5*60,
|
||||
builderNames=["MacOSX Darwin 8.8.1 talos try trunk"]))
|
||||
|
||||
c['schedulers'].append(Scheduler(name="Linux try perfrun scheduler",
|
||||
branch="linux",
|
||||
treeStableTimer=5*60,
|
||||
builderNames=["Linux talos try trunk"]))
|
||||
|
||||
# the 'builders' list defines the Builders. Each one is configured with a
|
||||
# dictionary, using the following keys:
|
||||
# name (required): the name used to describe this bilder
|
||||
# slavename (required): which slave to use, must appear in c['bots']
|
||||
# builddir (required): which subdirectory to run the builder in
|
||||
# factory (required): a BuildFactory to define how the build is run
|
||||
# periodicBuildTime (optional): if set, force a build every N seconds
|
||||
|
||||
win32_trunk_steps = factory.BuildFactory()
|
||||
win32_trunk_steps.addStep(MozillaChangePusher)
|
||||
win32_trunk_steps.addStep(ShellCommand,
|
||||
workdir=".",
|
||||
description="Cleanup",
|
||||
command=["rm", "-rf", "*.zip", "talos/", "firefox/"],
|
||||
env=MozillaEnvironments['vc8perf'])
|
||||
win32_trunk_steps.addStep(ShellCommand,
|
||||
command=["cvs", "-d", CVSROOT, "co", "-d", "talos",
|
||||
"mozilla/testing/performance/talos"],
|
||||
workdir=".",
|
||||
description="checking out talos",
|
||||
haltOnFailure=True,
|
||||
flunkOnFailure=True,
|
||||
env=MozillaEnvironments['vc8perf'])
|
||||
win32_trunk_steps.addStep(FileDownload,
|
||||
mastersrc="scripts/generate-tpcomponent.py",
|
||||
slavedest="generate-tpcomponent.py",
|
||||
workdir="talos/page_load_test")
|
||||
win32_trunk_steps.addStep(FileDownload,
|
||||
mastersrc="configs/sample.config",
|
||||
slavedest="sample.config",
|
||||
workdir="talos/")
|
||||
win32_trunk_steps.addStep(ShellCommand,
|
||||
command=["python", "generate-tpcomponent.py"],
|
||||
workdir="talos/page_load_test",
|
||||
description="setting up pageloader",
|
||||
haltOnFailure=True,
|
||||
flunkOnFailure=True,
|
||||
env=MozillaEnvironments['vc8perf'])
|
||||
win32_trunk_steps.addStep(MozillaTryServerWgetLatest,
|
||||
workdir=".",
|
||||
branch="1.9",
|
||||
url=TRUNK_BUILDDIR,
|
||||
filenameSearchString="win32.zip",
|
||||
env=MozillaEnvironments['vc8perf'])
|
||||
win32_trunk_steps.addStep(MozillaInstallZip,
|
||||
workdir=".",
|
||||
branch="1.9",
|
||||
haltOnFailure=True,
|
||||
env=MozillaEnvironments['vc8perf'])
|
||||
# TODO: remove this when msys bug is fixed
|
||||
win32_trunk_steps.addStep(ShellCommand,
|
||||
workdir="firefox/",
|
||||
description="chmod files (see msys bug)",
|
||||
command=["chmod", "-v", "-R", "a+x", "."],
|
||||
env=MozillaEnvironments['vc8perf'])
|
||||
win32_trunk_steps.addStep(MozillaUpdateConfig,
|
||||
workdir="talos/",
|
||||
branch="1.9",
|
||||
haltOnFailure=True,
|
||||
executablePath=r"C:\talos-slave\win32-trunk-mini\firefox\firefox.exe",
|
||||
configPath=".",
|
||||
env=MozillaEnvironments['vc8perf'])
|
||||
win32_trunk_steps.addStep(MozillaRunPerfTests,
|
||||
workdir="talos/",
|
||||
branch="1.9",
|
||||
timeout=14400,
|
||||
haltOnFailure=True,
|
||||
command=['python', 'run_tests.py', '--noisy'],
|
||||
env=MozillaEnvironments['vc8perf'])
|
||||
|
||||
linux_trunk_steps = factory.BuildFactory()
|
||||
linux_trunk_steps.addStep(MozillaChangePusher)
|
||||
linux_trunk_steps.addStep(ShellCommand,
|
||||
workdir=".",
|
||||
description="Cleanup",
|
||||
command=["rm", "-rf", "*.bz2", "talos/", "firefox/"],
|
||||
env=MozillaEnvironments['linux'])
|
||||
linux_trunk_steps.addStep(ShellCommand,
|
||||
command=["cvs", "-d", CVSROOT, "co", "-d", "talos",
|
||||
"mozilla/testing/performance/talos"],
|
||||
workdir=".",
|
||||
description="checking out talos",
|
||||
haltOnFailure=True,
|
||||
flunkOnFailure=True,
|
||||
env=MozillaEnvironments['linux'])
|
||||
linux_trunk_steps.addStep(FileDownload,
|
||||
mastersrc="scripts/generate-tpcomponent.py",
|
||||
slavedest="generate-tpcomponent.py",
|
||||
workdir="talos/page_load_test")
|
||||
linux_trunk_steps.addStep(FileDownload,
|
||||
mastersrc="configs/sample.config",
|
||||
slavedest="sample.config",
|
||||
workdir="talos/")
|
||||
linux_trunk_steps.addStep(ShellCommand,
|
||||
command=["python", "generate-tpcomponent.py"],
|
||||
workdir="talos/page_load_test",
|
||||
description="setting up pageloader",
|
||||
haltOnFailure=True,
|
||||
flunkOnFailure=True,
|
||||
env=MozillaEnvironments['linux'])
|
||||
linux_trunk_steps.addStep(MozillaTryServerWgetLatest,
|
||||
workdir=".",
|
||||
branch="1.9",
|
||||
url=TRUNK_BUILDDIR,
|
||||
filenameSearchString="try-linux.tar.bz2",
|
||||
env=MozillaEnvironments['linux'])
|
||||
linux_trunk_steps.addStep(MozillaInstallTarBz2,
|
||||
workdir=".",
|
||||
branch="LINUX-PL",
|
||||
haltOnFailure=True,
|
||||
env=MozillaEnvironments['linux'])
|
||||
linux_trunk_steps.addStep(MozillaUpdateConfig,
|
||||
workdir="talos/",
|
||||
branch="1.9",
|
||||
haltOnFailure=True,
|
||||
# TODO: when talos supports relative paths on win32
|
||||
# make this a relative path
|
||||
executablePath="../firefox/firefox",
|
||||
configPath=".",
|
||||
env=MozillaEnvironments['linux'])
|
||||
linux_trunk_steps.addStep(MozillaRunPerfTests,
|
||||
workdir="talos/",
|
||||
branch="LINUX-PL",
|
||||
timeout=21600,
|
||||
haltOnFailure=True,
|
||||
command=['python', 'run_tests.py', '--noisy'],
|
||||
env=MozillaEnvironments['linux'])
|
||||
|
||||
|
||||
mac_trunk_steps = factory.BuildFactory()
|
||||
mac_trunk_steps.addStep(MozillaChangePusher)
|
||||
mac_trunk_steps.addStep(ShellCommand,
|
||||
workdir=".",
|
||||
description="Cleanup",
|
||||
command="rm -vrf *",
|
||||
env=MozillaEnvironments['mac'])
|
||||
mac_trunk_steps.addStep(ShellCommand,
|
||||
command=["cvs", "-d", CVSROOT, "co", "-d", "talos",
|
||||
"mozilla/testing/performance/talos"],
|
||||
workdir=".",
|
||||
description="checking out talos",
|
||||
haltOnFailure=True,
|
||||
flunkOnFailure=True,
|
||||
env=MozillaEnvironments['mac'])
|
||||
mac_trunk_steps.addStep(FileDownload,
|
||||
mastersrc="scripts/generate-tpcomponent.py",
|
||||
slavedest="generate-tpcomponent.py",
|
||||
workdir="talos/page_load_test")
|
||||
mac_trunk_steps.addStep(FileDownload,
|
||||
mastersrc="scripts/installdmg.sh",
|
||||
slavedest="installdmg.sh",
|
||||
workdir=".")
|
||||
mac_trunk_steps.addStep(FileDownload,
|
||||
mastersrc="configs/sample.config",
|
||||
slavedest="sample.config",
|
||||
workdir="talos/")
|
||||
mac_trunk_steps.addStep(ShellCommand,
|
||||
command=["python", "generate-tpcomponent.py"],
|
||||
workdir="talos/page_load_test",
|
||||
description="setting up pageloader",
|
||||
haltOnFailure=True,
|
||||
flunkOnFailure=True,
|
||||
env=MozillaEnvironments['mac'])
|
||||
mac_trunk_steps.addStep(MozillaTryServerWgetLatest,
|
||||
workdir=".",
|
||||
branch="1.9",
|
||||
url=TRUNK_BUILDDIR,
|
||||
filenameSearchString="try-mac.dmg",
|
||||
env=MozillaEnvironments['mac'])
|
||||
mac_trunk_steps.addStep(MozillaInstallDmg,
|
||||
workdir=".",
|
||||
branch="MAC-PL",
|
||||
haltOnFailure=True,
|
||||
env=MozillaEnvironments['mac'])
|
||||
mac_trunk_steps.addStep(MozillaUpdateConfig,
|
||||
workdir="talos/",
|
||||
branch="1.9",
|
||||
haltOnFailure=True,
|
||||
executablePath="../Minefield.app/Contents/MacOS/firefox",
|
||||
configPath=".",
|
||||
env=MozillaEnvironments['mac'])
|
||||
mac_trunk_steps.addStep(MozillaRunPerfTests,
|
||||
workdir="talos/",
|
||||
branch="MAC-PL",
|
||||
timeout=21600,
|
||||
haltOnFailure=True,
|
||||
command=['python', 'run_tests.py', '--noisy'],
|
||||
env=MozillaEnvironments['mac'])
|
||||
|
||||
win32_trunk_builder = {
|
||||
'name': "WINNT 5.1 talos try trunk",
|
||||
'slavenames': ['qm-pxp-try01'],
|
||||
'builddir': "win32-trunk-mini",
|
||||
'factory': win32_trunk_steps,
|
||||
'category': "Firefox Trunk"
|
||||
}
|
||||
|
||||
linux_trunk_builder = {
|
||||
'name': "Linux talos try trunk",
|
||||
'slavenames': ['qm-pubuntu-try01'],
|
||||
'builddir': "linux-try-trunk",
|
||||
'factory': linux_trunk_steps,
|
||||
'category': "Firefox Trunk"
|
||||
}
|
||||
|
||||
mac_trunk_builder = {
|
||||
'name': "MacOSX Darwin 8.8.1 talos try trunk",
|
||||
'slavenames': ['qm-ptiger-try01'],
|
||||
'builddir': "mac-trunk",
|
||||
'factory': mac_trunk_steps,
|
||||
'category': "Firefox Trunk"
|
||||
}
|
||||
|
||||
c['builders'] = []
|
||||
|
||||
c['builders'].append(win32_trunk_builder)
|
||||
c['builders'].append(linux_trunk_builder)
|
||||
c['builders'].append(mac_trunk_builder)
|
||||
|
|
@ -0,0 +1,518 @@
|
|||
# -*- Python -*-
|
||||
|
||||
from buildbot.process.buildstep import BuildStep
|
||||
from buildbot.buildset import BuildSet
|
||||
from buildbot.sourcestamp import SourceStamp
|
||||
from buildbot.steps.shell import ShellCommand
|
||||
from buildbot.status.builder import SUCCESS, WARNINGS, FAILURE, SKIPPED, EXCEPTION
|
||||
|
||||
import re, urllib, sys, os
|
||||
from time import mktime, strptime, strftime, localtime
|
||||
from datetime import datetime
|
||||
from os import path
|
||||
import copy
|
||||
|
||||
MozillaEnvironments = { }
|
||||
|
||||
# platform SDK location. we can build both from one generic template.
|
||||
# modified from vc8 environment
|
||||
MozillaEnvironments['vc8perf'] = {
|
||||
"MOZ_CRASHREPORTER_NO_REPORT": '1',
|
||||
"MOZ_NO_REMOTE": '1',
|
||||
"NO_EM_RESTART": '1',
|
||||
"XPCOM_DEBUG_BREAK": 'warn',
|
||||
"CYGWINBASE": 'C:\\cygwin',
|
||||
"PATH": 'C:\\Python24;' + \
|
||||
'C:\\Python24\\Scripts;' + \
|
||||
'C:\\cygwin\\bin;' + \
|
||||
'C:\\WINDOWS\\System32;' + \
|
||||
'C:\\program files\\gnuwin32\\bin;' + \
|
||||
'C:\\WINDOWS;'
|
||||
}
|
||||
|
||||
MozillaEnvironments['linux'] = {
|
||||
"MOZ_CRASHREPORTER_NO_REPORT": '1',
|
||||
"MOZ_NO_REMOTE": '1',
|
||||
"NO_EM_RESTART": '1',
|
||||
"XPCOM_DEBUG_BREAK": 'warn',
|
||||
"DISPLAY": ":0",
|
||||
}
|
||||
|
||||
MozillaEnvironments['mac'] = {
|
||||
"MOZ_NO_REMOTE": '1',
|
||||
"NO_EM_RESTART": '1',
|
||||
"XPCOM_DEBUG_BREAK": 'warn',
|
||||
"MOZ_CRASHREPORTER_NO_REPORT": '1',
|
||||
# for extracting dmg's
|
||||
"PAGER": '/bin/cat',
|
||||
}
|
||||
|
||||
class ApacheDirectory:
|
||||
sortByDateString = "?C=M;O=A"
|
||||
lineParsingRegexp = r'<img.*?> <a.*?>(.*?)</a>[\s]+(\d\d-\w\w\w-\d\d\d\d \d\d:\d\d)\s+(.+?)\s'
|
||||
|
||||
def __init__(self, url):
|
||||
self.page = []
|
||||
self.url = url
|
||||
self.lineParserRE = re.compile(self.lineParsingRegexp)
|
||||
|
||||
def _retrievePageAt(self, urlString):
|
||||
content = []
|
||||
try:
|
||||
opener = urllib.URLopener()
|
||||
page = opener.open(urlString + self.sortByDateString)
|
||||
content = page.readlines()
|
||||
opener.close()
|
||||
except:
|
||||
print "unable to retrieve page at: " + self.url
|
||||
return content
|
||||
|
||||
def _buildPage(self, lines):
|
||||
for line in lines:
|
||||
if line.startswith("<img src="):
|
||||
match = self.lineParserRE.match(line)
|
||||
self.page.append(match.groups())
|
||||
|
||||
def _retrievePage(self):
|
||||
content = self._retrievePageAt(self.url)
|
||||
self._buildPage(content)
|
||||
|
||||
def _timeForString(self, dateString):
|
||||
t = mktime(strptime(dateString, "%d-%b-%Y %H:%M"))
|
||||
|
||||
def update(self):
|
||||
self._retrievePage()
|
||||
|
||||
def testrun(self):
|
||||
self.update()
|
||||
for entry in self.page:
|
||||
print entry
|
||||
|
||||
def getPage(self):
|
||||
return self.page
|
||||
|
||||
def getLatestEntry(self):
|
||||
return self.page.last()
|
||||
|
||||
def popLatestEntry(self):
|
||||
return self.page.pop()
|
||||
|
||||
|
||||
class LatestFileURL(ApacheDirectory):
|
||||
|
||||
def getLatestFilename(self):
|
||||
'''returns tuple with full url to file[0] and just filename[1]'''
|
||||
self.update()
|
||||
while self.page:
|
||||
entry = self.popLatestEntry()
|
||||
subdir = ApacheDirectory(self.url + entry[0])
|
||||
subdir.update()
|
||||
subPageItems = subdir.getPage()
|
||||
for item in subPageItems:
|
||||
if item[0].endswith(self.filenameSearchString):
|
||||
return (self.url + entry[0] + item[0], item[0])
|
||||
return ('', '')
|
||||
|
||||
def testrun(self):
|
||||
(fullURL, name) = self.getLatestFilename()
|
||||
print fullURL
|
||||
|
||||
def __init__(self, url, filenameSearchString):
|
||||
ApacheDirectory.__init__(self, url)
|
||||
self.filenameSearchString = filenameSearchString
|
||||
|
||||
|
||||
class MozillaChangePusher(BuildStep):
|
||||
warnOnFailure = True
|
||||
name = "resubmit extra changes"
|
||||
|
||||
def start(self):
|
||||
changes = self.step_status.build.getChanges()
|
||||
if len(changes) > 1:
|
||||
builderName = self.step_status.build.builder.name
|
||||
remainingChanges = changes[1:] # everything but the first
|
||||
# get rid of the rest of the changes in the Build and BuildStatus
|
||||
changes = changes[:1] # only the first one
|
||||
self.step_status.build.changes = changes
|
||||
bs = BuildSet([builderName], SourceStamp(changes=remainingChanges))
|
||||
# submit the buildset back to the BuildMaster
|
||||
self.build.builder.botmaster.parent.submitBuildSet(bs)
|
||||
self.finished(SUCCESS)
|
||||
return
|
||||
|
||||
self.finished(SKIPPED)
|
||||
return SKIPPED
|
||||
|
||||
|
||||
class MozillaWgetLatest(ShellCommand):
|
||||
"""Download built Firefox client from nightly staging directory."""
|
||||
haltOnFailure = True
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
assert kwargs['url'] != ""
|
||||
assert kwargs['filenameSearchString'] != ""
|
||||
self.url = kwargs['url']
|
||||
self.filenameSearchString = kwargs['filenameSearchString']
|
||||
self.branch = "HEAD"
|
||||
self.fileURL = ""
|
||||
if 'branch' in kwargs:
|
||||
self.branch = kwargs['branch']
|
||||
if not 'command' in kwargs:
|
||||
kwargs['command'] = ["wget"]
|
||||
ShellCommand.__init__(self, **kwargs)
|
||||
|
||||
def getFilename(self):
|
||||
return self.filename
|
||||
|
||||
def describe(self, done=False):
|
||||
return ["Wget Download"]
|
||||
|
||||
def start(self):
|
||||
urlGetter = LatestFileURL(self.url, self.filenameSearchString)
|
||||
(self.fileURL, self.filename) = urlGetter.getLatestFilename()
|
||||
if self.branch:
|
||||
self.setProperty("fileURL", self.fileURL)
|
||||
self.setProperty("filename", self.filename)
|
||||
self.setCommand(["wget", "-nv", "-N", "--no-check-certificate", self.fileURL])
|
||||
ShellCommand.start(self)
|
||||
|
||||
def evaluateCommand(self, cmd):
|
||||
superResult = ShellCommand.evaluateCommand(self, cmd)
|
||||
if SUCCESS != superResult:
|
||||
return FAILURE
|
||||
if None != re.search('ERROR', cmd.logs['stdio'].getText()):
|
||||
return FAILURE
|
||||
return SUCCESS
|
||||
|
||||
|
||||
class MozillaTryServerWgetLatest(MozillaWgetLatest):
|
||||
def evaluateCommand(self, cmd):
|
||||
who, rest = self.getProperty("filename").split('-', 1)
|
||||
identifier = rest.split("-firefox-try")[0]
|
||||
msg = 'TinderboxPrint: %s\n' % who
|
||||
msg += 'TinderboxPrint: %s\n' % identifier
|
||||
self.addCompleteLog("header", msg)
|
||||
|
||||
return MozillaWgetLatest.evaluateCommand(self, cmd)
|
||||
|
||||
|
||||
class MozillaInstallZip(ShellCommand):
|
||||
"""Install given file, unzipping to executablePath"""
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
self.filename = ""
|
||||
self.branch = ""
|
||||
if 'branch' in kwargs:
|
||||
self.branch = kwargs['branch']
|
||||
if 'filename' in kwargs:
|
||||
self.filename = kwargs['filename']
|
||||
if not 'command' in kwargs:
|
||||
kwargs['command'] = ["unzip", "-o"]
|
||||
ShellCommand.__init__(self, **kwargs)
|
||||
|
||||
def describe(self, done=False):
|
||||
return ["Install zip"]
|
||||
|
||||
def start(self):
|
||||
# removed the mkdir because this happens on the master, not the slave
|
||||
if not self.filename:
|
||||
if self.branch:
|
||||
self.filename = self.getProperty("filename")
|
||||
else:
|
||||
return FAILURE
|
||||
if self.filename:
|
||||
self.command.append(self.filename)
|
||||
ShellCommand.start(self)
|
||||
|
||||
def evaluateCommand(self, cmd):
|
||||
superResult = ShellCommand.evaluateCommand(self, cmd)
|
||||
if SUCCESS != superResult:
|
||||
return FAILURE
|
||||
if None != re.search('ERROR', cmd.logs['stdio'].getText()):
|
||||
return FAILURE
|
||||
if None != re.search('Usage:', cmd.logs['stdio'].getText()):
|
||||
return FAILURE
|
||||
return SUCCESS
|
||||
|
||||
|
||||
class MozillaUpdateConfig(ShellCommand):
|
||||
"""Configure YAML file for run_tests.py"""
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
self.title = "default"
|
||||
self.branch = ""
|
||||
self.currentDate = ""
|
||||
if 'build' in kwargs:
|
||||
self.title = kwargs['build'].slavename
|
||||
self.changes = kwargs['build'].source.changes
|
||||
self.buildid = strftime("%Y%m%d%H%M", localtime(self.changes[0].when))
|
||||
if 'branch' in kwargs:
|
||||
self.branch = kwargs['branch']
|
||||
assert 'configPath' in kwargs
|
||||
assert 'executablePath' in kwargs
|
||||
self.configPath = kwargs['configPath']
|
||||
self.exePath = kwargs['executablePath']
|
||||
if not 'command' in kwargs:
|
||||
kwargs['command'] = ["python", "PerfConfigurator.py", "-v",
|
||||
"-e", self.exePath, "-c", self.configPath,
|
||||
"-t", self.title, "-b", self.branch,
|
||||
"-i", self.buildid]
|
||||
ShellCommand.__init__(self, **kwargs)
|
||||
|
||||
def describe(self, done=False):
|
||||
return ["Update config"]
|
||||
|
||||
def evaluateCommand(self, cmd):
|
||||
superResult = ShellCommand.evaluateCommand(self, cmd)
|
||||
if SUCCESS != superResult:
|
||||
return FAILURE
|
||||
stdioText = cmd.logs['stdio'].getText()
|
||||
if None != re.search('ERROR', stdioText):
|
||||
return FAILURE
|
||||
if None != re.search('USAGE:', stdioText):
|
||||
return FAILURE
|
||||
configFileMatch = re.search('outputName\s*=\s*(\w*?.yml)', stdioText)
|
||||
if not configFileMatch:
|
||||
return FAILURE
|
||||
else:
|
||||
self.setProperty("configFile", configFileMatch.group(1))
|
||||
return SUCCESS
|
||||
|
||||
|
||||
class MozillaRunPerfTests(ShellCommand):
|
||||
"""Run the performance tests"""
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
if 'branch' in kwargs:
|
||||
self.branch = kwargs['branch']
|
||||
if not 'command' in kwargs:
|
||||
kwargs['command'] = ["python", "run_tests.py"]
|
||||
ShellCommand.__init__(self, **kwargs)
|
||||
|
||||
def describe(self, done=False):
|
||||
return ["Run performance tests"]
|
||||
|
||||
def createSummary(self, log):
|
||||
summary = []
|
||||
for line in log.readlines():
|
||||
if "RETURN:" in line:
|
||||
summary.append(line.replace("RETURN:", "TinderboxPrint:"))
|
||||
self.addCompleteLog('summary', "\n".join(summary))
|
||||
|
||||
def start(self):
|
||||
"""docstring for start"""
|
||||
self.command = copy.copy(self.command)
|
||||
self.command.append(self.getProperty("configFile"))
|
||||
ShellCommand.start(self)
|
||||
|
||||
def evaluateCommand(self, cmd):
|
||||
superResult = ShellCommand.evaluateCommand(self, cmd)
|
||||
stdioText = cmd.logs['stdio'].getText()
|
||||
if SUCCESS != superResult:
|
||||
return FAILURE
|
||||
if None != re.search('ERROR', stdioText):
|
||||
return FAILURE
|
||||
if None != re.search('USAGE:', stdioText):
|
||||
return FAILURE
|
||||
return SUCCESS
|
||||
|
||||
class MozillaInstallTarBz2(ShellCommand):
|
||||
"""Install given file, unzipping to executablePath"""
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
self.filename = ""
|
||||
self.branch = ""
|
||||
if 'branch' in kwargs:
|
||||
self.branch = kwargs['branch']
|
||||
if 'filename' in kwargs:
|
||||
self.filename = kwargs['filename']
|
||||
if not 'command' in kwargs:
|
||||
kwargs['command'] = ["tar", "-jvxf"]
|
||||
ShellCommand.__init__(self, **kwargs)
|
||||
|
||||
def describe(self, done=False):
|
||||
return ["Install tar.gz"]
|
||||
|
||||
def start(self):
|
||||
if not self.filename:
|
||||
if self.branch:
|
||||
self.filename = self.getProperty("filename")
|
||||
else:
|
||||
return FAILURE
|
||||
if self.filename:
|
||||
self.command.append(self.filename)
|
||||
ShellCommand.start(self)
|
||||
|
||||
def evaluateCommand(self, cmd):
|
||||
superResult = ShellCommand.evaluateCommand(self, cmd)
|
||||
if SUCCESS != superResult:
|
||||
return FAILURE
|
||||
return SUCCESS
|
||||
|
||||
class MozillaInstallTarGz(ShellCommand):
|
||||
"""Install given file, unzipping to executablePath"""
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
self.filename = ""
|
||||
self.branch = ""
|
||||
if 'branch' in kwargs:
|
||||
self.branch = kwargs['branch']
|
||||
if 'filename' in kwargs:
|
||||
self.filename = kwargs['filename']
|
||||
if not 'command' in kwargs:
|
||||
kwargs['command'] = ["tar", "-zvxf"]
|
||||
ShellCommand.__init__(self, **kwargs)
|
||||
|
||||
def describe(self, done=False):
|
||||
return ["Install tar.gz"]
|
||||
|
||||
def start(self):
|
||||
if not self.filename:
|
||||
if self.branch:
|
||||
self.filename = self.getProperty("filename")
|
||||
else:
|
||||
return FAILURE
|
||||
if self.filename:
|
||||
self.command.append(self.filename)
|
||||
ShellCommand.start(self)
|
||||
|
||||
def evaluateCommand(self, cmd):
|
||||
superResult = ShellCommand.evaluateCommand(self, cmd)
|
||||
if SUCCESS != superResult:
|
||||
return FAILURE
|
||||
return SUCCESS
|
||||
|
||||
class MozillaWgetFromChange(ShellCommand):
|
||||
"""Download built Firefox client from current change's filenames."""
|
||||
haltOnFailure = True
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
self.branch = "HEAD"
|
||||
self.fileURL = ""
|
||||
self.filename = ""
|
||||
self.filenameSearchString = "en-US.win32.zip"
|
||||
if 'filenameSearchString' in kwargs:
|
||||
self.filenameSearchString = kwargs['filenameSearchString']
|
||||
if 'url' in kwargs:
|
||||
self.url = kwargs['url']
|
||||
else:
|
||||
self.url = kwargs['build'].source.changes[0].files[0]
|
||||
if 'branch' in kwargs:
|
||||
self.branch = kwargs['branch']
|
||||
if not 'command' in kwargs:
|
||||
kwargs['command'] = ["wget"]
|
||||
ShellCommand.__init__(self, **kwargs)
|
||||
|
||||
def getFilename(self):
|
||||
return self.filename
|
||||
|
||||
def describe(self, done=False):
|
||||
return ["Wget Download"]
|
||||
|
||||
def start(self):
|
||||
urlGetter = LatestFileURL(self.url, self.filenameSearchString)
|
||||
self.filename = urlGetter.getLatestFilename()
|
||||
self.fileURL = self.url + self.filename
|
||||
if self.branch:
|
||||
self.setProperty("fileURL", self.fileURL)
|
||||
self.setProperty("filename", self.filename)
|
||||
self.setCommand(["wget", "-nv", "-N", self.fileURL])
|
||||
ShellCommand.start(self)
|
||||
|
||||
def evaluateCommand(self, cmd):
|
||||
superResult = ShellCommand.evaluateCommand(self, cmd)
|
||||
if SUCCESS != superResult:
|
||||
return FAILURE
|
||||
if None != re.search('ERROR', cmd.logs['stdio'].getText()):
|
||||
return FAILURE
|
||||
return SUCCESS
|
||||
|
||||
class MozillaUpdateConfigFromChange(ShellCommand):
|
||||
"""Configure YAML file for run_tests.py"""
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
self.title = "default"
|
||||
self.branch = ""
|
||||
self.currentDate = ""
|
||||
if 'build' in kwargs:
|
||||
self.title = kwargs['build'].slavename
|
||||
self.changes = kwargs['build'].source.changes
|
||||
self.buildid = self.changes[0].comments.split(',')[0]
|
||||
if 'branch' in kwargs:
|
||||
self.branch = kwargs['branch']
|
||||
assert 'configPath' in kwargs
|
||||
assert 'executablePath' in kwargs
|
||||
self.configPath = kwargs['configPath']
|
||||
self.exePath = kwargs['executablePath']
|
||||
if not 'command' in kwargs:
|
||||
kwargs['command'] = ["python", "PerfConfigurator.py", "-v", "-e",
|
||||
self.exePath, "-c", self.configPath,
|
||||
"-t", self.title, "-b", self.branch, "-d",
|
||||
"-i", self.buildid]
|
||||
ShellCommand.__init__(self, **kwargs)
|
||||
|
||||
def describe(self, done=False):
|
||||
return ["Update config"]
|
||||
|
||||
def evaluateCommand(self, cmd):
|
||||
superResult = ShellCommand.evaluateCommand(self, cmd)
|
||||
if SUCCESS != superResult:
|
||||
return FAILURE
|
||||
stdioText = cmd.logs['stdio'].getText()
|
||||
if None != re.search('ERROR', stdioText):
|
||||
return FAILURE
|
||||
if None != re.search('USAGE:', stdioText):
|
||||
return FAILURE
|
||||
configFileMatch = re.search('outputName\s*=\s*(\w*?.yml)', stdioText)
|
||||
if not configFileMatch:
|
||||
return FAILURE
|
||||
else:
|
||||
self.setProperty("configFile", configFileMatch.group(1))
|
||||
return SUCCESS
|
||||
|
||||
class MozillaInstallDmg(ShellCommand):
|
||||
"""Install given file, copying to workdir"""
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
self.filename = ""
|
||||
self.branch = ""
|
||||
if 'branch' in kwargs:
|
||||
self.branch = kwargs['branch']
|
||||
if 'filename' in kwargs:
|
||||
self.filename = kwargs['filename']
|
||||
if not 'command' in kwargs:
|
||||
kwargs['command'] = ["bash", "installdmg.sh", "$FILENAME"]
|
||||
ShellCommand.__init__(self, **kwargs)
|
||||
|
||||
def describe(self, done=False):
|
||||
return ["Install dmg"]
|
||||
|
||||
def start(self):
|
||||
if not self.filename:
|
||||
if self.branch:
|
||||
self.filename = self.getProperty("filename")
|
||||
else:
|
||||
return FAILURE
|
||||
|
||||
for i in range(len(self.command)):
|
||||
if self.command[i] == "$FILENAME":
|
||||
self.command[i] = self.filename
|
||||
ShellCommand.start(self)
|
||||
|
||||
def evaluateCommand(self, cmd):
|
||||
superResult = ShellCommand.evaluateCommand(self, cmd)
|
||||
if SUCCESS != superResult:
|
||||
return FAILURE
|
||||
return SUCCESS
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
if argv is None:
|
||||
argv = sys.argv
|
||||
# tester = LatestFileURL('https://build.mozilla.org/tryserver-builds/', "en-US.win32.zip")
|
||||
tester = LatestFileURL('https://build.mozilla.org/tryserver-builds/', 'win32.zip')
|
||||
tester.testrun()
|
||||
return 0
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
|
@ -0,0 +1,83 @@
|
|||
# Generates the proper chrome/ and component/ directories that can be dropped
|
||||
# into a profile to enable the pageloader
|
||||
# Original Author: Alice Nodelman (anodelman@mozilla.com)
|
||||
# Modified by: Ben Hearsum (bhearsum@mozilla.com)
|
||||
|
||||
import os
|
||||
import sys
|
||||
import zipfile
|
||||
import shutil
|
||||
import tempfile
|
||||
|
||||
# create the temp directory
|
||||
tmp_dir = tempfile.mkdtemp()
|
||||
pageloader_dir = os.path.join(tmp_dir, "pageloader")
|
||||
# where the chrome/ and component/ directories will be put
|
||||
working_dir = "."
|
||||
chrome_dir = os.path.join(working_dir, 'chrome')
|
||||
components_dir = os.path.join(working_dir, 'components')
|
||||
# where the pageloader will be checked out from
|
||||
cvsroot = ":pserver:anonymous@cvs-mirror.mozilla.org:/cvsroot"
|
||||
module = "mozilla/layout/tools/pageloader"
|
||||
|
||||
def removedir(rdir):
|
||||
if os.path.isdir(rdir):
|
||||
for root, dirs, files in os.walk(rdir, topdown=False):
|
||||
for name in files:
|
||||
os.remove(os.path.join(root, name))
|
||||
for name in dirs:
|
||||
os.rmdir(os.path.join(root, name))
|
||||
os.rmdir(rdir)
|
||||
|
||||
def zipdir(zip, zdir):
|
||||
if os.path.isdir(zdir):
|
||||
for root, dirs, files in os.walk(zdir, topdown=False):
|
||||
for name in files:
|
||||
zip.write(os.path.join(root, name), os.path.join(os.path.basename(zdir), name))
|
||||
else:
|
||||
zip.write(zdir)
|
||||
|
||||
|
||||
# bail if the directories already exist
|
||||
if os.path.exists(chrome_dir):
|
||||
print "chrome/ directory exists, bailing out"
|
||||
sys.exit(1)
|
||||
if os.path.exists(components_dir):
|
||||
print "components/ directory exists, bailing out"
|
||||
sys.exit(1)
|
||||
|
||||
oldcwd = os.getcwd()
|
||||
os.chdir(tmp_dir)
|
||||
# exit if cvs throws an error
|
||||
if os.system("cvs -d%s co -d pageloader %s" % (cvsroot, module)):
|
||||
print "could not retrieve pageloader, bailing out"
|
||||
sys.exit(1)
|
||||
os.chdir(oldcwd)
|
||||
|
||||
#create the directory structure in the working_dir
|
||||
os.mkdir(chrome_dir)
|
||||
os.mkdir(os.path.join(chrome_dir, 'content'))
|
||||
os.mkdir(components_dir)
|
||||
|
||||
#create the pageloader.manifest file
|
||||
f = open(os.path.join(chrome_dir, 'pageloader.manifest'), 'w')
|
||||
f.write('content pageloader jar:pageloader.jar!/content/\n')
|
||||
f.close()
|
||||
|
||||
shutil.copy(os.path.join(pageloader_dir, 'pageloader.xul'), os.path.join(chrome_dir, 'content', 'pageloader.xul'))
|
||||
shutil.copy(os.path.join(pageloader_dir, 'quit.js'), os.path.join(chrome_dir, 'content', 'quit.js'))
|
||||
shutil.copy(os.path.join(pageloader_dir, 'pageloader.js'), os.path.join(chrome_dir, 'content', 'pageloader.js'))
|
||||
shutil.copy(os.path.join(pageloader_dir, 'report.js'), os.path.join(chrome_dir, 'content', 'report.js'))
|
||||
|
||||
# create pageloader.jar
|
||||
jar = zipfile.ZipFile(os.path.join(chrome_dir, 'pageloader.jar'), 'w')
|
||||
zipdir(jar, os.path.join(chrome_dir, 'content'))
|
||||
jar.close()
|
||||
|
||||
removedir(os.path.join(chrome_dir, 'content'))
|
||||
|
||||
shutil.copy(os.path.join(pageloader_dir, 'tp-cmdline.js'), os.path.join(components_dir, 'tp-cmdline.js'))
|
||||
|
||||
#get rid of the temporary directory
|
||||
removedir(tmp_dir)
|
||||
|
||||
|
|
@ -0,0 +1,6 @@
|
|||
#!/bin/sh
|
||||
|
||||
mkdir -p mnt
|
||||
echo "y" | PAGER="/bin/cat" hdiutil attach -verbose -noautoopen -mountpoint ./mnt $1
|
||||
rsync -a ./mnt/* .
|
||||
hdiutil detach mnt
|
||||
|
|
@ -0,0 +1,216 @@
|
|||
import time
|
||||
|
||||
from urllib2 import urlopen
|
||||
from twisted.python import log, failure
|
||||
from twisted.internet import defer, reactor
|
||||
from twisted.internet.task import LoopingCall
|
||||
|
||||
from buildbot.changes import base, changes
|
||||
|
||||
class InvalidResultError(Exception):
|
||||
def __init__(self, value="InvalidResultError"):
|
||||
self.value = value
|
||||
def __str__(self):
|
||||
return repr(self.value)
|
||||
|
||||
class EmptyResult(Exception):
|
||||
pass
|
||||
|
||||
class NoMoreBuildNodes(Exception):
|
||||
pass
|
||||
|
||||
class NoMoreFileNodes(Exception):
|
||||
pass
|
||||
|
||||
class TinderboxResult:
|
||||
"""I hold a list of dictionaries representing build nodes
|
||||
items = hostname, status and date of change"""
|
||||
|
||||
nodes = []
|
||||
|
||||
def __init__(self, nodes):
|
||||
self.nodes = nodes
|
||||
|
||||
def __eq__(self, other):
|
||||
if len(self.nodes) != len(other.nodes):
|
||||
return False
|
||||
for i in range(len(self.nodes)):
|
||||
if self.nodes[i] != other.nodes[i]:
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
def nodeForHostname(self, nameString):
|
||||
"""returnt the node for a nameString"""
|
||||
for node in self.nodes:
|
||||
if nameString in node['hostname']:
|
||||
return node
|
||||
|
||||
return None
|
||||
|
||||
|
||||
class TinderboxParser:
|
||||
"""I parse the pipe-delimited result from a Tinderbox quickparse query."""
|
||||
|
||||
def __init__(self, tinderboxQuery):
|
||||
nodes = []
|
||||
f = urlopen(tinderboxQuery.geturl())
|
||||
s = f.read()
|
||||
f.close()
|
||||
lines = s.split('\n')
|
||||
for line in lines:
|
||||
if line == "": continue
|
||||
elements = line.split('|')
|
||||
if elements[0] == 'State': continue
|
||||
items = {'hostname': elements[2], 'status': elements[3], 'date': elements[4]}
|
||||
nodes.append(items)
|
||||
self.tinderboxResult = TinderboxResult(nodes)
|
||||
|
||||
def getData(self):
|
||||
return self.tinderboxResult
|
||||
|
||||
|
||||
class TinderboxPoller(base.ChangeSource):
|
||||
"""This source will poll a tinderbox server for changes and submit
|
||||
them to the change master."""
|
||||
|
||||
compare_attrs = ["tinderboxURL", "pollInterval", "tree", "branch"]
|
||||
|
||||
parent = None # filled in when we're added
|
||||
loop = None
|
||||
volatile = ['loop']
|
||||
working = False
|
||||
|
||||
def __init__(self, tinderboxURL, branch, tree="Firefox", machine="", pollInterval=30):
|
||||
"""
|
||||
@type tinderboxURL: string
|
||||
@param tinderboxURL: The base URL of the Tinderbox server
|
||||
(ie. http://tinderbox.mozilla.org)
|
||||
@type tree: string
|
||||
@param tree: The tree to look for changes in.
|
||||
For example, Firefox trunk is 'Firefox'
|
||||
@type branch: string
|
||||
@param branch: The branch to look for changes in. This must
|
||||
match the 'branch' option for the Scheduler.
|
||||
@type machine: string
|
||||
@param machine: A machine name to search for. Changes will
|
||||
only register for machines that match the
|
||||
substring "machine"
|
||||
@type pollInterval: int
|
||||
@param pollInterval: The time (in seconds) between queries for
|
||||
changes
|
||||
"""
|
||||
|
||||
self.tinderboxURL = tinderboxURL
|
||||
self.tree = tree
|
||||
self.branch = branch
|
||||
self.machine = machine
|
||||
self.pollInterval = pollInterval
|
||||
self.previousChange = ''
|
||||
self.lastPoll = time.time()
|
||||
self.lastChange = time.time()
|
||||
|
||||
def startService(self):
|
||||
self.loop = LoopingCall(self.poll)
|
||||
base.ChangeSource.startService(self)
|
||||
|
||||
reactor.callLater(0, self.loop.start, self.pollInterval)
|
||||
|
||||
def stopService(self):
|
||||
self.loop.stop()
|
||||
return base.ChangeSource.stopService(self)
|
||||
|
||||
def describe(self):
|
||||
str = ""
|
||||
str += "Getting changes from the Tinderbox service running at %s " \
|
||||
% self.tinderboxURL
|
||||
str += "<br>Using tree: %s, branch %s, hostname %s" % (self.tree, self.branch, self.machine)
|
||||
return str
|
||||
|
||||
def poll(self):
|
||||
if self.working:
|
||||
log.msg("Not polling Tinderbox because last poll is still working")
|
||||
else:
|
||||
self.working = True
|
||||
d = self._get_changes()
|
||||
d.addCallback(self._process_changes)
|
||||
d.addBoth(self._finished)
|
||||
return
|
||||
|
||||
def _finished(self, res):
|
||||
assert self.working
|
||||
self.working = False
|
||||
|
||||
# check for failure
|
||||
if isinstance(res, failure.Failure):
|
||||
log.msg("Tinderbox poll failed: %s" % res)
|
||||
return res
|
||||
|
||||
def _make_url(self):
|
||||
# build the tinderbox URL
|
||||
url = self.tinderboxURL
|
||||
url += "/" + self.tree
|
||||
url += "/" + "quickparse.txt"
|
||||
|
||||
return url
|
||||
|
||||
def _get_changes(self):
|
||||
url = self._make_url()
|
||||
log.msg("Polling Tinderbox tree at %s" % url)
|
||||
|
||||
self.lastPoll = time.time()
|
||||
# get the page, in pipe-delimited format
|
||||
return defer.maybeDeferred(urlopen, url)
|
||||
|
||||
def _process_changes(self, query):
|
||||
try:
|
||||
tp = TinderboxParser(query)
|
||||
result = tp.getData()
|
||||
except InvalidResultError, e:
|
||||
log.msg("Could not process Tinderbox query: " + e.value)
|
||||
return
|
||||
except EmptyResult:
|
||||
return
|
||||
|
||||
# check machine substring in result set
|
||||
if self.machine:
|
||||
node = result.nodeForHostname(self.machine)
|
||||
if node:
|
||||
result = TinderboxResult([node])
|
||||
else:
|
||||
return
|
||||
|
||||
# see if there are any new changes
|
||||
if self.previousChange:
|
||||
if (self.previousChange == result.nodes):
|
||||
return
|
||||
oldResults = result.nodes
|
||||
result.nodes = []
|
||||
for node in oldResults:
|
||||
if node not in self.previousChange:
|
||||
result.nodes.append(node)
|
||||
self.previousChange = oldResults
|
||||
else:
|
||||
self.previousChange = result.nodes
|
||||
return
|
||||
|
||||
allBuildDates = []
|
||||
for buildNode in result.nodes:
|
||||
buildDate = int(buildNode['date'])
|
||||
if self.lastChange > buildDate:
|
||||
# change too old
|
||||
continue
|
||||
allBuildDates.append(buildDate)
|
||||
c = changes.Change(who = buildNode['hostname'],
|
||||
files = ['TODO: filename goes here'],
|
||||
comments = buildNode['status'],
|
||||
branch = self.branch,
|
||||
when = buildDate)
|
||||
self.parent.addChange(c)
|
||||
|
||||
# do not allow repeats - count the last change as the largest
|
||||
# build start time that has been seen
|
||||
if allBuildDates:
|
||||
self.lastChange = max(allBuildDates)
|
||||
|
||||
|
||||
|
|
@ -0,0 +1,64 @@
|
|||
body {
|
||||
background: url("http://www.mozilla.org/images/mozilla-banner.gif")
|
||||
no-repeat top left;
|
||||
font-family: Verdana, Courier, sans-sarif;
|
||||
font-size: 80%;
|
||||
padding-top: 80px;
|
||||
}
|
||||
|
||||
a:link, a:visited {
|
||||
color: #000;
|
||||
}
|
||||
|
||||
td.Event, td.BuildStep, td.Activity, td.Change, td.Time, td.Builder, td.LastBuild {
|
||||
border: 1px solid #777;
|
||||
}
|
||||
|
||||
td.Event, td.Change, td.Time, td.Builder {
|
||||
background-color: #F3F3F3;
|
||||
}
|
||||
|
||||
td.green {
|
||||
background-color: #72ff75;
|
||||
}
|
||||
|
||||
.table {
|
||||
/* border-spacing: 2px; */
|
||||
border-spacing: 0px;
|
||||
width: 1%;
|
||||
border-collapse: seperate;
|
||||
}
|
||||
|
||||
td {
|
||||
padding: 0.5ex 1ex;
|
||||
text-align: center;
|
||||
border: 1px solid #DDD;
|
||||
}
|
||||
|
||||
/* Activity states */
|
||||
.idle {
|
||||
background-color: #F3F3F3;
|
||||
}
|
||||
|
||||
/* LastBuild, BuildStep states */
|
||||
.success {
|
||||
background-color: #72ff75;
|
||||
}
|
||||
.failure {
|
||||
background-color: red;
|
||||
}
|
||||
.warnings {
|
||||
background-color: orange;
|
||||
}
|
||||
.exception {
|
||||
background-color: #c000c0;
|
||||
}
|
||||
.start,.running {
|
||||
background-color: yellow;
|
||||
}
|
||||
|
||||
/* color trunk/branch builder links differently */
|
||||
|
||||
.Builder a[href*="trunk"] {
|
||||
color: #05e;
|
||||
}
|
||||
Загрузка…
Ссылка в новой задаче