Refactor to concentrate generate of jobs into one class with subclass
specialization. Call jenkins to create or update jobs unless files are specified.
This commit is contained in:
Syd Polk
Родитель
e30c5d34e6
Коммит
b47a427df1
|
|
@ -22,4 +22,5 @@ queue.xml
|
|||
scm-sync-configuration
|
||||
updates
|
||||
.idea
|
||||
*.pyc
|
||||
|
||||
|
|
|
|||
|
|
@ -1 +1 @@
|
|||
Release,Platform,PlatformJobLabel,Slave,Python,Bash
nightly,mac,mac,mac,System-CPython-2.7,bash
nightly,mac,mac10_9,mac10-9,System-CPython-2.7,bash
nightly,mac,mac10_8,mac10-8,System-CPython-2.7,bash
nightly,mac,mac10_7,mac10-7,System-CPython-2.7,bash
nightly,mac,mac10_6,mac10-6,Local-Python-2.7,bash
nightly,win32,win7_32,win32 && mse && win7,Win-Moz-Python,c:\\mozilla-build\\msys\\bin\\bash
nightly,win32,win7_32_64,win64 && mse && win7,Win-Moz-Python,c:\\mozilla-build\\msys\\bin\\bash
nightly,win32,win32,win32 && mse && win,Win-Moz-Python,c:\\mozilla-build\\msys\\bin\\bash
nightly,win32,win32_64,win64 && mse && win,Win-Moz-Python,c:\\mozilla-build\\msys\\bin\\bash
nightly,win64,win7_64,win64 && mse && win7,Win-Moz-Python,c:\\mozilla-build\\msys\\bin\\bash
nightly,win64,win64,win64 && mse && win,Win-Moz-Python,c:\\mozilla-build\\msys\\bin\\bash
aurora,mac,mac,mac,System-CPython-2.7,bash
aurora,mac,mac10_9,mac10-9,System-CPython-2.7,bash
aurora,mac,mac10_8,mac10-8,System-CPython-2.7,bash
aurora,mac,mac10_7,mac10-7,System-CPython-2.7,bash
aurora,mac,mac10_6,mac10-6,Local-Python-2.7,bash
mac,mac10_6,mac10-6,Local-Python-2.7,bash
nightly,win32,win7_32_64,win64 && mse && win7,Win-Moz-Python,c:\\mozilla-build\\msys\\bin\\bash
mac,mac10_6,mac10-6,Local-Python-2.7,bash
nightly,win32,win32,win32 && mse && win,Win-Moz-Python,c:\\mozilla-build\\msys\\bin\\bash
mac,mac10_6,mac10-6,Local-Python-2.7,bash
nightly,win32,win32_64,win64 && mse && win,Win-Moz-Python,c:\\mozilla-build\\msys\\bin\\bash
mac,mac10_6,mac10-6,Local-Python-2.7,bash
nightly,win64,win7_64,win64 && mse && win7,Win-Moz-Python,c:\\mozilla-build\\msys\\bin\\bash
mac,mac10_6,mac10-6,Local-Python-2.7,bash
nightly,win64,win64,win64 && mse && win,Win-Moz-Python,c:\\mozilla-build\\msys\\bin\\bash
mac,mac10_6,mac10-6,Local-Python-2.7,bash
aurora,mac,mac,mac,System-CPython-2.7,bash
beta,mac,mac,mac,System-CPython-2.7,bash
beta,mac,mac10_9,mac10-9,System-CPython-2.7,bash
beta,mac,mac10_8,mac10-8,System-CPython-2.7,bash
beta,mac,mac10_7,mac10-7,System-CPython-2.7,bash
beta,mac,mac10_6,mac10-6,Local-Python-2.7,bash
nightly,win32,win7_32,win32 && mse && win7,Win-Moz-Python,c:\\mozilla-build\\msys\\bin\\bash
nightly,win32,win32,win32 && mse && win,Win-Moz-Python,c:\\mozilla-build\\msys\\bin\\bash
beta,win32,win7_32_64,win64 && mse && win7,Win-Moz-Python,c:\\mozilla-build\\msys\\bin\\bash
nightly,win32,win7_32,win32 && mse && win7,Win-Moz-Python,c:\\mozilla-build\\msys\\bin\\bash
nightly,win64,win7_64,win64 && mse && win7,Win-Moz-Python,c:\\mozilla-build\\msys\\bin\\bash
beta,win32,win32_64,win64 && mse && win,Win-Moz-Python,c:\\mozilla-build\\msys\\bin\\bash
|
||||
Release,Platform,PlatformJobLabel,Slave,Python,Bash
nightly,mac,mac,mac,System-CPython-2.7,bash
nightly,mac,mac10_9,mac10-9,System-CPython-2.7,bash
nightly,mac,mac10_8,mac10-8,System-CPython-2.7,bash
nightly,mac,mac10_7,mac10-7,System-CPython-2.7,bash
nightly,win32,win7_32,win32 && win7,Win-Moz-Python,c:\\mozilla-build\\msys\\bin\\bash
nightly,win32,win7_32,win32 && mse && win7,Win-Moz-Python,c:\\mozilla-build\\msys\\bin\\bash
aurora,mac,mac10_9,mac10-9,System-CPython-2.7,bash
nightly,win32,win7_32_64,win64 && mse && win7,Win-Moz-Python,c:\\mozilla-build\\msys\\bin\\bash
nightly,win32,win32_64,win64 && win,Win-Moz-Python,c:\\mozilla-build\\msys\\bin\\bash
nightly,win64,win7_64,win64 && win7,Win-Moz-Python,c:\\mozilla-build\\msys\\bin\\bash
nightly,win64,win64,win64 && win,Win-Moz-Python,c:\\mozilla-build\\msys\\bin\\bash
aurora,mac,mac,mac,System-CPython-2.7,bash
aurora,mac,mac10_9,mac10-9,System-CPython-2.7,bash
aurora,mac,mac10_8,mac10-8,System-CPython-2.7,bash
aurora,mac,mac10_7,mac10-7,System-CPython-2.7,bash
nightly,win32,win7_32_64,win64 && mse && win7,Win-Moz-Python,c:\\mozilla-build\\msys\\bin\\bash
nightly,win32,win32,win32 && mse && win,Win-Moz-Python,c:\\mozilla-build\\msys\\bin\\bash
nightly,win32,win7_32_64,win64 && mse && win7,Win-Moz-Python,c:\\mozilla-build\\msys\\bin\\bash
nightly,win32,win32_64,win64 && mse && win,Win-Moz-Python,c:\\mozilla-build\\msys\\bin\\bash
nightly,win32,win7_32_64,win64 && mse && win7,Win-Moz-Python,c:\\mozilla-build\\msys\\bin\\bash
nightly,win64,win7_64,win64 && mse && win7,Win-Moz-Python,c:\\mozilla-build\\msys\\bin\\bash
nightly,win32,win7_32_64,win64 && mse && win7,Win-Moz-Python,c:\\mozilla-build\\msys\\bin\\bash
nightly,win64,win64,win64 && mse && win,Win-Moz-Python,c:\\mozilla-build\\msys\\bin\\bash
nightly,win32,win7_32_64,win64 && mse && win7,Win-Moz-Python,c:\\mozilla-build\\msys\\bin\\bash
aurora,mac,mac,mac,System-CPython-2.7,bash
aurora,win64,win64,win64 && win,Win-Moz-Python,c:\\mozilla-build\\msys\\bin\\bash
beta,mac,mac,mac,System-CPython-2.7,bash
beta,mac,mac10_9,mac10-9,System-CPython-2.7,bash
beta,mac,mac10_8,mac10-8,System-CPython-2.7,bash
beta,mac,mac10_7,mac10-7,System-CPython-2.7,bash
nightly,win32,win32,win32 && mse && win,Win-Moz-Python,c:\\mozilla-build\\msys\\bin\\bash
beta,win32,win7_32_64,win64 && win7,Win-Moz-Python,c:\\mozilla-build\\msys\\bin\\bash
nightly,win32,win32,win32 && mse && win,Win-Moz-Python,c:\\mozilla-build\\msys\\bin\\bash
nightly,win32,win7_32,win32 && mse && win7,Win-Moz-Python,c:\\mozilla-build\\msys\\bin\\bash
beta,win32,win32_64,win64 && win,Win-Moz-Python,c:\\mozilla-build\\msys\\bin\\bash
|
||||
|
|
|
@ -6,152 +6,63 @@
|
|||
|
||||
"""Generate jobs for jenkins based on a .csv file describing the parameters."""
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
import csv
|
||||
import os
|
||||
import os.path
|
||||
import re
|
||||
import sys
|
||||
|
||||
REGEXPS = {
|
||||
'url1': r'XXURL1XX',
|
||||
'package': r'XXPACKAGEXX',
|
||||
'url2': r'XXURL2XX',
|
||||
'common_tests': r'XXCOMMONTESTSXX',
|
||||
'web_platform_tests': r'XXWEBPLATFORMTESTSXX',
|
||||
'triggers': r'XXTRIGGERSXX',
|
||||
'expand': r'XXEXPANDXX',
|
||||
'slave': r'XXSLAVEXX',
|
||||
'platform': r'XXPLATFORMXX',
|
||||
'python': r'XXPYTHONXX',
|
||||
'bash': r'XXBASHXX',
|
||||
}
|
||||
from job_generator import JobGenerator
|
||||
|
||||
for key in REGEXPS:
|
||||
REGEXPS[key] = re.compile(REGEXPS[key])
|
||||
|
||||
class JobGenerator():
|
||||
def __init__(self):
|
||||
pass
|
||||
|
||||
class GenerateJobs():
|
||||
class GenerateMSEJobs(JobGenerator):
|
||||
def __init__(self, argv):
|
||||
self.argv = argv
|
||||
self.template = None
|
||||
super(GenerateMSEJobs, self).__init__(argv)
|
||||
self.REGEXPS = {
|
||||
'url1': r'XXURL1XX',
|
||||
'package': r'XXPACKAGEXX',
|
||||
'url2': r'XXURL2XX',
|
||||
'common_tests': r'XXCOMMONTESTSXX',
|
||||
'web_platform_tests': r'XXWEBPLATFORMTESTSXX',
|
||||
'triggers': r'XXTRIGGERSXX',
|
||||
'expand': r'XXEXPANDXX',
|
||||
'slave': r'XXSLAVEXX',
|
||||
'platform': r'XXPLATFORMXX',
|
||||
'python': r'XXPYTHONXX',
|
||||
'bash': r'XXBASHXX',
|
||||
}
|
||||
|
||||
def get_config(self):
|
||||
parser = argparse.ArgumentParser(description='Script for generating Jenkins jobs for MSE testing')
|
||||
parser.add_argument('--template', required=True)
|
||||
parser.add_argument('--table-csv', required=True, dest='csv')
|
||||
parser.add_argument('--jenkins-host', required=True, dest='host')
|
||||
args = parser.parse_args(self.argv)
|
||||
def process_row(self, row, template):
|
||||
jobname = 'mse-web-platform-%s-%s' % (row['Release'], row['PlatformJobLabel'])
|
||||
|
||||
self.config = {}
|
||||
self.config['template'] = args.template
|
||||
self.config['csv'] = args.csv
|
||||
self.config['host'] = args.host
|
||||
print 'jobname = %s' % jobname
|
||||
|
||||
'''Return a copy of the template so that it can be reused over and over again.'''
|
||||
def get_template(self):
|
||||
if not self.template:
|
||||
with open(self.config['template']) as myfile:
|
||||
self.template=myfile.read()
|
||||
data = self.template
|
||||
return data
|
||||
|
||||
def get_platform_extension(self, platform):
|
||||
if platform == 'linux64' or platform == 'linux32' or platform == 'linux':
|
||||
return 'tar.bz2'
|
||||
elif platform == 'win32' or platform == 'win64':
|
||||
return 'zip'
|
||||
elif platform == 'mac' or platform == 'mac64':
|
||||
return 'dmg'
|
||||
raise 'Unknown platform %s' % platform
|
||||
|
||||
def get_artifact_platform(self, platform):
|
||||
if platform == 'linux64':
|
||||
return 'linux-x86_64'
|
||||
elif platform == 'linux32':
|
||||
return 'linux-i686'
|
||||
elif platform == 'win64':
|
||||
return 'win64'
|
||||
if (row['Release'] != 'beta'):
|
||||
triggers = 'trigger-firefox-%s-%s,trigger-tests-%s-%s' % (row['Release'], row['Platform'], row['Release'], row['Platform'])
|
||||
else:
|
||||
return platform
|
||||
triggers = ''
|
||||
|
||||
def get_lowest_release(self, release1, release2):
|
||||
if release1 == 'esr' or release2 == 'esr':
|
||||
return 'esr'
|
||||
elif release1 == 'release' or release2 == 'release':
|
||||
return 'release'
|
||||
elif release1 == 'beta' or release2 == 'beta':
|
||||
return 'beta'
|
||||
elif release1 == 'aurora' or release2 == 'aurora':
|
||||
return 'aurora'
|
||||
else:
|
||||
return 'nightly'
|
||||
url1 = '%s/job/firefox-%s-%s/ws/releases' % (self.config['host'], row['Release'], row['Platform'])
|
||||
url2 = '%s/job/tests-%s-%s/ws/releases' % (self.config['host'], row['Release'], row['Platform'])
|
||||
|
||||
def generate(self):
|
||||
self.get_config()
|
||||
artifact_platform = self.get_artifact_platform(row['Platform'])
|
||||
|
||||
if not os.path.exists('jobs'):
|
||||
os.makedirs('jobs')
|
||||
package = 'firefox-latest-%s.en-US.%s.%s' % (row['Release'], artifact_platform, self.get_platform_extension(row['Platform']))
|
||||
common_tests = 'firefox-latest-%s.en-US.%s.common.tests.zip' % (row['Release'], artifact_platform)
|
||||
web_platform_tests = 'firefox-latest-%s.en-US.%s.web-platform.tests.zip' % (row['Release'], artifact_platform)
|
||||
|
||||
with open(self.config['csv'], 'rU') as csvfile:
|
||||
reader = csv.DictReader(csvfile)
|
||||
for row in reader:
|
||||
jobname = 'mse-web-platform-%s-%s' % (row['Release'], row['PlatformJobLabel'])
|
||||
|
||||
print 'jobname = %s' % jobname
|
||||
|
||||
triggers = 'trigger-firefox-%s-%s, trigger-tests-%s-%s' % (row['Release'], row['Platform'], row['Release'], row['Platform'])
|
||||
|
||||
url1 = '%s/job/firefox-%s-%s/ws/releases' % (self.config['host'], row['Release'], row['Platform'])
|
||||
url2 = '%s/job/tests-%s-%s/ws/releases' % (self.config['host'], row['Release'], row['Platform'])
|
||||
|
||||
artifact_platform = self.get_artifact_platform(row['Platform'])
|
||||
|
||||
package = 'firefox-latest-%s.en-US.%s.%s' % (row['Release'], artifact_platform, self.get_platform_extension(row['Platform']))
|
||||
common_tests = 'firefox-latest-%s.en-US.%s.common.tests.zip' % (row['Release'], artifact_platform)
|
||||
web_platform_tests = 'firefox-latest-%s.en-US.%s.web-platform.tests.zip' % (row['Release'], artifact_platform)
|
||||
|
||||
# We will need build_file for reporting results to treeherder
|
||||
# build_file = 'firefox-latest-%s.en-US.%s.txt' % (row['Release'], artifact_platform1)
|
||||
|
||||
template = self.get_template()
|
||||
|
||||
template = re.sub(REGEXPS['triggers'], triggers, template)
|
||||
template = re.sub(REGEXPS['url1'], url1, template)
|
||||
template = re.sub(REGEXPS['url2'], url2, template)
|
||||
template = re.sub(REGEXPS['package'], package, template)
|
||||
template = re.sub(REGEXPS['common_tests'], common_tests, template)
|
||||
template = re.sub(REGEXPS['web_platform_tests'], web_platform_tests, template)
|
||||
template = re.sub(REGEXPS['slave'], row['Slave'], template)
|
||||
template = re.sub(REGEXPS['platform'], row['Platform'], template)
|
||||
template = re.sub(REGEXPS['python'], row['Python'], template)
|
||||
template = re.sub(REGEXPS['bash'], row['Bash'], template)
|
||||
|
||||
# We will need build_file for reporting results to treeherder
|
||||
#template = re.sub(REGEXPS['build_file'], build_file, template)
|
||||
|
||||
# Generate directory
|
||||
dir = os.path.join('jobs', jobname)
|
||||
if not os.path.exists(dir):
|
||||
os.makedirs(dir)
|
||||
|
||||
# Write it out
|
||||
config_path = os.path.join('jobs', jobname, 'config.xml')
|
||||
config_file = open(config_path, 'w')
|
||||
config_file.write(template)
|
||||
config_file.close()
|
||||
|
||||
csvfile.close()
|
||||
|
||||
print "Finished processing."
|
||||
return 0
|
||||
template = re.sub(self.REGEXPS['triggers'], triggers, template)
|
||||
template = re.sub(self.REGEXPS['url1'], url1, template)
|
||||
template = re.sub(self.REGEXPS['url2'], url2, template)
|
||||
template = re.sub(self.REGEXPS['package'], package, template)
|
||||
template = re.sub(self.REGEXPS['common_tests'], common_tests, template)
|
||||
template = re.sub(self.REGEXPS['web_platform_tests'], web_platform_tests, template)
|
||||
template = re.sub(self.REGEXPS['slave'], row['Slave'], template)
|
||||
template = re.sub(self.REGEXPS['platform'], row['Platform'], template)
|
||||
template = re.sub(self.REGEXPS['python'], row['Python'], template)
|
||||
template = re.sub(self.REGEXPS['bash'], row['Bash'], template)
|
||||
|
||||
return jobname, template
|
||||
|
||||
def generate_jobs(argv):
|
||||
generator = GenerateJobs(argv)
|
||||
generator = GenerateMSEJobs(argv)
|
||||
return generator.generate()
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
|
|
|||
|
|
@ -6,35 +6,11 @@
|
|||
|
||||
"""Generate jobs for jenkins based on a .csv file describing the parameters."""
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
import csv
|
||||
import os
|
||||
import os.path
|
||||
import re
|
||||
import sys
|
||||
|
||||
REGEXPS = {
|
||||
'expand': r'XXEXPANDXX',
|
||||
'url1': r'XXURL1XX',
|
||||
'package1': r'XXPACKAGE1XX',
|
||||
'build_file1': r'XXBUILD_FILE1XX',
|
||||
'machine1': r'XXMACHINE1XX',
|
||||
'arch1': r'XXARCH1XX',
|
||||
'url2': r'XXURL2XX',
|
||||
'package2': r'XXPACKAGE2XX',
|
||||
'build_file2': r'XXBUILD_FILE2XX',
|
||||
'machine2': r'XXMACHINE2XX',
|
||||
'arch2': r'XXARCH2XX',
|
||||
'triggers': r'XXTRIGGERSXX',
|
||||
'slave': r'XXSLAVEXX',
|
||||
'sigserver': r'XXXSIGSERVERXXX',
|
||||
'tests_release': r'XXXTESTSRELEASEXXX',
|
||||
'os1': r'XXOS1XX',
|
||||
'os2': r'XXOS2XX',
|
||||
}
|
||||
|
||||
for key in REGEXPS:
|
||||
REGEXPS[key] = re.compile(REGEXPS[key])
|
||||
from job_generator import JobGenerator
|
||||
|
||||
platform_host_data = {
|
||||
'linux32': {
|
||||
|
|
@ -137,130 +113,91 @@ platform_host_data = {
|
|||
},
|
||||
}
|
||||
|
||||
class JobGenerator():
|
||||
def __init__(self):
|
||||
pass
|
||||
|
||||
|
||||
class GenerateJobs():
|
||||
class GenerateWebRTCJobs(JobGenerator):
|
||||
def __init__(self, argv):
|
||||
self.argv = argv
|
||||
self.template = None
|
||||
super(GenerateWebRTCJobs, self).__init__(argv)
|
||||
self.REGEXPS = {
|
||||
'url1': r'XXURL1XX',
|
||||
'package1': r'XXPACKAGE1XX',
|
||||
'build_file1': r'XXBUILD_FILE1XX',
|
||||
'machine1': r'XXMACHINE1XX',
|
||||
'arch1': r'XXARCH1XX',
|
||||
'url2': r'XXURL2XX',
|
||||
'package2': r'XXPACKAGE2XX',
|
||||
'build_file2': r'XXBUILD_FILE2XX',
|
||||
'machine2': r'XXMACHINE2XX',
|
||||
'arch2': r'XXARCH2XX',
|
||||
'triggers': r'XXTRIGGERSXX',
|
||||
'slave': r'XXSLAVEXX',
|
||||
'sigserver': r'XXXSIGSERVERXXX',
|
||||
'tests_release': r'XXXTESTSRELEASEXXX',
|
||||
'os1': r'XXOS1XX',
|
||||
'os2': r'XXOS2XX',
|
||||
}
|
||||
|
||||
def get_config(self):
|
||||
parser = argparse.ArgumentParser(description='Script for generating Jenkins jobs for WebRTC testing')
|
||||
parser.add_argument('--template', required=True)
|
||||
parser.add_argument('--table-csv', required=True, dest='csv')
|
||||
parser.add_argument('--jenkins-host', required=True, dest='host')
|
||||
args = parser.parse_args(self.argv)
|
||||
def process_row(self, row, template):
|
||||
jobname = 'webrtc-%s-%s' % (row['Release1'], row['PlatformJobLabel1'])
|
||||
if row['Release1'] != row['Release2']:
|
||||
jobname = '%s-%s' % (jobname, row['Release2'])
|
||||
jobname = '%s-%s' % (jobname, row['PlatformJobLabel2'])
|
||||
if row['Networking']:
|
||||
jobname = '%s-%s' % (jobname, row['Networking'])
|
||||
|
||||
self.config = {}
|
||||
self.config['template'] = args.template
|
||||
self.config['csv'] = args.csv
|
||||
self.config['host'] = args.host
|
||||
print 'jobname = %s' % jobname
|
||||
|
||||
'''Return a copy of the template so that it can be reused over and over again.'''
|
||||
def get_template(self):
|
||||
if not self.template:
|
||||
with open(self.config['template']) as myfile:
|
||||
self.template=myfile.read()
|
||||
data = self.template
|
||||
return data
|
||||
tests_release = self.get_lowest_release(row['Release1'], row['Release2'])
|
||||
|
||||
def get_lowest_release(self, release1, release2):
|
||||
if release1 == 'esr' or release2 == 'esr':
|
||||
return 'esr'
|
||||
elif release1 == 'release' or release2 == 'release':
|
||||
return 'release'
|
||||
elif release1 == 'beta' or release2 == 'beta':
|
||||
return 'beta'
|
||||
elif release1 == 'aurora' or release2 == 'aurora':
|
||||
return 'aurora'
|
||||
if tests_release == 'beta':
|
||||
triggers = ''
|
||||
else:
|
||||
return 'nightly'
|
||||
triggers = 'trigger-firefox-%s-%s' % (row['Release1'], row['Platform1'])
|
||||
if row['Release1'] != row['Release2'] or row['Platform1'] != row['Platform2']:
|
||||
triggers = '%s,trigger-firefox-%s-%s' % (triggers, row['Release2'], row['Platform2'])
|
||||
|
||||
def generate(self):
|
||||
self.get_config()
|
||||
expand = 'expand-tests-%s-linux64' % tests_release
|
||||
if row['Networking']:
|
||||
expand = '%s-network' % expand
|
||||
triggers = '%s,%s' % (triggers, expand)
|
||||
|
||||
if not os.path.exists('jobs'):
|
||||
os.makedirs('jobs')
|
||||
if row['Networking']:
|
||||
sigserver = "NETWORK_SIGNALLING_SERVER"
|
||||
else:
|
||||
sigserver = "SIGNALLING_SERVER"
|
||||
|
||||
with open(self.config['csv'], 'rU') as csvfile:
|
||||
reader = csv.DictReader(csvfile)
|
||||
for row in reader:
|
||||
jobname = 'webrtc-%s-%s' % (row['Release1'], row['PlatformJobLabel1'])
|
||||
if row['Release1'] != row['Release2']:
|
||||
jobname = '%s-%s' % (jobname, row['Release2'])
|
||||
jobname = '%s-%s' % (jobname, row['PlatformJobLabel2'])
|
||||
if row['Networking']:
|
||||
jobname = '%s-%s' % (jobname, row['Networking'])
|
||||
url1 = '%s/job/firefox-%s-%s/ws/releases' % (self.config['host'], row['Release1'], row['Platform1'])
|
||||
url2 = '%s/job/firefox-%s-%s/ws/releases' % (self.config['host'], row['Release2'], row['Platform2'])
|
||||
|
||||
print 'jobname = %s' % jobname
|
||||
artifact_platform1 = platform_host_data[row['Platform1']]['artifact_platform']
|
||||
artifact_platform2 = platform_host_data[row['Platform2']]['artifact_platform']
|
||||
|
||||
triggers = 'trigger-firefox-%s-%s' % (row['Release1'], row['Platform1'])
|
||||
if row['Release1'] != row['Release2'] or row['Platform1'] != row['Platform2']:
|
||||
triggers = '%s,trigger-firefox-%s-%s' % (triggers, row['Release2'], row['Platform2'])
|
||||
package1 = 'firefox-latest-%s.en-US.%s.%s' % (row['Release1'], artifact_platform1, platform_host_data[row['Platform1']]['extension'])
|
||||
package2 = 'firefox-latest-%s.en-US.%s.%s' % (row['Release2'], artifact_platform2, platform_host_data[row['Platform2']]['extension'])
|
||||
|
||||
tests_release = self.get_lowest_release(row['Release1'], row['Release2'])
|
||||
build_file1 = 'firefox-latest-%s.en-US.%s.txt' % (row['Release1'], artifact_platform1)
|
||||
build_file2 = 'firefox-latest-%s.en-US.%s.txt' % (row['Release2'], artifact_platform2)
|
||||
|
||||
expand = 'expand-tests-%s-linux64' % tests_release
|
||||
if row['Networking']:
|
||||
expand = '%s-network' % expand
|
||||
sigserver = "NETWORK_SIGNALLING_SERVER"
|
||||
else:
|
||||
sigserver = "SIGNALLING_SERVER"
|
||||
|
||||
url1 = '%s/job/firefox-%s-%s/ws/releases' % (self.config['host'], row['Release1'], row['Platform1'])
|
||||
url2 = '%s/job/firefox-%s-%s/ws/releases' % (self.config['host'], row['Release2'], row['Platform2'])
|
||||
|
||||
artifact_platform1 = platform_host_data[row['Platform1']]['artifact_platform']
|
||||
artifact_platform2 = platform_host_data[row['Platform2']]['artifact_platform']
|
||||
|
||||
package1 = 'firefox-latest-%s.en-US.%s.%s' % (row['Release1'], artifact_platform1, platform_host_data[row['Platform1']]['extension'])
|
||||
package2 = 'firefox-latest-%s.en-US.%s.%s' % (row['Release2'], artifact_platform2, platform_host_data[row['Platform2']]['extension'])
|
||||
|
||||
build_file1 = 'firefox-latest-%s.en-US.%s.txt' % (row['Release1'], artifact_platform1)
|
||||
build_file2 = 'firefox-latest-%s.en-US.%s.txt' % (row['Release2'], artifact_platform2)
|
||||
|
||||
template = self.get_template()
|
||||
|
||||
template = re.sub(REGEXPS['triggers'], triggers, template)
|
||||
template = re.sub(REGEXPS['expand'], expand, template)
|
||||
template = re.sub(REGEXPS['url1'], url1, template)
|
||||
template = re.sub(REGEXPS['url2'], url2, template)
|
||||
template = re.sub(REGEXPS['package1'], package1, template)
|
||||
template = re.sub(REGEXPS['package2'], package2, template)
|
||||
template = re.sub(REGEXPS['build_file1'], build_file1, template)
|
||||
template = re.sub(REGEXPS['build_file2'], build_file2, template)
|
||||
template = re.sub(REGEXPS['machine1'], row['Host1'], template)
|
||||
template = re.sub(REGEXPS['machine2'], row['Host2'], template)
|
||||
template = re.sub(REGEXPS['arch1'], row['Arch1'], template)
|
||||
template = re.sub(REGEXPS['arch2'], row['Arch2'], template)
|
||||
template = re.sub(REGEXPS['slave'], row['Slave'], template)
|
||||
template = re.sub(REGEXPS['sigserver'], sigserver, template)
|
||||
template = re.sub(REGEXPS['tests_release'], tests_release, template)
|
||||
template = re.sub(REGEXPS['os1'], platform_host_data[row['Platform1']]['os'], template)
|
||||
template = re.sub(REGEXPS['os2'], platform_host_data[row['Platform2']]['os'], template)
|
||||
|
||||
# Generate directory
|
||||
dir = os.path.join('jobs', jobname)
|
||||
if not os.path.exists(dir):
|
||||
os.makedirs(dir)
|
||||
|
||||
# Write it out
|
||||
config_path = os.path.join('jobs', jobname, 'config.xml')
|
||||
config_file = open(config_path, 'w')
|
||||
config_file.write(template)
|
||||
config_file.close()
|
||||
|
||||
csvfile.close()
|
||||
|
||||
print "Finished processing."
|
||||
return 0
|
||||
template = re.sub(self.REGEXPS['triggers'], triggers, template)
|
||||
template = re.sub(self.REGEXPS['url1'], url1, template)
|
||||
template = re.sub(self.REGEXPS['url2'], url2, template)
|
||||
template = re.sub(self.REGEXPS['package1'], package1, template)
|
||||
template = re.sub(self.REGEXPS['package2'], package2, template)
|
||||
template = re.sub(self.REGEXPS['build_file1'], build_file1, template)
|
||||
template = re.sub(self.REGEXPS['build_file2'], build_file2, template)
|
||||
template = re.sub(self.REGEXPS['machine1'], row['Host1'], template)
|
||||
template = re.sub(self.REGEXPS['machine2'], row['Host2'], template)
|
||||
template = re.sub(self.REGEXPS['arch1'], row['Arch1'], template)
|
||||
template = re.sub(self.REGEXPS['arch2'], row['Arch2'], template)
|
||||
template = re.sub(self.REGEXPS['slave'], row['Slave'], template)
|
||||
template = re.sub(self.REGEXPS['sigserver'], sigserver, template)
|
||||
template = re.sub(self.REGEXPS['tests_release'], tests_release, template)
|
||||
template = re.sub(self.REGEXPS['os1'], platform_host_data[row['Platform1']]['os'], template)
|
||||
template = re.sub(self.REGEXPS['os2'], platform_host_data[row['Platform2']]['os'], template)
|
||||
|
||||
return jobname, template
|
||||
|
||||
def generate_jobs(argv):
|
||||
generator = GenerateJobs(argv)
|
||||
generator = GenerateWebRTCJobs(argv)
|
||||
return generator.generate()
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
|
|
|||
|
|
@ -0,0 +1,127 @@
|
|||
#!/usr/bin/env python
|
||||
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
"""Generate jobs for jenkins based on a .csv file describing the parameters."""
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import os
|
||||
import os.path
|
||||
import re
|
||||
import jenkins
|
||||
|
||||
class JobGenerator(object):
|
||||
def __init__(self, argv):
|
||||
self.argv = argv
|
||||
self.template = None
|
||||
self.jenkins_instance = None
|
||||
self.REGEXPS = {}
|
||||
|
||||
@property
|
||||
def jenkins(self):
|
||||
if not self.jenkins_instance:
|
||||
# We will have to do work when we require authentication for
|
||||
# our Jenkins instances. Need to either pass in user/password
|
||||
# or read from a config file of some kind.
|
||||
self.jenkins_instance = jenkins.Jenkins(self.config['host'])
|
||||
return self.jenkins_instance
|
||||
|
||||
def initialize_regexps(self):
|
||||
for key in self.REGEXPS:
|
||||
self.REGEXPS[key] = re.compile(self.REGEXPS[key])
|
||||
|
||||
def get_config(self):
|
||||
parser = argparse.ArgumentParser(description='Script for generating Jenkins jobs for MSE testing')
|
||||
parser.add_argument('--template', required=True)
|
||||
parser.add_argument('--table-csv', required=True, dest='csv')
|
||||
parser.add_argument('--jenkins-host', required=True, dest='host')
|
||||
parser.add_argument('--generate-files', action='store_true')
|
||||
args = parser.parse_args(self.argv)
|
||||
|
||||
self.config = {}
|
||||
self.config['template'] = args.template
|
||||
self.config['csv'] = args.csv
|
||||
self.config['host'] = args.host
|
||||
if args.generate_files:
|
||||
self.config['generate_files'] = True
|
||||
else:
|
||||
self.config['generate_files'] = False
|
||||
|
||||
'''Return a copy of the template so that it can be reused over and over again.'''
|
||||
def get_template(self):
|
||||
if not self.template:
|
||||
with open(self.config['template']) as myfile:
|
||||
self.template=myfile.read()
|
||||
data = self.template
|
||||
return data
|
||||
|
||||
def get_platform_extension(self, platform):
|
||||
if platform == 'linux64' or platform == 'linux32' or platform == 'linux':
|
||||
return 'tar.bz2'
|
||||
elif platform == 'win32' or platform == 'win64':
|
||||
return 'zip'
|
||||
elif platform == 'mac' or platform == 'mac64':
|
||||
return 'dmg'
|
||||
raise 'Unknown platform %s' % platform
|
||||
|
||||
def get_artifact_platform(self, platform):
|
||||
if platform == 'linux64':
|
||||
return 'linux-x86_64'
|
||||
elif platform == 'linux32':
|
||||
return 'linux-i686'
|
||||
else:
|
||||
return platform
|
||||
|
||||
def get_lowest_release(self, release1, release2):
|
||||
if release1 == 'esr' or release2 == 'esr':
|
||||
return 'esr'
|
||||
elif release1 == 'release' or release2 == 'release':
|
||||
return 'release'
|
||||
elif release1 == 'beta' or release2 == 'beta':
|
||||
return 'beta'
|
||||
elif release1 == 'aurora' or release2 == 'aurora':
|
||||
return 'aurora'
|
||||
else:
|
||||
return 'nightly'
|
||||
|
||||
def process_row(self, row, template):
|
||||
return template
|
||||
|
||||
def generate(self):
|
||||
self.get_config()
|
||||
self.initialize_regexps()
|
||||
|
||||
with open(self.config['csv'], 'rU') as csvfile:
|
||||
reader = csv.DictReader(csvfile)
|
||||
for row in reader:
|
||||
template = self.get_template()
|
||||
jobname, template = self.process_row(row, template)
|
||||
|
||||
# Generate directory
|
||||
if self.config['generate_files']:
|
||||
if not os.path.exists('jobs'):
|
||||
os.makedirs('jobs')
|
||||
|
||||
dir = os.path.join('jobs', jobname)
|
||||
if not os.path.exists(dir):
|
||||
os.makedirs(dir)
|
||||
|
||||
# Write it out
|
||||
config_path = os.path.join('jobs', jobname, 'config.xml')
|
||||
config_file = open(config_path, 'w')
|
||||
config_file.write(template)
|
||||
config_file.close()
|
||||
else:
|
||||
# Tell Jenkins about it.
|
||||
if self.jenkins.job_exists(jobname):
|
||||
self.jenkins.reconfig_job(jobname, template)
|
||||
else:
|
||||
self.jenkins.create_job(jobname, template)
|
||||
|
||||
csvfile.close()
|
||||
|
||||
print "Finished processing."
|
||||
return 0
|
||||
|
|
@ -10,7 +10,7 @@
|
|||
</logRotator>
|
||||
<keepDependencies>false</keepDependencies>
|
||||
<properties/>
|
||||
<scm class="hudson.plugins.git.GitSCM" plugin="git@2.3.5">
|
||||
<scm class="hudson.plugins.git.GitSCM" plugin="git@2.4.0">
|
||||
<configVersion>2</configVersion>
|
||||
<userRemoteConfigs>
|
||||
<hudson.plugins.git.UserRemoteConfig>
|
||||
|
|
@ -47,7 +47,7 @@
|
|||
<spec>H/3 * * * *</spec>
|
||||
<ignorePostCommitHooks>false</ignorePostCommitHooks>
|
||||
</hudson.triggers.SCMTrigger>
|
||||
<org.jenkinsci.plugins.urltrigger.URLTrigger plugin="urltrigger@0.38">
|
||||
<org.jenkinsci.plugins.urltrigger.URLTrigger plugin="urltrigger@0.40">
|
||||
<spec>H * * * *</spec>
|
||||
<entries>
|
||||
<org.jenkinsci.plugins.urltrigger.URLTriggerEntry>
|
||||
|
|
|
|||
|
|
@ -63,7 +63,7 @@
|
|||
<triggers>
|
||||
<jenkins.triggers.ReverseBuildTrigger>
|
||||
<spec></spec>
|
||||
<upstreamProjects>XXTRIGGERSXX,XXEXPANDXX</upstreamProjects>
|
||||
<upstreamProjects>XXTRIGGERSXX</upstreamProjects>
|
||||
<threshold>
|
||||
<name>SUCCESS</name>
|
||||
<ordinal>0</ordinal>
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче