Refactor to concentrate generate of jobs into one class with subclass
specialization. Call jenkins to create or update jobs unless files are specified.
This commit is contained in:
Syd Polk
Родитель
e30c5d34e6
Коммит
b47a427df1
|
|
@ -22,4 +22,5 @@ queue.xml
|
||||||
scm-sync-configuration
|
scm-sync-configuration
|
||||||
updates
|
updates
|
||||||
.idea
|
.idea
|
||||||
|
*.pyc
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1 +1 @@
|
||||||
Release,Platform,PlatformJobLabel,Slave,Python,Bash
nightly,mac,mac,mac,System-CPython-2.7,bash
nightly,mac,mac10_9,mac10-9,System-CPython-2.7,bash
nightly,mac,mac10_8,mac10-8,System-CPython-2.7,bash
nightly,mac,mac10_7,mac10-7,System-CPython-2.7,bash
nightly,mac,mac10_6,mac10-6,Local-Python-2.7,bash
nightly,win32,win7_32,win32 && mse && win7,Win-Moz-Python,c:\\mozilla-build\\msys\\bin\\bash
nightly,win32,win7_32_64,win64 && mse && win7,Win-Moz-Python,c:\\mozilla-build\\msys\\bin\\bash
nightly,win32,win32,win32 && mse && win,Win-Moz-Python,c:\\mozilla-build\\msys\\bin\\bash
nightly,win32,win32_64,win64 && mse && win,Win-Moz-Python,c:\\mozilla-build\\msys\\bin\\bash
nightly,win64,win7_64,win64 && mse && win7,Win-Moz-Python,c:\\mozilla-build\\msys\\bin\\bash
nightly,win64,win64,win64 && mse && win,Win-Moz-Python,c:\\mozilla-build\\msys\\bin\\bash
aurora,mac,mac,mac,System-CPython-2.7,bash
aurora,mac,mac10_9,mac10-9,System-CPython-2.7,bash
aurora,mac,mac10_8,mac10-8,System-CPython-2.7,bash
aurora,mac,mac10_7,mac10-7,System-CPython-2.7,bash
aurora,mac,mac10_6,mac10-6,Local-Python-2.7,bash
mac,mac10_6,mac10-6,Local-Python-2.7,bash
nightly,win32,win7_32_64,win64 && mse && win7,Win-Moz-Python,c:\\mozilla-build\\msys\\bin\\bash
mac,mac10_6,mac10-6,Local-Python-2.7,bash
nightly,win32,win32,win32 && mse && win,Win-Moz-Python,c:\\mozilla-build\\msys\\bin\\bash
mac,mac10_6,mac10-6,Local-Python-2.7,bash
nightly,win32,win32_64,win64 && mse && win,Win-Moz-Python,c:\\mozilla-build\\msys\\bin\\bash
mac,mac10_6,mac10-6,Local-Python-2.7,bash
nightly,win64,win7_64,win64 && mse && win7,Win-Moz-Python,c:\\mozilla-build\\msys\\bin\\bash
mac,mac10_6,mac10-6,Local-Python-2.7,bash
nightly,win64,win64,win64 && mse && win,Win-Moz-Python,c:\\mozilla-build\\msys\\bin\\bash
mac,mac10_6,mac10-6,Local-Python-2.7,bash
aurora,mac,mac,mac,System-CPython-2.7,bash
beta,mac,mac,mac,System-CPython-2.7,bash
beta,mac,mac10_9,mac10-9,System-CPython-2.7,bash
beta,mac,mac10_8,mac10-8,System-CPython-2.7,bash
beta,mac,mac10_7,mac10-7,System-CPython-2.7,bash
beta,mac,mac10_6,mac10-6,Local-Python-2.7,bash
nightly,win32,win7_32,win32 && mse && win7,Win-Moz-Python,c:\\mozilla-build\\msys\\bin\\bash
nightly,win32,win32,win32 && mse && win,Win-Moz-Python,c:\\mozilla-build\\msys\\bin\\bash
beta,win32,win7_32_64,win64 && mse && win7,Win-Moz-Python,c:\\mozilla-build\\msys\\bin\\bash
nightly,win32,win7_32,win32 && mse && win7,Win-Moz-Python,c:\\mozilla-build\\msys\\bin\\bash
nightly,win64,win7_64,win64 && mse && win7,Win-Moz-Python,c:\\mozilla-build\\msys\\bin\\bash
beta,win32,win32_64,win64 && mse && win,Win-Moz-Python,c:\\mozilla-build\\msys\\bin\\bash
|
Release,Platform,PlatformJobLabel,Slave,Python,Bash
nightly,mac,mac,mac,System-CPython-2.7,bash
nightly,mac,mac10_9,mac10-9,System-CPython-2.7,bash
nightly,mac,mac10_8,mac10-8,System-CPython-2.7,bash
nightly,mac,mac10_7,mac10-7,System-CPython-2.7,bash
nightly,win32,win7_32,win32 && win7,Win-Moz-Python,c:\\mozilla-build\\msys\\bin\\bash
nightly,win32,win7_32,win32 && mse && win7,Win-Moz-Python,c:\\mozilla-build\\msys\\bin\\bash
aurora,mac,mac10_9,mac10-9,System-CPython-2.7,bash
nightly,win32,win7_32_64,win64 && mse && win7,Win-Moz-Python,c:\\mozilla-build\\msys\\bin\\bash
nightly,win32,win32_64,win64 && win,Win-Moz-Python,c:\\mozilla-build\\msys\\bin\\bash
nightly,win64,win7_64,win64 && win7,Win-Moz-Python,c:\\mozilla-build\\msys\\bin\\bash
nightly,win64,win64,win64 && win,Win-Moz-Python,c:\\mozilla-build\\msys\\bin\\bash
aurora,mac,mac,mac,System-CPython-2.7,bash
aurora,mac,mac10_9,mac10-9,System-CPython-2.7,bash
aurora,mac,mac10_8,mac10-8,System-CPython-2.7,bash
aurora,mac,mac10_7,mac10-7,System-CPython-2.7,bash
nightly,win32,win7_32_64,win64 && mse && win7,Win-Moz-Python,c:\\mozilla-build\\msys\\bin\\bash
nightly,win32,win32,win32 && mse && win,Win-Moz-Python,c:\\mozilla-build\\msys\\bin\\bash
nightly,win32,win7_32_64,win64 && mse && win7,Win-Moz-Python,c:\\mozilla-build\\msys\\bin\\bash
nightly,win32,win32_64,win64 && mse && win,Win-Moz-Python,c:\\mozilla-build\\msys\\bin\\bash
nightly,win32,win7_32_64,win64 && mse && win7,Win-Moz-Python,c:\\mozilla-build\\msys\\bin\\bash
nightly,win64,win7_64,win64 && mse && win7,Win-Moz-Python,c:\\mozilla-build\\msys\\bin\\bash
nightly,win32,win7_32_64,win64 && mse && win7,Win-Moz-Python,c:\\mozilla-build\\msys\\bin\\bash
nightly,win64,win64,win64 && mse && win,Win-Moz-Python,c:\\mozilla-build\\msys\\bin\\bash
nightly,win32,win7_32_64,win64 && mse && win7,Win-Moz-Python,c:\\mozilla-build\\msys\\bin\\bash
aurora,mac,mac,mac,System-CPython-2.7,bash
aurora,win64,win64,win64 && win,Win-Moz-Python,c:\\mozilla-build\\msys\\bin\\bash
beta,mac,mac,mac,System-CPython-2.7,bash
beta,mac,mac10_9,mac10-9,System-CPython-2.7,bash
beta,mac,mac10_8,mac10-8,System-CPython-2.7,bash
beta,mac,mac10_7,mac10-7,System-CPython-2.7,bash
nightly,win32,win32,win32 && mse && win,Win-Moz-Python,c:\\mozilla-build\\msys\\bin\\bash
beta,win32,win7_32_64,win64 && win7,Win-Moz-Python,c:\\mozilla-build\\msys\\bin\\bash
nightly,win32,win32,win32 && mse && win,Win-Moz-Python,c:\\mozilla-build\\msys\\bin\\bash
nightly,win32,win7_32,win32 && mse && win7,Win-Moz-Python,c:\\mozilla-build\\msys\\bin\\bash
beta,win32,win32_64,win64 && win,Win-Moz-Python,c:\\mozilla-build\\msys\\bin\\bash
|
||||||
|
|
|
@ -6,152 +6,63 @@
|
||||||
|
|
||||||
"""Generate jobs for jenkins based on a .csv file describing the parameters."""
|
"""Generate jobs for jenkins based on a .csv file describing the parameters."""
|
||||||
|
|
||||||
import argparse
|
|
||||||
import sys
|
|
||||||
import csv
|
|
||||||
import os
|
import os
|
||||||
import os.path
|
|
||||||
import re
|
import re
|
||||||
|
import sys
|
||||||
|
|
||||||
REGEXPS = {
|
from job_generator import JobGenerator
|
||||||
'url1': r'XXURL1XX',
|
|
||||||
'package': r'XXPACKAGEXX',
|
|
||||||
'url2': r'XXURL2XX',
|
|
||||||
'common_tests': r'XXCOMMONTESTSXX',
|
|
||||||
'web_platform_tests': r'XXWEBPLATFORMTESTSXX',
|
|
||||||
'triggers': r'XXTRIGGERSXX',
|
|
||||||
'expand': r'XXEXPANDXX',
|
|
||||||
'slave': r'XXSLAVEXX',
|
|
||||||
'platform': r'XXPLATFORMXX',
|
|
||||||
'python': r'XXPYTHONXX',
|
|
||||||
'bash': r'XXBASHXX',
|
|
||||||
}
|
|
||||||
|
|
||||||
for key in REGEXPS:
|
class GenerateMSEJobs(JobGenerator):
|
||||||
REGEXPS[key] = re.compile(REGEXPS[key])
|
|
||||||
|
|
||||||
class JobGenerator():
|
|
||||||
def __init__(self):
|
|
||||||
pass
|
|
||||||
|
|
||||||
class GenerateJobs():
|
|
||||||
def __init__(self, argv):
|
def __init__(self, argv):
|
||||||
self.argv = argv
|
super(GenerateMSEJobs, self).__init__(argv)
|
||||||
self.template = None
|
self.REGEXPS = {
|
||||||
|
'url1': r'XXURL1XX',
|
||||||
|
'package': r'XXPACKAGEXX',
|
||||||
|
'url2': r'XXURL2XX',
|
||||||
|
'common_tests': r'XXCOMMONTESTSXX',
|
||||||
|
'web_platform_tests': r'XXWEBPLATFORMTESTSXX',
|
||||||
|
'triggers': r'XXTRIGGERSXX',
|
||||||
|
'expand': r'XXEXPANDXX',
|
||||||
|
'slave': r'XXSLAVEXX',
|
||||||
|
'platform': r'XXPLATFORMXX',
|
||||||
|
'python': r'XXPYTHONXX',
|
||||||
|
'bash': r'XXBASHXX',
|
||||||
|
}
|
||||||
|
|
||||||
def get_config(self):
|
def process_row(self, row, template):
|
||||||
parser = argparse.ArgumentParser(description='Script for generating Jenkins jobs for MSE testing')
|
jobname = 'mse-web-platform-%s-%s' % (row['Release'], row['PlatformJobLabel'])
|
||||||
parser.add_argument('--template', required=True)
|
|
||||||
parser.add_argument('--table-csv', required=True, dest='csv')
|
|
||||||
parser.add_argument('--jenkins-host', required=True, dest='host')
|
|
||||||
args = parser.parse_args(self.argv)
|
|
||||||
|
|
||||||
self.config = {}
|
print 'jobname = %s' % jobname
|
||||||
self.config['template'] = args.template
|
|
||||||
self.config['csv'] = args.csv
|
|
||||||
self.config['host'] = args.host
|
|
||||||
|
|
||||||
'''Return a copy of the template so that it can be reused over and over again.'''
|
if (row['Release'] != 'beta'):
|
||||||
def get_template(self):
|
triggers = 'trigger-firefox-%s-%s,trigger-tests-%s-%s' % (row['Release'], row['Platform'], row['Release'], row['Platform'])
|
||||||
if not self.template:
|
|
||||||
with open(self.config['template']) as myfile:
|
|
||||||
self.template=myfile.read()
|
|
||||||
data = self.template
|
|
||||||
return data
|
|
||||||
|
|
||||||
def get_platform_extension(self, platform):
|
|
||||||
if platform == 'linux64' or platform == 'linux32' or platform == 'linux':
|
|
||||||
return 'tar.bz2'
|
|
||||||
elif platform == 'win32' or platform == 'win64':
|
|
||||||
return 'zip'
|
|
||||||
elif platform == 'mac' or platform == 'mac64':
|
|
||||||
return 'dmg'
|
|
||||||
raise 'Unknown platform %s' % platform
|
|
||||||
|
|
||||||
def get_artifact_platform(self, platform):
|
|
||||||
if platform == 'linux64':
|
|
||||||
return 'linux-x86_64'
|
|
||||||
elif platform == 'linux32':
|
|
||||||
return 'linux-i686'
|
|
||||||
elif platform == 'win64':
|
|
||||||
return 'win64'
|
|
||||||
else:
|
else:
|
||||||
return platform
|
triggers = ''
|
||||||
|
|
||||||
def get_lowest_release(self, release1, release2):
|
url1 = '%s/job/firefox-%s-%s/ws/releases' % (self.config['host'], row['Release'], row['Platform'])
|
||||||
if release1 == 'esr' or release2 == 'esr':
|
url2 = '%s/job/tests-%s-%s/ws/releases' % (self.config['host'], row['Release'], row['Platform'])
|
||||||
return 'esr'
|
|
||||||
elif release1 == 'release' or release2 == 'release':
|
|
||||||
return 'release'
|
|
||||||
elif release1 == 'beta' or release2 == 'beta':
|
|
||||||
return 'beta'
|
|
||||||
elif release1 == 'aurora' or release2 == 'aurora':
|
|
||||||
return 'aurora'
|
|
||||||
else:
|
|
||||||
return 'nightly'
|
|
||||||
|
|
||||||
def generate(self):
|
artifact_platform = self.get_artifact_platform(row['Platform'])
|
||||||
self.get_config()
|
|
||||||
|
|
||||||
if not os.path.exists('jobs'):
|
package = 'firefox-latest-%s.en-US.%s.%s' % (row['Release'], artifact_platform, self.get_platform_extension(row['Platform']))
|
||||||
os.makedirs('jobs')
|
common_tests = 'firefox-latest-%s.en-US.%s.common.tests.zip' % (row['Release'], artifact_platform)
|
||||||
|
web_platform_tests = 'firefox-latest-%s.en-US.%s.web-platform.tests.zip' % (row['Release'], artifact_platform)
|
||||||
|
|
||||||
with open(self.config['csv'], 'rU') as csvfile:
|
template = re.sub(self.REGEXPS['triggers'], triggers, template)
|
||||||
reader = csv.DictReader(csvfile)
|
template = re.sub(self.REGEXPS['url1'], url1, template)
|
||||||
for row in reader:
|
template = re.sub(self.REGEXPS['url2'], url2, template)
|
||||||
jobname = 'mse-web-platform-%s-%s' % (row['Release'], row['PlatformJobLabel'])
|
template = re.sub(self.REGEXPS['package'], package, template)
|
||||||
|
template = re.sub(self.REGEXPS['common_tests'], common_tests, template)
|
||||||
print 'jobname = %s' % jobname
|
template = re.sub(self.REGEXPS['web_platform_tests'], web_platform_tests, template)
|
||||||
|
template = re.sub(self.REGEXPS['slave'], row['Slave'], template)
|
||||||
triggers = 'trigger-firefox-%s-%s, trigger-tests-%s-%s' % (row['Release'], row['Platform'], row['Release'], row['Platform'])
|
template = re.sub(self.REGEXPS['platform'], row['Platform'], template)
|
||||||
|
template = re.sub(self.REGEXPS['python'], row['Python'], template)
|
||||||
url1 = '%s/job/firefox-%s-%s/ws/releases' % (self.config['host'], row['Release'], row['Platform'])
|
template = re.sub(self.REGEXPS['bash'], row['Bash'], template)
|
||||||
url2 = '%s/job/tests-%s-%s/ws/releases' % (self.config['host'], row['Release'], row['Platform'])
|
|
||||||
|
|
||||||
artifact_platform = self.get_artifact_platform(row['Platform'])
|
|
||||||
|
|
||||||
package = 'firefox-latest-%s.en-US.%s.%s' % (row['Release'], artifact_platform, self.get_platform_extension(row['Platform']))
|
|
||||||
common_tests = 'firefox-latest-%s.en-US.%s.common.tests.zip' % (row['Release'], artifact_platform)
|
|
||||||
web_platform_tests = 'firefox-latest-%s.en-US.%s.web-platform.tests.zip' % (row['Release'], artifact_platform)
|
|
||||||
|
|
||||||
# We will need build_file for reporting results to treeherder
|
|
||||||
# build_file = 'firefox-latest-%s.en-US.%s.txt' % (row['Release'], artifact_platform1)
|
|
||||||
|
|
||||||
template = self.get_template()
|
|
||||||
|
|
||||||
template = re.sub(REGEXPS['triggers'], triggers, template)
|
|
||||||
template = re.sub(REGEXPS['url1'], url1, template)
|
|
||||||
template = re.sub(REGEXPS['url2'], url2, template)
|
|
||||||
template = re.sub(REGEXPS['package'], package, template)
|
|
||||||
template = re.sub(REGEXPS['common_tests'], common_tests, template)
|
|
||||||
template = re.sub(REGEXPS['web_platform_tests'], web_platform_tests, template)
|
|
||||||
template = re.sub(REGEXPS['slave'], row['Slave'], template)
|
|
||||||
template = re.sub(REGEXPS['platform'], row['Platform'], template)
|
|
||||||
template = re.sub(REGEXPS['python'], row['Python'], template)
|
|
||||||
template = re.sub(REGEXPS['bash'], row['Bash'], template)
|
|
||||||
|
|
||||||
# We will need build_file for reporting results to treeherder
|
|
||||||
#template = re.sub(REGEXPS['build_file'], build_file, template)
|
|
||||||
|
|
||||||
# Generate directory
|
|
||||||
dir = os.path.join('jobs', jobname)
|
|
||||||
if not os.path.exists(dir):
|
|
||||||
os.makedirs(dir)
|
|
||||||
|
|
||||||
# Write it out
|
|
||||||
config_path = os.path.join('jobs', jobname, 'config.xml')
|
|
||||||
config_file = open(config_path, 'w')
|
|
||||||
config_file.write(template)
|
|
||||||
config_file.close()
|
|
||||||
|
|
||||||
csvfile.close()
|
|
||||||
|
|
||||||
print "Finished processing."
|
|
||||||
return 0
|
|
||||||
|
|
||||||
|
return jobname, template
|
||||||
|
|
||||||
def generate_jobs(argv):
|
def generate_jobs(argv):
|
||||||
generator = GenerateJobs(argv)
|
generator = GenerateMSEJobs(argv)
|
||||||
return generator.generate()
|
return generator.generate()
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
|
|
|
||||||
|
|
@ -6,35 +6,11 @@
|
||||||
|
|
||||||
"""Generate jobs for jenkins based on a .csv file describing the parameters."""
|
"""Generate jobs for jenkins based on a .csv file describing the parameters."""
|
||||||
|
|
||||||
import argparse
|
|
||||||
import sys
|
|
||||||
import csv
|
|
||||||
import os
|
import os
|
||||||
import os.path
|
|
||||||
import re
|
import re
|
||||||
|
import sys
|
||||||
|
|
||||||
REGEXPS = {
|
from job_generator import JobGenerator
|
||||||
'expand': r'XXEXPANDXX',
|
|
||||||
'url1': r'XXURL1XX',
|
|
||||||
'package1': r'XXPACKAGE1XX',
|
|
||||||
'build_file1': r'XXBUILD_FILE1XX',
|
|
||||||
'machine1': r'XXMACHINE1XX',
|
|
||||||
'arch1': r'XXARCH1XX',
|
|
||||||
'url2': r'XXURL2XX',
|
|
||||||
'package2': r'XXPACKAGE2XX',
|
|
||||||
'build_file2': r'XXBUILD_FILE2XX',
|
|
||||||
'machine2': r'XXMACHINE2XX',
|
|
||||||
'arch2': r'XXARCH2XX',
|
|
||||||
'triggers': r'XXTRIGGERSXX',
|
|
||||||
'slave': r'XXSLAVEXX',
|
|
||||||
'sigserver': r'XXXSIGSERVERXXX',
|
|
||||||
'tests_release': r'XXXTESTSRELEASEXXX',
|
|
||||||
'os1': r'XXOS1XX',
|
|
||||||
'os2': r'XXOS2XX',
|
|
||||||
}
|
|
||||||
|
|
||||||
for key in REGEXPS:
|
|
||||||
REGEXPS[key] = re.compile(REGEXPS[key])
|
|
||||||
|
|
||||||
platform_host_data = {
|
platform_host_data = {
|
||||||
'linux32': {
|
'linux32': {
|
||||||
|
|
@ -137,130 +113,91 @@ platform_host_data = {
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
class JobGenerator():
|
|
||||||
def __init__(self):
|
|
||||||
pass
|
|
||||||
|
|
||||||
|
class GenerateWebRTCJobs(JobGenerator):
|
||||||
class GenerateJobs():
|
|
||||||
def __init__(self, argv):
|
def __init__(self, argv):
|
||||||
self.argv = argv
|
super(GenerateWebRTCJobs, self).__init__(argv)
|
||||||
self.template = None
|
self.REGEXPS = {
|
||||||
|
'url1': r'XXURL1XX',
|
||||||
|
'package1': r'XXPACKAGE1XX',
|
||||||
|
'build_file1': r'XXBUILD_FILE1XX',
|
||||||
|
'machine1': r'XXMACHINE1XX',
|
||||||
|
'arch1': r'XXARCH1XX',
|
||||||
|
'url2': r'XXURL2XX',
|
||||||
|
'package2': r'XXPACKAGE2XX',
|
||||||
|
'build_file2': r'XXBUILD_FILE2XX',
|
||||||
|
'machine2': r'XXMACHINE2XX',
|
||||||
|
'arch2': r'XXARCH2XX',
|
||||||
|
'triggers': r'XXTRIGGERSXX',
|
||||||
|
'slave': r'XXSLAVEXX',
|
||||||
|
'sigserver': r'XXXSIGSERVERXXX',
|
||||||
|
'tests_release': r'XXXTESTSRELEASEXXX',
|
||||||
|
'os1': r'XXOS1XX',
|
||||||
|
'os2': r'XXOS2XX',
|
||||||
|
}
|
||||||
|
|
||||||
def get_config(self):
|
def process_row(self, row, template):
|
||||||
parser = argparse.ArgumentParser(description='Script for generating Jenkins jobs for WebRTC testing')
|
jobname = 'webrtc-%s-%s' % (row['Release1'], row['PlatformJobLabel1'])
|
||||||
parser.add_argument('--template', required=True)
|
if row['Release1'] != row['Release2']:
|
||||||
parser.add_argument('--table-csv', required=True, dest='csv')
|
jobname = '%s-%s' % (jobname, row['Release2'])
|
||||||
parser.add_argument('--jenkins-host', required=True, dest='host')
|
jobname = '%s-%s' % (jobname, row['PlatformJobLabel2'])
|
||||||
args = parser.parse_args(self.argv)
|
if row['Networking']:
|
||||||
|
jobname = '%s-%s' % (jobname, row['Networking'])
|
||||||
|
|
||||||
self.config = {}
|
print 'jobname = %s' % jobname
|
||||||
self.config['template'] = args.template
|
|
||||||
self.config['csv'] = args.csv
|
|
||||||
self.config['host'] = args.host
|
|
||||||
|
|
||||||
'''Return a copy of the template so that it can be reused over and over again.'''
|
tests_release = self.get_lowest_release(row['Release1'], row['Release2'])
|
||||||
def get_template(self):
|
|
||||||
if not self.template:
|
|
||||||
with open(self.config['template']) as myfile:
|
|
||||||
self.template=myfile.read()
|
|
||||||
data = self.template
|
|
||||||
return data
|
|
||||||
|
|
||||||
def get_lowest_release(self, release1, release2):
|
if tests_release == 'beta':
|
||||||
if release1 == 'esr' or release2 == 'esr':
|
triggers = ''
|
||||||
return 'esr'
|
|
||||||
elif release1 == 'release' or release2 == 'release':
|
|
||||||
return 'release'
|
|
||||||
elif release1 == 'beta' or release2 == 'beta':
|
|
||||||
return 'beta'
|
|
||||||
elif release1 == 'aurora' or release2 == 'aurora':
|
|
||||||
return 'aurora'
|
|
||||||
else:
|
else:
|
||||||
return 'nightly'
|
triggers = 'trigger-firefox-%s-%s' % (row['Release1'], row['Platform1'])
|
||||||
|
if row['Release1'] != row['Release2'] or row['Platform1'] != row['Platform2']:
|
||||||
|
triggers = '%s,trigger-firefox-%s-%s' % (triggers, row['Release2'], row['Platform2'])
|
||||||
|
|
||||||
def generate(self):
|
expand = 'expand-tests-%s-linux64' % tests_release
|
||||||
self.get_config()
|
if row['Networking']:
|
||||||
|
expand = '%s-network' % expand
|
||||||
|
triggers = '%s,%s' % (triggers, expand)
|
||||||
|
|
||||||
if not os.path.exists('jobs'):
|
if row['Networking']:
|
||||||
os.makedirs('jobs')
|
sigserver = "NETWORK_SIGNALLING_SERVER"
|
||||||
|
else:
|
||||||
|
sigserver = "SIGNALLING_SERVER"
|
||||||
|
|
||||||
with open(self.config['csv'], 'rU') as csvfile:
|
url1 = '%s/job/firefox-%s-%s/ws/releases' % (self.config['host'], row['Release1'], row['Platform1'])
|
||||||
reader = csv.DictReader(csvfile)
|
url2 = '%s/job/firefox-%s-%s/ws/releases' % (self.config['host'], row['Release2'], row['Platform2'])
|
||||||
for row in reader:
|
|
||||||
jobname = 'webrtc-%s-%s' % (row['Release1'], row['PlatformJobLabel1'])
|
|
||||||
if row['Release1'] != row['Release2']:
|
|
||||||
jobname = '%s-%s' % (jobname, row['Release2'])
|
|
||||||
jobname = '%s-%s' % (jobname, row['PlatformJobLabel2'])
|
|
||||||
if row['Networking']:
|
|
||||||
jobname = '%s-%s' % (jobname, row['Networking'])
|
|
||||||
|
|
||||||
print 'jobname = %s' % jobname
|
artifact_platform1 = platform_host_data[row['Platform1']]['artifact_platform']
|
||||||
|
artifact_platform2 = platform_host_data[row['Platform2']]['artifact_platform']
|
||||||
|
|
||||||
triggers = 'trigger-firefox-%s-%s' % (row['Release1'], row['Platform1'])
|
package1 = 'firefox-latest-%s.en-US.%s.%s' % (row['Release1'], artifact_platform1, platform_host_data[row['Platform1']]['extension'])
|
||||||
if row['Release1'] != row['Release2'] or row['Platform1'] != row['Platform2']:
|
package2 = 'firefox-latest-%s.en-US.%s.%s' % (row['Release2'], artifact_platform2, platform_host_data[row['Platform2']]['extension'])
|
||||||
triggers = '%s,trigger-firefox-%s-%s' % (triggers, row['Release2'], row['Platform2'])
|
|
||||||
|
|
||||||
tests_release = self.get_lowest_release(row['Release1'], row['Release2'])
|
build_file1 = 'firefox-latest-%s.en-US.%s.txt' % (row['Release1'], artifact_platform1)
|
||||||
|
build_file2 = 'firefox-latest-%s.en-US.%s.txt' % (row['Release2'], artifact_platform2)
|
||||||
|
|
||||||
expand = 'expand-tests-%s-linux64' % tests_release
|
template = re.sub(self.REGEXPS['triggers'], triggers, template)
|
||||||
if row['Networking']:
|
template = re.sub(self.REGEXPS['url1'], url1, template)
|
||||||
expand = '%s-network' % expand
|
template = re.sub(self.REGEXPS['url2'], url2, template)
|
||||||
sigserver = "NETWORK_SIGNALLING_SERVER"
|
template = re.sub(self.REGEXPS['package1'], package1, template)
|
||||||
else:
|
template = re.sub(self.REGEXPS['package2'], package2, template)
|
||||||
sigserver = "SIGNALLING_SERVER"
|
template = re.sub(self.REGEXPS['build_file1'], build_file1, template)
|
||||||
|
template = re.sub(self.REGEXPS['build_file2'], build_file2, template)
|
||||||
url1 = '%s/job/firefox-%s-%s/ws/releases' % (self.config['host'], row['Release1'], row['Platform1'])
|
template = re.sub(self.REGEXPS['machine1'], row['Host1'], template)
|
||||||
url2 = '%s/job/firefox-%s-%s/ws/releases' % (self.config['host'], row['Release2'], row['Platform2'])
|
template = re.sub(self.REGEXPS['machine2'], row['Host2'], template)
|
||||||
|
template = re.sub(self.REGEXPS['arch1'], row['Arch1'], template)
|
||||||
artifact_platform1 = platform_host_data[row['Platform1']]['artifact_platform']
|
template = re.sub(self.REGEXPS['arch2'], row['Arch2'], template)
|
||||||
artifact_platform2 = platform_host_data[row['Platform2']]['artifact_platform']
|
template = re.sub(self.REGEXPS['slave'], row['Slave'], template)
|
||||||
|
template = re.sub(self.REGEXPS['sigserver'], sigserver, template)
|
||||||
package1 = 'firefox-latest-%s.en-US.%s.%s' % (row['Release1'], artifact_platform1, platform_host_data[row['Platform1']]['extension'])
|
template = re.sub(self.REGEXPS['tests_release'], tests_release, template)
|
||||||
package2 = 'firefox-latest-%s.en-US.%s.%s' % (row['Release2'], artifact_platform2, platform_host_data[row['Platform2']]['extension'])
|
template = re.sub(self.REGEXPS['os1'], platform_host_data[row['Platform1']]['os'], template)
|
||||||
|
template = re.sub(self.REGEXPS['os2'], platform_host_data[row['Platform2']]['os'], template)
|
||||||
build_file1 = 'firefox-latest-%s.en-US.%s.txt' % (row['Release1'], artifact_platform1)
|
|
||||||
build_file2 = 'firefox-latest-%s.en-US.%s.txt' % (row['Release2'], artifact_platform2)
|
|
||||||
|
|
||||||
template = self.get_template()
|
|
||||||
|
|
||||||
template = re.sub(REGEXPS['triggers'], triggers, template)
|
|
||||||
template = re.sub(REGEXPS['expand'], expand, template)
|
|
||||||
template = re.sub(REGEXPS['url1'], url1, template)
|
|
||||||
template = re.sub(REGEXPS['url2'], url2, template)
|
|
||||||
template = re.sub(REGEXPS['package1'], package1, template)
|
|
||||||
template = re.sub(REGEXPS['package2'], package2, template)
|
|
||||||
template = re.sub(REGEXPS['build_file1'], build_file1, template)
|
|
||||||
template = re.sub(REGEXPS['build_file2'], build_file2, template)
|
|
||||||
template = re.sub(REGEXPS['machine1'], row['Host1'], template)
|
|
||||||
template = re.sub(REGEXPS['machine2'], row['Host2'], template)
|
|
||||||
template = re.sub(REGEXPS['arch1'], row['Arch1'], template)
|
|
||||||
template = re.sub(REGEXPS['arch2'], row['Arch2'], template)
|
|
||||||
template = re.sub(REGEXPS['slave'], row['Slave'], template)
|
|
||||||
template = re.sub(REGEXPS['sigserver'], sigserver, template)
|
|
||||||
template = re.sub(REGEXPS['tests_release'], tests_release, template)
|
|
||||||
template = re.sub(REGEXPS['os1'], platform_host_data[row['Platform1']]['os'], template)
|
|
||||||
template = re.sub(REGEXPS['os2'], platform_host_data[row['Platform2']]['os'], template)
|
|
||||||
|
|
||||||
# Generate directory
|
|
||||||
dir = os.path.join('jobs', jobname)
|
|
||||||
if not os.path.exists(dir):
|
|
||||||
os.makedirs(dir)
|
|
||||||
|
|
||||||
# Write it out
|
|
||||||
config_path = os.path.join('jobs', jobname, 'config.xml')
|
|
||||||
config_file = open(config_path, 'w')
|
|
||||||
config_file.write(template)
|
|
||||||
config_file.close()
|
|
||||||
|
|
||||||
csvfile.close()
|
|
||||||
|
|
||||||
print "Finished processing."
|
|
||||||
return 0
|
|
||||||
|
|
||||||
|
return jobname, template
|
||||||
|
|
||||||
def generate_jobs(argv):
|
def generate_jobs(argv):
|
||||||
generator = GenerateJobs(argv)
|
generator = GenerateWebRTCJobs(argv)
|
||||||
return generator.generate()
|
return generator.generate()
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,127 @@
|
||||||
|
#!/usr/bin/env python
|
||||||
|
|
||||||
|
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||||
|
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||||
|
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||||
|
|
||||||
|
"""Generate jobs for jenkins based on a .csv file describing the parameters."""
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import csv
|
||||||
|
import os
|
||||||
|
import os.path
|
||||||
|
import re
|
||||||
|
import jenkins
|
||||||
|
|
||||||
|
class JobGenerator(object):
|
||||||
|
def __init__(self, argv):
|
||||||
|
self.argv = argv
|
||||||
|
self.template = None
|
||||||
|
self.jenkins_instance = None
|
||||||
|
self.REGEXPS = {}
|
||||||
|
|
||||||
|
@property
|
||||||
|
def jenkins(self):
|
||||||
|
if not self.jenkins_instance:
|
||||||
|
# We will have to do work when we require authentication for
|
||||||
|
# our Jenkins instances. Need to either pass in user/password
|
||||||
|
# or read from a config file of some kind.
|
||||||
|
self.jenkins_instance = jenkins.Jenkins(self.config['host'])
|
||||||
|
return self.jenkins_instance
|
||||||
|
|
||||||
|
def initialize_regexps(self):
|
||||||
|
for key in self.REGEXPS:
|
||||||
|
self.REGEXPS[key] = re.compile(self.REGEXPS[key])
|
||||||
|
|
||||||
|
def get_config(self):
|
||||||
|
parser = argparse.ArgumentParser(description='Script for generating Jenkins jobs for MSE testing')
|
||||||
|
parser.add_argument('--template', required=True)
|
||||||
|
parser.add_argument('--table-csv', required=True, dest='csv')
|
||||||
|
parser.add_argument('--jenkins-host', required=True, dest='host')
|
||||||
|
parser.add_argument('--generate-files', action='store_true')
|
||||||
|
args = parser.parse_args(self.argv)
|
||||||
|
|
||||||
|
self.config = {}
|
||||||
|
self.config['template'] = args.template
|
||||||
|
self.config['csv'] = args.csv
|
||||||
|
self.config['host'] = args.host
|
||||||
|
if args.generate_files:
|
||||||
|
self.config['generate_files'] = True
|
||||||
|
else:
|
||||||
|
self.config['generate_files'] = False
|
||||||
|
|
||||||
|
'''Return a copy of the template so that it can be reused over and over again.'''
|
||||||
|
def get_template(self):
|
||||||
|
if not self.template:
|
||||||
|
with open(self.config['template']) as myfile:
|
||||||
|
self.template=myfile.read()
|
||||||
|
data = self.template
|
||||||
|
return data
|
||||||
|
|
||||||
|
def get_platform_extension(self, platform):
|
||||||
|
if platform == 'linux64' or platform == 'linux32' or platform == 'linux':
|
||||||
|
return 'tar.bz2'
|
||||||
|
elif platform == 'win32' or platform == 'win64':
|
||||||
|
return 'zip'
|
||||||
|
elif platform == 'mac' or platform == 'mac64':
|
||||||
|
return 'dmg'
|
||||||
|
raise 'Unknown platform %s' % platform
|
||||||
|
|
||||||
|
def get_artifact_platform(self, platform):
|
||||||
|
if platform == 'linux64':
|
||||||
|
return 'linux-x86_64'
|
||||||
|
elif platform == 'linux32':
|
||||||
|
return 'linux-i686'
|
||||||
|
else:
|
||||||
|
return platform
|
||||||
|
|
||||||
|
def get_lowest_release(self, release1, release2):
|
||||||
|
if release1 == 'esr' or release2 == 'esr':
|
||||||
|
return 'esr'
|
||||||
|
elif release1 == 'release' or release2 == 'release':
|
||||||
|
return 'release'
|
||||||
|
elif release1 == 'beta' or release2 == 'beta':
|
||||||
|
return 'beta'
|
||||||
|
elif release1 == 'aurora' or release2 == 'aurora':
|
||||||
|
return 'aurora'
|
||||||
|
else:
|
||||||
|
return 'nightly'
|
||||||
|
|
||||||
|
def process_row(self, row, template):
|
||||||
|
return template
|
||||||
|
|
||||||
|
def generate(self):
|
||||||
|
self.get_config()
|
||||||
|
self.initialize_regexps()
|
||||||
|
|
||||||
|
with open(self.config['csv'], 'rU') as csvfile:
|
||||||
|
reader = csv.DictReader(csvfile)
|
||||||
|
for row in reader:
|
||||||
|
template = self.get_template()
|
||||||
|
jobname, template = self.process_row(row, template)
|
||||||
|
|
||||||
|
# Generate directory
|
||||||
|
if self.config['generate_files']:
|
||||||
|
if not os.path.exists('jobs'):
|
||||||
|
os.makedirs('jobs')
|
||||||
|
|
||||||
|
dir = os.path.join('jobs', jobname)
|
||||||
|
if not os.path.exists(dir):
|
||||||
|
os.makedirs(dir)
|
||||||
|
|
||||||
|
# Write it out
|
||||||
|
config_path = os.path.join('jobs', jobname, 'config.xml')
|
||||||
|
config_file = open(config_path, 'w')
|
||||||
|
config_file.write(template)
|
||||||
|
config_file.close()
|
||||||
|
else:
|
||||||
|
# Tell Jenkins about it.
|
||||||
|
if self.jenkins.job_exists(jobname):
|
||||||
|
self.jenkins.reconfig_job(jobname, template)
|
||||||
|
else:
|
||||||
|
self.jenkins.create_job(jobname, template)
|
||||||
|
|
||||||
|
csvfile.close()
|
||||||
|
|
||||||
|
print "Finished processing."
|
||||||
|
return 0
|
||||||
|
|
@ -10,7 +10,7 @@
|
||||||
</logRotator>
|
</logRotator>
|
||||||
<keepDependencies>false</keepDependencies>
|
<keepDependencies>false</keepDependencies>
|
||||||
<properties/>
|
<properties/>
|
||||||
<scm class="hudson.plugins.git.GitSCM" plugin="git@2.3.5">
|
<scm class="hudson.plugins.git.GitSCM" plugin="git@2.4.0">
|
||||||
<configVersion>2</configVersion>
|
<configVersion>2</configVersion>
|
||||||
<userRemoteConfigs>
|
<userRemoteConfigs>
|
||||||
<hudson.plugins.git.UserRemoteConfig>
|
<hudson.plugins.git.UserRemoteConfig>
|
||||||
|
|
@ -47,7 +47,7 @@
|
||||||
<spec>H/3 * * * *</spec>
|
<spec>H/3 * * * *</spec>
|
||||||
<ignorePostCommitHooks>false</ignorePostCommitHooks>
|
<ignorePostCommitHooks>false</ignorePostCommitHooks>
|
||||||
</hudson.triggers.SCMTrigger>
|
</hudson.triggers.SCMTrigger>
|
||||||
<org.jenkinsci.plugins.urltrigger.URLTrigger plugin="urltrigger@0.38">
|
<org.jenkinsci.plugins.urltrigger.URLTrigger plugin="urltrigger@0.40">
|
||||||
<spec>H * * * *</spec>
|
<spec>H * * * *</spec>
|
||||||
<entries>
|
<entries>
|
||||||
<org.jenkinsci.plugins.urltrigger.URLTriggerEntry>
|
<org.jenkinsci.plugins.urltrigger.URLTriggerEntry>
|
||||||
|
|
|
||||||
|
|
@ -63,7 +63,7 @@
|
||||||
<triggers>
|
<triggers>
|
||||||
<jenkins.triggers.ReverseBuildTrigger>
|
<jenkins.triggers.ReverseBuildTrigger>
|
||||||
<spec></spec>
|
<spec></spec>
|
||||||
<upstreamProjects>XXTRIGGERSXX,XXEXPANDXX</upstreamProjects>
|
<upstreamProjects>XXTRIGGERSXX</upstreamProjects>
|
||||||
<threshold>
|
<threshold>
|
||||||
<name>SUCCESS</name>
|
<name>SUCCESS</name>
|
||||||
<ordinal>0</ordinal>
|
<ordinal>0</ordinal>
|
||||||
|
|
|
||||||
Загрузка…
Ссылка в новой задаче