diff --git a/.gitignore b/.gitignore index b1dc6a1..20528f2 100644 --- a/.gitignore +++ b/.gitignore @@ -22,4 +22,5 @@ queue.xml scm-sync-configuration updates .idea +*.pyc diff --git a/MSE_Web_Platform_Test_Matrix.csv b/MSE_Web_Platform_Test_Matrix.csv index 143fa2d..29ebdb5 100644 --- a/MSE_Web_Platform_Test_Matrix.csv +++ b/MSE_Web_Platform_Test_Matrix.csv @@ -1 +1 @@ -Release,Platform,PlatformJobLabel,Slave,Python,Bash nightly,mac,mac,mac,System-CPython-2.7,bash nightly,mac,mac10_9,mac10-9,System-CPython-2.7,bash nightly,mac,mac10_8,mac10-8,System-CPython-2.7,bash nightly,mac,mac10_7,mac10-7,System-CPython-2.7,bash nightly,mac,mac10_6,mac10-6,Local-Python-2.7,bash nightly,win32,win7_32,win32 && mse && win7,Win-Moz-Python,c:\\mozilla-build\\msys\\bin\\bash nightly,win32,win7_32_64,win64 && mse && win7,Win-Moz-Python,c:\\mozilla-build\\msys\\bin\\bash nightly,win32,win32,win32 && mse && win,Win-Moz-Python,c:\\mozilla-build\\msys\\bin\\bash nightly,win32,win32_64,win64 && mse && win,Win-Moz-Python,c:\\mozilla-build\\msys\\bin\\bash nightly,win64,win7_64,win64 && mse && win7,Win-Moz-Python,c:\\mozilla-build\\msys\\bin\\bash nightly,win64,win64,win64 && mse && win,Win-Moz-Python,c:\\mozilla-build\\msys\\bin\\bash aurora,mac,mac,mac,System-CPython-2.7,bash aurora,mac,mac10_9,mac10-9,System-CPython-2.7,bash aurora,mac,mac10_8,mac10-8,System-CPython-2.7,bash aurora,mac,mac10_7,mac10-7,System-CPython-2.7,bash aurora,mac,mac10_6,mac10-6,Local-Python-2.7,bash aurora,win32,win7_32,win32 && mse && win7,Win-Moz-Python,c:\\mozilla-build\\msys\\bin\\bash aurora,win32,win7_32_64,win64 && mse && win7,Win-Moz-Python,c:\\mozilla-build\\msys\\bin\\bash aurora,win32,win32,win32 && mse && win,Win-Moz-Python,c:\\mozilla-build\\msys\\bin\\bash aurora,win32,win32_64,win64 && mse && win,Win-Moz-Python,c:\\mozilla-build\\msys\\bin\\bash aurora,win64,win7_64,win64 && mse && win7,Win-Moz-Python,c:\\mozilla-build\\msys\\bin\\bash aurora,win64,win64,win64 && mse && win,Win-Moz-Python,c:\\mozilla-build\\msys\\bin\\bash beta,mac,mac,mac,System-CPython-2.7,bash beta,mac,mac10_9,mac10-9,System-CPython-2.7,bash beta,mac,mac10_8,mac10-8,System-CPython-2.7,bash beta,mac,mac10_7,mac10-7,System-CPython-2.7,bash beta,mac,mac10_6,mac10-6,Local-Python-2.7,bash beta,win32,win7_32,win32 && mse && win7,Win-Moz-Python,c:\\mozilla-build\\msys\\bin\\bash beta,win32,win7_32_64,win64 && mse && win7,Win-Moz-Python,c:\\mozilla-build\\msys\\bin\\bash beta,win32,win32,win32 && mse && win,Win-Moz-Python,c:\\mozilla-build\\msys\\bin\\bash beta,win32,win32_64,win64 && mse && win,Win-Moz-Python,c:\\mozilla-build\\msys\\bin\\bash \ No newline at end of file +Release,Platform,PlatformJobLabel,Slave,Python,Bash nightly,mac,mac,mac,System-CPython-2.7,bash nightly,mac,mac10_9,mac10-9,System-CPython-2.7,bash nightly,mac,mac10_8,mac10-8,System-CPython-2.7,bash nightly,mac,mac10_7,mac10-7,System-CPython-2.7,bash nightly,win32,win7_32,win32 && win7,Win-Moz-Python,c:\\mozilla-build\\msys\\bin\\bash nightly,win32,win7_32_64,win64 && win7,Win-Moz-Python,c:\\mozilla-build\\msys\\bin\\bash nightly,win32,win32,win32 && win,Win-Moz-Python,c:\\mozilla-build\\msys\\bin\\bash nightly,win32,win32_64,win64 && win,Win-Moz-Python,c:\\mozilla-build\\msys\\bin\\bash nightly,win64,win7_64,win64 && win7,Win-Moz-Python,c:\\mozilla-build\\msys\\bin\\bash nightly,win64,win64,win64 && win,Win-Moz-Python,c:\\mozilla-build\\msys\\bin\\bash aurora,mac,mac,mac,System-CPython-2.7,bash aurora,mac,mac10_9,mac10-9,System-CPython-2.7,bash aurora,mac,mac10_8,mac10-8,System-CPython-2.7,bash aurora,mac,mac10_7,mac10-7,System-CPython-2.7,bash aurora,win32,win7_32,win32 && win7,Win-Moz-Python,c:\\mozilla-build\\msys\\bin\\bash aurora,win32,win7_32_64,win64 && win7,Win-Moz-Python,c:\\mozilla-build\\msys\\bin\\bash aurora,win32,win32,win32 && win,Win-Moz-Python,c:\\mozilla-build\\msys\\bin\\bash aurora,win32,win32_64,win64 && win,Win-Moz-Python,c:\\mozilla-build\\msys\\bin\\bash aurora,win64,win7_64,win64 && win7,Win-Moz-Python,c:\\mozilla-build\\msys\\bin\\bash aurora,win64,win64,win64 && win,Win-Moz-Python,c:\\mozilla-build\\msys\\bin\\bash beta,mac,mac,mac,System-CPython-2.7,bash beta,mac,mac10_9,mac10-9,System-CPython-2.7,bash beta,mac,mac10_8,mac10-8,System-CPython-2.7,bash beta,mac,mac10_7,mac10-7,System-CPython-2.7,bash beta,win32,win7_32,win32 && win7,Win-Moz-Python,c:\\mozilla-build\\msys\\bin\\bash beta,win32,win7_32_64,win64 && win7,Win-Moz-Python,c:\\mozilla-build\\msys\\bin\\bash beta,win32,win32,win32 && win,Win-Moz-Python,c:\\mozilla-build\\msys\\bin\\bash beta,win32,win32_64,win64 && win,Win-Moz-Python,c:\\mozilla-build\\msys\\bin\\bash \ No newline at end of file diff --git a/generate_mse_jobs.py b/generate_mse_jobs.py index bd555be..0e4dd2e 100755 --- a/generate_mse_jobs.py +++ b/generate_mse_jobs.py @@ -6,152 +6,63 @@ """Generate jobs for jenkins based on a .csv file describing the parameters.""" -import argparse -import sys -import csv import os -import os.path import re +import sys -REGEXPS = { - 'url1': r'XXURL1XX', - 'package': r'XXPACKAGEXX', - 'url2': r'XXURL2XX', - 'common_tests': r'XXCOMMONTESTSXX', - 'web_platform_tests': r'XXWEBPLATFORMTESTSXX', - 'triggers': r'XXTRIGGERSXX', - 'expand': r'XXEXPANDXX', - 'slave': r'XXSLAVEXX', - 'platform': r'XXPLATFORMXX', - 'python': r'XXPYTHONXX', - 'bash': r'XXBASHXX', -} +from job_generator import JobGenerator -for key in REGEXPS: - REGEXPS[key] = re.compile(REGEXPS[key]) - -class JobGenerator(): - def __init__(self): - pass - -class GenerateJobs(): +class GenerateMSEJobs(JobGenerator): def __init__(self, argv): - self.argv = argv - self.template = None + super(GenerateMSEJobs, self).__init__(argv) + self.REGEXPS = { + 'url1': r'XXURL1XX', + 'package': r'XXPACKAGEXX', + 'url2': r'XXURL2XX', + 'common_tests': r'XXCOMMONTESTSXX', + 'web_platform_tests': r'XXWEBPLATFORMTESTSXX', + 'triggers': r'XXTRIGGERSXX', + 'expand': r'XXEXPANDXX', + 'slave': r'XXSLAVEXX', + 'platform': r'XXPLATFORMXX', + 'python': r'XXPYTHONXX', + 'bash': r'XXBASHXX', + } - def get_config(self): - parser = argparse.ArgumentParser(description='Script for generating Jenkins jobs for MSE testing') - parser.add_argument('--template', required=True) - parser.add_argument('--table-csv', required=True, dest='csv') - parser.add_argument('--jenkins-host', required=True, dest='host') - args = parser.parse_args(self.argv) + def process_row(self, row, template): + jobname = 'mse-web-platform-%s-%s' % (row['Release'], row['PlatformJobLabel']) - self.config = {} - self.config['template'] = args.template - self.config['csv'] = args.csv - self.config['host'] = args.host + print 'jobname = %s' % jobname - '''Return a copy of the template so that it can be reused over and over again.''' - def get_template(self): - if not self.template: - with open(self.config['template']) as myfile: - self.template=myfile.read() - data = self.template - return data - - def get_platform_extension(self, platform): - if platform == 'linux64' or platform == 'linux32' or platform == 'linux': - return 'tar.bz2' - elif platform == 'win32' or platform == 'win64': - return 'zip' - elif platform == 'mac' or platform == 'mac64': - return 'dmg' - raise 'Unknown platform %s' % platform - - def get_artifact_platform(self, platform): - if platform == 'linux64': - return 'linux-x86_64' - elif platform == 'linux32': - return 'linux-i686' - elif platform == 'win64': - return 'win64' + if (row['Release'] != 'beta'): + triggers = 'trigger-firefox-%s-%s,trigger-tests-%s-%s' % (row['Release'], row['Platform'], row['Release'], row['Platform']) else: - return platform + triggers = '' - def get_lowest_release(self, release1, release2): - if release1 == 'esr' or release2 == 'esr': - return 'esr' - elif release1 == 'release' or release2 == 'release': - return 'release' - elif release1 == 'beta' or release2 == 'beta': - return 'beta' - elif release1 == 'aurora' or release2 == 'aurora': - return 'aurora' - else: - return 'nightly' + url1 = '%s/job/firefox-%s-%s/ws/releases' % (self.config['host'], row['Release'], row['Platform']) + url2 = '%s/job/tests-%s-%s/ws/releases' % (self.config['host'], row['Release'], row['Platform']) - def generate(self): - self.get_config() + artifact_platform = self.get_artifact_platform(row['Platform']) - if not os.path.exists('jobs'): - os.makedirs('jobs') + package = 'firefox-latest-%s.en-US.%s.%s' % (row['Release'], artifact_platform, self.get_platform_extension(row['Platform'])) + common_tests = 'firefox-latest-%s.en-US.%s.common.tests.zip' % (row['Release'], artifact_platform) + web_platform_tests = 'firefox-latest-%s.en-US.%s.web-platform.tests.zip' % (row['Release'], artifact_platform) - with open(self.config['csv'], 'rU') as csvfile: - reader = csv.DictReader(csvfile) - for row in reader: - jobname = 'mse-web-platform-%s-%s' % (row['Release'], row['PlatformJobLabel']) - - print 'jobname = %s' % jobname - - triggers = 'trigger-firefox-%s-%s, trigger-tests-%s-%s' % (row['Release'], row['Platform'], row['Release'], row['Platform']) - - url1 = '%s/job/firefox-%s-%s/ws/releases' % (self.config['host'], row['Release'], row['Platform']) - url2 = '%s/job/tests-%s-%s/ws/releases' % (self.config['host'], row['Release'], row['Platform']) - - artifact_platform = self.get_artifact_platform(row['Platform']) - - package = 'firefox-latest-%s.en-US.%s.%s' % (row['Release'], artifact_platform, self.get_platform_extension(row['Platform'])) - common_tests = 'firefox-latest-%s.en-US.%s.common.tests.zip' % (row['Release'], artifact_platform) - web_platform_tests = 'firefox-latest-%s.en-US.%s.web-platform.tests.zip' % (row['Release'], artifact_platform) - - # We will need build_file for reporting results to treeherder - # build_file = 'firefox-latest-%s.en-US.%s.txt' % (row['Release'], artifact_platform1) - - template = self.get_template() - - template = re.sub(REGEXPS['triggers'], triggers, template) - template = re.sub(REGEXPS['url1'], url1, template) - template = re.sub(REGEXPS['url2'], url2, template) - template = re.sub(REGEXPS['package'], package, template) - template = re.sub(REGEXPS['common_tests'], common_tests, template) - template = re.sub(REGEXPS['web_platform_tests'], web_platform_tests, template) - template = re.sub(REGEXPS['slave'], row['Slave'], template) - template = re.sub(REGEXPS['platform'], row['Platform'], template) - template = re.sub(REGEXPS['python'], row['Python'], template) - template = re.sub(REGEXPS['bash'], row['Bash'], template) - - # We will need build_file for reporting results to treeherder - #template = re.sub(REGEXPS['build_file'], build_file, template) - - # Generate directory - dir = os.path.join('jobs', jobname) - if not os.path.exists(dir): - os.makedirs(dir) - - # Write it out - config_path = os.path.join('jobs', jobname, 'config.xml') - config_file = open(config_path, 'w') - config_file.write(template) - config_file.close() - - csvfile.close() - - print "Finished processing." - return 0 + template = re.sub(self.REGEXPS['triggers'], triggers, template) + template = re.sub(self.REGEXPS['url1'], url1, template) + template = re.sub(self.REGEXPS['url2'], url2, template) + template = re.sub(self.REGEXPS['package'], package, template) + template = re.sub(self.REGEXPS['common_tests'], common_tests, template) + template = re.sub(self.REGEXPS['web_platform_tests'], web_platform_tests, template) + template = re.sub(self.REGEXPS['slave'], row['Slave'], template) + template = re.sub(self.REGEXPS['platform'], row['Platform'], template) + template = re.sub(self.REGEXPS['python'], row['Python'], template) + template = re.sub(self.REGEXPS['bash'], row['Bash'], template) + return jobname, template def generate_jobs(argv): - generator = GenerateJobs(argv) + generator = GenerateMSEJobs(argv) return generator.generate() if __name__ == '__main__': diff --git a/generate_webrtc_jobs.py b/generate_webrtc_jobs.py index 5e67aa9..bd81326 100755 --- a/generate_webrtc_jobs.py +++ b/generate_webrtc_jobs.py @@ -6,35 +6,11 @@ """Generate jobs for jenkins based on a .csv file describing the parameters.""" -import argparse -import sys -import csv import os -import os.path import re +import sys -REGEXPS = { - 'expand': r'XXEXPANDXX', - 'url1': r'XXURL1XX', - 'package1': r'XXPACKAGE1XX', - 'build_file1': r'XXBUILD_FILE1XX', - 'machine1': r'XXMACHINE1XX', - 'arch1': r'XXARCH1XX', - 'url2': r'XXURL2XX', - 'package2': r'XXPACKAGE2XX', - 'build_file2': r'XXBUILD_FILE2XX', - 'machine2': r'XXMACHINE2XX', - 'arch2': r'XXARCH2XX', - 'triggers': r'XXTRIGGERSXX', - 'slave': r'XXSLAVEXX', - 'sigserver': r'XXXSIGSERVERXXX', - 'tests_release': r'XXXTESTSRELEASEXXX', - 'os1': r'XXOS1XX', - 'os2': r'XXOS2XX', -} - -for key in REGEXPS: - REGEXPS[key] = re.compile(REGEXPS[key]) +from job_generator import JobGenerator platform_host_data = { 'linux32': { @@ -137,130 +113,91 @@ platform_host_data = { }, } -class JobGenerator(): - def __init__(self): - pass - -class GenerateJobs(): +class GenerateWebRTCJobs(JobGenerator): def __init__(self, argv): - self.argv = argv - self.template = None + super(GenerateWebRTCJobs, self).__init__(argv) + self.REGEXPS = { + 'url1': r'XXURL1XX', + 'package1': r'XXPACKAGE1XX', + 'build_file1': r'XXBUILD_FILE1XX', + 'machine1': r'XXMACHINE1XX', + 'arch1': r'XXARCH1XX', + 'url2': r'XXURL2XX', + 'package2': r'XXPACKAGE2XX', + 'build_file2': r'XXBUILD_FILE2XX', + 'machine2': r'XXMACHINE2XX', + 'arch2': r'XXARCH2XX', + 'triggers': r'XXTRIGGERSXX', + 'slave': r'XXSLAVEXX', + 'sigserver': r'XXXSIGSERVERXXX', + 'tests_release': r'XXXTESTSRELEASEXXX', + 'os1': r'XXOS1XX', + 'os2': r'XXOS2XX', + } - def get_config(self): - parser = argparse.ArgumentParser(description='Script for generating Jenkins jobs for WebRTC testing') - parser.add_argument('--template', required=True) - parser.add_argument('--table-csv', required=True, dest='csv') - parser.add_argument('--jenkins-host', required=True, dest='host') - args = parser.parse_args(self.argv) + def process_row(self, row, template): + jobname = 'webrtc-%s-%s' % (row['Release1'], row['PlatformJobLabel1']) + if row['Release1'] != row['Release2']: + jobname = '%s-%s' % (jobname, row['Release2']) + jobname = '%s-%s' % (jobname, row['PlatformJobLabel2']) + if row['Networking']: + jobname = '%s-%s' % (jobname, row['Networking']) - self.config = {} - self.config['template'] = args.template - self.config['csv'] = args.csv - self.config['host'] = args.host + print 'jobname = %s' % jobname - '''Return a copy of the template so that it can be reused over and over again.''' - def get_template(self): - if not self.template: - with open(self.config['template']) as myfile: - self.template=myfile.read() - data = self.template - return data + tests_release = self.get_lowest_release(row['Release1'], row['Release2']) - def get_lowest_release(self, release1, release2): - if release1 == 'esr' or release2 == 'esr': - return 'esr' - elif release1 == 'release' or release2 == 'release': - return 'release' - elif release1 == 'beta' or release2 == 'beta': - return 'beta' - elif release1 == 'aurora' or release2 == 'aurora': - return 'aurora' + if tests_release == 'beta': + triggers = '' else: - return 'nightly' + triggers = 'trigger-firefox-%s-%s' % (row['Release1'], row['Platform1']) + if row['Release1'] != row['Release2'] or row['Platform1'] != row['Platform2']: + triggers = '%s,trigger-firefox-%s-%s' % (triggers, row['Release2'], row['Platform2']) - def generate(self): - self.get_config() + expand = 'expand-tests-%s-linux64' % tests_release + if row['Networking']: + expand = '%s-network' % expand + triggers = '%s,%s' % (triggers, expand) - if not os.path.exists('jobs'): - os.makedirs('jobs') + if row['Networking']: + sigserver = "NETWORK_SIGNALLING_SERVER" + else: + sigserver = "SIGNALLING_SERVER" - with open(self.config['csv'], 'rU') as csvfile: - reader = csv.DictReader(csvfile) - for row in reader: - jobname = 'webrtc-%s-%s' % (row['Release1'], row['PlatformJobLabel1']) - if row['Release1'] != row['Release2']: - jobname = '%s-%s' % (jobname, row['Release2']) - jobname = '%s-%s' % (jobname, row['PlatformJobLabel2']) - if row['Networking']: - jobname = '%s-%s' % (jobname, row['Networking']) + url1 = '%s/job/firefox-%s-%s/ws/releases' % (self.config['host'], row['Release1'], row['Platform1']) + url2 = '%s/job/firefox-%s-%s/ws/releases' % (self.config['host'], row['Release2'], row['Platform2']) - print 'jobname = %s' % jobname + artifact_platform1 = platform_host_data[row['Platform1']]['artifact_platform'] + artifact_platform2 = platform_host_data[row['Platform2']]['artifact_platform'] - triggers = 'trigger-firefox-%s-%s' % (row['Release1'], row['Platform1']) - if row['Release1'] != row['Release2'] or row['Platform1'] != row['Platform2']: - triggers = '%s,trigger-firefox-%s-%s' % (triggers, row['Release2'], row['Platform2']) + package1 = 'firefox-latest-%s.en-US.%s.%s' % (row['Release1'], artifact_platform1, platform_host_data[row['Platform1']]['extension']) + package2 = 'firefox-latest-%s.en-US.%s.%s' % (row['Release2'], artifact_platform2, platform_host_data[row['Platform2']]['extension']) - tests_release = self.get_lowest_release(row['Release1'], row['Release2']) + build_file1 = 'firefox-latest-%s.en-US.%s.txt' % (row['Release1'], artifact_platform1) + build_file2 = 'firefox-latest-%s.en-US.%s.txt' % (row['Release2'], artifact_platform2) - expand = 'expand-tests-%s-linux64' % tests_release - if row['Networking']: - expand = '%s-network' % expand - sigserver = "NETWORK_SIGNALLING_SERVER" - else: - sigserver = "SIGNALLING_SERVER" - - url1 = '%s/job/firefox-%s-%s/ws/releases' % (self.config['host'], row['Release1'], row['Platform1']) - url2 = '%s/job/firefox-%s-%s/ws/releases' % (self.config['host'], row['Release2'], row['Platform2']) - - artifact_platform1 = platform_host_data[row['Platform1']]['artifact_platform'] - artifact_platform2 = platform_host_data[row['Platform2']]['artifact_platform'] - - package1 = 'firefox-latest-%s.en-US.%s.%s' % (row['Release1'], artifact_platform1, platform_host_data[row['Platform1']]['extension']) - package2 = 'firefox-latest-%s.en-US.%s.%s' % (row['Release2'], artifact_platform2, platform_host_data[row['Platform2']]['extension']) - - build_file1 = 'firefox-latest-%s.en-US.%s.txt' % (row['Release1'], artifact_platform1) - build_file2 = 'firefox-latest-%s.en-US.%s.txt' % (row['Release2'], artifact_platform2) - - template = self.get_template() - - template = re.sub(REGEXPS['triggers'], triggers, template) - template = re.sub(REGEXPS['expand'], expand, template) - template = re.sub(REGEXPS['url1'], url1, template) - template = re.sub(REGEXPS['url2'], url2, template) - template = re.sub(REGEXPS['package1'], package1, template) - template = re.sub(REGEXPS['package2'], package2, template) - template = re.sub(REGEXPS['build_file1'], build_file1, template) - template = re.sub(REGEXPS['build_file2'], build_file2, template) - template = re.sub(REGEXPS['machine1'], row['Host1'], template) - template = re.sub(REGEXPS['machine2'], row['Host2'], template) - template = re.sub(REGEXPS['arch1'], row['Arch1'], template) - template = re.sub(REGEXPS['arch2'], row['Arch2'], template) - template = re.sub(REGEXPS['slave'], row['Slave'], template) - template = re.sub(REGEXPS['sigserver'], sigserver, template) - template = re.sub(REGEXPS['tests_release'], tests_release, template) - template = re.sub(REGEXPS['os1'], platform_host_data[row['Platform1']]['os'], template) - template = re.sub(REGEXPS['os2'], platform_host_data[row['Platform2']]['os'], template) - - # Generate directory - dir = os.path.join('jobs', jobname) - if not os.path.exists(dir): - os.makedirs(dir) - - # Write it out - config_path = os.path.join('jobs', jobname, 'config.xml') - config_file = open(config_path, 'w') - config_file.write(template) - config_file.close() - - csvfile.close() - - print "Finished processing." - return 0 + template = re.sub(self.REGEXPS['triggers'], triggers, template) + template = re.sub(self.REGEXPS['url1'], url1, template) + template = re.sub(self.REGEXPS['url2'], url2, template) + template = re.sub(self.REGEXPS['package1'], package1, template) + template = re.sub(self.REGEXPS['package2'], package2, template) + template = re.sub(self.REGEXPS['build_file1'], build_file1, template) + template = re.sub(self.REGEXPS['build_file2'], build_file2, template) + template = re.sub(self.REGEXPS['machine1'], row['Host1'], template) + template = re.sub(self.REGEXPS['machine2'], row['Host2'], template) + template = re.sub(self.REGEXPS['arch1'], row['Arch1'], template) + template = re.sub(self.REGEXPS['arch2'], row['Arch2'], template) + template = re.sub(self.REGEXPS['slave'], row['Slave'], template) + template = re.sub(self.REGEXPS['sigserver'], sigserver, template) + template = re.sub(self.REGEXPS['tests_release'], tests_release, template) + template = re.sub(self.REGEXPS['os1'], platform_host_data[row['Platform1']]['os'], template) + template = re.sub(self.REGEXPS['os2'], platform_host_data[row['Platform2']]['os'], template) + return jobname, template def generate_jobs(argv): - generator = GenerateJobs(argv) + generator = GenerateWebRTCJobs(argv) return generator.generate() if __name__ == '__main__': diff --git a/job_generator.py b/job_generator.py new file mode 100755 index 0000000..c7f2495 --- /dev/null +++ b/job_generator.py @@ -0,0 +1,127 @@ +#!/usr/bin/env python + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +"""Generate jobs for jenkins based on a .csv file describing the parameters.""" + +import argparse +import csv +import os +import os.path +import re +import jenkins + +class JobGenerator(object): + def __init__(self, argv): + self.argv = argv + self.template = None + self.jenkins_instance = None + self.REGEXPS = {} + + @property + def jenkins(self): + if not self.jenkins_instance: + # We will have to do work when we require authentication for + # our Jenkins instances. Need to either pass in user/password + # or read from a config file of some kind. + self.jenkins_instance = jenkins.Jenkins(self.config['host']) + return self.jenkins_instance + + def initialize_regexps(self): + for key in self.REGEXPS: + self.REGEXPS[key] = re.compile(self.REGEXPS[key]) + + def get_config(self): + parser = argparse.ArgumentParser(description='Script for generating Jenkins jobs for MSE testing') + parser.add_argument('--template', required=True) + parser.add_argument('--table-csv', required=True, dest='csv') + parser.add_argument('--jenkins-host', required=True, dest='host') + parser.add_argument('--generate-files', action='store_true') + args = parser.parse_args(self.argv) + + self.config = {} + self.config['template'] = args.template + self.config['csv'] = args.csv + self.config['host'] = args.host + if args.generate_files: + self.config['generate_files'] = True + else: + self.config['generate_files'] = False + + '''Return a copy of the template so that it can be reused over and over again.''' + def get_template(self): + if not self.template: + with open(self.config['template']) as myfile: + self.template=myfile.read() + data = self.template + return data + + def get_platform_extension(self, platform): + if platform == 'linux64' or platform == 'linux32' or platform == 'linux': + return 'tar.bz2' + elif platform == 'win32' or platform == 'win64': + return 'zip' + elif platform == 'mac' or platform == 'mac64': + return 'dmg' + raise 'Unknown platform %s' % platform + + def get_artifact_platform(self, platform): + if platform == 'linux64': + return 'linux-x86_64' + elif platform == 'linux32': + return 'linux-i686' + else: + return platform + + def get_lowest_release(self, release1, release2): + if release1 == 'esr' or release2 == 'esr': + return 'esr' + elif release1 == 'release' or release2 == 'release': + return 'release' + elif release1 == 'beta' or release2 == 'beta': + return 'beta' + elif release1 == 'aurora' or release2 == 'aurora': + return 'aurora' + else: + return 'nightly' + + def process_row(self, row, template): + return template + + def generate(self): + self.get_config() + self.initialize_regexps() + + with open(self.config['csv'], 'rU') as csvfile: + reader = csv.DictReader(csvfile) + for row in reader: + template = self.get_template() + jobname, template = self.process_row(row, template) + + # Generate directory + if self.config['generate_files']: + if not os.path.exists('jobs'): + os.makedirs('jobs') + + dir = os.path.join('jobs', jobname) + if not os.path.exists(dir): + os.makedirs(dir) + + # Write it out + config_path = os.path.join('jobs', jobname, 'config.xml') + config_file = open(config_path, 'w') + config_file.write(template) + config_file.close() + else: + # Tell Jenkins about it. + if self.jenkins.job_exists(jobname): + self.jenkins.reconfig_job(jobname, template) + else: + self.jenkins.create_job(jobname, template) + + csvfile.close() + + print "Finished processing." + return 0 diff --git a/mse-web-platform-job-template.xml b/mse-web-platform-job-template.xml index cad61b1..e813d7f 100644 --- a/mse-web-platform-job-template.xml +++ b/mse-web-platform-job-template.xml @@ -10,7 +10,7 @@ false - + 2 @@ -47,7 +47,7 @@ H/3 * * * * false - + H * * * * diff --git a/webrtc-job-template.xml b/webrtc-job-template.xml index 74003d3..af4cfe7 100644 --- a/webrtc-job-template.xml +++ b/webrtc-job-template.xml @@ -63,7 +63,7 @@ - XXTRIGGERSXX,XXEXPANDXX + XXTRIGGERSXX SUCCESS 0