Merged PR 528: Add CI, linting, and unit test support
This commit is contained in:
Sean Brogan
Matthew Carlson
Родитель
d356e7c7a0
Коммит
e12a8e1a37
|
|
@ -0,0 +1,2 @@
|
|||
[run]
|
||||
omit = tests/*
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
[flake8]
|
||||
#E501 line too long
|
||||
#E266 too many leading '#' for block comment
|
||||
#E722 do not use bare 'except'
|
||||
ignore = E501,E266,E722
|
||||
|
|
@ -4,3 +4,9 @@ Lib
|
|||
dist
|
||||
*.egg-info
|
||||
build.
|
||||
/cov_html
|
||||
/.pytest_cache
|
||||
/pytest_MuBuild_report.html
|
||||
/.coverage
|
||||
/cov.xml
|
||||
/test.junit.xml
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
## @file ConfigValidator.py
|
||||
# @file ConfigValidator.py
|
||||
# This module contains support for validating .mu.json config files
|
||||
#
|
||||
# Used to support CI/CD and exporting test results for other tools.
|
||||
|
|
@ -26,10 +26,8 @@
|
|||
# OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
|
||||
# ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
##
|
||||
import logging
|
||||
import os
|
||||
import urllib.request as req
|
||||
import urllib.parse as p
|
||||
|
||||
'''
|
||||
Example_MU_CONFIG_FILE:
|
||||
|
|
@ -65,101 +63,100 @@ Example_MU_CONFIG_FILE:
|
|||
'''
|
||||
|
||||
|
||||
#Checks the top level MU Config
|
||||
def check_mu_confg(config,edk2path,pluginList):
|
||||
# Checks the top level MU Config
|
||||
def check_mu_confg(config, edk2path, pluginList):
|
||||
workspace = edk2path.WorkspacePath
|
||||
|
||||
def _mu_error(message):
|
||||
raise Exception("Mu Config Error: {0}".format(message))
|
||||
|
||||
def _is_valid_dir(path,name):
|
||||
path = os.path.join(workspace,path)
|
||||
def _is_valid_dir(path, name):
|
||||
path = os.path.join(workspace, path)
|
||||
if not os.path.isdir(path):
|
||||
_mu_error("{0} isn't a valid directory".format(path))
|
||||
|
||||
def _check_url(url):
|
||||
request = req.Request(url)
|
||||
try:
|
||||
response = req.urlopen(request)
|
||||
req.urlopen(request)
|
||||
return True
|
||||
except:
|
||||
#The url wasn't valid
|
||||
# The url wasn't valid
|
||||
return False
|
||||
|
||||
def _check_packages(packages,name):
|
||||
def _check_packages(packages, name):
|
||||
for package in packages:
|
||||
path = edk2path.GetAbsolutePathOnThisSytemFromEdk2RelativePath(package)
|
||||
if path is None or not os.path.isdir(path):
|
||||
_mu_error("{0} isn't a valid package to build".format(package))
|
||||
return True
|
||||
|
||||
def _is_valid_arch(targets,name):
|
||||
valid_targets = ["AARCH64","IA32","X64"]
|
||||
def _is_valid_arch(targets, name):
|
||||
valid_targets = ["AARCH64", "IA32", "X64"]
|
||||
for target in targets:
|
||||
if not target in valid_targets:
|
||||
if target not in valid_targets:
|
||||
_mu_error("{0} is not a valid target".format(target))
|
||||
|
||||
def _check_dependencies(dependencies,name):
|
||||
valid_attributes = ["Path","Url","Branch","Commit"]
|
||||
def _check_dependencies(dependencies, name):
|
||||
valid_attributes = ["Path", "Url", "Branch", "Commit"]
|
||||
for dependency in dependencies:
|
||||
# check to make sure we have a path
|
||||
if not "Path" in dependency:
|
||||
if "Path" not in dependency:
|
||||
_mu_error("Path not found in dependency {0}".format(dependency))
|
||||
# check to sure we have a valid url and we can reach it
|
||||
if not "Url" in dependency:
|
||||
if "Url" not in dependency:
|
||||
_mu_error("Url not found in dependency {0}".format(dependency))
|
||||
if not _check_url(dependency["Url"]):
|
||||
_mu_error("Invalid URL {0}".format(dependency["Url"]))
|
||||
# make sure we have a valid branch or commit
|
||||
if not "Branch" in dependency and not "Commit" in dependency:
|
||||
if "Branch" not in dependency and "Commit" not in dependency:
|
||||
_mu_error("You must have a commit or a branch dependency {0}".format(dependency))
|
||||
if "Branch" in dependency and "Commit" in dependency:
|
||||
_mu_error("You cannot have both a commit or a branch dependency {0}".format(dependency))
|
||||
#check to make sure we don't have something else in there
|
||||
# check to make sure we don't have something else in there
|
||||
for attribute in dependency:
|
||||
if not attribute in valid_attributes:
|
||||
if attribute not in valid_attributes:
|
||||
_mu_error("Unknown attribute {0} in dependecy".format(attribute))
|
||||
|
||||
return True
|
||||
|
||||
|
||||
|
||||
config_rules = {
|
||||
"required": {
|
||||
"Name": {
|
||||
"type":"str"
|
||||
"type": "str"
|
||||
},
|
||||
"GroupName": {
|
||||
"type":"str"
|
||||
"type": "str"
|
||||
},
|
||||
"Scopes": {
|
||||
"type":"list",
|
||||
"items":"str"
|
||||
"type": "list",
|
||||
"items": "str"
|
||||
},
|
||||
"ArchSupported": {
|
||||
"type":"list",
|
||||
"validator":_is_valid_arch
|
||||
"type": "list",
|
||||
"validator": _is_valid_arch
|
||||
},
|
||||
"RelativeWorkspaceRoot": {
|
||||
"type":"str",
|
||||
"validator":_is_valid_dir
|
||||
"type": "str",
|
||||
"validator": _is_valid_dir
|
||||
},
|
||||
"Targets": {
|
||||
"type":"list"
|
||||
"type": "list"
|
||||
}
|
||||
},
|
||||
"optional": {
|
||||
"Packages":{
|
||||
"type":"list",
|
||||
"items":"str",
|
||||
"validator":_check_packages
|
||||
"Packages": {
|
||||
"type": "list",
|
||||
"items": "str",
|
||||
"validator": _check_packages
|
||||
},
|
||||
"PackagesPath": {
|
||||
"type":"list",
|
||||
"items":"str"
|
||||
"type": "list",
|
||||
"items": "str"
|
||||
},
|
||||
"Dependencies": {
|
||||
"type":"list",
|
||||
"validator":_check_dependencies
|
||||
"type": "list",
|
||||
"validator": _check_dependencies
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -170,19 +167,19 @@ def check_mu_confg(config,edk2path,pluginList):
|
|||
if "config_name" in plugin.descriptor:
|
||||
plugin_name = plugin.descriptor["config_name"]
|
||||
config_rules["optional"][plugin_name] = {
|
||||
"validator" : plugin.Obj.ValidateConfig
|
||||
"validator": plugin.Obj.ValidateConfig
|
||||
}
|
||||
|
||||
#check if all the requires are satisified
|
||||
# check if all the requires are satisified
|
||||
for rule in config_rules["required"]:
|
||||
if not rule in config:
|
||||
if rule not in config:
|
||||
_mu_error("{0} is a required attribute in your MU Config".format(rule))
|
||||
|
||||
if "type" in config_rules["required"][rule]:
|
||||
config_type = str(type(config[rule]).__name__)
|
||||
wanted_type = config_rules["required"][rule]["type"]
|
||||
if config_type != wanted_type:
|
||||
_mu_error("{0} is a required attribute and is not the correct type. We are expecting a {1} and got a {2}".format(rule,config_type,wanted_type))
|
||||
_mu_error("{0} is a required attribute and is not the correct type. We are expecting a {1} and got a {2}".format(rule, config_type, wanted_type))
|
||||
|
||||
if "validator" in config_rules["required"][rule]:
|
||||
validator = config_rules["required"][rule]["validator"]
|
||||
|
|
@ -190,26 +187,27 @@ def check_mu_confg(config,edk2path,pluginList):
|
|||
|
||||
# check optional types
|
||||
for rule in config_rules["optional"]:
|
||||
if not rule in config:
|
||||
if rule not in config:
|
||||
continue
|
||||
|
||||
if "type" in config_rules["optional"][rule]:
|
||||
config_type = str(type(config[rule]).__name__)
|
||||
wanted_type = config_rules["optional"][rule]["type"]
|
||||
if config_type != wanted_type:
|
||||
_mu_error("{0} is a optional attribute and is not the correct type. We are expecting a {1} and got a {2}".format(rule,config_type,wanted_type))
|
||||
_mu_error("{0} is a optional attribute and is not the correct type. We are expecting a {1} and got a {2}".format(rule, config_type, wanted_type))
|
||||
|
||||
if "validator" in config_rules["optional"][rule]:
|
||||
validator = config_rules["optional"][rule]["validator"]
|
||||
validator(config[rule],"Base mu.json")
|
||||
validator(config[rule], "Base mu.json")
|
||||
|
||||
#check to make sure we don't have any stray keys in there
|
||||
# check to make sure we don't have any stray keys in there
|
||||
for rule in config:
|
||||
if not rule in config_rules["optional"] and not rule in config_rules["required"]:
|
||||
if rule not in config_rules["optional"] and rule not in config_rules["required"]:
|
||||
_mu_error("Unknown parameter {0} is unexpected".format(rule))
|
||||
|
||||
return True
|
||||
|
||||
|
||||
'''
|
||||
{
|
||||
"Defines": {
|
||||
|
|
@ -238,17 +236,19 @@ def check_mu_confg(config,edk2path,pluginList):
|
|||
##
|
||||
# Checks the package configuration for errors
|
||||
##
|
||||
def check_package_confg(name,config,pluginList):
|
||||
|
||||
|
||||
def check_package_confg(name, config, pluginList):
|
||||
def _mu_error(message):
|
||||
raise Exception("Package {0} Config Error: {1}".format(name,message))
|
||||
raise Exception("Package {0} Config Error: {1}".format(name, message))
|
||||
|
||||
config_rules = {
|
||||
"required": {
|
||||
},
|
||||
"optional": {
|
||||
"Defines": {
|
||||
"type":"dict",
|
||||
"items":"str"
|
||||
"type": "dict",
|
||||
"items": "str"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -260,41 +260,40 @@ def check_package_confg(name,config,pluginList):
|
|||
plugin_name = plugin.descriptor["config_name"]
|
||||
# add the validator
|
||||
config_rules["optional"][plugin_name] = {
|
||||
"validator" : plugin.Obj.ValidateConfig
|
||||
"validator": plugin.Obj.ValidateConfig
|
||||
}
|
||||
|
||||
#check if all the requires are satisified
|
||||
# check if all the requires are satisified
|
||||
for rule in config_rules["required"]:
|
||||
if not rule in config:
|
||||
if rule not in config:
|
||||
_mu_error("{0} is a required attribute in your MU Config".format(rule))
|
||||
|
||||
if "type" in config_rules["required"][rule]:
|
||||
config_type = str(type(config[rule]).__name__)
|
||||
wanted_type = config_rules["required"][rule]["type"]
|
||||
if config_type != wanted_type:
|
||||
_mu_error("{0} is a required attribute and is not the correct type. We are expecting a {1} and got a {2}".format(rule,config_type,wanted_type))
|
||||
_mu_error("{0} is a required attribute and is not the correct type. We are expecting a {1} and got a {2}".format(rule, config_type, wanted_type))
|
||||
|
||||
if "validator" in config_rules["required"][rule]:
|
||||
validator = config_rules["required"][rule]["validator"]
|
||||
validator(config[rule],name)
|
||||
validator(config[rule], name)
|
||||
|
||||
# check optional types
|
||||
for rule in config_rules["optional"]:
|
||||
if not rule in config:
|
||||
if rule not in config:
|
||||
continue
|
||||
|
||||
if "type" in config_rules["optional"][rule]:
|
||||
config_type = str(type(config[rule]).__name__)
|
||||
wanted_type = config_rules["optional"][rule]["type"]
|
||||
if config_type != wanted_type:
|
||||
_mu_error("{0} is a optional attribute and is not the correct type. We are expecting a {1} and got a {2}".format(rule,config_type,wanted_type))
|
||||
_mu_error("{0} is a optional attribute and is not the correct type. We are expecting a {1} and got a {2}".format(rule, config_type, wanted_type))
|
||||
|
||||
if "validator" in config_rules["optional"][rule]:
|
||||
validator = config_rules["optional"][rule]["validator"]
|
||||
validator(config[rule],name)
|
||||
validator(config[rule], name)
|
||||
|
||||
#check to make sure we don't have any stray keys in there
|
||||
# check to make sure we don't have any stray keys in there
|
||||
for rule in config:
|
||||
if not rule in config_rules["optional"] and not rule in config_rules["required"]:
|
||||
if rule not in config_rules["optional"] and rule not in config_rules["required"]:
|
||||
_mu_error("Unknown parameter {0} is unexpected".format(rule))
|
||||
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
## @file MuBuild.py
|
||||
# @file MuBuild.py
|
||||
# This module contains code that supports Project Mu CI/CD
|
||||
# This is the main entry for the build and test process
|
||||
# of Non-Product builds
|
||||
|
|
@ -47,15 +47,16 @@ import pkg_resources
|
|||
|
||||
def get_mu_config():
|
||||
parser = argparse.ArgumentParser(description='Run the Mu Build')
|
||||
parser.add_argument ('-c', '--mu_config', dest = 'mu_config', required = True, type=str, help ='Provide the Mu config relative to the current working directory')
|
||||
parser.add_argument ('-p', '--pkg','--pkg-dir', dest='pkglist', nargs="+", type=str, help = 'A package or folder you want to test (abs path or cwd relative). Can list multiple by doing -p <pkg1> <pkg2> <pkg3>', default=[])
|
||||
parser.add_argument('-ignore','--ignore-git', dest="git_ignore",action="store_true",help="Whether to ignore errors in the git cloing process", default=False)
|
||||
parser.add_argument('-force','--force-git', dest="git_force",action="store_true",help="Whether to force git repos to clone in the git cloing process", default=False)
|
||||
parser.add_argument('-update-git','--update-git', dest="git_update",action="store_true",help="Whether to update git repos as needed in the git cloing process", default=False)
|
||||
parser.add_argument('-c', '--mu_config', dest='mu_config', required=True, type=str, help='Provide the Mu config relative to the current working directory')
|
||||
parser.add_argument('-p', '--pkg', '--pkg-dir', dest='pkglist', nargs="+", type=str, help='A package or folder you want to test (abs path or cwd relative). Can list multiple by doing -p <pkg1> <pkg2> <pkg3>', default=[])
|
||||
parser.add_argument('-ignore', '--ignore-git', dest="git_ignore", action="store_true", help="Whether to ignore errors in the git cloing process", default=False)
|
||||
parser.add_argument('-force', '--force-git', dest="git_force", action="store_true", help="Whether to force git repos to clone in the git cloing process", default=False)
|
||||
parser.add_argument('-update-git', '--update-git', dest="git_update", action="store_true", help="Whether to update git repos as needed in the git cloing process", default=False)
|
||||
args, sys.argv = parser.parse_known_args()
|
||||
return args
|
||||
|
||||
def merge_config(mu_config,pkg_config,descriptor={}):
|
||||
|
||||
def merge_config(mu_config, pkg_config, descriptor={}):
|
||||
plugin_name = ""
|
||||
config = dict()
|
||||
if "module" in descriptor:
|
||||
|
|
@ -77,8 +78,10 @@ def merge_config(mu_config,pkg_config,descriptor={}):
|
|||
#
|
||||
# Main driver of Project Mu Builds
|
||||
#
|
||||
|
||||
|
||||
def main():
|
||||
#Parse command line arguments
|
||||
# Parse command line arguments
|
||||
PROJECT_SCOPES = ("project_mu",)
|
||||
buildArgs = get_mu_config()
|
||||
mu_config_filepath = os.path.abspath(buildArgs.mu_config)
|
||||
|
|
@ -86,16 +89,16 @@ def main():
|
|||
if mu_config_filepath is None or not os.path.isfile(mu_config_filepath):
|
||||
raise Exception("Invalid path to mu.json file for build: ", mu_config_filepath)
|
||||
|
||||
#have a build config file
|
||||
# have a build config file
|
||||
with open(mu_config_filepath, 'r') as mu_config_file:
|
||||
mu_config = yaml.safe_load(mu_config_file)
|
||||
WORKSPACE_PATH = os.path.realpath(os.path.join(os.path.dirname(mu_config_filepath), mu_config["RelativeWorkspaceRoot"]))
|
||||
|
||||
#Setup the logging to the file as well as the console
|
||||
# Setup the logging to the file as well as the console
|
||||
MuLogging.clean_build_logs(WORKSPACE_PATH)
|
||||
MuLogging.setup_logging(WORKSPACE_PATH)
|
||||
|
||||
#Get scopes from config file
|
||||
# Get scopes from config file
|
||||
if "Scopes" in mu_config:
|
||||
PROJECT_SCOPES += tuple(mu_config["Scopes"])
|
||||
|
||||
|
|
@ -104,22 +107,20 @@ def main():
|
|||
# Get Package Path from config file
|
||||
pplist = list()
|
||||
if(mu_config["RelativeWorkspaceRoot"] != ""):
|
||||
#this package is not at workspace root.
|
||||
# this package is not at workspace root.
|
||||
# Add self
|
||||
pplist.append(os.path.dirname(mu_config_filepath))
|
||||
|
||||
#Include packages from the config file
|
||||
# Include packages from the config file
|
||||
if "PackagesPath" in mu_config:
|
||||
for a in mu_config["PackagesPath"]:
|
||||
pplist.append(a)
|
||||
|
||||
#Check Dependencies for Repo
|
||||
# Check Dependencies for Repo
|
||||
if "Dependencies" in mu_config:
|
||||
pplist.extend(RepoResolver.resolve_all(WORKSPACE_PATH,mu_config["Dependencies"], ignore=buildArgs.git_ignore, force=buildArgs.git_force, update_ok=buildArgs.git_update))
|
||||
pplist.extend(RepoResolver.resolve_all(WORKSPACE_PATH, mu_config["Dependencies"], ignore=buildArgs.git_ignore, force=buildArgs.git_force, update_ok=buildArgs.git_update))
|
||||
|
||||
|
||||
|
||||
#make Edk2Path object to handle all path operations
|
||||
# make Edk2Path object to handle all path operations
|
||||
edk2path = Edk2Path(WORKSPACE_PATH, pplist)
|
||||
|
||||
logging.info("Running ProjectMu Build: {0}".format(mu_config["Name"]))
|
||||
|
|
@ -129,7 +130,7 @@ def main():
|
|||
logging.info("mu_python_library version: " + pkg_resources.get_distribution("mu_python_library").version)
|
||||
logging.info("mu_environment version: " + pkg_resources.get_distribution("mu_environment").version)
|
||||
|
||||
#which package to build
|
||||
# which package to build
|
||||
packageList = mu_config["Packages"]
|
||||
#
|
||||
# If mu pk list supplied lets see if they are a file system path
|
||||
|
|
@ -137,10 +138,10 @@ def main():
|
|||
#
|
||||
#
|
||||
if(len(buildArgs.pkglist) > 0):
|
||||
packageList = [] #clear it
|
||||
packageList = [] # clear it
|
||||
|
||||
for mu_pk_path in buildArgs.pkglist:
|
||||
#if abs path lets convert
|
||||
# if abs path lets convert
|
||||
if os.path.isabs(mu_pk_path):
|
||||
temp = edk2path.GetEdk2RelativePathFromAbsolutePath(mu_pk_path)
|
||||
if(temp is not None):
|
||||
|
|
@ -149,7 +150,7 @@ def main():
|
|||
logging.critical("pkg-dir invalid absolute path: {0}".format(mu_pk_path))
|
||||
raise Exception("Invalid Package Path")
|
||||
else:
|
||||
#Check if relative path
|
||||
# Check if relative path
|
||||
temp = os.path.join(os.getcwd(), mu_pk_path)
|
||||
temp = edk2path.GetEdk2RelativePathFromAbsolutePath(temp)
|
||||
if(temp is not None):
|
||||
|
|
@ -163,19 +164,17 @@ def main():
|
|||
CommonBuildEntry.update_process(edk2path.WorkspacePath, PROJECT_SCOPES)
|
||||
env = ShellEnvironment.GetBuildVars()
|
||||
|
||||
|
||||
archSupported = " ".join(mu_config["ArchSupported"])
|
||||
env.SetValue("TARGET_ARCH", archSupported, "Platform Hardcoded")
|
||||
|
||||
|
||||
#Generate consumable XML object- junit format
|
||||
# Generate consumable XML object- junit format
|
||||
JunitReport = MuJunitReport()
|
||||
|
||||
#Keep track of failures
|
||||
# Keep track of failures
|
||||
failure_num = 0
|
||||
total_num = 0
|
||||
|
||||
#Load plugins
|
||||
# Load plugins
|
||||
pluginManager = PluginManager.PluginManager()
|
||||
failedPlugins = pluginManager.SetListOfEnvironmentDescriptors(build_env.plugins)
|
||||
if failedPlugins:
|
||||
|
|
@ -185,20 +184,20 @@ def main():
|
|||
raise Exception("One or more plugins failed to load.")
|
||||
|
||||
helper = PluginManager.HelperFunctions()
|
||||
if( helper.LoadFromPluginManager(pluginManager) > 0):
|
||||
if(helper.LoadFromPluginManager(pluginManager) > 0):
|
||||
raise Exception("One or more helper plugins failed to load.")
|
||||
|
||||
pluginList = pluginManager.GetPluginsOfClass(PluginManager.IMuBuildPlugin)
|
||||
|
||||
# Check to make sure our configuration is valid
|
||||
ConfigValidator.check_mu_confg(mu_config,edk2path,pluginList)
|
||||
ConfigValidator.check_mu_confg(mu_config, edk2path, pluginList)
|
||||
|
||||
for pkgToRunOn in packageList:
|
||||
#
|
||||
# run all loaded MuBuild Plugins/Tests
|
||||
#
|
||||
ts = JunitReport.create_new_testsuite(pkgToRunOn, "MuBuild.{0}.{1}".format( mu_config["GroupName"], pkgToRunOn) )
|
||||
_, loghandle = MuLogging.setup_logging(WORKSPACE_PATH,"BUILDLOG_{0}.txt".format(pkgToRunOn))
|
||||
ts = JunitReport.create_new_testsuite(pkgToRunOn, "MuBuild.{0}.{1}".format(mu_config["GroupName"], pkgToRunOn))
|
||||
_, loghandle = MuLogging.setup_logging(WORKSPACE_PATH, "BUILDLOG_{0}.txt".format(pkgToRunOn))
|
||||
logging.info("Package Running: {0}".format(pkgToRunOn))
|
||||
ShellEnvironment.CheckpointBuildVars()
|
||||
env = ShellEnvironment.GetBuildVars()
|
||||
|
|
@ -212,19 +211,18 @@ def main():
|
|||
logging.info("No Pkg Config file for {0}".format(pkgToRunOn))
|
||||
pkg_config = dict()
|
||||
|
||||
#check the resulting configuration
|
||||
ConfigValidator.check_package_confg(pkgToRunOn,pkg_config,pluginList)
|
||||
# check the resulting configuration
|
||||
ConfigValidator.check_package_confg(pkgToRunOn, pkg_config, pluginList)
|
||||
|
||||
for Descriptor in pluginList:
|
||||
#Get our targets
|
||||
# Get our targets
|
||||
targets = ["DEBUG"]
|
||||
if Descriptor.Obj.IsTargetDependent() and "Targets" in mu_config:
|
||||
targets = mu_config["Targets"]
|
||||
|
||||
|
||||
for target in targets:
|
||||
logging.critical("---Running {2}: {0} {1}".format(Descriptor.Name,target,pkgToRunOn))
|
||||
total_num +=1
|
||||
logging.critical("---Running {2}: {0} {1}".format(Descriptor.Name, target, pkgToRunOn))
|
||||
total_num += 1
|
||||
ShellEnvironment.CheckpointBuildVars()
|
||||
env = ShellEnvironment.GetBuildVars()
|
||||
|
||||
|
|
@ -232,12 +230,12 @@ def main():
|
|||
(testcasename, testclassname) = Descriptor.Obj.GetTestName(pkgToRunOn, env)
|
||||
tc = ts.create_new_testcase(testcasename, testclassname)
|
||||
|
||||
#merge the repo level and package level for this specific plugin
|
||||
pkg_plugin_configuration = merge_config(mu_config,pkg_config,Descriptor.descriptor)
|
||||
# merge the repo level and package level for this specific plugin
|
||||
pkg_plugin_configuration = merge_config(mu_config, pkg_config, Descriptor.descriptor)
|
||||
|
||||
#perhaps we should ask the validator to run on the
|
||||
# perhaps we should ask the validator to run on the
|
||||
|
||||
#Check if need to skip this particular plugin
|
||||
# Check if need to skip this particular plugin
|
||||
if "skip" in pkg_plugin_configuration and pkg_plugin_configuration["skip"]:
|
||||
tc.SetSkipped()
|
||||
logging.critical(" ->Test Skipped! %s" % Descriptor.Name)
|
||||
|
|
@ -256,12 +254,11 @@ def main():
|
|||
except Exception as exp:
|
||||
exc_type, exc_value, exc_traceback = sys.exc_info()
|
||||
logging.critical("EXCEPTION: {0}".format(exp))
|
||||
exceptionPrint = traceback.format_exception(type(exp), exp,exc_traceback)
|
||||
exceptionPrint = traceback.format_exception(type(exp), exp, exc_traceback)
|
||||
logging.critical(" ".join(exceptionPrint))
|
||||
tc.SetError("Exception: {0}".format(exp), "UNEXPECTED EXCEPTION")
|
||||
rc = 1
|
||||
|
||||
|
||||
if(rc != 0):
|
||||
failure_num += 1
|
||||
if(rc is None):
|
||||
|
|
@ -269,28 +266,28 @@ def main():
|
|||
else:
|
||||
logging.error("Test Failed: %s returned %d" % (Descriptor.Name, rc))
|
||||
else:
|
||||
logging.info("Test Success {0} {1}".format(Descriptor.Name,target))
|
||||
logging.info("Test Success {0} {1}".format(Descriptor.Name, target))
|
||||
|
||||
#revert to the checkpoint we created previously
|
||||
# revert to the checkpoint we created previously
|
||||
ShellEnvironment.RevertBuildVars()
|
||||
#finished target loop
|
||||
#Finished plugin loop
|
||||
# finished target loop
|
||||
# Finished plugin loop
|
||||
|
||||
MuLogging.stop_logging(loghandle) #stop the logging for this particularbuild file
|
||||
MuLogging.stop_logging(loghandle) # stop the logging for this particularbuild file
|
||||
ShellEnvironment.RevertBuildVars()
|
||||
#Finished buildable file loop
|
||||
|
||||
# Finished buildable file loop
|
||||
|
||||
JunitReport.Output(os.path.join(WORKSPACE_PATH, "Build", "BuildLogs", "TestSuites.xml"))
|
||||
|
||||
#Print Overall Success
|
||||
# Print Overall Success
|
||||
if(failure_num != 0):
|
||||
logging.critical("Overall Build Status: Error")
|
||||
logging.critical("There were {0} failures out of {1} attempts".format(failure_num,total_num))
|
||||
logging.critical("There were {0} failures out of {1} attempts".format(failure_num, total_num))
|
||||
else:
|
||||
logging.critical("Overall Build Status: Success")
|
||||
|
||||
sys.exit(failure_num)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
## @file MuLogging.py
|
||||
# @file MuLogging.py
|
||||
# Handle basic logging config for Project Mu Builds
|
||||
# MuBuild splits logs into a master log and per package.
|
||||
#
|
||||
|
|
@ -28,16 +28,17 @@
|
|||
import logging
|
||||
import sys
|
||||
from datetime import datetime
|
||||
from datetime import date
|
||||
import os
|
||||
import shutil
|
||||
|
||||
|
||||
def clean_build_logs(ws):
|
||||
# Make sure that we have a clean environment.
|
||||
if os.path.isdir(os.path.join(ws, "Build", "BuildLogs")):
|
||||
shutil.rmtree(os.path.join(ws, "Build", "BuildLogs"))
|
||||
|
||||
def setup_logging(workspace, filename=None, loghandle = None):
|
||||
|
||||
def setup_logging(workspace, filename=None, loghandle=None):
|
||||
|
||||
if loghandle is not None:
|
||||
stop_logging(loghandle)
|
||||
|
|
@ -48,33 +49,33 @@ def setup_logging(workspace, filename=None, loghandle = None):
|
|||
filename = "BUILDLOG_MASTER.txt"
|
||||
logging_level = logging.DEBUG
|
||||
|
||||
#setup logger
|
||||
# setup logger
|
||||
logger = logging.getLogger('')
|
||||
logger.setLevel(logging.DEBUG)
|
||||
|
||||
if len(logger.handlers) == 0:
|
||||
#Create the main console as logger
|
||||
# Create the main console as logger
|
||||
formatter = logging.Formatter("%(levelname)s- %(message)s")
|
||||
console = logging.StreamHandler()
|
||||
console.setLevel(logging.DEBUG)
|
||||
console.setFormatter(formatter)
|
||||
logger.addHandler(console)
|
||||
|
||||
|
||||
logfile = os.path.join(workspace, "Build", "BuildLogs", filename)
|
||||
if(not os.path.isdir(os.path.dirname(logfile))):
|
||||
os.makedirs(os.path.dirname(logfile))
|
||||
|
||||
#Create master file logger
|
||||
# Create master file logger
|
||||
fileformatter = logging.Formatter("%(levelname)s - %(message)s")
|
||||
filelogger = logging.FileHandler(filename=(logfile), mode='w')
|
||||
filelogger.setLevel(logging_level)
|
||||
filelogger.setFormatter(fileformatter)
|
||||
logger.addHandler(filelogger)
|
||||
logging.info("Log Started: " + datetime.strftime(datetime.now(), "%A, %B %d, %Y %I:%M%p" ))
|
||||
logging.info("Log Started: " + datetime.strftime(datetime.now(), "%A, %B %d, %Y %I:%M%p"))
|
||||
logging.info("Running Python version: " + str(sys.version_info))
|
||||
|
||||
return logfile,filelogger
|
||||
return logfile, filelogger
|
||||
|
||||
|
||||
def stop_logging(loghandle):
|
||||
loghandle.close()
|
||||
|
|
|
|||
|
|
@ -0,0 +1,199 @@
|
|||
import tempfile
|
||||
import logging
|
||||
import os
|
||||
import unittest
|
||||
from MuBuild import ConfigValidator
|
||||
import yaml
|
||||
try:
|
||||
from StringIO import StringIO
|
||||
except ImportError:
|
||||
from io import StringIO
|
||||
|
||||
test_dir = None
|
||||
plugin_list = []
|
||||
|
||||
|
||||
class Edk2Path_Injected(object):
|
||||
def __init__(self):
|
||||
self.WorkspacePath = None
|
||||
|
||||
def GetAbsolutePathOnThisSytemFromEdk2RelativePath(self, package):
|
||||
return os.path.abspath(self.WorkspacePath)
|
||||
|
||||
|
||||
class PluginList_Injected(object):
|
||||
def __init__(self, name):
|
||||
self.descriptor = dict()
|
||||
self.Obj = None
|
||||
self.Name = name
|
||||
|
||||
|
||||
class TestConfigValidator(unittest.TestCase):
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
global test_dir
|
||||
global valid_config
|
||||
global plugin_list
|
||||
|
||||
logger = logging.getLogger('')
|
||||
logger.addHandler(logging.NullHandler())
|
||||
unittest.installHandler()
|
||||
# get a temporary directory that we can create the right folders
|
||||
test_dir = Edk2Path_Injected()
|
||||
test_dir.WorkspacePath = tempfile.mkdtemp()
|
||||
|
||||
def test_valid_config(self):
|
||||
global test_dir
|
||||
global pluginList
|
||||
yaml_string = StringIO("""
|
||||
{
|
||||
"Name": "Project Mu Plus Repo CI Build",
|
||||
"GroupName": "MuPlus",
|
||||
|
||||
# Workspace path relative to this file
|
||||
"RelativeWorkspaceRoot": "",
|
||||
"Scopes": [ "corebuild" ],
|
||||
|
||||
# Other Repos that are dependencies
|
||||
"Dependencies": [
|
||||
# FileSystem Path relative to workspace
|
||||
# Url
|
||||
# Branch
|
||||
# Commit
|
||||
{
|
||||
"Path": "MU_BASECORE",
|
||||
"Url": "https://github.com/Microsoft/mu_basecore.git",
|
||||
"Branch": "release/201808"
|
||||
},
|
||||
],
|
||||
|
||||
# Edk2 style PackagesPath for resolving dependencies.
|
||||
# Only needed if it isn't this package and isn't a dependency
|
||||
"PackagesPath": [],
|
||||
|
||||
# Packages in this repo
|
||||
"Packages": [
|
||||
"UefiTestingPkg"
|
||||
],
|
||||
"ArchSupported": [
|
||||
"IA32",
|
||||
"X64",
|
||||
"AARCH64"
|
||||
],
|
||||
"Targets": [
|
||||
"DEBUG",
|
||||
"RELEASE"
|
||||
]
|
||||
}
|
||||
""")
|
||||
|
||||
valid_config = yaml.safe_load(yaml_string)
|
||||
# make sure the valid configuration is read just fine
|
||||
try:
|
||||
ConfigValidator.check_mu_confg(valid_config, test_dir, plugin_list)
|
||||
except Exception as e:
|
||||
self.fail("We shouldn't throw an exception", e)
|
||||
|
||||
def test_invalid_configs(self):
|
||||
global test_dir
|
||||
global pluginList
|
||||
bad_yaml_string = StringIO("""
|
||||
{
|
||||
"Name": "Project Mu Plus Repo CI Build",
|
||||
"GroupName": "MuPlus",
|
||||
|
||||
# Workspace path relative to this file
|
||||
"RelativeWorkspaceRoot": "",
|
||||
"InvalidAttribute": "this will throw an error",
|
||||
"Scopes": [ "corebuild" ],
|
||||
|
||||
# Other Repos that are dependencies
|
||||
"Dependencies": [
|
||||
# FileSystem Path relative to workspace
|
||||
# Url
|
||||
# Branch
|
||||
# Commit
|
||||
{
|
||||
"Path": "MU_BASECORE",
|
||||
"Url": "https://github.com/Microsoft/mu_basecore.git",
|
||||
"Branch": "release/201808"
|
||||
},
|
||||
],
|
||||
|
||||
# Edk2 style PackagesPath for resolving dependencies.
|
||||
# Only needed if it isn't this package and isn't a dependency
|
||||
"PackagesPath": [],
|
||||
|
||||
# Packages in this repo
|
||||
"Packages": [
|
||||
"UefiTestingPkg"
|
||||
],
|
||||
"ArchSupported": [
|
||||
"IA32",
|
||||
"X64",
|
||||
"AARCH64"
|
||||
],
|
||||
"Targets": [
|
||||
"DEBUG",
|
||||
"RELEASE"
|
||||
]
|
||||
}
|
||||
""")
|
||||
invalid_config = yaml.safe_load(bad_yaml_string)
|
||||
with self.assertRaises(Exception):
|
||||
ConfigValidator.check_mu_confg(invalid_config, test_dir, plugin_list)
|
||||
|
||||
def test_invalid_url_config(self):
|
||||
global test_dir
|
||||
global pluginList
|
||||
|
||||
bad_url_yaml_string = StringIO("""
|
||||
{
|
||||
"Name": "Project Mu Plus Repo CI Build",
|
||||
"GroupName": "MuPlus",
|
||||
|
||||
# Workspace path relative to this file
|
||||
"RelativeWorkspaceRoot": "",
|
||||
"Scopes": [ "corebuild" ],
|
||||
|
||||
# Other Repos that are dependencies
|
||||
"Dependencies": [
|
||||
# FileSystem Path relative to workspace
|
||||
# Url
|
||||
# Branch
|
||||
# Commit
|
||||
{
|
||||
"Path": "MU_BASECORE",
|
||||
"Url": "https://github.com/InvalidRepo",
|
||||
"Branch": "release/201808"
|
||||
},
|
||||
],
|
||||
|
||||
# Edk2 style PackagesPath for resolving dependencies.
|
||||
# Only needed if it isn't this package and isn't a dependency
|
||||
"PackagesPath": [],
|
||||
|
||||
# Packages in this repo
|
||||
"Packages": [
|
||||
"UefiTestingPkg"
|
||||
],
|
||||
"ArchSupported": [
|
||||
"IA32",
|
||||
"X64",
|
||||
"AARCH64"
|
||||
],
|
||||
"Targets": [
|
||||
"DEBUG",
|
||||
"RELEASE"
|
||||
]
|
||||
}
|
||||
""")
|
||||
|
||||
invalid_config = yaml.safe_load(bad_url_yaml_string)
|
||||
with self.assertRaises(Exception):
|
||||
ConfigValidator.check_mu_confg(invalid_config, test_dir, plugin_list)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
|
|
@ -1,9 +1,10 @@
|
|||
=========
|
||||
|
||||
========
|
||||
MU Build
|
||||
=========
|
||||
========
|
||||
|
||||
About
|
||||
==============
|
||||
=====
|
||||
|
||||
Provided with config file, mu_build fetches/clones dependencies then compiles every module in each package.
|
||||
This is the entrypoint into the CI / Pull Request build and test infrastructure.
|
||||
|
|
|
|||
|
|
@ -24,11 +24,19 @@ Please open any issues in the Project Mu GitHub tracker. [More Details](https://
|
|||
## Contributing Code or Docs
|
||||
|
||||
Please follow the general Project Mu Pull Request process. [More Details](https://microsoft.github.io/mu/How/contributing/)
|
||||
Additionally make sure all testing described in the "Development" section passes.
|
||||
|
||||
## Installing
|
||||
## Using
|
||||
|
||||
Install from pip with `pip install mu_build`
|
||||
Install from source with `pip install -e .`
|
||||
[Usage Details](using.md)
|
||||
|
||||
## Development
|
||||
|
||||
[Development Details](developing.md)
|
||||
|
||||
## Publish
|
||||
|
||||
[Publish Details](publishing.md)
|
||||
|
||||
## Copyright & License
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,70 @@
|
|||
workspace:
|
||||
clean: all
|
||||
|
||||
steps:
|
||||
- checkout: self
|
||||
clean: true
|
||||
|
||||
- task: UsePythonVersion@0
|
||||
inputs:
|
||||
versionSpec: '3.7.x'
|
||||
architecture: 'x64'
|
||||
|
||||
- script: python -m pip install --upgrade pip
|
||||
displayName: 'Install/Upgrade pip'
|
||||
|
||||
- script: pip uninstall -y mu_build
|
||||
displayName: 'Remove existing version of self'
|
||||
|
||||
- script: pip install --upgrade -r requirements.txt
|
||||
displayName: 'Install requirements'
|
||||
|
||||
- script: pip install -e .
|
||||
displayName: 'Install from Source'
|
||||
|
||||
- script: pytest -v --junitxml=test.junit.xml --html=pytest_MuBuild_report.html --self-contained-html --cov=MuBuild --cov-report html:cov_html --cov-report xml:cov.xml --cov-config .coveragerc
|
||||
displayName: 'Run UnitTests'
|
||||
|
||||
# Publish Test Results to Azure Pipelines/TFS
|
||||
- task: PublishTestResults@2
|
||||
displayName: 'Publish junit test results'
|
||||
continueOnError: true
|
||||
condition: succeededOrFailed()
|
||||
inputs:
|
||||
testResultsFormat: 'JUnit' # Options: JUnit, NUnit, VSTest, xUnit
|
||||
testResultsFiles: 'test.junit.xml'
|
||||
mergeTestResults: true # Optional
|
||||
publishRunAttachments: true # Optional
|
||||
|
||||
# Publish Build Artifacts
|
||||
# Publish build artifacts to Azure Pipelines/TFS or a file share
|
||||
- task: PublishBuildArtifacts@1
|
||||
inputs:
|
||||
pathtoPublish: 'pytest_MuBuild_report.html'
|
||||
artifactName: 'MuBuild unit test report'
|
||||
continueOnError: true
|
||||
condition: succeededOrFailed()
|
||||
|
||||
# Publish Code Coverage Results
|
||||
# Publish Cobertura code coverage results
|
||||
- task: PublishCodeCoverageResults@1
|
||||
inputs:
|
||||
codeCoverageTool: 'cobertura' # Options: cobertura, jaCoCo
|
||||
summaryFileLocation: $(System.DefaultWorkingDirectory)/cov.xml
|
||||
reportDirectory: $(System.DefaultWorkingDirectory)/cov_html
|
||||
condition: succeededOrFailed()
|
||||
|
||||
- script: flake8 . > flake8.err.log
|
||||
displayName: 'Run flake8'
|
||||
condition: succeededOrFailed()
|
||||
|
||||
- task: PublishBuildArtifacts@1
|
||||
inputs:
|
||||
pathtoPublish: 'flake8.err.log'
|
||||
artifactName: 'Flake8 Error log file'
|
||||
continueOnError: true
|
||||
condition: Failed()
|
||||
|
||||
|
||||
#- script: python setup.py sdist bdist_wheel
|
||||
# displayName: 'Build a wheel'
|
||||
|
|
@ -0,0 +1,45 @@
|
|||
# Developing Project Mu Pip Build
|
||||
|
||||
## Pre-Requisites
|
||||
|
||||
1. Get the code
|
||||
|
||||
``` cmd
|
||||
git clone https://github.com/Microsoft/mu_pip_build.git
|
||||
```
|
||||
|
||||
2. Install development dependencies
|
||||
|
||||
``` cmd
|
||||
pip install --upgrade -r requirements.txt
|
||||
```
|
||||
|
||||
3. Uninstall any copy of mu_build
|
||||
|
||||
``` cmd
|
||||
pip uninstall mu_build
|
||||
```
|
||||
|
||||
4. Install from local source (run command from root of repo)
|
||||
|
||||
``` cmd
|
||||
pip install -e .
|
||||
```
|
||||
|
||||
## Testing
|
||||
|
||||
1. Run a Basic Syntax/Lint Check (using flake8) and resolve any issues
|
||||
|
||||
``` cmd
|
||||
flake8 MuBuild
|
||||
```
|
||||
|
||||
2. Run pytest with coverage data collected
|
||||
|
||||
``` cmd
|
||||
pytest -v --junitxml=test.junit.xml --html=pytest_MuBuild_report.html --self-contained-html --cov=MuBuild --cov-report html:cov_html --cov-report xml:cov.xml --cov-config .coveragerc
|
||||
```
|
||||
|
||||
3. Look at the reports
|
||||
* pytest_MuBuild_report.html
|
||||
* cov_html/index.html
|
||||
|
|
@ -0,0 +1,34 @@
|
|||
# Publishing Project Mu Pip Build
|
||||
|
||||
The MuBuild is published as a pypi (pip) module. The pip module is named __mu_build__. Pypi allows for easy version management, dependency management, and sharing.
|
||||
|
||||
Publishing/releasing a new version is generally handled thru a server based build process but for completeness the process is documented here.
|
||||
|
||||
## Steps
|
||||
|
||||
!!! Info
|
||||
These directions assume you have already configured your workspace for developing. If not please first do that. Directions on the [developing](developing.md) page.
|
||||
|
||||
1. Install tools for publishing
|
||||
|
||||
``` cmd
|
||||
pip install --upgrade -r requirements.publisher.txt
|
||||
```
|
||||
|
||||
2. Build a wheel
|
||||
|
||||
``` cmd
|
||||
python setup.py sdist bdist_wheel
|
||||
```
|
||||
|
||||
3. Publish the wheel/distribution to pypi
|
||||
|
||||
``` cmd
|
||||
twine upload dist/*
|
||||
```
|
||||
|
||||
## Server Build
|
||||
|
||||
## Release Checklist
|
||||
|
||||
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
setuptools
|
||||
wheel
|
||||
twine
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
pytest
|
||||
pytest-html
|
||||
pytest-cov
|
||||
flake8
|
||||
12
setup.py
12
setup.py
|
|
@ -1,4 +1,4 @@
|
|||
## @file setup.py
|
||||
# @file setup.py
|
||||
# This contains setup info for mu_build pip module
|
||||
#
|
||||
##
|
||||
|
|
@ -40,12 +40,18 @@ setuptools.setup(
|
|||
url="https://github.com/microsoft/mu_pip_build",
|
||||
license='BSD2',
|
||||
packages=setuptools.find_packages(),
|
||||
entry_points = {
|
||||
'console_scripts': ['mu_build=MuBuild.MuBuild:main'],
|
||||
entry_points={
|
||||
'console_scripts': ['mu_build=MuBuild.MuBuild:main']
|
||||
},
|
||||
install_requires=[
|
||||
'pyyaml',
|
||||
'mu_python_library>=0.2.0',
|
||||
'mu_environment>=0.2.1'
|
||||
],
|
||||
classifiers=[
|
||||
"Programming Language :: Python :: 3",
|
||||
"License :: OSI Approved :: BSD License",
|
||||
"Operating System :: OS Independent",
|
||||
"Development Status :: 3 - Alpha"
|
||||
]
|
||||
)
|
||||
|
|
@ -0,0 +1,10 @@
|
|||
# Using Project Mu Pip Build
|
||||
|
||||
Install from pip
|
||||
|
||||
```cmd
|
||||
pip install mu_build
|
||||
```
|
||||
## Usage Docs
|
||||
|
||||
__TBD__
|
||||
Загрузка…
Ссылка в новой задаче