Merged PR 528: Add CI, linting, and unit test support

This commit is contained in:
Sean Brogan 2018-11-30 23:25:18 +00:00 коммит произвёл Matthew Carlson
Родитель d356e7c7a0
Коммит e12a8e1a37
16 изменённых файлов: 573 добавлений и 183 удалений

2
.coveragerc Normal file
Просмотреть файл

@ -0,0 +1,2 @@
[run]
omit = tests/*

5
.flake8 Normal file
Просмотреть файл

@ -0,0 +1,5 @@
[flake8]
#E501 line too long
#E266 too many leading '#' for block comment
#E722 do not use bare 'except'
ignore = E501,E266,E722

8
.gitignore поставляемый
Просмотреть файл

@ -3,4 +3,10 @@
Lib
dist
*.egg-info
build.
build.
/cov_html
/.pytest_cache
/pytest_MuBuild_report.html
/.coverage
/cov.xml
/test.junit.xml

Просмотреть файл

@ -1,8 +1,8 @@
## @file ConfigValidator.py
# @file ConfigValidator.py
# This module contains support for validating .mu.json config files
#
# Used to support CI/CD and exporting test results for other tools.
# This does test report generation without being a test runner.
# This does test report generation without being a test runner.
##
# Copyright (c) 2018, Microsoft Corporation
#
@ -26,10 +26,8 @@
# OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
# ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
##
import logging
import os
import urllib.request as req
import urllib.parse as p
'''
Example_MU_CONFIG_FILE:
@ -65,101 +63,100 @@ Example_MU_CONFIG_FILE:
'''
#Checks the top level MU Config
def check_mu_confg(config,edk2path,pluginList):
# Checks the top level MU Config
def check_mu_confg(config, edk2path, pluginList):
workspace = edk2path.WorkspacePath
def _mu_error(message):
raise Exception("Mu Config Error: {0}".format(message))
def _is_valid_dir(path,name):
path = os.path.join(workspace,path)
def _is_valid_dir(path, name):
path = os.path.join(workspace, path)
if not os.path.isdir(path):
_mu_error("{0} isn't a valid directory".format(path))
def _check_url(url):
request = req.Request(url)
try:
response = req.urlopen(request)
req.urlopen(request)
return True
except:
#The url wasn't valid
# The url wasn't valid
return False
def _check_packages(packages,name):
def _check_packages(packages, name):
for package in packages:
path = edk2path.GetAbsolutePathOnThisSytemFromEdk2RelativePath(package)
if path is None or not os.path.isdir(path):
_mu_error("{0} isn't a valid package to build".format(package))
return True
def _is_valid_arch(targets,name):
valid_targets = ["AARCH64","IA32","X64"]
def _is_valid_arch(targets, name):
valid_targets = ["AARCH64", "IA32", "X64"]
for target in targets:
if not target in valid_targets:
if target not in valid_targets:
_mu_error("{0} is not a valid target".format(target))
def _check_dependencies(dependencies,name):
valid_attributes = ["Path","Url","Branch","Commit"]
def _check_dependencies(dependencies, name):
valid_attributes = ["Path", "Url", "Branch", "Commit"]
for dependency in dependencies:
# check to make sure we have a path
if not "Path" in dependency:
if "Path" not in dependency:
_mu_error("Path not found in dependency {0}".format(dependency))
# check to sure we have a valid url and we can reach it
if not "Url" in dependency:
if "Url" not in dependency:
_mu_error("Url not found in dependency {0}".format(dependency))
if not _check_url(dependency["Url"]):
_mu_error("Invalid URL {0}".format(dependency["Url"]))
_mu_error("Invalid URL {0}".format(dependency["Url"]))
# make sure we have a valid branch or commit
if not "Branch" in dependency and not "Commit" in dependency:
if "Branch" not in dependency and "Commit" not in dependency:
_mu_error("You must have a commit or a branch dependency {0}".format(dependency))
if "Branch" in dependency and "Commit" in dependency:
if "Branch" in dependency and "Commit" in dependency:
_mu_error("You cannot have both a commit or a branch dependency {0}".format(dependency))
#check to make sure we don't have something else in there
# check to make sure we don't have something else in there
for attribute in dependency:
if not attribute in valid_attributes:
if attribute not in valid_attributes:
_mu_error("Unknown attribute {0} in dependecy".format(attribute))
return True
config_rules = {
"required": {
"Name": {
"type":"str"
},
"type": "str"
},
"GroupName": {
"type":"str"
},
"type": "str"
},
"Scopes": {
"type":"list",
"items":"str"
"type": "list",
"items": "str"
},
"ArchSupported": {
"type":"list",
"validator":_is_valid_arch
"type": "list",
"validator": _is_valid_arch
},
"RelativeWorkspaceRoot": {
"type":"str",
"validator":_is_valid_dir
"type": "str",
"validator": _is_valid_dir
},
"Targets": {
"type":"list"
"type": "list"
}
},
"optional": {
"Packages":{
"type":"list",
"items":"str",
"validator":_check_packages
"Packages": {
"type": "list",
"items": "str",
"validator": _check_packages
},
"PackagesPath": {
"type":"list",
"items":"str"
"type": "list",
"items": "str"
},
"Dependencies": {
"type":"list",
"validator":_check_dependencies
"type": "list",
"validator": _check_dependencies
}
}
}
@ -170,19 +167,19 @@ def check_mu_confg(config,edk2path,pluginList):
if "config_name" in plugin.descriptor:
plugin_name = plugin.descriptor["config_name"]
config_rules["optional"][plugin_name] = {
"validator" : plugin.Obj.ValidateConfig
"validator": plugin.Obj.ValidateConfig
}
#check if all the requires are satisified
# check if all the requires are satisified
for rule in config_rules["required"]:
if not rule in config:
if rule not in config:
_mu_error("{0} is a required attribute in your MU Config".format(rule))
if "type" in config_rules["required"][rule]:
config_type = str(type(config[rule]).__name__)
wanted_type = config_rules["required"][rule]["type"]
if config_type != wanted_type:
_mu_error("{0} is a required attribute and is not the correct type. We are expecting a {1} and got a {2}".format(rule,config_type,wanted_type))
_mu_error("{0} is a required attribute and is not the correct type. We are expecting a {1} and got a {2}".format(rule, config_type, wanted_type))
if "validator" in config_rules["required"][rule]:
validator = config_rules["required"][rule]["validator"]
@ -190,33 +187,34 @@ def check_mu_confg(config,edk2path,pluginList):
# check optional types
for rule in config_rules["optional"]:
if not rule in config:
continue
if rule not in config:
continue
if "type" in config_rules["optional"][rule]:
config_type = str(type(config[rule]).__name__)
wanted_type = config_rules["optional"][rule]["type"]
if config_type != wanted_type:
_mu_error("{0} is a optional attribute and is not the correct type. We are expecting a {1} and got a {2}".format(rule,config_type,wanted_type))
_mu_error("{0} is a optional attribute and is not the correct type. We are expecting a {1} and got a {2}".format(rule, config_type, wanted_type))
if "validator" in config_rules["optional"][rule]:
validator = config_rules["optional"][rule]["validator"]
validator(config[rule],"Base mu.json")
#check to make sure we don't have any stray keys in there
validator(config[rule], "Base mu.json")
# check to make sure we don't have any stray keys in there
for rule in config:
if not rule in config_rules["optional"] and not rule in config_rules["required"]:
if rule not in config_rules["optional"] and rule not in config_rules["required"]:
_mu_error("Unknown parameter {0} is unexpected".format(rule))
return True
'''
{
"Defines": {
"PLATFORM_NAME": "MdeModule",
"DSC_SPECIFICATION": "0x00010005",
"SUPPORTED_ARCHITECTURES": "IA32|X64|ARM|AARCH64",
"BUILD_TARGETS": "DEBUG|RELEASE"
"BUILD_TARGETS": "DEBUG|RELEASE"
},
"CompilerPlugin": {
"skip":false,
@ -227,7 +225,7 @@ def check_mu_confg(config,edk2path,pluginList):
"MdePkg/MdePkg.dec",
"MdeModulePkg/MdeModulePkg.dec",
"MsUnitTestPkg/MsUnitTestPkg.dec"
],
],
"IgnoreInf": {
},
@ -238,17 +236,19 @@ def check_mu_confg(config,edk2path,pluginList):
##
# Checks the package configuration for errors
##
def check_package_confg(name,config,pluginList):
def check_package_confg(name, config, pluginList):
def _mu_error(message):
raise Exception("Package {0} Config Error: {1}".format(name,message))
raise Exception("Package {0} Config Error: {1}".format(name, message))
config_rules = {
"required": {
},
"optional": {
"optional": {
"Defines": {
"type":"dict",
"items":"str"
"type": "dict",
"items": "str"
}
}
}
@ -260,41 +260,40 @@ def check_package_confg(name,config,pluginList):
plugin_name = plugin.descriptor["config_name"]
# add the validator
config_rules["optional"][plugin_name] = {
"validator" : plugin.Obj.ValidateConfig
"validator": plugin.Obj.ValidateConfig
}
#check if all the requires are satisified
# check if all the requires are satisified
for rule in config_rules["required"]:
if not rule in config:
if rule not in config:
_mu_error("{0} is a required attribute in your MU Config".format(rule))
if "type" in config_rules["required"][rule]:
config_type = str(type(config[rule]).__name__)
wanted_type = config_rules["required"][rule]["type"]
if config_type != wanted_type:
_mu_error("{0} is a required attribute and is not the correct type. We are expecting a {1} and got a {2}".format(rule,config_type,wanted_type))
_mu_error("{0} is a required attribute and is not the correct type. We are expecting a {1} and got a {2}".format(rule, config_type, wanted_type))
if "validator" in config_rules["required"][rule]:
validator = config_rules["required"][rule]["validator"]
validator(config[rule],name)
validator(config[rule], name)
# check optional types
for rule in config_rules["optional"]:
if not rule in config:
continue
if rule not in config:
continue
if "type" in config_rules["optional"][rule]:
config_type = str(type(config[rule]).__name__)
wanted_type = config_rules["optional"][rule]["type"]
if config_type != wanted_type:
_mu_error("{0} is a optional attribute and is not the correct type. We are expecting a {1} and got a {2}".format(rule,config_type,wanted_type))
_mu_error("{0} is a optional attribute and is not the correct type. We are expecting a {1} and got a {2}".format(rule, config_type, wanted_type))
if "validator" in config_rules["optional"][rule]:
validator = config_rules["optional"][rule]["validator"]
validator(config[rule],name)
#check to make sure we don't have any stray keys in there
validator(config[rule], name)
# check to make sure we don't have any stray keys in there
for rule in config:
if not rule in config_rules["optional"] and not rule in config_rules["required"]:
if rule not in config_rules["optional"] and rule not in config_rules["required"]:
_mu_error("Unknown parameter {0} is unexpected".format(rule))

Просмотреть файл

@ -1,4 +1,4 @@
## @file MuBuild.py
# @file MuBuild.py
# This module contains code that supports Project Mu CI/CD
# This is the main entry for the build and test process
# of Non-Product builds
@ -47,28 +47,29 @@ import pkg_resources
def get_mu_config():
parser = argparse.ArgumentParser(description='Run the Mu Build')
parser.add_argument ('-c', '--mu_config', dest = 'mu_config', required = True, type=str, help ='Provide the Mu config relative to the current working directory')
parser.add_argument ('-p', '--pkg','--pkg-dir', dest='pkglist', nargs="+", type=str, help = 'A package or folder you want to test (abs path or cwd relative). Can list multiple by doing -p <pkg1> <pkg2> <pkg3>', default=[])
parser.add_argument('-ignore','--ignore-git', dest="git_ignore",action="store_true",help="Whether to ignore errors in the git cloing process", default=False)
parser.add_argument('-force','--force-git', dest="git_force",action="store_true",help="Whether to force git repos to clone in the git cloing process", default=False)
parser.add_argument('-update-git','--update-git', dest="git_update",action="store_true",help="Whether to update git repos as needed in the git cloing process", default=False)
args, sys.argv = parser.parse_known_args()
parser.add_argument('-c', '--mu_config', dest='mu_config', required=True, type=str, help='Provide the Mu config relative to the current working directory')
parser.add_argument('-p', '--pkg', '--pkg-dir', dest='pkglist', nargs="+", type=str, help='A package or folder you want to test (abs path or cwd relative). Can list multiple by doing -p <pkg1> <pkg2> <pkg3>', default=[])
parser.add_argument('-ignore', '--ignore-git', dest="git_ignore", action="store_true", help="Whether to ignore errors in the git cloing process", default=False)
parser.add_argument('-force', '--force-git', dest="git_force", action="store_true", help="Whether to force git repos to clone in the git cloing process", default=False)
parser.add_argument('-update-git', '--update-git', dest="git_update", action="store_true", help="Whether to update git repos as needed in the git cloing process", default=False)
args, sys.argv = parser.parse_known_args()
return args
def merge_config(mu_config,pkg_config,descriptor={}):
def merge_config(mu_config, pkg_config, descriptor={}):
plugin_name = ""
config = dict()
if "module" in descriptor:
plugin_name = descriptor["module"]
if "config_name" in descriptor:
plugin_name = descriptor["config_name"]
if plugin_name == "":
return config
if plugin_name in mu_config:
config.update(mu_config[plugin_name])
if plugin_name in pkg_config:
config.update(pkg_config[plugin_name])
@ -77,49 +78,49 @@ def merge_config(mu_config,pkg_config,descriptor={}):
#
# Main driver of Project Mu Builds
#
def main():
#Parse command line arguments
# Parse command line arguments
PROJECT_SCOPES = ("project_mu",)
buildArgs = get_mu_config()
mu_config_filepath = os.path.abspath(buildArgs.mu_config)
if mu_config_filepath is None or not os.path.isfile(mu_config_filepath):
raise Exception("Invalid path to mu.json file for build: ", mu_config_filepath)
#have a build config file
# have a build config file
with open(mu_config_filepath, 'r') as mu_config_file:
mu_config = yaml.safe_load(mu_config_file)
WORKSPACE_PATH = os.path.realpath(os.path.join(os.path.dirname(mu_config_filepath), mu_config["RelativeWorkspaceRoot"]))
#Setup the logging to the file as well as the console
# Setup the logging to the file as well as the console
MuLogging.clean_build_logs(WORKSPACE_PATH)
MuLogging.setup_logging(WORKSPACE_PATH)
#Get scopes from config file
# Get scopes from config file
if "Scopes" in mu_config:
PROJECT_SCOPES += tuple(mu_config["Scopes"])
# SET PACKAGE PATH
#
#
# Get Package Path from config file
pplist = list()
if(mu_config["RelativeWorkspaceRoot"] != ""):
#this package is not at workspace root.
# this package is not at workspace root.
# Add self
pplist.append(os.path.dirname(mu_config_filepath))
#Include packages from the config file
# Include packages from the config file
if "PackagesPath" in mu_config:
for a in mu_config["PackagesPath"]:
pplist.append(a)
#Check Dependencies for Repo
# Check Dependencies for Repo
if "Dependencies" in mu_config:
pplist.extend(RepoResolver.resolve_all(WORKSPACE_PATH,mu_config["Dependencies"], ignore=buildArgs.git_ignore, force=buildArgs.git_force, update_ok=buildArgs.git_update))
pplist.extend(RepoResolver.resolve_all(WORKSPACE_PATH, mu_config["Dependencies"], ignore=buildArgs.git_ignore, force=buildArgs.git_force, update_ok=buildArgs.git_update))
#make Edk2Path object to handle all path operations
# make Edk2Path object to handle all path operations
edk2path = Edk2Path(WORKSPACE_PATH, pplist)
logging.info("Running ProjectMu Build: {0}".format(mu_config["Name"]))
@ -129,7 +130,7 @@ def main():
logging.info("mu_python_library version: " + pkg_resources.get_distribution("mu_python_library").version)
logging.info("mu_environment version: " + pkg_resources.get_distribution("mu_environment").version)
#which package to build
# which package to build
packageList = mu_config["Packages"]
#
# If mu pk list supplied lets see if they are a file system path
@ -137,10 +138,10 @@ def main():
#
#
if(len(buildArgs.pkglist) > 0):
packageList = [] #clear it
packageList = [] # clear it
for mu_pk_path in buildArgs.pkglist:
#if abs path lets convert
# if abs path lets convert
if os.path.isabs(mu_pk_path):
temp = edk2path.GetEdk2RelativePathFromAbsolutePath(mu_pk_path)
if(temp is not None):
@ -148,8 +149,8 @@ def main():
else:
logging.critical("pkg-dir invalid absolute path: {0}".format(mu_pk_path))
raise Exception("Invalid Package Path")
else:
#Check if relative path
else:
# Check if relative path
temp = os.path.join(os.getcwd(), mu_pk_path)
temp = edk2path.GetEdk2RelativePathFromAbsolutePath(temp)
if(temp is not None):
@ -157,25 +158,23 @@ def main():
else:
logging.critical("pkg-dir invalid relative path: {0}".format(mu_pk_path))
raise Exception("Invalid Package Path")
# Bring up the common minimum environment.
(build_env, shell_env) = SelfDescribingEnvironment.BootstrapEnvironment(edk2path.WorkspacePath, PROJECT_SCOPES)
CommonBuildEntry.update_process(edk2path.WorkspacePath, PROJECT_SCOPES)
env = ShellEnvironment.GetBuildVars()
archSupported = " ".join(mu_config["ArchSupported"])
env.SetValue("TARGET_ARCH", archSupported, "Platform Hardcoded")
#Generate consumable XML object- junit format
# Generate consumable XML object- junit format
JunitReport = MuJunitReport()
#Keep track of failures
# Keep track of failures
failure_num = 0
total_num = 0
#Load plugins
# Load plugins
pluginManager = PluginManager.PluginManager()
failedPlugins = pluginManager.SetListOfEnvironmentDescriptors(build_env.plugins)
if failedPlugins:
@ -185,20 +184,20 @@ def main():
raise Exception("One or more plugins failed to load.")
helper = PluginManager.HelperFunctions()
if( helper.LoadFromPluginManager(pluginManager) > 0):
if(helper.LoadFromPluginManager(pluginManager) > 0):
raise Exception("One or more helper plugins failed to load.")
pluginList = pluginManager.GetPluginsOfClass(PluginManager.IMuBuildPlugin)
# Check to make sure our configuration is valid
ConfigValidator.check_mu_confg(mu_config,edk2path,pluginList)
ConfigValidator.check_mu_confg(mu_config, edk2path, pluginList)
for pkgToRunOn in packageList:
#
# run all loaded MuBuild Plugins/Tests
#
ts = JunitReport.create_new_testsuite(pkgToRunOn, "MuBuild.{0}.{1}".format( mu_config["GroupName"], pkgToRunOn) )
_, loghandle = MuLogging.setup_logging(WORKSPACE_PATH,"BUILDLOG_{0}.txt".format(pkgToRunOn))
ts = JunitReport.create_new_testsuite(pkgToRunOn, "MuBuild.{0}.{1}".format(mu_config["GroupName"], pkgToRunOn))
_, loghandle = MuLogging.setup_logging(WORKSPACE_PATH, "BUILDLOG_{0}.txt".format(pkgToRunOn))
logging.info("Package Running: {0}".format(pkgToRunOn))
ShellEnvironment.CheckpointBuildVars()
env = ShellEnvironment.GetBuildVars()
@ -212,43 +211,42 @@ def main():
logging.info("No Pkg Config file for {0}".format(pkgToRunOn))
pkg_config = dict()
#check the resulting configuration
ConfigValidator.check_package_confg(pkgToRunOn,pkg_config,pluginList)
# check the resulting configuration
ConfigValidator.check_package_confg(pkgToRunOn, pkg_config, pluginList)
for Descriptor in pluginList:
#Get our targets
# Get our targets
targets = ["DEBUG"]
if Descriptor.Obj.IsTargetDependent() and "Targets" in mu_config:
targets = mu_config["Targets"]
for target in targets:
logging.critical("---Running {2}: {0} {1}".format(Descriptor.Name,target,pkgToRunOn))
total_num +=1
logging.critical("---Running {2}: {0} {1}".format(Descriptor.Name, target, pkgToRunOn))
total_num += 1
ShellEnvironment.CheckpointBuildVars()
env = ShellEnvironment.GetBuildVars()
env.SetValue("TARGET", target, "MuBuild.py before RunBuildPlugin")
(testcasename, testclassname) = Descriptor.Obj.GetTestName(pkgToRunOn, env)
tc = ts.create_new_testcase(testcasename, testclassname)
#merge the repo level and package level for this specific plugin
pkg_plugin_configuration = merge_config(mu_config,pkg_config,Descriptor.descriptor)
# merge the repo level and package level for this specific plugin
pkg_plugin_configuration = merge_config(mu_config, pkg_config, Descriptor.descriptor)
#perhaps we should ask the validator to run on the
# perhaps we should ask the validator to run on the
#Check if need to skip this particular plugin
# Check if need to skip this particular plugin
if "skip" in pkg_plugin_configuration and pkg_plugin_configuration["skip"]:
tc.SetSkipped()
logging.critical(" ->Test Skipped! %s" % Descriptor.Name)
else:
try:
# - package is the edk2 path to package. This means workspace/packagepath relative.
# - package is the edk2 path to package. This means workspace/packagepath relative.
# - edk2path object configured with workspace and packages path
# - any additional command line args
# - RepoConfig Object (dict) for the build
# - PkgConfig Object (dict)
# - EnvConfig Object
# - EnvConfig Object
# - Plugin Manager Instance
# - Plugin Helper Obj Instance
# - testcase Object used for outputing junit results
@ -256,11 +254,10 @@ def main():
except Exception as exp:
exc_type, exc_value, exc_traceback = sys.exc_info()
logging.critical("EXCEPTION: {0}".format(exp))
exceptionPrint = traceback.format_exception(type(exp), exp,exc_traceback)
exceptionPrint = traceback.format_exception(type(exp), exp, exc_traceback)
logging.critical(" ".join(exceptionPrint))
tc.SetError("Exception: {0}".format(exp), "UNEXPECTED EXCEPTION")
rc = 1
if(rc != 0):
failure_num += 1
@ -269,28 +266,28 @@ def main():
else:
logging.error("Test Failed: %s returned %d" % (Descriptor.Name, rc))
else:
logging.info("Test Success {0} {1}".format(Descriptor.Name,target))
#revert to the checkpoint we created previously
ShellEnvironment.RevertBuildVars()
#finished target loop
#Finished plugin loop
MuLogging.stop_logging(loghandle) #stop the logging for this particularbuild file
ShellEnvironment.RevertBuildVars()
#Finished buildable file loop
logging.info("Test Success {0} {1}".format(Descriptor.Name, target))
# revert to the checkpoint we created previously
ShellEnvironment.RevertBuildVars()
# finished target loop
# Finished plugin loop
MuLogging.stop_logging(loghandle) # stop the logging for this particularbuild file
ShellEnvironment.RevertBuildVars()
# Finished buildable file loop
JunitReport.Output(os.path.join(WORKSPACE_PATH, "Build", "BuildLogs", "TestSuites.xml"))
#Print Overall Success
# Print Overall Success
if(failure_num != 0):
logging.critical("Overall Build Status: Error")
logging.critical("There were {0} failures out of {1} attempts".format(failure_num,total_num))
logging.critical("There were {0} failures out of {1} attempts".format(failure_num, total_num))
else:
logging.critical("Overall Build Status: Success")
sys.exit(failure_num)
if __name__ == '__main__':
main()
main()

Просмотреть файл

@ -1,6 +1,6 @@
## @file MuLogging.py
# @file MuLogging.py
# Handle basic logging config for Project Mu Builds
# MuBuild splits logs into a master log and per package.
# MuBuild splits logs into a master log and per package.
#
##
# Copyright (c) 2018, Microsoft Corporation
@ -28,53 +28,54 @@
import logging
import sys
from datetime import datetime
from datetime import date
import os
import shutil
def clean_build_logs(ws):
# Make sure that we have a clean environment.
# Make sure that we have a clean environment.
if os.path.isdir(os.path.join(ws, "Build", "BuildLogs")):
shutil.rmtree(os.path.join(ws, "Build", "BuildLogs"))
def setup_logging(workspace, filename=None, loghandle = None):
def setup_logging(workspace, filename=None, loghandle=None):
if loghandle is not None:
stop_logging(loghandle)
logging_level = logging.DEBUG
if filename is None:
filename = "BUILDLOG_MASTER.txt"
logging_level = logging.DEBUG
#setup logger
# setup logger
logger = logging.getLogger('')
logger.setLevel(logging.DEBUG)
if len(logger.handlers) == 0:
#Create the main console as logger
# Create the main console as logger
formatter = logging.Formatter("%(levelname)s- %(message)s")
console = logging.StreamHandler()
console.setLevel(logging.DEBUG)
console.setFormatter(formatter)
logger.addHandler(console)
logfile = os.path.join(workspace, "Build", "BuildLogs", filename)
if(not os.path.isdir(os.path.dirname(logfile))):
os.makedirs(os.path.dirname(logfile))
#Create master file logger
# Create master file logger
fileformatter = logging.Formatter("%(levelname)s - %(message)s")
filelogger = logging.FileHandler(filename=(logfile), mode='w')
filelogger.setLevel(logging_level)
filelogger.setFormatter(fileformatter)
logger.addHandler(filelogger)
logging.info("Log Started: " + datetime.strftime(datetime.now(), "%A, %B %d, %Y %I:%M%p" ))
logging.info("Log Started: " + datetime.strftime(datetime.now(), "%A, %B %d, %Y %I:%M%p"))
logging.info("Running Python version: " + str(sys.version_info))
return logfile,filelogger
return logfile, filelogger
def stop_logging(loghandle):
loghandle.close()

Просмотреть файл

@ -0,0 +1,199 @@
import tempfile
import logging
import os
import unittest
from MuBuild import ConfigValidator
import yaml
try:
from StringIO import StringIO
except ImportError:
from io import StringIO
test_dir = None
plugin_list = []
class Edk2Path_Injected(object):
def __init__(self):
self.WorkspacePath = None
def GetAbsolutePathOnThisSytemFromEdk2RelativePath(self, package):
return os.path.abspath(self.WorkspacePath)
class PluginList_Injected(object):
def __init__(self, name):
self.descriptor = dict()
self.Obj = None
self.Name = name
class TestConfigValidator(unittest.TestCase):
@classmethod
def setUpClass(cls):
global test_dir
global valid_config
global plugin_list
logger = logging.getLogger('')
logger.addHandler(logging.NullHandler())
unittest.installHandler()
# get a temporary directory that we can create the right folders
test_dir = Edk2Path_Injected()
test_dir.WorkspacePath = tempfile.mkdtemp()
def test_valid_config(self):
global test_dir
global pluginList
yaml_string = StringIO("""
{
"Name": "Project Mu Plus Repo CI Build",
"GroupName": "MuPlus",
# Workspace path relative to this file
"RelativeWorkspaceRoot": "",
"Scopes": [ "corebuild" ],
# Other Repos that are dependencies
"Dependencies": [
# FileSystem Path relative to workspace
# Url
# Branch
# Commit
{
"Path": "MU_BASECORE",
"Url": "https://github.com/Microsoft/mu_basecore.git",
"Branch": "release/201808"
},
],
# Edk2 style PackagesPath for resolving dependencies.
# Only needed if it isn't this package and isn't a dependency
"PackagesPath": [],
# Packages in this repo
"Packages": [
"UefiTestingPkg"
],
"ArchSupported": [
"IA32",
"X64",
"AARCH64"
],
"Targets": [
"DEBUG",
"RELEASE"
]
}
""")
valid_config = yaml.safe_load(yaml_string)
# make sure the valid configuration is read just fine
try:
ConfigValidator.check_mu_confg(valid_config, test_dir, plugin_list)
except Exception as e:
self.fail("We shouldn't throw an exception", e)
def test_invalid_configs(self):
global test_dir
global pluginList
bad_yaml_string = StringIO("""
{
"Name": "Project Mu Plus Repo CI Build",
"GroupName": "MuPlus",
# Workspace path relative to this file
"RelativeWorkspaceRoot": "",
"InvalidAttribute": "this will throw an error",
"Scopes": [ "corebuild" ],
# Other Repos that are dependencies
"Dependencies": [
# FileSystem Path relative to workspace
# Url
# Branch
# Commit
{
"Path": "MU_BASECORE",
"Url": "https://github.com/Microsoft/mu_basecore.git",
"Branch": "release/201808"
},
],
# Edk2 style PackagesPath for resolving dependencies.
# Only needed if it isn't this package and isn't a dependency
"PackagesPath": [],
# Packages in this repo
"Packages": [
"UefiTestingPkg"
],
"ArchSupported": [
"IA32",
"X64",
"AARCH64"
],
"Targets": [
"DEBUG",
"RELEASE"
]
}
""")
invalid_config = yaml.safe_load(bad_yaml_string)
with self.assertRaises(Exception):
ConfigValidator.check_mu_confg(invalid_config, test_dir, plugin_list)
def test_invalid_url_config(self):
global test_dir
global pluginList
bad_url_yaml_string = StringIO("""
{
"Name": "Project Mu Plus Repo CI Build",
"GroupName": "MuPlus",
# Workspace path relative to this file
"RelativeWorkspaceRoot": "",
"Scopes": [ "corebuild" ],
# Other Repos that are dependencies
"Dependencies": [
# FileSystem Path relative to workspace
# Url
# Branch
# Commit
{
"Path": "MU_BASECORE",
"Url": "https://github.com/InvalidRepo",
"Branch": "release/201808"
},
],
# Edk2 style PackagesPath for resolving dependencies.
# Only needed if it isn't this package and isn't a dependency
"PackagesPath": [],
# Packages in this repo
"Packages": [
"UefiTestingPkg"
],
"ArchSupported": [
"IA32",
"X64",
"AARCH64"
],
"Targets": [
"DEBUG",
"RELEASE"
]
}
""")
invalid_config = yaml.safe_load(bad_url_yaml_string)
with self.assertRaises(Exception):
ConfigValidator.check_mu_confg(invalid_config, test_dir, plugin_list)
if __name__ == '__main__':
unittest.main()

Просмотреть файл

@ -1,9 +1,10 @@
=========
========
MU Build
=========
========
About
==============
=====
Provided with config file, mu_build fetches/clones dependencies then compiles every module in each package.
This is the entrypoint into the CI / Pull Request build and test infrastructure.

Просмотреть файл

@ -24,11 +24,19 @@ Please open any issues in the Project Mu GitHub tracker. [More Details](https://
## Contributing Code or Docs
Please follow the general Project Mu Pull Request process. [More Details](https://microsoft.github.io/mu/How/contributing/)
Additionally make sure all testing described in the "Development" section passes.
## Installing
## Using
Install from pip with `pip install mu_build`
Install from source with `pip install -e .`
[Usage Details](using.md)
## Development
[Development Details](developing.md)
## Publish
[Publish Details](publishing.md)
## Copyright & License

Просмотреть файл

@ -0,0 +1,70 @@
workspace:
clean: all
steps:
- checkout: self
clean: true
- task: UsePythonVersion@0
inputs:
versionSpec: '3.7.x'
architecture: 'x64'
- script: python -m pip install --upgrade pip
displayName: 'Install/Upgrade pip'
- script: pip uninstall -y mu_build
displayName: 'Remove existing version of self'
- script: pip install --upgrade -r requirements.txt
displayName: 'Install requirements'
- script: pip install -e .
displayName: 'Install from Source'
- script: pytest -v --junitxml=test.junit.xml --html=pytest_MuBuild_report.html --self-contained-html --cov=MuBuild --cov-report html:cov_html --cov-report xml:cov.xml --cov-config .coveragerc
displayName: 'Run UnitTests'
# Publish Test Results to Azure Pipelines/TFS
- task: PublishTestResults@2
displayName: 'Publish junit test results'
continueOnError: true
condition: succeededOrFailed()
inputs:
testResultsFormat: 'JUnit' # Options: JUnit, NUnit, VSTest, xUnit
testResultsFiles: 'test.junit.xml'
mergeTestResults: true # Optional
publishRunAttachments: true # Optional
# Publish Build Artifacts
# Publish build artifacts to Azure Pipelines/TFS or a file share
- task: PublishBuildArtifacts@1
inputs:
pathtoPublish: 'pytest_MuBuild_report.html'
artifactName: 'MuBuild unit test report'
continueOnError: true
condition: succeededOrFailed()
# Publish Code Coverage Results
# Publish Cobertura code coverage results
- task: PublishCodeCoverageResults@1
inputs:
codeCoverageTool: 'cobertura' # Options: cobertura, jaCoCo
summaryFileLocation: $(System.DefaultWorkingDirectory)/cov.xml
reportDirectory: $(System.DefaultWorkingDirectory)/cov_html
condition: succeededOrFailed()
- script: flake8 . > flake8.err.log
displayName: 'Run flake8'
condition: succeededOrFailed()
- task: PublishBuildArtifacts@1
inputs:
pathtoPublish: 'flake8.err.log'
artifactName: 'Flake8 Error log file'
continueOnError: true
condition: Failed()
#- script: python setup.py sdist bdist_wheel
# displayName: 'Build a wheel'

45
developing.md Normal file
Просмотреть файл

@ -0,0 +1,45 @@
# Developing Project Mu Pip Build
## Pre-Requisites
1. Get the code
``` cmd
git clone https://github.com/Microsoft/mu_pip_build.git
```
2. Install development dependencies
``` cmd
pip install --upgrade -r requirements.txt
```
3. Uninstall any copy of mu_build
``` cmd
pip uninstall mu_build
```
4. Install from local source (run command from root of repo)
``` cmd
pip install -e .
```
## Testing
1. Run a Basic Syntax/Lint Check (using flake8) and resolve any issues
``` cmd
flake8 MuBuild
```
2. Run pytest with coverage data collected
``` cmd
pytest -v --junitxml=test.junit.xml --html=pytest_MuBuild_report.html --self-contained-html --cov=MuBuild --cov-report html:cov_html --cov-report xml:cov.xml --cov-config .coveragerc
```
3. Look at the reports
* pytest_MuBuild_report.html
* cov_html/index.html

34
publishing.md Normal file
Просмотреть файл

@ -0,0 +1,34 @@
# Publishing Project Mu Pip Build
The MuBuild is published as a pypi (pip) module. The pip module is named __mu_build__. Pypi allows for easy version management, dependency management, and sharing.
Publishing/releasing a new version is generally handled thru a server based build process but for completeness the process is documented here.
## Steps
!!! Info
These directions assume you have already configured your workspace for developing. If not please first do that. Directions on the [developing](developing.md) page.
1. Install tools for publishing
``` cmd
pip install --upgrade -r requirements.publisher.txt
```
2. Build a wheel
``` cmd
python setup.py sdist bdist_wheel
```
3. Publish the wheel/distribution to pypi
``` cmd
twine upload dist/*
```
## Server Build
## Release Checklist

Просмотреть файл

@ -0,0 +1,3 @@
setuptools
wheel
twine

4
requirements.txt Normal file
Просмотреть файл

@ -0,0 +1,4 @@
pytest
pytest-html
pytest-cov
flake8

Просмотреть файл

@ -1,4 +1,4 @@
## @file setup.py
# @file setup.py
# This contains setup info for mu_build pip module
#
##
@ -40,12 +40,18 @@ setuptools.setup(
url="https://github.com/microsoft/mu_pip_build",
license='BSD2',
packages=setuptools.find_packages(),
entry_points = {
'console_scripts': ['mu_build=MuBuild.MuBuild:main'],
entry_points={
'console_scripts': ['mu_build=MuBuild.MuBuild:main']
},
install_requires=[
'pyyaml',
'mu_python_library>=0.2.0',
'mu_environment>=0.2.1'
'pyyaml',
'mu_python_library>=0.2.0',
'mu_environment>=0.2.1'
],
classifiers=[
"Programming Language :: Python :: 3",
"License :: OSI Approved :: BSD License",
"Operating System :: OS Independent",
"Development Status :: 3 - Alpha"
]
)
)

10
using.md Normal file
Просмотреть файл

@ -0,0 +1,10 @@
# Using Project Mu Pip Build
Install from pip
```cmd
pip install mu_build
```
## Usage Docs
__TBD__