From e12a8e1a37f60cb42701245d4c1015ca8e712887 Mon Sep 17 00:00:00 2001 From: Sean Brogan Date: Fri, 30 Nov 2018 23:25:18 +0000 Subject: [PATCH] Merged PR 528: Add CI, linting, and unit test support --- .coveragerc | 2 + .flake8 | 5 + .gitignore | 8 +- MuBuild/ConfigValidator.py | 163 +++++++++++---------- MuBuild/MuBuild.py | 143 +++++++++--------- MuBuild/MuLogging.py | 29 ++-- MuBuild/tests/test_ConfigValidator.py | 199 ++++++++++++++++++++++++++ README.rst | 7 +- RepoDetails.md | 14 +- azure-pipelines-pr-gate.yml | 70 +++++++++ developing.md | 45 ++++++ publishing.md | 34 +++++ requirements.publisher.txt | 3 + requirements.txt | 4 + setup.py | 20 ++- using.md | 10 ++ 16 files changed, 573 insertions(+), 183 deletions(-) create mode 100644 .coveragerc create mode 100644 .flake8 create mode 100644 MuBuild/tests/test_ConfigValidator.py create mode 100644 azure-pipelines-pr-gate.yml create mode 100644 developing.md create mode 100644 publishing.md create mode 100644 requirements.publisher.txt create mode 100644 requirements.txt create mode 100644 using.md diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 0000000..16bb903 --- /dev/null +++ b/.coveragerc @@ -0,0 +1,2 @@ +[run] +omit = tests/* \ No newline at end of file diff --git a/.flake8 b/.flake8 new file mode 100644 index 0000000..f16ca21 --- /dev/null +++ b/.flake8 @@ -0,0 +1,5 @@ +[flake8] +#E501 line too long +#E266 too many leading '#' for block comment +#E722 do not use bare 'except' +ignore = E501,E266,E722 \ No newline at end of file diff --git a/.gitignore b/.gitignore index 3b8b8ea..bc43486 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,10 @@ Lib dist *.egg-info -build. \ No newline at end of file +build. +/cov_html +/.pytest_cache +/pytest_MuBuild_report.html +/.coverage +/cov.xml +/test.junit.xml \ No newline at end of file diff --git a/MuBuild/ConfigValidator.py b/MuBuild/ConfigValidator.py index 5575660..a943568 100644 --- a/MuBuild/ConfigValidator.py +++ b/MuBuild/ConfigValidator.py @@ -1,8 +1,8 @@ -## @file ConfigValidator.py +# @file ConfigValidator.py # This module contains support for validating .mu.json config files # # Used to support CI/CD and exporting test results for other tools. -# This does test report generation without being a test runner. +# This does test report generation without being a test runner. ## # Copyright (c) 2018, Microsoft Corporation # @@ -26,10 +26,8 @@ # OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF # ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ## -import logging import os import urllib.request as req -import urllib.parse as p ''' Example_MU_CONFIG_FILE: @@ -65,101 +63,100 @@ Example_MU_CONFIG_FILE: ''' -#Checks the top level MU Config -def check_mu_confg(config,edk2path,pluginList): +# Checks the top level MU Config +def check_mu_confg(config, edk2path, pluginList): workspace = edk2path.WorkspacePath + def _mu_error(message): raise Exception("Mu Config Error: {0}".format(message)) - def _is_valid_dir(path,name): - path = os.path.join(workspace,path) + def _is_valid_dir(path, name): + path = os.path.join(workspace, path) if not os.path.isdir(path): _mu_error("{0} isn't a valid directory".format(path)) def _check_url(url): request = req.Request(url) try: - response = req.urlopen(request) + req.urlopen(request) return True except: - #The url wasn't valid + # The url wasn't valid return False - - def _check_packages(packages,name): + + def _check_packages(packages, name): for package in packages: path = edk2path.GetAbsolutePathOnThisSytemFromEdk2RelativePath(package) if path is None or not os.path.isdir(path): _mu_error("{0} isn't a valid package to build".format(package)) return True - def _is_valid_arch(targets,name): - valid_targets = ["AARCH64","IA32","X64"] + def _is_valid_arch(targets, name): + valid_targets = ["AARCH64", "IA32", "X64"] for target in targets: - if not target in valid_targets: + if target not in valid_targets: _mu_error("{0} is not a valid target".format(target)) - def _check_dependencies(dependencies,name): - valid_attributes = ["Path","Url","Branch","Commit"] + def _check_dependencies(dependencies, name): + valid_attributes = ["Path", "Url", "Branch", "Commit"] for dependency in dependencies: # check to make sure we have a path - if not "Path" in dependency: + if "Path" not in dependency: _mu_error("Path not found in dependency {0}".format(dependency)) # check to sure we have a valid url and we can reach it - if not "Url" in dependency: + if "Url" not in dependency: _mu_error("Url not found in dependency {0}".format(dependency)) if not _check_url(dependency["Url"]): - _mu_error("Invalid URL {0}".format(dependency["Url"])) + _mu_error("Invalid URL {0}".format(dependency["Url"])) # make sure we have a valid branch or commit - if not "Branch" in dependency and not "Commit" in dependency: + if "Branch" not in dependency and "Commit" not in dependency: _mu_error("You must have a commit or a branch dependency {0}".format(dependency)) - if "Branch" in dependency and "Commit" in dependency: + if "Branch" in dependency and "Commit" in dependency: _mu_error("You cannot have both a commit or a branch dependency {0}".format(dependency)) - #check to make sure we don't have something else in there + # check to make sure we don't have something else in there for attribute in dependency: - if not attribute in valid_attributes: + if attribute not in valid_attributes: _mu_error("Unknown attribute {0} in dependecy".format(attribute)) - + return True - - config_rules = { "required": { "Name": { - "type":"str" - }, + "type": "str" + }, "GroupName": { - "type":"str" - }, + "type": "str" + }, "Scopes": { - "type":"list", - "items":"str" + "type": "list", + "items": "str" }, "ArchSupported": { - "type":"list", - "validator":_is_valid_arch + "type": "list", + "validator": _is_valid_arch }, "RelativeWorkspaceRoot": { - "type":"str", - "validator":_is_valid_dir + "type": "str", + "validator": _is_valid_dir }, "Targets": { - "type":"list" + "type": "list" } }, "optional": { - "Packages":{ - "type":"list", - "items":"str", - "validator":_check_packages + "Packages": { + "type": "list", + "items": "str", + "validator": _check_packages }, "PackagesPath": { - "type":"list", - "items":"str" + "type": "list", + "items": "str" }, "Dependencies": { - "type":"list", - "validator":_check_dependencies + "type": "list", + "validator": _check_dependencies } } } @@ -170,19 +167,19 @@ def check_mu_confg(config,edk2path,pluginList): if "config_name" in plugin.descriptor: plugin_name = plugin.descriptor["config_name"] config_rules["optional"][plugin_name] = { - "validator" : plugin.Obj.ValidateConfig + "validator": plugin.Obj.ValidateConfig } - #check if all the requires are satisified + # check if all the requires are satisified for rule in config_rules["required"]: - if not rule in config: + if rule not in config: _mu_error("{0} is a required attribute in your MU Config".format(rule)) - + if "type" in config_rules["required"][rule]: config_type = str(type(config[rule]).__name__) wanted_type = config_rules["required"][rule]["type"] if config_type != wanted_type: - _mu_error("{0} is a required attribute and is not the correct type. We are expecting a {1} and got a {2}".format(rule,config_type,wanted_type)) + _mu_error("{0} is a required attribute and is not the correct type. We are expecting a {1} and got a {2}".format(rule, config_type, wanted_type)) if "validator" in config_rules["required"][rule]: validator = config_rules["required"][rule]["validator"] @@ -190,33 +187,34 @@ def check_mu_confg(config,edk2path,pluginList): # check optional types for rule in config_rules["optional"]: - if not rule in config: - continue - + if rule not in config: + continue + if "type" in config_rules["optional"][rule]: config_type = str(type(config[rule]).__name__) wanted_type = config_rules["optional"][rule]["type"] if config_type != wanted_type: - _mu_error("{0} is a optional attribute and is not the correct type. We are expecting a {1} and got a {2}".format(rule,config_type,wanted_type)) + _mu_error("{0} is a optional attribute and is not the correct type. We are expecting a {1} and got a {2}".format(rule, config_type, wanted_type)) if "validator" in config_rules["optional"][rule]: validator = config_rules["optional"][rule]["validator"] - validator(config[rule],"Base mu.json") - - #check to make sure we don't have any stray keys in there + validator(config[rule], "Base mu.json") + + # check to make sure we don't have any stray keys in there for rule in config: - if not rule in config_rules["optional"] and not rule in config_rules["required"]: + if rule not in config_rules["optional"] and rule not in config_rules["required"]: _mu_error("Unknown parameter {0} is unexpected".format(rule)) - + return True + ''' { "Defines": { "PLATFORM_NAME": "MdeModule", "DSC_SPECIFICATION": "0x00010005", "SUPPORTED_ARCHITECTURES": "IA32|X64|ARM|AARCH64", - "BUILD_TARGETS": "DEBUG|RELEASE" + "BUILD_TARGETS": "DEBUG|RELEASE" }, "CompilerPlugin": { "skip":false, @@ -227,7 +225,7 @@ def check_mu_confg(config,edk2path,pluginList): "MdePkg/MdePkg.dec", "MdeModulePkg/MdeModulePkg.dec", "MsUnitTestPkg/MsUnitTestPkg.dec" - ], + ], "IgnoreInf": { }, @@ -238,17 +236,19 @@ def check_mu_confg(config,edk2path,pluginList): ## # Checks the package configuration for errors ## -def check_package_confg(name,config,pluginList): + + +def check_package_confg(name, config, pluginList): def _mu_error(message): - raise Exception("Package {0} Config Error: {1}".format(name,message)) + raise Exception("Package {0} Config Error: {1}".format(name, message)) config_rules = { "required": { }, - "optional": { + "optional": { "Defines": { - "type":"dict", - "items":"str" + "type": "dict", + "items": "str" } } } @@ -260,41 +260,40 @@ def check_package_confg(name,config,pluginList): plugin_name = plugin.descriptor["config_name"] # add the validator config_rules["optional"][plugin_name] = { - "validator" : plugin.Obj.ValidateConfig + "validator": plugin.Obj.ValidateConfig } - #check if all the requires are satisified + # check if all the requires are satisified for rule in config_rules["required"]: - if not rule in config: + if rule not in config: _mu_error("{0} is a required attribute in your MU Config".format(rule)) - + if "type" in config_rules["required"][rule]: config_type = str(type(config[rule]).__name__) wanted_type = config_rules["required"][rule]["type"] if config_type != wanted_type: - _mu_error("{0} is a required attribute and is not the correct type. We are expecting a {1} and got a {2}".format(rule,config_type,wanted_type)) + _mu_error("{0} is a required attribute and is not the correct type. We are expecting a {1} and got a {2}".format(rule, config_type, wanted_type)) if "validator" in config_rules["required"][rule]: validator = config_rules["required"][rule]["validator"] - validator(config[rule],name) + validator(config[rule], name) # check optional types for rule in config_rules["optional"]: - if not rule in config: - continue - + if rule not in config: + continue + if "type" in config_rules["optional"][rule]: config_type = str(type(config[rule]).__name__) wanted_type = config_rules["optional"][rule]["type"] if config_type != wanted_type: - _mu_error("{0} is a optional attribute and is not the correct type. We are expecting a {1} and got a {2}".format(rule,config_type,wanted_type)) + _mu_error("{0} is a optional attribute and is not the correct type. We are expecting a {1} and got a {2}".format(rule, config_type, wanted_type)) if "validator" in config_rules["optional"][rule]: validator = config_rules["optional"][rule]["validator"] - validator(config[rule],name) - - #check to make sure we don't have any stray keys in there + validator(config[rule], name) + + # check to make sure we don't have any stray keys in there for rule in config: - if not rule in config_rules["optional"] and not rule in config_rules["required"]: + if rule not in config_rules["optional"] and rule not in config_rules["required"]: _mu_error("Unknown parameter {0} is unexpected".format(rule)) - \ No newline at end of file diff --git a/MuBuild/MuBuild.py b/MuBuild/MuBuild.py index edfdbad..6572167 100644 --- a/MuBuild/MuBuild.py +++ b/MuBuild/MuBuild.py @@ -1,4 +1,4 @@ -## @file MuBuild.py +# @file MuBuild.py # This module contains code that supports Project Mu CI/CD # This is the main entry for the build and test process # of Non-Product builds @@ -47,28 +47,29 @@ import pkg_resources def get_mu_config(): parser = argparse.ArgumentParser(description='Run the Mu Build') - parser.add_argument ('-c', '--mu_config', dest = 'mu_config', required = True, type=str, help ='Provide the Mu config relative to the current working directory') - parser.add_argument ('-p', '--pkg','--pkg-dir', dest='pkglist', nargs="+", type=str, help = 'A package or folder you want to test (abs path or cwd relative). Can list multiple by doing -p ', default=[]) - parser.add_argument('-ignore','--ignore-git', dest="git_ignore",action="store_true",help="Whether to ignore errors in the git cloing process", default=False) - parser.add_argument('-force','--force-git', dest="git_force",action="store_true",help="Whether to force git repos to clone in the git cloing process", default=False) - parser.add_argument('-update-git','--update-git', dest="git_update",action="store_true",help="Whether to update git repos as needed in the git cloing process", default=False) - args, sys.argv = parser.parse_known_args() + parser.add_argument('-c', '--mu_config', dest='mu_config', required=True, type=str, help='Provide the Mu config relative to the current working directory') + parser.add_argument('-p', '--pkg', '--pkg-dir', dest='pkglist', nargs="+", type=str, help='A package or folder you want to test (abs path or cwd relative). Can list multiple by doing -p ', default=[]) + parser.add_argument('-ignore', '--ignore-git', dest="git_ignore", action="store_true", help="Whether to ignore errors in the git cloing process", default=False) + parser.add_argument('-force', '--force-git', dest="git_force", action="store_true", help="Whether to force git repos to clone in the git cloing process", default=False) + parser.add_argument('-update-git', '--update-git', dest="git_update", action="store_true", help="Whether to update git repos as needed in the git cloing process", default=False) + args, sys.argv = parser.parse_known_args() return args -def merge_config(mu_config,pkg_config,descriptor={}): + +def merge_config(mu_config, pkg_config, descriptor={}): plugin_name = "" config = dict() if "module" in descriptor: plugin_name = descriptor["module"] if "config_name" in descriptor: plugin_name = descriptor["config_name"] - + if plugin_name == "": return config if plugin_name in mu_config: config.update(mu_config[plugin_name]) - + if plugin_name in pkg_config: config.update(pkg_config[plugin_name]) @@ -77,49 +78,49 @@ def merge_config(mu_config,pkg_config,descriptor={}): # # Main driver of Project Mu Builds # + + def main(): - #Parse command line arguments + # Parse command line arguments PROJECT_SCOPES = ("project_mu",) buildArgs = get_mu_config() mu_config_filepath = os.path.abspath(buildArgs.mu_config) - + if mu_config_filepath is None or not os.path.isfile(mu_config_filepath): raise Exception("Invalid path to mu.json file for build: ", mu_config_filepath) - - #have a build config file + + # have a build config file with open(mu_config_filepath, 'r') as mu_config_file: mu_config = yaml.safe_load(mu_config_file) WORKSPACE_PATH = os.path.realpath(os.path.join(os.path.dirname(mu_config_filepath), mu_config["RelativeWorkspaceRoot"])) - #Setup the logging to the file as well as the console + # Setup the logging to the file as well as the console MuLogging.clean_build_logs(WORKSPACE_PATH) MuLogging.setup_logging(WORKSPACE_PATH) - #Get scopes from config file + # Get scopes from config file if "Scopes" in mu_config: PROJECT_SCOPES += tuple(mu_config["Scopes"]) # SET PACKAGE PATH - # + # # Get Package Path from config file pplist = list() if(mu_config["RelativeWorkspaceRoot"] != ""): - #this package is not at workspace root. + # this package is not at workspace root. # Add self pplist.append(os.path.dirname(mu_config_filepath)) - - #Include packages from the config file + + # Include packages from the config file if "PackagesPath" in mu_config: for a in mu_config["PackagesPath"]: pplist.append(a) - #Check Dependencies for Repo + # Check Dependencies for Repo if "Dependencies" in mu_config: - pplist.extend(RepoResolver.resolve_all(WORKSPACE_PATH,mu_config["Dependencies"], ignore=buildArgs.git_ignore, force=buildArgs.git_force, update_ok=buildArgs.git_update)) + pplist.extend(RepoResolver.resolve_all(WORKSPACE_PATH, mu_config["Dependencies"], ignore=buildArgs.git_ignore, force=buildArgs.git_force, update_ok=buildArgs.git_update)) - - - #make Edk2Path object to handle all path operations + # make Edk2Path object to handle all path operations edk2path = Edk2Path(WORKSPACE_PATH, pplist) logging.info("Running ProjectMu Build: {0}".format(mu_config["Name"])) @@ -129,7 +130,7 @@ def main(): logging.info("mu_python_library version: " + pkg_resources.get_distribution("mu_python_library").version) logging.info("mu_environment version: " + pkg_resources.get_distribution("mu_environment").version) - #which package to build + # which package to build packageList = mu_config["Packages"] # # If mu pk list supplied lets see if they are a file system path @@ -137,10 +138,10 @@ def main(): # # if(len(buildArgs.pkglist) > 0): - packageList = [] #clear it + packageList = [] # clear it for mu_pk_path in buildArgs.pkglist: - #if abs path lets convert + # if abs path lets convert if os.path.isabs(mu_pk_path): temp = edk2path.GetEdk2RelativePathFromAbsolutePath(mu_pk_path) if(temp is not None): @@ -148,8 +149,8 @@ def main(): else: logging.critical("pkg-dir invalid absolute path: {0}".format(mu_pk_path)) raise Exception("Invalid Package Path") - else: - #Check if relative path + else: + # Check if relative path temp = os.path.join(os.getcwd(), mu_pk_path) temp = edk2path.GetEdk2RelativePathFromAbsolutePath(temp) if(temp is not None): @@ -157,25 +158,23 @@ def main(): else: logging.critical("pkg-dir invalid relative path: {0}".format(mu_pk_path)) raise Exception("Invalid Package Path") - + # Bring up the common minimum environment. (build_env, shell_env) = SelfDescribingEnvironment.BootstrapEnvironment(edk2path.WorkspacePath, PROJECT_SCOPES) CommonBuildEntry.update_process(edk2path.WorkspacePath, PROJECT_SCOPES) env = ShellEnvironment.GetBuildVars() - archSupported = " ".join(mu_config["ArchSupported"]) env.SetValue("TARGET_ARCH", archSupported, "Platform Hardcoded") - - - #Generate consumable XML object- junit format + + # Generate consumable XML object- junit format JunitReport = MuJunitReport() - #Keep track of failures + # Keep track of failures failure_num = 0 total_num = 0 - #Load plugins + # Load plugins pluginManager = PluginManager.PluginManager() failedPlugins = pluginManager.SetListOfEnvironmentDescriptors(build_env.plugins) if failedPlugins: @@ -185,20 +184,20 @@ def main(): raise Exception("One or more plugins failed to load.") helper = PluginManager.HelperFunctions() - if( helper.LoadFromPluginManager(pluginManager) > 0): + if(helper.LoadFromPluginManager(pluginManager) > 0): raise Exception("One or more helper plugins failed to load.") - + pluginList = pluginManager.GetPluginsOfClass(PluginManager.IMuBuildPlugin) - + # Check to make sure our configuration is valid - ConfigValidator.check_mu_confg(mu_config,edk2path,pluginList) + ConfigValidator.check_mu_confg(mu_config, edk2path, pluginList) for pkgToRunOn in packageList: # # run all loaded MuBuild Plugins/Tests # - ts = JunitReport.create_new_testsuite(pkgToRunOn, "MuBuild.{0}.{1}".format( mu_config["GroupName"], pkgToRunOn) ) - _, loghandle = MuLogging.setup_logging(WORKSPACE_PATH,"BUILDLOG_{0}.txt".format(pkgToRunOn)) + ts = JunitReport.create_new_testsuite(pkgToRunOn, "MuBuild.{0}.{1}".format(mu_config["GroupName"], pkgToRunOn)) + _, loghandle = MuLogging.setup_logging(WORKSPACE_PATH, "BUILDLOG_{0}.txt".format(pkgToRunOn)) logging.info("Package Running: {0}".format(pkgToRunOn)) ShellEnvironment.CheckpointBuildVars() env = ShellEnvironment.GetBuildVars() @@ -212,43 +211,42 @@ def main(): logging.info("No Pkg Config file for {0}".format(pkgToRunOn)) pkg_config = dict() - #check the resulting configuration - ConfigValidator.check_package_confg(pkgToRunOn,pkg_config,pluginList) + # check the resulting configuration + ConfigValidator.check_package_confg(pkgToRunOn, pkg_config, pluginList) for Descriptor in pluginList: - #Get our targets + # Get our targets targets = ["DEBUG"] if Descriptor.Obj.IsTargetDependent() and "Targets" in mu_config: targets = mu_config["Targets"] - - + for target in targets: - logging.critical("---Running {2}: {0} {1}".format(Descriptor.Name,target,pkgToRunOn)) - total_num +=1 + logging.critical("---Running {2}: {0} {1}".format(Descriptor.Name, target, pkgToRunOn)) + total_num += 1 ShellEnvironment.CheckpointBuildVars() env = ShellEnvironment.GetBuildVars() - + env.SetValue("TARGET", target, "MuBuild.py before RunBuildPlugin") (testcasename, testclassname) = Descriptor.Obj.GetTestName(pkgToRunOn, env) tc = ts.create_new_testcase(testcasename, testclassname) - #merge the repo level and package level for this specific plugin - pkg_plugin_configuration = merge_config(mu_config,pkg_config,Descriptor.descriptor) + # merge the repo level and package level for this specific plugin + pkg_plugin_configuration = merge_config(mu_config, pkg_config, Descriptor.descriptor) - #perhaps we should ask the validator to run on the + # perhaps we should ask the validator to run on the - #Check if need to skip this particular plugin + # Check if need to skip this particular plugin if "skip" in pkg_plugin_configuration and pkg_plugin_configuration["skip"]: tc.SetSkipped() logging.critical(" ->Test Skipped! %s" % Descriptor.Name) else: try: - # - package is the edk2 path to package. This means workspace/packagepath relative. + # - package is the edk2 path to package. This means workspace/packagepath relative. # - edk2path object configured with workspace and packages path # - any additional command line args # - RepoConfig Object (dict) for the build # - PkgConfig Object (dict) - # - EnvConfig Object + # - EnvConfig Object # - Plugin Manager Instance # - Plugin Helper Obj Instance # - testcase Object used for outputing junit results @@ -256,11 +254,10 @@ def main(): except Exception as exp: exc_type, exc_value, exc_traceback = sys.exc_info() logging.critical("EXCEPTION: {0}".format(exp)) - exceptionPrint = traceback.format_exception(type(exp), exp,exc_traceback) + exceptionPrint = traceback.format_exception(type(exp), exp, exc_traceback) logging.critical(" ".join(exceptionPrint)) tc.SetError("Exception: {0}".format(exp), "UNEXPECTED EXCEPTION") rc = 1 - if(rc != 0): failure_num += 1 @@ -269,28 +266,28 @@ def main(): else: logging.error("Test Failed: %s returned %d" % (Descriptor.Name, rc)) else: - logging.info("Test Success {0} {1}".format(Descriptor.Name,target)) - - #revert to the checkpoint we created previously - ShellEnvironment.RevertBuildVars() - #finished target loop - #Finished plugin loop - - MuLogging.stop_logging(loghandle) #stop the logging for this particularbuild file - ShellEnvironment.RevertBuildVars() - #Finished buildable file loop + logging.info("Test Success {0} {1}".format(Descriptor.Name, target)) + # revert to the checkpoint we created previously + ShellEnvironment.RevertBuildVars() + # finished target loop + # Finished plugin loop + + MuLogging.stop_logging(loghandle) # stop the logging for this particularbuild file + ShellEnvironment.RevertBuildVars() + # Finished buildable file loop JunitReport.Output(os.path.join(WORKSPACE_PATH, "Build", "BuildLogs", "TestSuites.xml")) - #Print Overall Success + # Print Overall Success if(failure_num != 0): logging.critical("Overall Build Status: Error") - logging.critical("There were {0} failures out of {1} attempts".format(failure_num,total_num)) + logging.critical("There were {0} failures out of {1} attempts".format(failure_num, total_num)) else: logging.critical("Overall Build Status: Success") sys.exit(failure_num) - + + if __name__ == '__main__': - main() \ No newline at end of file + main() diff --git a/MuBuild/MuLogging.py b/MuBuild/MuLogging.py index d1b34dc..89368ff 100644 --- a/MuBuild/MuLogging.py +++ b/MuBuild/MuLogging.py @@ -1,6 +1,6 @@ -## @file MuLogging.py +# @file MuLogging.py # Handle basic logging config for Project Mu Builds -# MuBuild splits logs into a master log and per package. +# MuBuild splits logs into a master log and per package. # ## # Copyright (c) 2018, Microsoft Corporation @@ -28,53 +28,54 @@ import logging import sys from datetime import datetime -from datetime import date import os import shutil + def clean_build_logs(ws): - # Make sure that we have a clean environment. + # Make sure that we have a clean environment. if os.path.isdir(os.path.join(ws, "Build", "BuildLogs")): shutil.rmtree(os.path.join(ws, "Build", "BuildLogs")) -def setup_logging(workspace, filename=None, loghandle = None): + +def setup_logging(workspace, filename=None, loghandle=None): if loghandle is not None: stop_logging(loghandle) logging_level = logging.DEBUG - + if filename is None: filename = "BUILDLOG_MASTER.txt" logging_level = logging.DEBUG - - #setup logger + + # setup logger logger = logging.getLogger('') logger.setLevel(logging.DEBUG) if len(logger.handlers) == 0: - #Create the main console as logger + # Create the main console as logger formatter = logging.Formatter("%(levelname)s- %(message)s") console = logging.StreamHandler() console.setLevel(logging.DEBUG) console.setFormatter(formatter) logger.addHandler(console) - logfile = os.path.join(workspace, "Build", "BuildLogs", filename) if(not os.path.isdir(os.path.dirname(logfile))): os.makedirs(os.path.dirname(logfile)) - - #Create master file logger + + # Create master file logger fileformatter = logging.Formatter("%(levelname)s - %(message)s") filelogger = logging.FileHandler(filename=(logfile), mode='w') filelogger.setLevel(logging_level) filelogger.setFormatter(fileformatter) logger.addHandler(filelogger) - logging.info("Log Started: " + datetime.strftime(datetime.now(), "%A, %B %d, %Y %I:%M%p" )) + logging.info("Log Started: " + datetime.strftime(datetime.now(), "%A, %B %d, %Y %I:%M%p")) logging.info("Running Python version: " + str(sys.version_info)) - return logfile,filelogger + return logfile, filelogger + def stop_logging(loghandle): loghandle.close() diff --git a/MuBuild/tests/test_ConfigValidator.py b/MuBuild/tests/test_ConfigValidator.py new file mode 100644 index 0000000..5e2b93f --- /dev/null +++ b/MuBuild/tests/test_ConfigValidator.py @@ -0,0 +1,199 @@ +import tempfile +import logging +import os +import unittest +from MuBuild import ConfigValidator +import yaml +try: + from StringIO import StringIO +except ImportError: + from io import StringIO + +test_dir = None +plugin_list = [] + + +class Edk2Path_Injected(object): + def __init__(self): + self.WorkspacePath = None + + def GetAbsolutePathOnThisSytemFromEdk2RelativePath(self, package): + return os.path.abspath(self.WorkspacePath) + + +class PluginList_Injected(object): + def __init__(self, name): + self.descriptor = dict() + self.Obj = None + self.Name = name + + +class TestConfigValidator(unittest.TestCase): + + @classmethod + def setUpClass(cls): + global test_dir + global valid_config + global plugin_list + + logger = logging.getLogger('') + logger.addHandler(logging.NullHandler()) + unittest.installHandler() + # get a temporary directory that we can create the right folders + test_dir = Edk2Path_Injected() + test_dir.WorkspacePath = tempfile.mkdtemp() + + def test_valid_config(self): + global test_dir + global pluginList + yaml_string = StringIO(""" + { + "Name": "Project Mu Plus Repo CI Build", + "GroupName": "MuPlus", + + # Workspace path relative to this file + "RelativeWorkspaceRoot": "", + "Scopes": [ "corebuild" ], + + # Other Repos that are dependencies + "Dependencies": [ + # FileSystem Path relative to workspace + # Url + # Branch + # Commit + { + "Path": "MU_BASECORE", + "Url": "https://github.com/Microsoft/mu_basecore.git", + "Branch": "release/201808" + }, + ], + + # Edk2 style PackagesPath for resolving dependencies. + # Only needed if it isn't this package and isn't a dependency + "PackagesPath": [], + + # Packages in this repo + "Packages": [ + "UefiTestingPkg" + ], + "ArchSupported": [ + "IA32", + "X64", + "AARCH64" + ], + "Targets": [ + "DEBUG", + "RELEASE" + ] + } + """) + + valid_config = yaml.safe_load(yaml_string) + # make sure the valid configuration is read just fine + try: + ConfigValidator.check_mu_confg(valid_config, test_dir, plugin_list) + except Exception as e: + self.fail("We shouldn't throw an exception", e) + + def test_invalid_configs(self): + global test_dir + global pluginList + bad_yaml_string = StringIO(""" + { + "Name": "Project Mu Plus Repo CI Build", + "GroupName": "MuPlus", + + # Workspace path relative to this file + "RelativeWorkspaceRoot": "", + "InvalidAttribute": "this will throw an error", + "Scopes": [ "corebuild" ], + + # Other Repos that are dependencies + "Dependencies": [ + # FileSystem Path relative to workspace + # Url + # Branch + # Commit + { + "Path": "MU_BASECORE", + "Url": "https://github.com/Microsoft/mu_basecore.git", + "Branch": "release/201808" + }, + ], + + # Edk2 style PackagesPath for resolving dependencies. + # Only needed if it isn't this package and isn't a dependency + "PackagesPath": [], + + # Packages in this repo + "Packages": [ + "UefiTestingPkg" + ], + "ArchSupported": [ + "IA32", + "X64", + "AARCH64" + ], + "Targets": [ + "DEBUG", + "RELEASE" + ] + } + """) + invalid_config = yaml.safe_load(bad_yaml_string) + with self.assertRaises(Exception): + ConfigValidator.check_mu_confg(invalid_config, test_dir, plugin_list) + + def test_invalid_url_config(self): + global test_dir + global pluginList + + bad_url_yaml_string = StringIO(""" + { + "Name": "Project Mu Plus Repo CI Build", + "GroupName": "MuPlus", + + # Workspace path relative to this file + "RelativeWorkspaceRoot": "", + "Scopes": [ "corebuild" ], + + # Other Repos that are dependencies + "Dependencies": [ + # FileSystem Path relative to workspace + # Url + # Branch + # Commit + { + "Path": "MU_BASECORE", + "Url": "https://github.com/InvalidRepo", + "Branch": "release/201808" + }, + ], + + # Edk2 style PackagesPath for resolving dependencies. + # Only needed if it isn't this package and isn't a dependency + "PackagesPath": [], + + # Packages in this repo + "Packages": [ + "UefiTestingPkg" + ], + "ArchSupported": [ + "IA32", + "X64", + "AARCH64" + ], + "Targets": [ + "DEBUG", + "RELEASE" + ] + } + """) + + invalid_config = yaml.safe_load(bad_url_yaml_string) + with self.assertRaises(Exception): + ConfigValidator.check_mu_confg(invalid_config, test_dir, plugin_list) + + +if __name__ == '__main__': + unittest.main() diff --git a/README.rst b/README.rst index 8f77911..1543133 100644 --- a/README.rst +++ b/README.rst @@ -1,9 +1,10 @@ -========= + +======== MU Build -========= +======== About -============== +===== Provided with config file, mu_build fetches/clones dependencies then compiles every module in each package. This is the entrypoint into the CI / Pull Request build and test infrastructure. diff --git a/RepoDetails.md b/RepoDetails.md index 2df4eba..233d599 100644 --- a/RepoDetails.md +++ b/RepoDetails.md @@ -24,11 +24,19 @@ Please open any issues in the Project Mu GitHub tracker. [More Details](https:// ## Contributing Code or Docs Please follow the general Project Mu Pull Request process. [More Details](https://microsoft.github.io/mu/How/contributing/) +Additionally make sure all testing described in the "Development" section passes. -## Installing +## Using -Install from pip with `pip install mu_build` -Install from source with `pip install -e .` +[Usage Details](using.md) + +## Development + +[Development Details](developing.md) + +## Publish + +[Publish Details](publishing.md) ## Copyright & License diff --git a/azure-pipelines-pr-gate.yml b/azure-pipelines-pr-gate.yml new file mode 100644 index 0000000..8bb63fb --- /dev/null +++ b/azure-pipelines-pr-gate.yml @@ -0,0 +1,70 @@ +workspace: + clean: all + +steps: +- checkout: self + clean: true + +- task: UsePythonVersion@0 + inputs: + versionSpec: '3.7.x' + architecture: 'x64' + +- script: python -m pip install --upgrade pip + displayName: 'Install/Upgrade pip' + +- script: pip uninstall -y mu_build + displayName: 'Remove existing version of self' + +- script: pip install --upgrade -r requirements.txt + displayName: 'Install requirements' + +- script: pip install -e . + displayName: 'Install from Source' + +- script: pytest -v --junitxml=test.junit.xml --html=pytest_MuBuild_report.html --self-contained-html --cov=MuBuild --cov-report html:cov_html --cov-report xml:cov.xml --cov-config .coveragerc + displayName: 'Run UnitTests' + +# Publish Test Results to Azure Pipelines/TFS +- task: PublishTestResults@2 + displayName: 'Publish junit test results' + continueOnError: true + condition: succeededOrFailed() + inputs: + testResultsFormat: 'JUnit' # Options: JUnit, NUnit, VSTest, xUnit + testResultsFiles: 'test.junit.xml' + mergeTestResults: true # Optional + publishRunAttachments: true # Optional + +# Publish Build Artifacts +# Publish build artifacts to Azure Pipelines/TFS or a file share +- task: PublishBuildArtifacts@1 + inputs: + pathtoPublish: 'pytest_MuBuild_report.html' + artifactName: 'MuBuild unit test report' + continueOnError: true + condition: succeededOrFailed() + +# Publish Code Coverage Results +# Publish Cobertura code coverage results +- task: PublishCodeCoverageResults@1 + inputs: + codeCoverageTool: 'cobertura' # Options: cobertura, jaCoCo + summaryFileLocation: $(System.DefaultWorkingDirectory)/cov.xml + reportDirectory: $(System.DefaultWorkingDirectory)/cov_html + condition: succeededOrFailed() + +- script: flake8 . > flake8.err.log + displayName: 'Run flake8' + condition: succeededOrFailed() + +- task: PublishBuildArtifacts@1 + inputs: + pathtoPublish: 'flake8.err.log' + artifactName: 'Flake8 Error log file' + continueOnError: true + condition: Failed() + + +#- script: python setup.py sdist bdist_wheel +# displayName: 'Build a wheel' \ No newline at end of file diff --git a/developing.md b/developing.md new file mode 100644 index 0000000..41033c1 --- /dev/null +++ b/developing.md @@ -0,0 +1,45 @@ +# Developing Project Mu Pip Build + +## Pre-Requisites + +1. Get the code + +``` cmd +git clone https://github.com/Microsoft/mu_pip_build.git +``` + +2. Install development dependencies + +``` cmd +pip install --upgrade -r requirements.txt +``` + +3. Uninstall any copy of mu_build + +``` cmd +pip uninstall mu_build +``` + +4. Install from local source (run command from root of repo) + +``` cmd +pip install -e . +``` + +## Testing + +1. Run a Basic Syntax/Lint Check (using flake8) and resolve any issues + +``` cmd +flake8 MuBuild +``` + +2. Run pytest with coverage data collected + +``` cmd +pytest -v --junitxml=test.junit.xml --html=pytest_MuBuild_report.html --self-contained-html --cov=MuBuild --cov-report html:cov_html --cov-report xml:cov.xml --cov-config .coveragerc +``` + +3. Look at the reports + * pytest_MuBuild_report.html + * cov_html/index.html diff --git a/publishing.md b/publishing.md new file mode 100644 index 0000000..05a4f8f --- /dev/null +++ b/publishing.md @@ -0,0 +1,34 @@ +# Publishing Project Mu Pip Build + +The MuBuild is published as a pypi (pip) module. The pip module is named __mu_build__. Pypi allows for easy version management, dependency management, and sharing. + +Publishing/releasing a new version is generally handled thru a server based build process but for completeness the process is documented here. + +## Steps + +!!! Info + These directions assume you have already configured your workspace for developing. If not please first do that. Directions on the [developing](developing.md) page. + +1. Install tools for publishing + +``` cmd +pip install --upgrade -r requirements.publisher.txt +``` + +2. Build a wheel + +``` cmd +python setup.py sdist bdist_wheel +``` + +3. Publish the wheel/distribution to pypi + +``` cmd +twine upload dist/* +``` + +## Server Build + +## Release Checklist + + diff --git a/requirements.publisher.txt b/requirements.publisher.txt new file mode 100644 index 0000000..8f3da39 --- /dev/null +++ b/requirements.publisher.txt @@ -0,0 +1,3 @@ +setuptools +wheel +twine diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..f33faf0 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,4 @@ +pytest +pytest-html +pytest-cov +flake8 diff --git a/setup.py b/setup.py index 4decae6..3a201c2 100644 --- a/setup.py +++ b/setup.py @@ -1,4 +1,4 @@ -## @file setup.py +# @file setup.py # This contains setup info for mu_build pip module # ## @@ -40,12 +40,18 @@ setuptools.setup( url="https://github.com/microsoft/mu_pip_build", license='BSD2', packages=setuptools.find_packages(), - entry_points = { - 'console_scripts': ['mu_build=MuBuild.MuBuild:main'], + entry_points={ + 'console_scripts': ['mu_build=MuBuild.MuBuild:main'] }, install_requires=[ - 'pyyaml', - 'mu_python_library>=0.2.0', - 'mu_environment>=0.2.1' + 'pyyaml', + 'mu_python_library>=0.2.0', + 'mu_environment>=0.2.1' + ], + classifiers=[ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: BSD License", + "Operating System :: OS Independent", + "Development Status :: 3 - Alpha" ] -) \ No newline at end of file +) diff --git a/using.md b/using.md new file mode 100644 index 0000000..5b8e2bf --- /dev/null +++ b/using.md @@ -0,0 +1,10 @@ +# Using Project Mu Pip Build + +Install from pip + +```cmd +pip install mu_build +``` +## Usage Docs + +__TBD__