From a93e56de11fb885251db8644c8a517af8428edf0 Mon Sep 17 00:00:00 2001 From: Andres Paz Date: Wed, 22 Jul 2020 13:21:54 -0700 Subject: [PATCH] Separate CI build and validation scripts for different areas (#410) * Separate building projects versions of the katas, Jupyter Notebooks validation and building projects that create, validate and pack binaries into separate CI jobs. * Include DumpMachine unit tests to the CI. --- azure-pipelines.yml | 67 ++++++++++++++++------- scripts/build.ps1 | 14 ++--- scripts/manifest.ps1 | 16 ++++++ scripts/{steps.yml => steps-binaries.yml} | 25 +-------- scripts/steps-init.yml | 23 ++++++++ scripts/test.ps1 | 34 +++++++++--- scripts/validate-projects.ps1 | 27 +++++++++ 7 files changed, 147 insertions(+), 59 deletions(-) create mode 100644 scripts/manifest.ps1 rename scripts/{steps.yml => steps-binaries.yml} (54%) create mode 100644 scripts/steps-init.yml create mode 100644 scripts/validate-projects.ps1 diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 63768f6b..cc64e64e 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -1,7 +1,3 @@ -# .NET Desktop -# Build and run tests for .NET Desktop or Windows classic desktop solutions. -# Add steps that publish symbols, save build artifacts, and more: -# https://docs.microsoft.com/azure/devops/pipelines/apps/windows/dot-net name: $(Build.Major).$(Build.Minor).$(BuildId) trigger: - master @@ -15,19 +11,52 @@ variables: IQSharp.Hosting.Env: 'build-agent-katas' jobs: -- job: Windows - pool: - vmImage: 'windows-2019' - steps: - - template: scripts/steps.yml -- job: macOS - pool: - vmImage: 'macOS-latest' - steps: - - template: scripts/steps.yml -- job: Linux - pool: - vmImage: 'ubuntu-latest' - steps: - - template: scripts/steps.yml + - job: binaries + displayName: 'Build Binary Packages' + steps: + - template: scripts/steps-init.yml + - template: scripts/steps-binaries.yml + + - job: validate_projects + displayName: 'Validate C# Projects' + strategy: + matrix: + linux: + imageName: 'ubuntu-latest' + mac: + imageName: 'macOS-latest' + windows: + imageName: 'windows-latest' + pool: + vmImage: $(imageName) + steps: + - template: scripts/steps-init.yml + + - powershell: ./validate-unicode.ps1 + displayName: "Validating unicode characters" + workingDirectory: $(System.DefaultWorkingDirectory)/scripts + + - powershell: ./validate-projects.ps1 + displayName: "Validating C# projects" + workingDirectory: $(System.DefaultWorkingDirectory)/scripts + + - job: validate_notebooks + displayName: 'Validate Notebooks' + strategy: + matrix: + linux: + imageName: 'ubuntu-latest' + mac: + imageName: 'macOS-latest' + windows: + imageName: 'windows-latest' + pool: + vmImage: $(imageName) + steps: + - template: scripts/steps-init.yml + + - powershell: ./validate-notebooks.ps1 + displayName: "Validating notebooks" + workingDirectory: $(System.DefaultWorkingDirectory)/scripts + diff --git a/scripts/build.ps1 b/scripts/build.ps1 index b49b255d..d4cc21d2 100644 --- a/scripts/build.ps1 +++ b/scripts/build.ps1 @@ -22,17 +22,11 @@ function Build-One { $script:all_ok = ($LastExitCode -eq 0) -and $script:all_ok } -# Microsoft.Quantum.Katas.sln should always be built: +Write-Host "Building Katas binaries:" + +Build-One '..\utilities\DumpUnitary\DumpUnitary.sln' Build-One '..\utilities\Microsoft.Quantum.Katas\Microsoft.Quantum.Katas.sln' -# Building all katas projects can be disabled with the ENABLE_KATAS -if ($Env:ENABLE_KATAS -ne "false") { - Get-ChildItem (Join-Path $PSScriptRoot '..') -Recurse -Include '*.sln' -Exclude 'Microsoft.Quantum.Katas.sln' ` - | ForEach-Object { Build-One $_.FullName } -} else { - Write-Host "##vso[task.logissue type=warning;]Skipping building Katas solutions. Env:ENABLE_KATAS is '$Env:ENABLE_KATAS'." -} - if (-not $all_ok) { - throw "At least one test failed execution. Check the logs." + throw "At least one project failed building. Check the logs." } \ No newline at end of file diff --git a/scripts/manifest.ps1 b/scripts/manifest.ps1 new file mode 100644 index 00000000..68a94911 --- /dev/null +++ b/scripts/manifest.ps1 @@ -0,0 +1,16 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +#!/usr/bin/env pwsh + +& "$PSScriptRoot/set-env.ps1" + +@{ + Packages = @( + "Microsoft.Quantum.Katas" + ); + Assemblies = @( + "..\utilities\Common\bin\$Env:BUILD_CONFIGURATION\netstandard2.1\Microsoft.Quantum.Katas.Common.dll", + "..\utilities\Microsoft.Quantum.Katas\bin\$Env:BUILD_CONFIGURATION\netstandard2.1\Microsoft.Quantum.Katas.dll" + ) | ForEach-Object { Get-Item (Join-Path $PSScriptRoot $_) }; +} | Write-Output; diff --git a/scripts/steps.yml b/scripts/steps-binaries.yml similarity index 54% rename from scripts/steps.yml rename to scripts/steps-binaries.yml index 9baff9ee..7804dd81 100644 --- a/scripts/steps.yml +++ b/scripts/steps-binaries.yml @@ -1,27 +1,5 @@ steps: -## -# Pre-reqs -## -- task: UseDotNet@2 - displayName: 'Use .NET Core SDK 3.1.201' - inputs: - packageType: sdk - version: '3.1.201' - -- task: UsePythonVersion@0 - inputs: - versionSpec: '3.6' - architecture: 'x64' - displayName: 'Use Python 3.6' - -- script: pip install setuptools wheel pytest jupyter - displayName: 'Install Python tools' - -- powershell: ./install-iqsharp.ps1 - displayName: "Installing IQ#" - workingDirectory: $(System.DefaultWorkingDirectory)/scripts - ## # Build, test & pack ## @@ -38,3 +16,6 @@ steps: displayName: "Packing" workingDirectory: $(System.DefaultWorkingDirectory)/scripts +- powershell: ./manifest.ps1 + displayName: "List built assemblies" + workingDirectory: $(System.DefaultWorkingDirectory)/scripts diff --git a/scripts/steps-init.yml b/scripts/steps-init.yml new file mode 100644 index 00000000..2cd5e70d --- /dev/null +++ b/scripts/steps-init.yml @@ -0,0 +1,23 @@ +steps: + +## +# Pre-reqs +## +- task: UseDotNet@2 + displayName: 'Use .NET Core SDK 3.1.201' + inputs: + packageType: sdk + version: '3.1.201' + +- task: UsePythonVersion@0 + inputs: + versionSpec: '3.6' + architecture: 'x64' + displayName: 'Use Python 3.6' + +- script: pip install setuptools wheel pytest jupyter + displayName: 'Install Python tools' + +- powershell: ./install-iqsharp.ps1 + displayName: "Installing IQ#" + workingDirectory: $(System.DefaultWorkingDirectory)/scripts diff --git a/scripts/test.ps1 b/scripts/test.ps1 index 466024d5..ca02780e 100644 --- a/scripts/test.ps1 +++ b/scripts/test.ps1 @@ -1,15 +1,33 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. -$ErrorActionPreference = 'Stop' - & "$PSScriptRoot/set-env.ps1" +$all_ok = $True -& "$PSScriptRoot/validate-unicode.ps1" +function Test-One { + Param($project) -# Validating all katas projects can be disables with the ENABLE_KATAS -if ($Env:ENABLE_KATAS -ne "false") { - & "$PSScriptRoot/validate-notebooks.ps1" -} else { - Write-Host "##vso[task.logissue type=warning;]Skipping testing Katas notebooks. Env:ENABLE_KATAS is '$Env:ENABLE_KATAS'." + Write-Host "##[info]Testing $project" + dotnet test $project ` + -c $Env:BUILD_CONFIGURATION ` + -v $Env:BUILD_VERBOSITY ` + --no-build ` + --logger trx ` + /property:DefineConstants=$Env:ASSEMBLY_CONSTANTS ` + /property:InformationalVersion=$Env:SEMVER_VERSION ` + /property:Version=$Env:ASSEMBLY_VERSION + + if ($LastExitCode -ne 0) { + Write-Host "##vso[task.logissue type=error;]Failed to test $project" + $script:all_ok = $False + } } + +Write-Host "Testing Katas binaries:" + +Test-One '..\utilities\DumpUnitary\DumpUnitary.sln' +Test-One '..\utilities\Microsoft.Quantum.Katas\Microsoft.Quantum.Katas.sln' + +if (-not $all_ok) { + throw "At least one test failed execution. Check the logs." +} \ No newline at end of file diff --git a/scripts/validate-projects.ps1 b/scripts/validate-projects.ps1 new file mode 100644 index 00000000..8dc426b3 --- /dev/null +++ b/scripts/validate-projects.ps1 @@ -0,0 +1,27 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +$ErrorActionPreference = 'Stop' + +& "$PSScriptRoot/set-env.ps1" + + +function Build-One { + param( + $project + ); + + Write-Host "##[info]Building $project..." + dotnet build $project ` + -c $Env:BUILD_CONFIGURATION ` + -v $Env:BUILD_VERBOSITY ` + /property:DefineConstants=$Env:ASSEMBLY_CONSTANTS ` + /property:Version=$Env:ASSEMBLY_VERSION ` + /property:QsharpDocsOutDir=$Env:DOCS_OUTDIR + + $script:all_ok = ($LastExitCode -eq 0) -and $script:all_ok +} + +# Build all Katas solutions: +Get-ChildItem (Join-Path $PSScriptRoot '..') -Recurse -Include '*.sln' -Exclude 'Microsoft.Quantum.Katas.sln' ` +| ForEach-Object { Build-One $_.FullName }