added Azure Pipelines build system (#3794)

2019-05-18 16:05:31 -05:00 · 2019-05-18 16:05:31 -05:00 · a85e63e38c
--- a/.artifactignore
+++ b/.artifactignore
@ -0,0 +1,3 @@
+**/*
+!TestResults/*
+!bin/incrementalist.txt
--- a/build-system/README.md
+++ b/build-system/README.md
@ -0,0 +1,8 @@
+# Azure Pipelines Build Files
+These `.yaml` files are used by Windows Azure DevOps Pipelines to help execute the following types of builds:
+
+- Pull request validation on Linux (Mono / .NET Core)
+- Pull request validation on Windows (.NET Framework / .NET Core)
+- NuGet releases with automatic release notes posted to a Github Release repository.
+
+**NOTE**: you will need to change some of the pipeline variables inside the `windows-release.yaml` for your specific project and you will also want to create variable groups with your signing and NuGet push information.
--- a/build-system/azure-pipeline.template.yaml
+++ b/build-system/azure-pipeline.template.yaml
@ -0,0 +1,54 @@
+parameters:
+  name: ''
+  displayName: ''
+  vmImage: ''
+  dependsOn: 'WindowsBuild'
+  artifactName: 'akkaBuild'
+  scriptFileName: ''
+  scriptArgs: 'all'
+  outputDirectory: ''
+  timeoutInMinutes: 120
+
+jobs:
+  - job: ${{ parameters.name }}
+    displayName: ${{ parameters.displayName }}
+    timeoutInMinutes: ${{ parameters.timeoutInMinutes }}
+    pool:
+      vmImage: ${{ parameters.vmImage }}
+    steps:
+      - task: Bash@3 
+        displayName: Linux / OSX Build
+        inputs:
+          filePath: ${{ parameters.scriptFileName }}
+          arguments: ${{ parameters.scriptArgs }}
+        continueOnError: true
+        condition: in( variables['Agent.OS'], 'Linux', 'Darwin' )
+      # Windows
+      - task: BatchScript@1
+        displayName: Windows Build
+        inputs:
+          filename: ${{ parameters.scriptFileName }}
+          arguments: ${{ parameters.scriptArgs }}
+        continueOnError: true
+        condition: eq( variables['Agent.OS'], 'Windows_NT' )
+      - task: PublishTestResults@2
+        inputs:
+          testRunner: VSTest
+          testResultsFiles: '**/*.trx' #TestResults folder usually
+          testRunTitle: ${{ parameters.name }}
+          mergeTestResults: true
+      - script: 'echo 1>&2'
+        failOnStderr: true
+        displayName: 'If above is partially succeeded, then fail'
+        condition: eq(variables['Agent.JobStatus'], 'SucceededWithIssues')
+      - task: CopyFiles@2
+        displayName: 'Copy Build Output'
+        inputs:
+          sourceFolder: ${{ parameters.outputDirectory }}
+          contents: '*'
+          targetFolder: $(Build.ArtifactStagingDirectory)
+          continueOnError: boolean  # 'true' if future steps should run even if this step fails; defaults to 'false'
+      - task: PublishBuildArtifacts@1
+        inputs:
+          pathtoPublish: '$(Build.ArtifactStagingDirectory)'
+          artifactName: ${{ parameters.name }}
--- a/build-system/linux-pr-validation.yaml
+++ b/build-system/linux-pr-validation.yaml
@ -0,0 +1,22 @@
+# Pull request validation for Linux against the `dev` and `master` branches
+# See https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema for reference
+trigger:
+  branches:
+    include:
+      - dev
+      - master
+
+name: $(Year:yyyy).$(Month).$(DayOfMonth)$(Rev:.r)
+
+pr:
+  autoCancel: true # indicates whether additional pushes to a PR should cancel in-progress runs for the same PR. Defaults to true
+  branches:
+    include: [ dev, master ] # branch names which will trigger a build
+
+jobs:
+- template: azure-pipeline.template.yaml
+  parameters:
+    name: Ubuntu
+    vmImage: 'ubuntu-16.04'
+    scriptFileName: ./build.sh
+    scriptArgs: all
--- a/build-system/pr-validation.yaml
+++ b/build-system/pr-validation.yaml
@ -0,0 +1,103 @@
+# Pull request validation for Windows against the `dev` and `master` branches
+# See https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema for reference
+trigger:
+  branches:
+    include:
+      - dev
+      - test-dev
+      - master
+
+pr:
+  autoCancel: true # indicates whether additional pushes to a PR should cancel in-progress runs for the same PR. Defaults to true
+  branches:
+    include: [ dev, test-dev, master ] # branch names which will trigger a build
+
+name: $(Year:yyyy).$(Month).$(DayOfMonth)$(Rev:.r)
+
+jobs:
+  - job: WindowsBuild
+    displayName: Windows Build
+    pool:
+      vmImage: vs2017-win2016
+      demands: Cmd
+    steps:
+      - checkout: self  # self represents the repo where the initial Pipelines YAML file was found
+        clean: false  # whether to fetch clean each time
+        submodules: recursive  # set to 'true' for a single level of submodules or 'recursive' to get submodules of submodules
+        persistCredentials: true
+      - task: BatchScript@1
+        displayName: Windows Build
+        inputs:
+          filename: build.cmd
+          arguments: 'buildRelease incremental' # Run an incremental build
+        continueOnError: true
+        condition: eq( variables['Agent.OS'], 'Windows_NT' )
+      - task: CopyFiles@2
+        displayName: 'Copy Build Output'
+        inputs:
+          sourceFolder: bin
+          contents: '*'
+          targetFolder: $(Build.ArtifactStagingDirectory)
+          continueOnError: boolean  # 'true' if future steps should run even if this step fails; defaults to 'false'
+      - task: PublishBuildArtifacts@1
+        inputs:
+          pathtoPublish: $(Build.ArtifactStagingDirectory)
+          artifactName: incrementalistOutput
+      - script: 'echo 1>&2'
+        failOnStderr: true
+        displayName: 'If above is partially succeeded, then fail'
+        condition: eq(variables['Agent.JobStatus'], 'SucceededWithIssues')
+
+  - template: azure-pipeline.template.yaml
+    parameters:
+      name: 'netfx_tests_windows'
+      displayName: '.NET Framework Unit Tests (Windows)'
+      vmImage: 'vs2017-win2016'
+      scriptFileName: build.cmd
+      scriptArgs: runTests incremental
+      outputDirectory: 'TestResults'
+
+  - template: azure-pipeline.template.yaml
+    parameters:
+      name: 'net_core_tests_windows'
+      displayName: '.NET Core Unit Tests (Windows)'
+      vmImage: 'vs2017-win2016'
+      scriptFileName: build.cmd
+      scriptArgs: runTestsNetCore incremental
+      outputDirectory: 'TestResults'
+
+  - template: azure-pipeline.template.yaml
+    parameters:
+      name: 'net_core_tests_linux'
+      displayName: '.NET Core Unit Tests (Linux)'
+      vmImage: 'ubuntu-16.04'
+      scriptFileName: './build.sh'
+      scriptArgs: runTestsNetCore incremental
+      outputDirectory: 'TestResults'
+
+  - template: azure-pipeline.template.yaml
+    parameters:
+      name: 'net_core_mntr_windows'
+      displayName: '.NET Core Multi-Node Tests (Windows)'
+      vmImage: 'vs2017-win2016'
+      scriptFileName: 'build.cmd'
+      scriptArgs: MultiNodeTestsNetCore incremental
+      outputDirectory: 'TestResults'
+
+  - template: azure-pipeline.template.yaml
+    parameters:
+      name: 'net_fx_mntr_windows'
+      displayName: '.NET Framework Multi-Node Tests (Windows)'
+      vmImage: 'vs2017-win2016'
+      scriptFileName: 'build.cmd'
+      scriptArgs: MultiNodeTests incremental
+      outputDirectory: 'TestResults'
+
+  - template: azure-pipeline.template.yaml
+    parameters:
+      name: 'nuget_pack'
+      displayName: 'NuGet Pack'
+      vmImage: 'vs2017-win2016'
+      scriptFileName: build.cmd
+      scriptArgs: CreateNuget nugetprerelease=dev incremental
+      outputDirectory: 'bin/nuget'
--- a/build-system/windows-release.yaml
+++ b/build-system/windows-release.yaml
@ -0,0 +1,118 @@
+# Release task for PbLib projects
+# See https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema for reference
+trigger:
+  branches:
+    include:
+      - refs/tags/*
+
+variables:
+  - name: githubConnectionName
+    value: releases
+  - name: projectName
+    value: lighthouse 
+  - name: githubRepositoryName
+    value: petabridge/lighthouse 
+  - name: dockerConnectionName
+    value: PetabridgeDocker
+
+jobs:
+  - job: publishNetCore
+    pool:
+      vmImage: windows-2019
+      demands: Cmd
+    steps:
+    - task: BatchScript@1
+      displayName: 'dotnet publish'
+      inputs:
+        filename: build.cmd
+        arguments: 'PublishCode'
+    - task: CopyFiles@2
+      displayName: 'Copy dotnet publish Output'
+      inputs:
+        sourceFolder: src/Lighthouse/bin/Release/netcoreapp2.1/publish/
+        targetFolder: $(Build.ArtifactStagingDirectory)/bin/Release/netcoreapp2.1/publish/
+    - task: CopyFiles@2
+      displayName: 'Copy Dockerfiles'
+      inputs:
+        sourceFolder: src/Lighthouse
+        contents: Dockerfile-*
+        targetFolder: $(Build.ArtifactStagingDirectory)
+    - task: PublishBuildArtifacts@1
+      inputs:
+        pathtoPublish: '$(Build.ArtifactStagingDirectory)'
+        artifactName: drop
+    - task: GitHubRelease@0
+      displayName: 'GitHub release (create)'
+      inputs:
+        gitHubConnection: $(githubConnectionName)
+        repositoryName: $(githubRepositoryName)
+        title: '$(projectName) v$(Build.SourceBranchName)'
+        releaseNotesFile: 'RELEASE_NOTES.md'
+        assets: |
+         bin\nuget\*.nupkg
+
+  - job:  linuxImageDeploy
+    pool:
+      vmImage: ubuntu-16.04
+    dependsOn: publishNetCore
+    steps:
+      - task: DownloadBuildArtifacts@0
+        inputs:
+          buildType: 'current'
+          downloadType: 'single'
+          artifactName: 'drop'
+          itemPattern: drop/**
+          downloadPath: '$(Agent.BuildDirectory)'
+      - task: Docker@2
+        displayName: "Login to Docker Hub"
+        inputs:
+          command: login
+          containerRegistry: $(dockerConnectionName)
+      - task: Docker@2
+        displayName: Docker Build (Linux)
+        inputs:
+          command: buildAndPush
+          Dockerfile: $(Agent.BuildDirectory)/drop/Dockerfile-linux
+          repository: petabridge/lighthouse
+          tags: |
+            latest
+            linux-latest
+            $(Build.SourceBranchName)
+            $(Build.SourceBranchName)-linux
+      - task: Docker@2
+        displayName: Logout of Docker Hub
+        inputs:
+          command: logout
+          containerRegistry: $(dockerConnectionName)
+
+  - job:  windowsImageDeploy
+    pool:
+      vmImage: windows-2019
+    dependsOn: publishNetCore
+    steps:
+      - task: DownloadBuildArtifacts@0
+        inputs:
+          buildType: 'current'
+          downloadType: 'single'
+          artifactName: 'drop'
+          itemPattern: drop/**
+          downloadPath: '$(Agent.BuildDirectory)'
+      - task: Docker@2
+        displayName: "Login to Docker Hub"
+        inputs:
+          command: login
+          containerRegistry: $(dockerConnectionName)
+      - task: Docker@2
+        displayName: Docker Build (Windows)
+        inputs:
+          command: buildAndPush
+          Dockerfile: $(Agent.BuildDirectory)/drop/Dockerfile-windows
+          repository: petabridge/lighthouse
+          tags: |
+            windows-latest
+            $(Build.SourceBranchName)-nanoserver
+      - task: Docker@2
+        displayName: Logout of Docker Hub
+        inputs:
+          command: logout
+          containerRegistry: $(dockerConnectionName)
--- a/build.fsx
+++ b/build.fsx
@ -68,7 +68,7 @@ Target "Clean" (fun _ ->
 // Incrementalist targets 
 //--------------------------------------------------------------------------------
 // Pulls the set of all affected projects detected by Incrementalist from the cached file
-let getAffectedProjects = 
+let getAffectedProjectsTopology = 
    lazy(
        log (sprintf "Checking inside %s for changes" incrementalistReport)

@ -77,12 +77,22 @@ let getAffectedProjects =
        log (sprintf "Found changes via Incrementalist? %b - searched inside %s" incrementalistFoundChanges incrementalistReport)
        if not incrementalistFoundChanges then None
        else
-            Some ((File.ReadAllText incrementalistReport).Split ',')
+            let sortedItems = (File.ReadAllLines incrementalistReport) |> Seq.map (fun x -> (x.Split ','))
+                              |> Seq.map (fun items -> (items.[0], items))
+            let d = dict sortedItems
+            Some(d)
+    )
+
+let getAffectedProjects = 
+    lazy(
+        let finalProjects = getAffectedProjectsTopology.Value
+        match finalProjects with
+        | None -> None
+        | Some p -> Some (p.Values |> Seq.concat)
    )

 Target "ComputeIncrementalChanges" (fun _ ->
    if runIncrementally then
-        log (sprintf "(Debug) .NET Core Root found at %s" (Environment.GetEnvironmentVariable "DOTNET_ROOT"))
        let targetBranch = match getBuildParam "targetBranch" with
                            | "" -> "dev"
                            | null -> "dev"
@ -145,6 +155,14 @@ let skipBuild =
        | _ -> false
    )

+let headProjects =
+    lazy(
+        match getAffectedProjectsTopology.Value with
+        | None when runIncrementally -> [||]
+        | None -> [|solution|]
+        | Some p -> p.Keys |> Seq.toArray
+    )
+
 Target "AssemblyInfo" (fun _ ->
    XmlPokeInnerText "./src/common.props" "//Project/PropertyGroup/VersionPrefix" releaseNotes.AssemblyVersion    
    XmlPokeInnerText "./src/common.props" "//Project/PropertyGroup/PackageReleaseNotes" (releaseNotes.Notes |> String.concat "\n")
@ -153,13 +171,15 @@ Target "AssemblyInfo" (fun _ ->
 Target "Build" (fun _ ->   
    if not skipBuild.Value then
        let additionalArgs = if versionSuffix.Length > 0 then [sprintf "/p:VersionSuffix=%s" versionSuffix] else []  
+        let buildProject proj =
+            DotNetCli.Build
+                (fun p -> 
+                    { p with
+                        Project = proj
+                        Configuration = configuration
+                        AdditionalArgs = additionalArgs })

-        DotNetCli.Build
-            (fun p -> 
-                { p with
-                    Project = solution
-                    Configuration = configuration
-                    AdditionalArgs = additionalArgs })
+        getAffectedProjects.Value.Value|> Seq.iter buildProject
 )

 //--------------------------------------------------------------------------------
@ -215,7 +235,7 @@ Target "RunTests" (fun _ ->
            info.WorkingDirectory <- (Directory.GetParent project).FullName
            info.Arguments <- arguments) (TimeSpan.FromMinutes 30.0) 
        
-        ResultHandling.failBuildIfXUnitReportedError TestRunnerErrorLevel.DontFailBuild result
+        ResultHandling.failBuildIfXUnitReportedError TestRunnerErrorLevel.Error result

    CreateDir outputTests
    projects |> Seq.iter (runSingleProject)
@ -240,7 +260,7 @@ Target "RunTestsNetCore" (fun _ ->
                info.WorkingDirectory <- (Directory.GetParent project).FullName
                info.Arguments <- arguments) (TimeSpan.FromMinutes 30.0) 
        
-            ResultHandling.failBuildIfXUnitReportedError TestRunnerErrorLevel.DontFailBuild result
+            ResultHandling.failBuildIfXUnitReportedError TestRunnerErrorLevel.Error result

        CreateDir outputTests
        projects |> Seq.iter (runSingleProject)
@ -378,60 +398,64 @@ Target "NBench" <| fun _ ->
 //--------------------------------------------------------------------------------

 Target "CreateNuget" (fun _ ->    
-    let projects = !! "src/**/*.*sproj"
-                   -- "src/**/*.Tests*.*sproj"
-                   -- "src/benchmark/**/*.*sproj"
-                   -- "src/examples/**/*.*sproj"
-                   -- "src/**/*.MultiNodeTestRunner.csproj"
-                   -- "src/**/*.MultiNodeTestRunner.Shared.csproj"
-                   -- "src/**/*.NodeTestRunner.csproj"
+    if not skipBuild.Value then
+        let projects = 
+            let rawProjects = !! "src/**/*.*sproj"
+                            -- "src/**/*.Tests*.*sproj"
+                            -- "src/benchmark/**/*.*sproj"
+                            -- "src/examples/**/*.*sproj"
+                            -- "src/**/*.MultiNodeTestRunner.csproj"
+                            -- "src/**/*.MultiNodeTestRunner.Shared.csproj"
+                            -- "src/**/*.NodeTestRunner.csproj"
+            rawProjects |> Seq.choose filterProjects

-    let runSingleProject project =
-        DotNetCli.Pack
-            (fun p -> 
-                { p with
-                    Project = project
-                    Configuration = configuration
-                    AdditionalArgs = ["--include-symbols"]
-                    VersionSuffix = versionSuffix
-                    OutputPath = outputNuGet })
+        let runSingleProject project =
+            DotNetCli.Pack
+                (fun p -> 
+                    { p with
+                        Project = project
+                        Configuration = configuration
+                        AdditionalArgs = ["--include-symbols --no-build"]
+                        VersionSuffix = versionSuffix
+                        OutputPath = outputNuGet })

-    projects |> Seq.iter (runSingleProject)
+        projects |> Seq.iter (runSingleProject)
 )
-open Fake.TemplateHelper
+
 Target "PublishMntr" (fun _ ->
-    let executableProjects = !! "./src/**/Akka.MultiNodeTestRunner.csproj"
+    if not skipBuild.Value then
+        let executableProjects = !! "./src/**/Akka.MultiNodeTestRunner.csproj"

-    // Windows .NET 4.5.2
-    executableProjects |> Seq.iter (fun project ->
-        DotNetCli.Restore
-            (fun p -> 
-                { p with
-                    Project = project                  
-                    AdditionalArgs = ["-r win7-x64"; sprintf "/p:VersionSuffix=%s" versionSuffix] })
-    )
+        // Windows .NET 4.5.2
+        executableProjects |> Seq.iter (fun project ->
+            DotNetCli.Restore
+                (fun p -> 
+                    { p with
+                        Project = project                  
+                        AdditionalArgs = ["-r win7-x64"; sprintf "/p:VersionSuffix=%s" versionSuffix] })
+        )

-    // Windows .NET 4.5.2
-    executableProjects |> Seq.iter (fun project ->  
-        DotNetCli.Publish
-            (fun p ->
-                { p with
-                    Project = project
-                    Configuration = configuration
-                    Runtime = "win7-x64"
-                    Framework = testNetFrameworkVersion
-                    VersionSuffix = versionSuffix }))
+        // Windows .NET 4.5.2
+        executableProjects |> Seq.iter (fun project ->  
+            DotNetCli.Publish
+                (fun p ->
+                    { p with
+                        Project = project
+                        Configuration = configuration
+                        Runtime = "win7-x64"
+                        Framework = testNetFrameworkVersion
+                        VersionSuffix = versionSuffix }))

-    // Windows .NET Core
-    executableProjects |> Seq.iter (fun project ->  
-        DotNetCli.Publish
-            (fun p ->
-                { p with
-                    Project = project
-                    Configuration = configuration
-                    Runtime = "win7-x64"
-                    Framework = testNetCoreVersion
-                    VersionSuffix = versionSuffix }))
+        // Windows .NET Core
+        executableProjects |> Seq.iter (fun project ->  
+            DotNetCli.Publish
+                (fun p ->
+                    { p with
+                        Project = project
+                        Configuration = configuration
+                        Runtime = "win7-x64"
+                        Framework = testNetCoreVersion
+                        VersionSuffix = versionSuffix }))
 )

 Target "CreateMntrNuget" (fun _ -> 
@ -637,13 +661,16 @@ Target "RunTestsFull" DoNothing
 Target "RunTestsNetCoreFull" DoNothing

 // build dependencies
-"Clean" ==> "AssemblyInfo" ==> "Build" ==> "PublishMntr" ==> "BuildRelease"
+"Clean" ==> "AssemblyInfo" ==> "Build"
+"Build" ==> "PublishMntr" ==> "BuildRelease"
 "ComputeIncrementalChanges" ==> "Build" // compute incremental changes

 // tests dependencies
-// "RunTests" and "RunTestsNetCore" don't use clean / build so they can be run multiple times, successively, without rebuilding
-"Build" ==> "RunTests" ==> "RunTestsFull"
-"Build" ==> "RunTestsNetCore" ==> "RunTestsNetCoreFull"
+"Build" ==> "RunTests"
+"Build" ==> "RunTestsNetCore"
+
+"BuildRelease" ==> "MultiNodeTestsNetCore"
+"BuildRelease" ==> "MultiNodeTests"

 // nuget dependencies
 "BuildRelease" ==> "CreateMntrNuget" ==> "CreateNuget" ==> "PublishNuget" ==> "Nuget"
@ -656,6 +683,7 @@ Target "RunTestsNetCoreFull" DoNothing
 "RunTests" ==> "All"
 "RunTestsNetCore" ==> "All"
 "MultiNodeTests" ==> "All"
+"MultiNodeTestsNetCore" ==> "All"
 "NBench" ==> "All"

 RunTargetOrDefault "Help"
--- a/build.ps1
+++ b/build.ps1
@ -39,7 +39,7 @@ $NugetUrl = "https://dist.nuget.org/win-x86-commandline/v$NugetVersion/nuget.exe
 $ProtobufVersion = "3.4.0"
 $DocfxVersion = "2.40.5"

-$IncrementalistVersion = "0.1.3";
+$IncrementalistVersion = "0.1.4";

 # Make sure tools folder exists
 $PSScriptRoot = Split-Path $MyInvocation.MyCommand.Path -Parent
@ -90,8 +90,6 @@ if($FoundDotNetCliVersion -ne $DotNetVersion) {
    $env:PATH = "$InstallPath;$env:PATH"
    $env:DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1
    $env:DOTNET_CLI_TELEMETRY_OPTOUT=1
-    $env:DOTNET_ROOT="$InstallPath" #workaround for https://github.com/dotnet/cli/issues/9114
-    Write-Host "$env:DOTNET_ROOT"
 }

 ###########################################################################
--- a/build.sh
+++ b/build.sh
@ -17,7 +17,7 @@ DOTNET_VERSION=2.1.500
 DOTNET_INSTALLER_URL=https://raw.githubusercontent.com/dotnet/cli/v$DOTNET_VERSION/scripts/obtain/dotnet-install.sh
 DOTNET_CHANNEL=LTS
 PROTOBUF_VERSION=3.4.0
-INCREMENTALIST_VERSION=0.1.3
+INCREMENTALIST_VERSION=0.1.4

 # Define default arguments.
 TARGET="Default"
@ -57,7 +57,6 @@ bash "$SCRIPT_DIR/.dotnet/dotnet-install.sh" --version $DOTNET_VERSION --channel
 export PATH="$SCRIPT_DIR/.dotnet":$PATH
 export DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1
 export DOTNET_CLI_TELEMETRY_OPTOUT=1
-export DOTNET_ROOT="$SCRIPT_DIR/.dotnet"
 chmod -R 0755 ".dotnet"
 "$SCRIPT_DIR/.dotnet/dotnet" --info