Convert build pipeline to 1ES (#1061)

- Convert build pipeline to 1ES
- Remove old pipeline
- Changes to build.ps1 for new pipeline

azure-pipelines.yml

@@ -1,127 +0,0 @@
-# Starter pipeline
-# Start with a minimal pipeline that you can customize to build and deploy your code.
-# Add steps that build, run tests, deploy, and more:
-# https://aka.ms/yaml
-
-#trigger:
-#- master
-#- dev
-
-strategy:
-  matrix:
-    linux:
-      imageName: 'MMSUbuntu20.04TLS'
-    windows:
-      imageName: 'MMS2019TLS'
-
-pool:
-    name: '1ES-Hosted-AzFunc'
-    demands:
-      - ImageOverride -equals $(imageName)
-
-variables:
-  Configuration: Release
-  buildNumber: $[ counter('build', 001) ] # Start higher than our AppVeyor versions. Every build (pr or branch) will increment.
-
-steps:
-- task: AzureKeyVault@2
-  inputs:
-    azureSubscription: 'Simple Batch(0b894477-1614-4c8d-8a9b-a697a24596b8)'
-    KeyVaultName: 'powershell-worker'
-    SecretsFilter: '*'
-    RunAsPreJob: true
-
-- pwsh: |
-    $releaseBranches = @('v4.x/ps7.4', 'v4.x/ps7.2', 'v4.x/ps7.0', 'v3.x/ps7', 'v3.x/ps6', 'v2.x')
-
-    Write-Host "BuildSourceBranch: $($env:BuildSourceBranch)"
-    $branchName = $env:BuildSourceBranch.Replace("refs/heads/", "")
-    Write-Host "BranchName: $branchName"
-
-    $isReleaseBuild = ($releaseBranches -contains $branchName)
-    Write-Host "##vso[task.setvariable variable=IsReleaseBuild]$isReleaseBuild"
-    Write-Host "IsReleaseBuild: $isReleaseBuild"
-  displayName: 'Set IsReleaseBuild variable'
-  env:
-    BuildSourceBranch: $(Build.SourceBranch)
-  condition: eq(variables['UPLOADPACKAGETOPRERELEASEFEED'], 'false')
-
-- pwsh: ./build.ps1 -NoBuild -Bootstrap
-  displayName: 'Running ./build.ps1 -NoBuild -Bootstrap'
-
-- pwsh: ./Check-CsprojVulnerabilities.ps1
-  displayName: 'Check for security vulnerabilities'
-
-- pwsh: |
-      $ErrorActionPreference = "Stop"
-
-      $shouldAddSBOM = $null
-      if ([string]::IsNullOrEmpty($IsReleaseBuild))
-      {
-        Write-Host "IsReleaseBuild is null or empty. Setting shouldAddSBOM to false"
-        $shouldAddSBOM = $false
-      }
-      else
-      {
-        Write-Host "IsReleaseBuild: $IsReleaseBuild"
-        $shouldAddSBOM = ($IsReleaseBuild -eq "true")
-      }
-
-      Write-Host "shouldAddSBOM: $shouldAddSBOM"
-
-      ./build.ps1 -Clean -Configuration Release -BuildNumber "$(buildNumber)" -AddSBOM:$shouldAddSBOM -SBOMUtilSASUrl "$(SBOMUtilSASUrl)"
-  displayName: 'Build worker code'
-
-- pwsh: ./build.ps1 -NoBuild -Test
-  displayName: 'Running UnitTest'
-
-- pwsh: ./test/E2E/Start-E2ETest.ps1
-  env:
-    AzureWebJobsStorage: $(AzureWebJobsStorage)
-    AzureWebJobsCosmosDBConnectionString: $(AzureWebJobsCosmosDBConnectionString)
-    AzureWebJobsServiceBus: $(AzureWebJobsServiceBus)
-    AzureWebJobsEventHubSender: $(AzureWebJobsEventHubSender)
-    FUNCTIONS_WORKER_RUNTIME : "powershell"
-  displayName: 'Running E2ETest'
-
-- task: PublishTestResults@2
-  inputs:
-    testResultsFormat: 'VSTest'
-    testResultsFiles: '**/*.trx'
-    failTaskOnFailedTests: true
-  condition: succeededOrFailed()
-  displayName: 'Publish tests results'
-
-- task: CopyFiles@2
-  inputs:
-    SourceFolder: '$(System.DefaultWorkingDirectory)/package'
-    Contents: '**/*.nupkg'
-    TargetFolder: '$(Build.ArtifactStagingDirectory)'
-  displayName: 'Copy package to artifacts directory'
-
-- task: NuGetCommand@2
-  condition: and(ne(variables['Build.Reason'], 'PullRequest'), eq(variables['IsReleaseBuild'], 'true'), eq(variables['UPLOADPACKAGETOPRERELEASEFEED'], 'false'))
-  inputs:
-    command: 'push'
-    packagesToPush: '$(Build.ArtifactStagingDirectory)/**/*.nupkg;!$(Build.ArtifactStagingDirectory)/**/*.symbols.nupkg'
-    nuGetFeedType: 'internal'
-    publishVstsFeed: 'e6a70c92-4128-439f-8012-382fe78d6396/c0493cce-bc63-4e11-9fc9-e7c45291f151'
-    allowPackageConflicts: true
-  displayName: 'Push NuGet package'
-
-- task: NuGetCommand@2
-  condition: eq(variables['UPLOADPACKAGETOPRERELEASEFEED'], 'true')
-  inputs:
-    command: 'push'
-    packagesToPush: '$(Build.ArtifactStagingDirectory)/**/*.nupkg;!$(Build.ArtifactStagingDirectory)/**/*.symbols.nupkg'
-    nuGetFeedType: 'internal'
-    publishVstsFeed: 'e6a70c92-4128-439f-8012-382fe78d6396/f37f760c-aebd-443e-9714-ce725cd427df'
-    allowPackageConflicts: true
-  displayName: 'Push NuGet package to the AzureFunctionsPreRelease feed'
-
-- task: PublishBuildArtifacts@1
-  inputs:
-    PathtoPublish: '$(Build.ArtifactStagingDirectory)'
-    ArtifactName: 'drop'
-    publishLocation: 'Container'
-  displayName: 'Publish build artifacts'

build.ps1

@@ -28,13 +28,7 @@ param(
     $Configuration = "Debug",
 
     [string]
-    $BuildNumber = '0',
+    $BuildNumber = '0'
-
-    [switch]
-    $AddSBOM,
-
-    [string]
-    $SBOMUtilSASUrl
 )
 
 #Requires -Version 7.0
@@ -73,35 +67,6 @@ function Get-FunctionsCoreToolsDir {
     }
 }
 
-function Install-SBOMUtil
-{
-    if ([string]::IsNullOrEmpty($SBOMUtilSASUrl))
-    {
-        throw "The `$SBOMUtilSASUrl parameter cannot be null or empty when specifying the `$AddSBOM switch"
-    }
-
-    $MANIFESTOOLNAME = "ManifestTool"
-    Write-Log "Installing $MANIFESTOOLNAME..."
-
-    $MANIFESTOOL_DIRECTORY = Join-Path $PSScriptRoot $MANIFESTOOLNAME
-    Remove-Item -Recurse -Force $MANIFESTOOL_DIRECTORY -ErrorAction Ignore
-
-    Invoke-RestMethod -Uri $SBOMUtilSASUrl -OutFile "$MANIFESTOOL_DIRECTORY.zip"
-    Expand-Archive "$MANIFESTOOL_DIRECTORY.zip" -DestinationPath $MANIFESTOOL_DIRECTORY
-
-    $dllName = "Microsoft.ManifestTool.dll"
-    $manifestToolPath = "$MANIFESTOOL_DIRECTORY/$dllName"
-
-    if (-not (Test-Path $manifestToolPath))
-    {
-        throw "$MANIFESTOOL_DIRECTORY does not contain '$dllName'"
-    }
-
-    Write-Log 'Done.'
-
-    return $manifestToolPath
-}
-
 function Deploy-PowerShellWorker {
     $ErrorActionPreference = 'Stop'
 
@@ -179,28 +144,6 @@ if (!$NoBuild.IsPresent) {
 
     dotnet publish -c $Configuration "/p:BuildNumber=$BuildNumber" $PSScriptRoot
 
-    if ($AddSBOM)
-    {
-        # Install manifest tool
-        $manifestTool = Install-SBOMUtil
-        Write-Log "manifestTool: $manifestTool "
-
-        # Generate manifest
-        $buildPath = "$PSScriptRoot/src/bin/$Configuration/$TargetFramework/publish"
-        $telemetryFilePath = Join-Path $PSScriptRoot ((New-Guid).Guid + ".json")
-        $packageName = "Microsoft.Azure.Functions.PowerShellWorker.nuspec"
-
-        # Delete the manifest folder if it exists
-        $manifestFolderPath = Join-Path $buildPath "_manifest"
-        if (Test-Path $manifestFolderPath)
-        {
-            Remove-Item $manifestFolderPath -Recurse -Force -ErrorAction Ignore
-        }
-
-        Write-Log "Running: dotnet $manifestTool generate -BuildDropPath $buildPath -BuildComponentPath $buildPath -Verbosity Information -t $telemetryFilePath"
-        & { dotnet $manifestTool generate -BuildDropPath $buildPath -BuildComponentPath $buildPath -Verbosity Information -t $telemetryFilePath -PackageName $packageName }
-    }
-
     dotnet pack -c $Configuration "/p:BuildNumber=$BuildNumber" "$PSScriptRoot/package"
 }
 

eng/ci/official.yml

@@ -0,0 +1,48 @@
+trigger:
+  batch: true
+  branches:
+    include:
+      - v4.x/*
+      - v3.x/*
+
+# CI only, does not trigger on PRs.
+pr: none
+
+resources:
+  repositories:
+    - repository: 1es
+      type: git
+      name: 1ESPipelineTemplates/1ESPipelineTemplates
+      ref: refs/tags/release
+
+variables:
+  Configuration: Release
+  buildNumber: $[ counter('build', 4000) ] # Start higher than the versions from the previous pipeline. Every build (pr or branch) will increment.
+
+extends:
+  template: v1/1ES.Official.PipelineTemplate.yml@1es
+  parameters:
+    pool:
+      name: 1es-pool-azfunc
+      image: 1es-windows-2022
+      os: windows
+
+    stages:
+      - stage: WindowsUnitTests
+        dependsOn: []
+        jobs:
+          - template: /eng/ci/templates/test.yml@self
+
+      - stage: LinuxUnitTests
+        dependsOn: []
+        jobs:
+          - template: /eng/ci/templates/test.yml@self
+        pool:
+          name: 1es-pool-azfunc
+          image: 1es-ubuntu-22.04
+          os: linux
+
+      - stage: Build
+        dependsOn: [WindowsUnitTests, LinuxUnitTests]
+        jobs:
+          - template: /eng/ci/templates/build.yml@self

eng/ci/public.yml

@@ -0,0 +1,45 @@
+trigger:
+  batch: true
+  branches:
+    include:
+    - dev
+    - v4.x/*
+    - v3.x/*
+
+# Only run the tests on PR to official branches - do we want to run them on all PRs?
+pr:
+  branches:
+    include:
+    - '*'
+
+resources:
+  repositories:
+    - repository: 1es
+      type: git
+      name: 1ESPipelineTemplates/1ESPipelineTemplates
+      ref: refs/tags/release
+
+extends:
+  template: v1/1ES.Unofficial.PipelineTemplate.yml
+  parameters:
+    pool:
+      name: 1es-pool-azfunc-public
+      image: 1es-windows-2022
+      os: windows
+
+    stages:
+      - stage: WindowsUnitTests
+        dependsOn: []
+        jobs:
+          - template: /eng/ci/templates/test.yml@self
+        pool:
+          name: 1es-pool-azfunc-public
+
+      - stage: LinuxUnitTests
+        dependsOn: []
+        jobs:
+          - template: /eng/ci/templates/test.yml@self
+        pool:
+          name: 1es-pool-azfunc-public
+          image: 1es-ubuntu-22.04
+          os: linux

eng/ci/templates/build.yml

@@ -0,0 +1,38 @@
+jobs:
+  - job:
+    templateContext:
+      outputs:
+        - output: nuget
+          packagesToPush: "$(Build.ArtifactStagingDirectory)/*.nupkg"
+          packageParentPath: "$(Build.ArtifactStagingDirectory)"
+          nuGetFeedType: internal
+          publishVstsFeed: "e6a70c92-4128-439f-8012-382fe78d6396/c0493cce-bc63-4e11-9fc9-e7c45291f151"
+          sbomPackageName: "Azure Functions PowerShell Worker"
+          sbomBuildComponentPath: "$(Build.SourcesDirectory)"
+          allowPackageConflicts: true
+        # - output: nuget
+        #   condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/dev'), eq(variables['UPLOADPACKAGETOPRERELEASEFEED'], true))
+        #   packagesToPush: '$(Build.ArtifactStagingDirectory)/*.nupkg'
+        #   packageParentPath: '$(Build.ArtifactStagingDirectory)'
+        #   nuGetFeedType: 'internal'
+        #   publishVstsFeed: 'e6a70c92-4128-439f-8012-382fe78d6396/f37f760c-aebd-443e-9714-ce725cd427df' # AzureFunctionsPreRelease feed
+        #   allowPackageConflicts: true
+    steps:
+      - pwsh: ./build.ps1 -NoBuild -Bootstrap
+        displayName: "Running ./build.ps1 -NoBuild -Bootstrap"
+
+      - pwsh: |
+          $ErrorActionPreference = "Stop"
+
+          ./build.ps1 -Clean -Configuration Release -BuildNumber "$(buildNumber)"
+        displayName: "Build worker code"
+
+      - task: CopyFiles@2
+        inputs:
+          SourceFolder: "$(System.DefaultWorkingDirectory)/package"
+          Contents: |
+            **/*.nuspec
+            **/*.nupkg
+          TargetFolder: "$(Build.ArtifactStagingDirectory)"
+          flattenFolders: true
+        displayName: "Copy package to artifacts directory"

eng/ci/templates/test.yml

@@ -0,0 +1,34 @@
+jobs:
+  - job: UnitTests
+    steps:
+      - pwsh: ./build.ps1 -NoBuild -Bootstrap
+        displayName: "Running ./build.ps1 -NoBuild -Bootstrap"
+
+      - pwsh: ./Check-CsprojVulnerabilities.ps1
+        displayName: "Check for security vulnerabilities"
+
+      - pwsh: |
+          $ErrorActionPreference = "Stop"
+
+          ./build.ps1 -Clean -Configuration Release -BuildNumber "$(buildNumber)"
+        displayName: "Build worker code"
+
+      - pwsh: ./build.ps1 -NoBuild -Test
+        displayName: "Running UnitTest"
+
+      # - pwsh: ./test/E2E/Start-E2ETest.ps1
+      #   env:
+      #     AzureWebJobsStorage: $(AzureWebJobsStorage)
+      #     AzureWebJobsCosmosDBConnectionString: $(AzureWebJobsCosmosDBConnectionString)
+      #     AzureWebJobsServiceBus: $(AzureWebJobsServiceBus)
+      #     AzureWebJobsEventHubSender: $(AzureWebJobsEventHubSender)
+      #     FUNCTIONS_WORKER_RUNTIME: "powershell"
+      #   displayName: "Running E2ETest"
+
+      - task: PublishTestResults@2
+        inputs:
+          testResultsFormat: "VSTest"
+          testResultsFiles: "**/*.trx"
+          failTaskOnFailedTests: true
+        condition: succeededOrFailed()
+        displayName: "Publish tests results"