Add a job for experimental Framework MSBuild insertion to a pipeline (#9309)

Context We need to add an extra job to .exp-insertions.yml pipeline for creating experimental Framework MSBuild. Changes Made Pipeline refactored, a job added Fixed bug in deploy script code path for Framework MSBuild Testing Manual run of the pipeline
2023-10-12 10:03:38 +02:00 · 2023-10-12 10:03:38 +02:00 · c36a54ed33
--- a/.exp-insertions.yml
+++ b/.exp-insertions.yml
@ -1,13 +1,19 @@
-# Pipeline creates a dotnet with experimental msbuild dlls.
+# Pipeline creates experimental msbuild insertions.

 trigger: none # Prevents this pipeline from triggering on check-ins
 pr: none # don't run this on PR as well

 parameters:
-  # Dotnet installer channel from where to take the latest dotnet bits.
+  # Dotnet installer channel from which to take the latest dotnet bits.
  - name: DotnetInstallerChannel
    displayName: Dotnet installer channel
+    type: string  
+    default: 'none'
+  # VS version for which to take the latest Retail MSBuild bits.
+  - name: VSVersionName
+    displayName: VS Version
    type: string
+    default: 'none'
  # Branch from the MSBuild Build CI pipeline. Default: main
  # Top run for the branch would be used to create an experimental insertion. 
  - name: MSBuildBranch
@ -23,105 +29,192 @@ parameters:
 variables:
  - name: _MsBuildCiPipelineId
    value: 9434
+  - name: _MSBuildConfigFilePath
+    value: "config/batmon/Q-Prod-Co3/Coordinator/ToolsReleaseConfig-GeneralPublic.json"
+  - name: VSVersion
+    value: ${{parameters.VSVersionName}}

 pool:
  vmImage: windows-latest

-steps:
- powershell: |
-    mkdir '$(System.ArtifactsDirectory)/installer'
+resources:
+  repositories: 
+    - repository: CloudBuildConfig
+      type: git
+      name: CloudBuild/CloudBuildConfig
+      endpoint: CloudBuild_Test
+      ref: refs/heads/main 

-    $dotnetChannel = '${{parameters.DotnetInstallerChannel}}'
-    $sdks = "dotnet-sdk-win-x64.zip", "dotnet-sdk-linux-x64.tar.gz"
+jobs:
+- job: CreateExpDotnet
+  displayName: Create Experimental Dotnet
+  condition: ne('${{ parameters.DotnetInstallerChannel }}', 'none')
+  steps:
+  - powershell: |
+      mkdir '$(System.ArtifactsDirectory)/installer'

-    foreach ($sdk in $sdks)
-    {
-      Write-Host "Downloading dotnet $sdk from channel $dotnetChannel"
-      Invoke-WebRequest `
-        -Uri "https://aka.ms/dotnet/$dotnetChannel/daily/$sdk" `
-        -OutFile "$(System.ArtifactsDirectory)/installer/$sdk"
-    }
-    mkdir '$(Pipeline.Workspace)/artifacts'
-    
-  displayName: Download latest dotnet sdks
+      $dotnetChannel = '${{parameters.DotnetInstallerChannel}}'
+      $sdks = "dotnet-sdk-win-x64.zip", "dotnet-sdk-linux-x64.tar.gz"
+
+      foreach ($sdk in $sdks)
+      {
+        Write-Host "Downloading dotnet $sdk from channel $dotnetChannel"
+        Invoke-WebRequest `
+          -Uri "https://aka.ms/dotnet/$dotnetChannel/daily/$sdk" `
+          -OutFile "$(System.ArtifactsDirectory)/installer/$sdk"
+      }
+      mkdir '$(Pipeline.Workspace)/artifacts'
+    displayName: Download latest dotnet sdks

-# Download latest build artifacts for a branch from MSBuild Build CI
- ${{ if eq(parameters.MSBuildBuildID, 'default') }}:
  - task: DownloadBuildArtifacts@1
    inputs:
      buildType: specific
      project: DevDiv
      pipeline: $(_MsBuildCiPipelineId) 
-      buildVersionToDownload: latestFromBranch
-      branchName: '${{parameters.MSBuildBranch}}'  
+      ${{ if eq(parameters.MSBuildBuildID, 'default') }}: 
+        buildVersionToDownload: latestFromBranch
+        branchName: '${{parameters.MSBuildBranch}}'
+      ${{ else }}:
+        buildVersionToDownload: specific
+        buildId: ${{parameters.MSBuildBuildID}} 
      artifactName: bin
      downloadPath: '$(System.ArtifactsDirectory)/msbuild/artifacts/bin'
-      itemPattern: "MSBuild.Bootstrap/**"  
-    displayName: Download latest msbuild from branch
+    displayName: Download msbuild artifacts

-# Download build artifacts for MSBuild Build CI specific build
- ${{ if ne(parameters.MSBuildBuildID, 'default') }}:
+  - powershell: |
+      $sdk = "dotnet-sdk-win-x64"
+
+      Write-Host "Extracting $(System.ArtifactsDirectory)/installer/$sdk.zip"
+      Expand-Archive "$(System.ArtifactsDirectory)/installer/$sdk.zip" -DestinationPath "$(Pipeline.Workspace)/exp-dotnet/$sdk"
+    
+      $dotnetDirectory = Get-ChildItem -Directory -Path "$(Pipeline.Workspace)/exp-dotnet/$sdk/sdk"
+      $dotnetVersion = $dotnetDirectory.Name
+      Write-Host "Detected dotnet version: $dotnetVersion"
+    
+      Write-Host "Updating MSBuild dlls."
+      $(Build.SourcesDirectory)/scripts/Deploy-MSBuild.ps1 `
+        -destination "$(Pipeline.Workspace)/exp-dotnet/$sdk/sdk/$dotnetVersion" `
+        -binDirectory "$(System.ArtifactsDirectory)/msbuild/artifacts/bin" `
+        -configuration Release `
+        -makeBackup $false
+    
+      Write-Host "Compressing dotnet sdk files"
+      Get-ChildItem -Path "$(Pipeline.Workspace)/exp-dotnet/$sdk" | Compress-Archive -DestinationPath "$(Pipeline.Workspace)/artifacts/$sdk.zip"
+
+    displayName: Dogfood msbuild dlls to dotnet sdk win-x64
+
+  - powershell: |
+      $sdk = "dotnet-sdk-linux-x64"
+    
+      mkdir "$(Pipeline.Workspace)/exp-dotnet/$sdk"
+
+      Write-Host "Extracting $(System.ArtifactsDirectory)/installer/$sdk.tar.gz"
+      tar -xzvf "$(System.ArtifactsDirectory)/installer/$sdk.tar.gz" -C "$(Pipeline.Workspace)/exp-dotnet/$sdk"
+
+      $dotnetDirectory = Get-ChildItem -Directory -Path $(Pipeline.Workspace)/exp-dotnet/$sdk/sdk
+      $dotnetVersion = $dotnetDirectory.Name
+      Write-Host "Detected dotnet version: $dotnetVersion"
+    
+      Write-Host "Updating MSBuild dlls."
+      $(Build.SourcesDirectory)/scripts/Deploy-MSBuild.ps1 `
+        -destination "$(Pipeline.Workspace)/exp-dotnet/$sdk/sdk/$dotnetVersion" `
+        -binDirectory "$(System.ArtifactsDirectory)/msbuild/artifacts/bin" `
+        -configuration Release `
+        -makeBackup $false
+    
+      Write-Host "Compressing dotnet sdk files"
+      tar -czvf "$(Pipeline.Workspace)/artifacts/$sdk.tar.gz" -C "$(Pipeline.Workspace)/exp-dotnet/$sdk" .
+    displayName: Dogfood msbuild dlls to dotnet sdk linux-x64
+
+  - task: PublishPipelineArtifact@1
+    inputs:
+      targetPath: '$(Pipeline.Workspace)/artifacts'
+      artifactName: ExperimentalDotnet
+      parallel: true
+    condition: always()
+    displayName: Publish crank assests artifacts
+
+
+- job: CreateExpMSBuild
+  displayName: "Create Experimental MSBuild"
+  condition: ne('${{ parameters.VSVersionName }}', 'none')
+  steps:
+  - checkout: self
+
+  - checkout: CloudBuildConfig
+
+  - powershell: |
+      $json = (Get-Content "$(Build.SourcesDirectory)/CloudBuildConfig/$(_MSBuildConfigFilePath)" -Raw) | ConvertFrom-Json 
+      $MSBuildDropPath = $json.Tools.MSBuild.Locations
+      Write-Host "##vso[task.setvariable variable=MSBuildDropPath]$MSBuildDropPath"
+      Write-Host "MSBuild Drop Path directory: $MSBuildDropPath"
+    displayName: Get Retail MSBuild Drop Path
+  
+  - task: NuGetToolInstaller@1
+    displayName: 'Install NuGet.exe'
+
+  - task: NuGetCommand@2
+    displayName: Restore internal tools
+    inputs:
+      command: restore
+      feedsToUse: config
+      restoreSolution: '$(Build.SourcesDirectory)\DotNet-msbuild-Trusted\eng\common\internal\Tools.csproj'
+      nugetConfigPath: '$(Build.SourcesDirectory)\DotNet-msbuild-Trusted\NuGet.config'
+      restoreDirectory: '$(Build.SourcesDirectory)\DotNet-msbuild-Trusted\.packages'
+  
+  - powershell: |
+      mkdir "$(Pipeline.Workspace)/artifacts"
+      
+      $dropAppDirectory = Get-ChildItem -Directory -Path "$(Build.SourcesDirectory)/DotNet-msbuild-Trusted/.packages/drop.app"
+      $dropAppVersion = $dropAppDirectory.Name
+      Write-Host "Detected drop.exe version: $dropAppVersion"
+
+      $dropExePath = "$(Build.SourcesDirectory)/DotNet-msbuild-Trusted/.packages/drop.app/$dropAppVersion/lib/net45/drop.exe"
+      Write-Host "Detected drop.exe path: $dropExePath"
+
+      Write-Host "Downloading VS msbuild"
+      & "$dropExePath" get --patAuthEnvVar 'cloudbuild-token' -u "$(MSBuildDropPath)\$(VSVersion)" -d "$(System.ArtifactsDirectory)/VSMSBuildDrop"
+      Write-Host "Download of VS msbuild finished"
+
+      Write-Host "Copying VS msbuild to $(Pipeline.Workspace)/VSMSBuild"
+      Copy-Item -Path "$(System.ArtifactsDirectory)/VSMSBuildDrop/*" -Destination "$(Pipeline.Workspace)/VSMSBuild" -Recurse
+      Write-Host "Copy of VS msbuild finished"
+    displayName: Download msbuild vs drop
+    env:
+      cloudbuild-token: $(cloudbuild-token)
+      
  - task: DownloadBuildArtifacts@1
    inputs:
      buildType: specific
      project: DevDiv
      pipeline: $(_MsBuildCiPipelineId) 
-      buildVersionToDownload: specific
-      buildId: ${{parameters.MSBuildBuildID}} 
+      ${{ if eq(parameters.MSBuildBuildID, 'default') }}: 
+        buildVersionToDownload: latestFromBranch
+        branchName: '${{parameters.MSBuildBranch}}'
+      ${{ else }}:
+        buildVersionToDownload: specific
+        buildId: ${{parameters.MSBuildBuildID}} 
      artifactName: bin
      downloadPath: '$(System.ArtifactsDirectory)/msbuild/artifacts/bin'
-      itemPattern: "MSBuild.Bootstrap/**"
-    displayName: Download specified msbuild build
-    
- powershell: |
-    $sdk = "dotnet-sdk-win-x64"
+    displayName: Download msbuild artifacts

-    Write-Host "Extracting $(System.ArtifactsDirectory)/installer/$sdk.zip"
-    Expand-Archive "$(System.ArtifactsDirectory)/installer/$sdk.zip" -DestinationPath "$(Pipeline.Workspace)/exp-dotnet/$sdk"
-    
-    $dotnetDirectory = Get-ChildItem -Directory -Path "$(Pipeline.Workspace)/exp-dotnet/$sdk/sdk"
-    $dotnetVersion = $dotnetDirectory.Name
-    Write-Host "Detected dotnet version: $dotnetVersion"
-    
-    Write-Host "Updating MSBuild dlls."
-    $(Build.SourcesDirectory)/scripts/Deploy-MSBuild.ps1 `
-      -destination "$(Pipeline.Workspace)/exp-dotnet/$sdk/sdk/$dotnetVersion" `
-      -bootstrapDirectory "$(System.ArtifactsDirectory)/msbuild/artifacts/bin/MSBuild.Bootstrap" `
-      -configuration Release `
-      -makeBackup $false
-    
-    Write-Host "Compressing dotnet sdk files"
-    Get-ChildItem -Path "$(Pipeline.Workspace)/exp-dotnet/$sdk" | Compress-Archive -DestinationPath "$(Pipeline.Workspace)/artifacts/$sdk.zip"
+  - powershell: |
+      Write-Host "Updating MSBuild dlls."
+      $(Build.SourcesDirectory)/DotNet-msbuild-Trusted/scripts/Deploy-MSBuild.ps1 `
+        -destination "$(Pipeline.Workspace)/VSMSBuild/$(VSVersion)/MSBuild/Current/Bin" `
+        -binDirectory "$(System.ArtifactsDirectory)/msbuild/artifacts/bin" `
+        -configuration Release `
+        -makeBackup $false

-  displayName: Dogfood msbuild dlls to dotnet sdk win-x64
+      ls "$(Pipeline.Workspace)/VSMSBuild/$(VSVersion)"
+      Write-Host "Compressing msbuild files"
+      Get-ChildItem -Path "$(Pipeline.Workspace)/VSMSBuild/$(VSVersion)" | Compress-Archive -DestinationPath "$(Pipeline.Workspace)/artifacts/MSBuild.zip"
+    displayName: Dogfood msbuild dlls 

- powershell: |
-    $sdk = "dotnet-sdk-linux-x64"
-    
-    mkdir "$(Pipeline.Workspace)/exp-dotnet/$sdk"
-
-    Write-Host "Extracting $(System.ArtifactsDirectory)/installer/$sdk.tar.gz"
-    tar -xzvf "$(System.ArtifactsDirectory)/installer/$sdk.tar.gz" -C "$(Pipeline.Workspace)/exp-dotnet/$sdk"
-
-    $dotnetDirectory = Get-ChildItem -Directory -Path $(Pipeline.Workspace)/exp-dotnet/$sdk/sdk
-    $dotnetVersion = $dotnetDirectory.Name
-    Write-Host "Detected dotnet version: $dotnetVersion"
-    
-    Write-Host "Updating MSBuild dlls."
-    $(Build.SourcesDirectory)/scripts/Deploy-MSBuild.ps1 `
-      -destination "$(Pipeline.Workspace)/exp-dotnet/$sdk/sdk/$dotnetVersion" `
-      -bootstrapDirectory "$(System.ArtifactsDirectory)/msbuild/artifacts/bin/MSBuild.Bootstrap" `
-      -configuration Release `
-      -makeBackup $false
-    
-    Write-Host "Compressing dotnet sdk files"
-    tar -czvf "$(Pipeline.Workspace)/artifacts/$sdk.tar.gz" -C "$(Pipeline.Workspace)/exp-dotnet/$sdk" .
-  displayName: Dogfood msbuild dlls to dotnet sdk linux-x64
-
- task: PublishPipelineArtifact@1
-  inputs:
-    targetPath: '$(Pipeline.Workspace)/artifacts'
-    artifactName: ExperimentalDotnet
-    parallel: true
-  condition: always()
-  displayName: Publish crank assests artifacts
+  - task: PublishPipelineArtifact@1
+    inputs:
+      targetPath: '$(Pipeline.Workspace)/artifacts'
+      artifactName: ExperimentalMSBuild
+      parallel: true
+    condition: always()
+    displayName: Publish crank assests artifacts
--- a/scripts/Deploy-MSBuild.ps1
+++ b/scripts/Deploy-MSBuild.ps1
@ -6,7 +6,7 @@ Param(
  [string] $configuration = "Debug",
  [ValidateSet('Core','Desktop', 'Detect', 'Full')]
  [string] $runtime = "Detect",
-  [string] $bootstrapDirectory = "",
+  [string] $binDirectory = "",
  [bool] $makeBackup = $true
 )

@ -80,11 +80,11 @@ if ($runtime -eq "Desktop") {
 }

 # If bootstrap directory is not defined in parameters, use the default location
-if ($bootstrapDirectory -eq "") {
-    $bootstrapDirectory = "artifacts\bin\MSBuild.Bootstrap" 
+if ($binDirectory -eq "") {
+    $binDirectory = "artifacts\bin" 
 }

-$bootstrapBinDirectory = "$bootstrapDirectory\$configuration\$targetFramework"
+$bootstrapBinDirectory = "$binDirectory\MSBuild.Bootstrap\$configuration\$targetFramework"

 $filesToCopyToBin = @(
    FileToCopy "$bootstrapBinDirectory\Microsoft.Build.dll"
@ -116,8 +116,8 @@ $filesToCopyToBin = @(

 if ($runtime -eq "Desktop") {
    $runtimeSpecificFiles = @(
-        FileToCopy "artifacts\bin\Microsoft.Build.Conversion\$configuration\$targetFramework\Microsoft.Build.Conversion.Core.dll"
-        FileToCopy "artifacts\bin\Microsoft.Build.Engine\$configuration\$targetFramework\Microsoft.Build.Engine.dll"
+        FileToCopy "$binDirectory\Microsoft.Build.Conversion\$configuration\$targetFramework\Microsoft.Build.Conversion.Core.dll"
+        FileToCopy "$binDirectory\Microsoft.Build.Engine\$configuration\$targetFramework\Microsoft.Build.Engine.dll"

        FileToCopy "$bootstrapBinDirectory\Microsoft.Bcl.AsyncInterfaces.dll"
        FileToCopy "$bootstrapBinDirectory\Microsoft.Data.Entity.targets"
@ -152,14 +152,14 @@ if ($runtime -eq "Desktop") {
    $x86files = @(
        FileToCopy "$bootstrapBinDirectory\MSBuild.exe"
        FileToCopy "$bootstrapBinDirectory\MSBuild.exe.config"
-        FileToCopy "artifacts\bin\MSBuildTaskHost\$configuration\net35\MSBuildTaskHost.exe"
-        FileToCopy "artifacts\bin\MSBuildTaskHost\$configuration\net35\MSBuildTaskHost.pdb"
+        FileToCopy "$binDirectory\MSBuildTaskHost\$configuration\net35\MSBuildTaskHost.exe"
+        FileToCopy "$binDirectory\MSBuildTaskHost\$configuration\net35\MSBuildTaskHost.pdb"
    )
    $amd64files = @(
-        FileToCopy "artifacts\bin\MSBuild\x64\$configuration\$targetFramework\MSBuild.exe"
-        FileToCopy "artifacts\bin\MSBuild\x64\$configuration\$targetFramework\MSBuild.exe.config"
-        FileToCopy "artifacts\bin\MSBuildTaskHost\x64\$configuration\net35\MSBuildTaskHost.exe"
-        FileToCopy "artifacts\bin\MSBuildTaskHost\x64\$configuration\net35\MSBuildTaskHost.pdb"
+        FileToCopy "$binDirectory\MSBuild\x64\$configuration\$targetFramework\MSBuild.exe"
+        FileToCopy "$binDirectory\MSBuild\x64\$configuration\$targetFramework\MSBuild.exe.config"
+        FileToCopy "$binDirectory\MSBuildTaskHost\x64\$configuration\net35\MSBuildTaskHost.exe"
+        FileToCopy "$binDirectory\MSBuildTaskHost\x64\$configuration\net35\MSBuildTaskHost.pdb"
    )
 }