Create pipeline for exp insertions (#9231)

* update deloyment script

* Address PR comments

.exp-insertions.yml

@@ -0,0 +1,127 @@
+# Pipeline creates a dotnet with experimental msbuild dlls.
+
+trigger: none # Prevents this pipeline from triggering on check-ins
+pr: none # don't run this on PR as well
+
+parameters:
+  # Dotnet installer channel from where to take the latest dotnet bits.
+  - name: DotnetInstallerChannel
+    displayName: Dotnet installer channel
+    type: string
+  # Branch from the MSBuild Build CI pipeline. Default: main
+  # Top run for the branch would be used to create an experimental insertion. 
+  - name: MSBuildBranch
+    displayName: MSBuild Branch
+    type: string
+    default: 'refs/heads/main'
+  # BuildID from the MSBuild Build CI pipeline. Overrides the choice of MSBuildBranch parameter 
+  - name: MSBuildBuildID
+    displayName: MSBuild CI Run Override
+    type: string
+    default: 'default'
+
+variables:
+  - name: _MsBuildCiPipelineId
+    value: 9434
+
+pool:
+  vmImage: windows-latest
+
+steps:
+- powershell: |
+    mkdir '$(System.ArtifactsDirectory)/installer'
+
+    $dotnetChannel = '${{parameters.DotnetInstallerChannel}}'
+    $sdks = "dotnet-sdk-win-x64.zip", "dotnet-sdk-linux-x64.tar.gz"
+
+    foreach ($sdk in $sdks)
+    {
+      Write-Host "Downloading dotnet $sdk from channel $dotnetChannel"
+      Invoke-WebRequest `
+        -Uri "https://aka.ms/dotnet/$dotnetChannel/daily/$sdk" `
+        -OutFile "$(System.ArtifactsDirectory)/installer/$sdk"
+    }
+    mkdir '$(Pipeline.Workspace)/artifacts'
+    
+  displayName: Download latest dotnet sdks
+
+# Download latest build artifacts for a branch from MSBuild Build CI
+- ${{ if eq(parameters.MSBuildBuildID, 'default') }}:
+  - task: DownloadBuildArtifacts@1
+    inputs:
+      buildType: specific
+      project: DevDiv
+      pipeline: $(_MsBuildCiPipelineId) 
+      buildVersionToDownload: latestFromBranch
+      branchName: '${{parameters.MSBuildBranch}}'  
+      artifactName: bin
+      downloadPath: '$(System.ArtifactsDirectory)/msbuild/artifacts/bin'
+      itemPattern: "MSBuild.Bootstrap/**"  
+    displayName: Download latest msbuild from branch
+
+# Download build artifacts for MSBuild Build CI specific build
+- ${{ if ne(parameters.MSBuildBuildID, 'default') }}:
+  - task: DownloadBuildArtifacts@1
+    inputs:
+      buildType: specific
+      project: DevDiv
+      pipeline: $(_MsBuildCiPipelineId) 
+      buildVersionToDownload: specific
+      buildId: ${{parameters.MSBuildBuildID}} 
+      artifactName: bin
+      downloadPath: '$(System.ArtifactsDirectory)/msbuild/artifacts/bin'
+      itemPattern: "MSBuild.Bootstrap/**"
+    displayName: Download specified msbuild build
+    
+- powershell: |
+    $sdk = "dotnet-sdk-win-x64"
+
+    Write-Host "Extracting $(System.ArtifactsDirectory)/installer/$sdk.zip"
+    Expand-Archive "$(System.ArtifactsDirectory)/installer/$sdk.zip" -DestinationPath "$(Pipeline.Workspace)/exp-dotnet/$sdk"
+    
+    $dotnetDirectory = Get-ChildItem -Directory -Path "$(Pipeline.Workspace)/exp-dotnet/$sdk/sdk"
+    $dotnetVersion = $dotnetDirectory.Name
+    Write-Host "Detected dotnet version: $dotnetVersion"
+    
+    Write-Host "Updating MSBuild dlls."
+    $(Build.SourcesDirectory)/scripts/Deploy-MSBuild.ps1 `
+      -destination "$(Pipeline.Workspace)/exp-dotnet/$sdk/sdk/$dotnetVersion" `
+      -bootstrapDirectory "$(System.ArtifactsDirectory)/msbuild/artifacts/bin/MSBuild.Bootstrap" `
+      -configuration Release `
+      -makeBackup $false
+    
+    Write-Host "Compressing dotnet sdk files"
+    Get-ChildItem -Path "$(Pipeline.Workspace)/exp-dotnet/$sdk" | Compress-Archive -DestinationPath "$(Pipeline.Workspace)/artifacts/$sdk.zip"
+
+  displayName: Dogfood msbuild dlls to dotnet sdk win-x64
+
+- powershell: |
+    $sdk = "dotnet-sdk-linux-x64"
+    
+    mkdir "$(Pipeline.Workspace)/exp-dotnet/$sdk"
+
+    Write-Host "Extracting $(System.ArtifactsDirectory)/installer/$sdk.tar.gz"
+    tar -xzvf "$(System.ArtifactsDirectory)/installer/$sdk.tar.gz" -C "$(Pipeline.Workspace)/exp-dotnet/$sdk"
+
+    $dotnetDirectory = Get-ChildItem -Directory -Path $(Pipeline.Workspace)/exp-dotnet/$sdk/sdk
+    $dotnetVersion = $dotnetDirectory.Name
+    Write-Host "Detected dotnet version: $dotnetVersion"
+    
+    Write-Host "Updating MSBuild dlls."
+    $(Build.SourcesDirectory)/scripts/Deploy-MSBuild.ps1 `
+      -destination "$(Pipeline.Workspace)/exp-dotnet/$sdk/sdk/$dotnetVersion" `
+      -bootstrapDirectory "$(System.ArtifactsDirectory)/msbuild/artifacts/bin/MSBuild.Bootstrap" `
+      -configuration Release `
+      -makeBackup $false
+    
+    Write-Host "Compressing dotnet sdk files"
+    tar -czvf "$(Pipeline.Workspace)/artifacts/$sdk.tar.gz" -C "$(Pipeline.Workspace)/exp-dotnet/$sdk" .
+  displayName: Dogfood msbuild dlls to dotnet sdk linux-x64
+
+- task: PublishPipelineArtifact@1
+  inputs:
+    targetPath: '$(Pipeline.Workspace)/artifacts'
+    artifactName: ExperimentalDotnet
+    parallel: true
+  condition: always()
+  displayName: Publish crank assests artifacts

scripts/Deploy-MSBuild.ps1

@@ -5,7 +5,9 @@ Param(
   [ValidateSet('Debug','Release')]
   [string] $configuration = "Debug",
   [ValidateSet('Core','Desktop', 'Detect', 'Full')]
-  [string] $runtime = "Detect"
+  [string] $runtime = "Detect",
+  [string] $bootstrapDirectory = "",
+  [bool] $makeBackup = $true
 )
 
 Set-StrictMode -Version "Latest"
@@ -15,9 +17,9 @@ function Copy-WithBackup ($origin, $destinationSubFolder = "") {
     $directoryPart = [IO.Path]::Combine($destination, $destinationSubFolder, $origin.IntermediaryDirectories)
     $destinationPath = Join-Path -Path $directoryPart (Split-Path $origin.SourceFile -leaf)
 
-    $backupInto = [IO.Path]::Combine($BackupFolder, $destinationSubFolder)
+    if (($makeBackup) -and (Test-Path $destinationPath -PathType Leaf)) {
+        $backupInto = [IO.Path]::Combine($BackupFolder, $destinationSubFolder)
 
-    if (Test-Path $destinationPath -PathType Leaf) {
         # Back up previous copy of the file
         if (!(Test-Path $backupInto)) {
             [system.io.directory]::CreateDirectory($backupInto)
@@ -45,10 +47,15 @@ function FileToCopy([string] $sourceFileRelativeToRepoRoot, [string] $intermedia
 
 # TODO: find most-recently-built MSBuild and make it default $configuration
 
-$BackupFolder = New-Item (Join-Path $destination -ChildPath "Backup-$(Get-Date -Format FileDateTime)") -itemType directory -ErrorAction Stop
-
 Write-Verbose "Copying $configuration MSBuild to $destination"
-Write-Host "Existing MSBuild assemblies backed up to $BackupFolder"
+
+if ($makeBackup) {
+    $BackupFolder = New-Item (Join-Path $destination -ChildPath "Backup-$(Get-Date -Format FileDateTime)") -itemType directory -ErrorAction Stop
+    Write-Host "Existing MSBuild assemblies backed up to $BackupFolder"
+}
+else {
+    Write-Host "Existing MSBuild assemblies won't be backed up"
+}
 
 if ($runtime -eq "Detect") {
     if ($destination -like "*dotnet*sdk*") {
@@ -72,7 +79,12 @@ if ($runtime -eq "Desktop") {
     $targetFramework = "net8.0"
 }
 
-$bootstrapBinDirectory = "artifacts\bin\MSBuild.Bootstrap\$configuration\$targetFramework"
+# If bootstrap directory is not defined in parameters, use the default location
+if ($bootstrapDirectory -eq "") {
+    $bootstrapDirectory = "artifacts\bin\MSBuild.Bootstrap" 
+}
+
+$bootstrapBinDirectory = "$bootstrapDirectory\$configuration\$targetFramework"
 
 $filesToCopyToBin = @(
     FileToCopy "$bootstrapBinDirectory\Microsoft.Build.dll"