Backport maestro and artifact drop infra improvements from net9.0 (#24247)

* [ci] Set OfficialBuildId when building asset manifests (#23102) Context: 812807c5cb The maestro build promotion step has been failing with: PublishArtifactsInManifest.proj(130,5): error : Asset 'D:\a\_work\1\a\3fecf55a-18fb-414d-b980-84c0f56a3856\MergedManifest.xml' already exists with different contents at 'https://dotnetbuilds.blob.core.windows.net/public/assets/manifests/dotnet-maui/9.0.0-ci-dev/MergedManifest.xml' The manifest version information passed to the `PublishBuildAssets.proj` build does not contain revision information, causing asset publishing to fail as it will not overwrite existing assets with the same version. We should be able to fix this by setting the `OfficialBuildId` property when building the asset manifests, similar to what is currently being done when [packing the NuGets][0]. [0]: 5d3e788f29/eng/cake/dotnet.cake (L285) * [ci] Use drop service for SDK insertion artifacts (#23658) Context: 8759ec9edd Steps to upload release artifacts to custom blob storage have been replaced with azure-artifacts-drop (aka.ms/drop). A new version of nuget-msi-convert has been added that will create a set of artifact drops for the following shipping artifacts: * nugets * vs-components * vs-packs The nugets drop contains all shipping packages that should be pushed to various feeds or NuGet.org. The components and packs drops are used for VS insertions. * [ci] Simplify shipping drop metadata names (#24103) Using the $(System.JobAttempt) variable in the drop metadata artifact name is problematic. In some cases the drop artifacts created by the nuget-msi-convert job will be used by a different job, and the job attempt number will not necessarily match if any jobs are re-ran. * [ci] Use passwordless auth for darc/maestro (#24220) Fixes: https://github.com/dotnet/maui/issues/23974 Migrates darc/maestro commands to use a passwordless auth flow, as token based authentication is deprecated and will be removed in the future. * Update bar manifest build step * Update bar manifest build working dir * Use new task name * Run add build to channel outside sources dir
2024-08-20 11:17:21 -04:00 · 2024-08-20 11:17:21 -04:00 · b57d6642da
--- a/eng/Version.Details.xml
+++ b/eng/Version.Details.xml
@ -14,9 +14,9 @@
    </Dependency>
  </ProductDependencies>
  <ToolsetDependencies>
-    <Dependency Name="Microsoft.DotNet.Build.Tasks.Feed" Version="8.0.0-beta.24310.5">
+    <Dependency Name="Microsoft.DotNet.Build.Tasks.Feed" Version="9.0.0-beta.24408.2">
      <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>9f6799fdc16ae19b3e9478c55b997a6aab839d09</Sha>
+      <Sha>60ae233c3d77f11c5fdb53e570b64d503b13ba59</Sha>
    </Dependency>
    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="8.0.0-beta.24310.5">
      <Uri>https://github.com/dotnet/arcade</Uri>
--- a/eng/Versions.props
+++ b/eng/Versions.props
@ -114,7 +114,7 @@
    <TizenUIExtensionsVersion>0.9.0</TizenUIExtensionsVersion>
    <ExCSSPackageVersion>4.2.3</ExCSSPackageVersion>
    <SystemDrawingCommonPackageVersion>8.0.3</SystemDrawingCommonPackageVersion>
-    <MicrosoftDotNetBuildTasksFeedVersion>8.0.0-beta.24310.5</MicrosoftDotNetBuildTasksFeedVersion>
+    <MicrosoftDotNetBuildTasksFeedVersion>9.0.0-beta.24408.2</MicrosoftDotNetBuildTasksFeedVersion>
  </PropertyGroup>
  <PropertyGroup>
    <MicrosoftNETTestSdkPackageVersion>17.6.0</MicrosoftNETTestSdkPackageVersion>
--- a/eng/pipelines/common/sdk-insertion.yml
+++ b/eng/pipelines/common/sdk-insertion.yml
@ -5,80 +5,92 @@ parameters:
  pushMauiPackagesToMaestro: false
  nugetArtifactName: nuget-signed
  nugetArtifactPath: $(Build.StagingDirectory)\nuget-signed
+  dropRetentionDays: 120

 jobs:
 - job: create_artifact_statuses
-  displayName: Create GitHub Artifact Status and Push to Maestro
+  displayName: Publish symbols and Push to Maestro
  timeoutInMinutes: 60
  pool:
    name: ${{ parameters.poolName }}
    image: ${{ parameters.vmImage }}
    os: ${{ parameters.os }}
-  variables:
-  - group: Publish-Build-Assets
+  templateContext:
+    outputs:
+    - output: artifactsDrop
+      dropServiceURI: https://devdiv.artifacts.visualstudio.com/DefaultCollection
+      buildNumber: $(ReleaseDropPrefix)/symbols
+      dropMetadataContainerName: DropMetadata-shipping-symbols
+      sourcePath: $(Build.StagingDirectory)\symbols
+      retentionDays: ${{ parameters.dropRetentionDays }}
+      toLowerCase: false
  steps:
  - checkout: self
+
+  # Download symbols to be published to the symbols artifact drop declared above
  - task: DownloadPipelineArtifact@2
    inputs:
-      artifactName: ${{ parameters.nugetArtifactName }}
-      downloadPath: ${{ parameters.nugetArtifactPath }}
+      artifactName: nuget
+      downloadPath: $(Build.StagingDirectory)\symbols
      patterns: |
-        *.nupkg
        **/*.snupkg
        **/additional-assets.zip
+    displayName: Download symbols
+
  - task: DownloadPipelineArtifact@2
    inputs:
-      artifactName: vs-msi-nugets
-      downloadPath: ${{ parameters.nugetArtifactPath }}
-  - template: templates\common\upload-vs-insertion-artifacts.yml@sdk-insertions
-    parameters:
-      githubToken: $(github--pat--vs-mobiletools-engineering-service2)
-      githubContext: $(NupkgCommitStatusName)
-      blobName: $(NupkgCommitStatusName)
-      packagePrefix: maui
-      artifactsPath: ${{ parameters.nugetArtifactPath }}
-      yamlResourceName: yaml-templates
-  - template: templates\common\upload-vs-insertion-artifacts.yml@sdk-insertions
-    parameters:
-      githubToken: $(github--pat--vs-mobiletools-engineering-service2)
-      githubContext: $(VSDropCommitStatusName)
-      blobName: $(VSDropCommitStatusName)
-      packagePrefix: maui
-      artifactsPath: $(Build.StagingDirectory)/$(VSDropCommitStatusName)
-      yamlResourceName: yaml-templates
-      downloadSteps:
-      - task: DownloadPipelineArtifact@2
-        inputs:
-          artifactName: vsdrop-signed
-          downloadPath: $(Build.StagingDirectory)/$(VSDropCommitStatusName)
-  - template: templates\common\upload-vs-insertion-artifacts.yml@sdk-insertions
-    parameters:
-      githubToken: $(github--pat--vs-mobiletools-engineering-service2)
-      githubContext: $(MultiTargetVSDropCommitStatusName)
-      blobName: $(MultiTargetVSDropCommitStatusName)
-      packagePrefix: maui
-      artifactsPath: $(Build.StagingDirectory)/$(MultiTargetVSDropCommitStatusName)
-      yamlResourceName: yaml-templates
-      downloadSteps:
-      - task: DownloadPipelineArtifact@2
-        inputs:
-          artifactName: vsdrop-multitarget-signed
-          downloadPath: $(Build.StagingDirectory)/$(MultiTargetVSDropCommitStatusName)
-  - task: DotNetCoreCLI@2
+      artifactName: DropMetadata-shipping-nugets
+      downloadPath: $(Build.StagingDirectory)\metadata
+    displayName: Download nugets drop metadata
+
+  - powershell: |
+      $jsonContent = Get-Content -Path "$(Build.StagingDirectory)\metadata\VSTSDrop.json" -Raw | ConvertFrom-Json
+      $dropPrefix = $jsonContent.VstsDropBuildArtifact.VstsDropUrl -replace 'https://devdiv.artifacts.visualstudio.com/DefaultCollection/_apis/drop/drops/' -replace '/nugets'
+      Write-Host "##vso[task.setvariable variable=ReleaseDropPrefix]$dropPrefix"
+    displayName: Set variable ReleaseDropPrefix
+
+  # Download nugets drop created by nuget-msi-convert/job/v4.yml and publish to maestro
+  - task: ms-vscs-artifact.build-tasks.artifactDropDownloadTask-1.artifactDropDownloadTask@1
+    displayName: Download $(ReleaseDropPrefix)/nugets
+    inputs:
+      dropServiceURI: https://devdiv.artifacts.visualstudio.com/DefaultCollection
+      buildNumber: $(ReleaseDropPrefix)/nugets
+      destinationPath: ${{ parameters.nugetArtifactPath }}
+
+  - task: UseDotNet@2
+    displayName: Install .NET 9.x
+    inputs:
+      version: 9.x
+      includePreviewVersions: true
+
+  - task: AzureCLI@2
    displayName: Generate and publish BAR manifest
    inputs:
-      projects: $(Build.SourcesDirectory)\src\Workload\Microsoft.Maui.Sdk\Microsoft.Maui.Sdk.csproj
-      arguments: >-
+      azureSubscription: "Darc: Maestro Production"
+      scriptType: ps
+      scriptLocation: inlineScript
+      inlineScript: >-
+        & dotnet build $(Build.SourcesDirectory)\src\Workload\Microsoft.Maui.Sdk\Microsoft.Maui.Sdk.csproj
        -t:PushManifestToBuildAssetRegistry
-        -p:BuildAssetRegistryToken=$(MaestroAccessToken)
-        -p:OutputPath=$(Build.StagingDirectory)\nuget-signed\
+        -p:OfficialBuildId=$(_BuildOfficalId)
+        -p:OutputPath=${{ parameters.nugetArtifactPath }}
        -v:n -bl:$(Build.StagingDirectory)\binlogs\push-bar-manifest.binlog
+    # Execute outside of the source directory to avoid any conflicting global.json file
+      workingDirectory: $(Build.SourcesDirectory)\..
    condition: and(succeeded(), eq('${{ parameters.pushMauiPackagesToMaestro }}', 'true'))
-  - powershell: |
-      $versionEndpoint = 'https://maestro.dot.net/api/assets/darc-version?api-version=2019-01-16'
-      $darcVersion = $(Invoke-WebRequest -Uri $versionEndpoint -UseBasicParsing).Content
-      $arcadeServicesSource = 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json'
-      & dotnet tool update microsoft.dotnet.darc --version "$darcVersion" --add-source "$arcadeServicesSource" --tool-path $(Agent.ToolsDirectory)\darc -v n
-      & $(Agent.ToolsDirectory)\darc\darc add-build-to-channel --default-channels --id $(BARBuildId) --publishing-infra-version 3 --password $(MaestroAccessToken) --azdev-pat $(publishing-dnceng-devdiv-code-r-build-re)
+
+  - task: AzureCLI@2
+    inputs:
+      azureSubscription: "Darc: Maestro Production"
+      scriptType: ps
+      scriptLocation: inlineScript
+      inlineScript: |
+        $versionEndpoint = 'https://maestro.dot.net/api/assets/darc-version?api-version=2019-01-16'
+        $darcVersion = $(Invoke-WebRequest -Uri $versionEndpoint -UseBasicParsing).Content
+        $arcadeServicesSource = 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json'
+        & dotnet tool update microsoft.dotnet.darc --version "$darcVersion" --add-source "$arcadeServicesSource" --tool-path $(Agent.ToolsDirectory)\darc -v n
+        & $(Agent.ToolsDirectory)\darc\darc add-build-to-channel --default-channels --id $(BARBuildId) --ci --publishing-infra-version 3 --azdev-pat $(System.AccessToken)
+    # Execute outside of the source directory to avoid any conflicting global.json file
+      workingDirectory: $(Build.SourcesDirectory)\..
    displayName: Add build to default darc channel
    condition: and(succeeded(), eq('${{ parameters.pushMauiPackagesToMaestro }}', 'true'))
--- a/eng/pipelines/common/sign.yml
+++ b/eng/pipelines/common/sign.yml
@ -22,7 +22,7 @@ stages:
          use1ESTemplate: true
          usePipelineArtifactTasks: true
        
-      - template: nuget-msi-convert/job/v3.yml@yaml-templates
+      - template: nuget-msi-convert/job/v4.yml@yaml-templates
        parameters:
          yamlResourceName: yaml-templates
          artifactName: nuget-signed
@ -42,7 +42,7 @@ stages:
            inputs:
              TargetFolders: |
                $(Build.ArtifactStagingDirectory)\bin\manifests
-                $(Build.ArtifactStagingDirectory)\bin\manifests-multitarget
+                $(Build.ArtifactStagingDirectory)\bin\manifests-packs
                $(Build.ArtifactStagingDirectory)\bin\msi-nupkgs
              ExcludeSNVerify: true
              ApprovalListPathForCerts: $(Build.ArtifactStagingDirectory)\sign-verify\SignVerifyIgnore.txt
--- a/eng/pipelines/common/variables.yml
+++ b/eng/pipelines/common/variables.yml
@ -1,6 +1,8 @@
 variables:
 - name: BuildVersion
  value: $[counter('buildversion-counter', 5000)]
+- name: _BuildOfficalId
+  value: $[ format('{0}.{1}', format('{0:yyyyMMdd}', pipeline.startTime), counter(format('{0:yyyyMMdd}', pipeline.startTime), 1) )]
 - name: NUGET_VERSION
  value: 6.4.0
 - name: DOTNET_SKIP_FIRST_TIME_EXPERIENCE
--- a/eng/pipelines/maui-release.yml
+++ b/eng/pipelines/maui-release.yml
@ -144,9 +144,7 @@ extends:
                value: $(Build.SourcesDirectory)/build.cmd -ci
              - name: _BuildConfig
                value: Release
-              - name: _BuildOfficalId
-                value: $[ format('{0}.{1}', format('{0:yyyyMMdd}', pipeline.startTime), counter(format('{0:yyyyMMdd}', pipeline.startTime), 1) )]
-            
+
            steps:
              - template: /eng/pipelines/common/pack.yml@self
                parameters:
--- a/src/Workload/Shared/Maestro.targets
+++ b/src/Workload/Shared/Maestro.targets
@ -36,7 +36,7 @@
      <ManifestBuildData Include="AzureDevOpsBranch=$(BUILD_SOURCEBRANCH)" />
    </ItemGroup>

-    <PushToAzureDevOpsArtifacts
+    <PushToBuildStorage
        ItemsToPush="@(ItemsToPush)"
        IsStableBuild="$(StabilizePackageVersion)"
        ManifestBuildData="@(ManifestBuildData)"
@ -47,15 +47,17 @@
        AssetManifestPath="$(AssetManifestPath)"
        PublishingVersion="3" />

+    <Message Text="BAR manifest version: $(PackageReferenceVersion)" />
+
    <MSBuild
        Targets="Restore"
        Projects="$(PkgMicrosoft_DotNet_Arcade_Sdk)\tools\SdkTasks\PublishBuildAssets.proj"
-        Properties="Configuration=$(Configuration);RepoRoot=$(MauiRootDirectory);VersionPrefix=$(Version)"
+        Properties="Configuration=$(Configuration);RepoRoot=$(MauiRootDirectory);VersionPrefix=$(PackageReferenceVersion)"
    />

    <MSBuild
        Projects="$(PkgMicrosoft_DotNet_Arcade_Sdk)\tools\SdkTasks\PublishBuildAssets.proj"
-        Properties="Configuration=$(Configuration);RepoRoot=$(MauiRootDirectory);VersionPrefix=$(Version);ManifestsPath=$(ArtifactsLogDir)AssetManifest;MaestroApiEndpoint=https://maestro.dot.net"
+        Properties="Configuration=$(Configuration);RepoRoot=$(MauiRootDirectory);VersionPrefix=$(PackageReferenceVersion);ManifestsPath=$(ArtifactsLogDir)AssetManifest;MaestroApiEndpoint=https://maestro.dot.net"
    />
  </Target>