Retargeting to .NET 9 (#10484)

.editorconfig

@@ -415,3 +415,13 @@ dotnet_diagnostic.IDE0305.severity = suggestion
 
 # Temporarily disable SA1010 "Opening square brackets should not be preceded by a space" until https://github.com/DotNetAnalyzers/StyleCopAnalyzers/issues/3687 is fixed
 dotnet_diagnostic.SA1010.severity = none
+
+# Remove unnecessary equality operator  IDE0100 (https://learn.microsoft.com/en-gb/dotnet/fundamentals/code-analysis/style-rules/ide0100)
+# Making it a suggestion to have a flexibility for future
+dotnet_diagnostic.IDE0100.severity = suggestion
+
+# File header should match the template, making it error since couple of files met in the code base without any header
+dotnet_diagnostic.IDE0073.severity = error
+
+# Use 'System.Threading.Lock'
+dotnet_diagnostic.IDE0330.severity = suggestion

Directory.Build.props

@@ -27,7 +27,7 @@
      -->
      <!-- When building in source-only modes, let the TFM float based on what arcade is in use.
           When building in normal modes (independent build or a dotnet product build), set explicitly. -->
-    <LatestDotNetCoreForMSBuild>net8.0</LatestDotNetCoreForMSBuild>
+    <LatestDotNetCoreForMSBuild>net9.0</LatestDotNetCoreForMSBuild>
     <LatestDotNetCoreForMSBuild Condition="'$(DotNetBuildFromSource)' == 'true' or '$(DotNetBuildSourceOnly)' == 'true'">$(NetCurrent)</LatestDotNetCoreForMSBuild>
   </PropertyGroup>
 
@@ -58,6 +58,9 @@
     -->
 
     <NoWarn>$(NoWarn);NU1507;NU1603;NU5105;1701;1702;SYSLIB0011;SYSLIB0037;SYSLIB0044;RS0016;RS0017;</NoWarn>
+    <!-- Loading certificate data through the constructor or Import is obsolete. Use X509CertificateLoader instead to load certific
+ates https://learn.microsoft.com/en-gb/dotnet/fundamentals/syslib-diagnostics/syslib0057 -->
+    <NoWarn>$(NoWarn);SYSLIB0057;</NoWarn>
   </PropertyGroup>
 
   <!-- Configuration MSBuild for portable (xcopy-install) toolsets: works on WinNT and linux/mac via Mono, isolates from machine environment:

Directory.Build.targets

@@ -37,15 +37,4 @@
     <RemoveDir Directories="$(_PackageFolderInGlobalPackages)"
                Condition="Exists('$(_PackageFolderInGlobalPackages)')" />
   </Target>
-
-  <!-- Make APICompat use roslyn from the toolset SDK instead of from the toolset package. This avoids unification issues on desktop msbuild.
-       TODO: Remove when Visual Studio with an 8.0.200 or 9.0 SDK is consumed. -->
-  <Target Name="FixAPICompatWorkAroundRoslynMove"
-          AfterTargets="CollectApiCompatInputs"
-          Condition="'$(MSBuildRuntimeType)' != 'Core'">
-    <PropertyGroup>
-      <RoslynAssembliesPath>$(RoslynTargetsPath)</RoslynAssembliesPath>
-    </PropertyGroup>
-  </Target>
-
 </Project>

NuGet.config

@@ -17,6 +17,7 @@
     <add key="dotnet6" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet6/nuget/v3/index.json" />
     <add key="dotnet8" value="https://dnceng.pkgs.visualstudio.com/public/_packaging/dotnet8/nuget/v3/index.json" />
     <add key="dotnet8-transport" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet8-transport/nuget/v3/index.json" />
+    <add key="dotnet9" value="https://dnceng.pkgs.visualstudio.com/public/_packaging/dotnet9/nuget/v3/index.json" />
     <add key="BuildXL" value="https://pkgs.dev.azure.com/ms/BuildXL/_packaging/BuildXL/nuget/v3/index.json" />
   </packageSources>
   <disabledPackageSources>

eng/Version.Details.xml

@@ -71,14 +71,14 @@
     </Dependency>
   </ProductDependencies>
   <ToolsetDependencies>
-    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="8.0.0-beta.24421.4">
+    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="9.0.0-beta.24423.2">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>4460b755f3c7c89e9660d9580ff79afc4218dd85</Sha>
+      <Sha>91599268652b51969b8d8088d4f2f2ba7b3ebb19</Sha>
     </Dependency>
     <!-- Intermediate is necessary for source build. -->
-    <Dependency Name="Microsoft.SourceBuild.Intermediate.arcade" Version="8.0.0-beta.24421.4">
+    <Dependency Name="Microsoft.SourceBuild.Intermediate.arcade" Version="9.0.0-beta.24423.2">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>4460b755f3c7c89e9660d9580ff79afc4218dd85</Sha>
+      <Sha>91599268652b51969b8d8088d4f2f2ba7b3ebb19</Sha>
       <SourceBuild RepoName="arcade" ManagedOnly="true" />
     </Dependency>
     <Dependency Name="Microsoft.DotNet.XliffTasks" Version="1.0.0-beta.23475.1" CoherentParentDependency="Microsoft.DotNet.Arcade.Sdk">
@@ -98,6 +98,10 @@
     <Dependency Name="Microsoft.Net.Compilers.Toolset" Version="4.12.0-2.24425.4">
       <Uri>https://github.com/dotnet/roslyn</Uri>
       <Sha>13b65d3b5638457d94c83b545235f47f0d11e483</Sha>
+    </Dependency>
+    <Dependency Name="Microsoft.SourceBuild.Intermediate.roslyn" Version="4.12.0-2.24425.4">
+      <Uri>https://github.com/dotnet/roslyn</Uri>
+      <Sha>13b65d3b5638457d94c83b545235f47f0d11e483</Sha>
       <SourceBuild RepoName="roslyn" ManagedOnly="true" />
     </Dependency>
     <Dependency Name="Microsoft.DotNet.XUnitExtensions" Version="8.0.0-beta.24421.4">

eng/Versions.props

@@ -57,7 +57,7 @@
     <SystemRuntimeCompilerServicesUnsafeVersion>6.0.0</SystemRuntimeCompilerServicesUnsafeVersion>
   </PropertyGroup>
   <PropertyGroup Condition="!$(TargetFramework.StartsWith('net4'))">
-    <BootstrapSdkVersion>8.0.302</BootstrapSdkVersion>
+    <BootstrapSdkVersion>9.0.100-preview.7.24407.12</BootstrapSdkVersion>
   </PropertyGroup>
   <Target Name="OverrideArcadeFileVersion" AfterTargets="_InitializeAssemblyVersion">
     <!-- See https://github.com/dotnet/arcade/issues/3386
@@ -75,4 +75,13 @@
       <FileVersion>$(VersionPrefix).$(FileVersion.Split('.')[3])</FileVersion>
     </PropertyGroup>
   </Target>
+  <!--  Freeze the version for the current upgrade of the sdk. 
+        TODO:Delete the lines below once the Retargeting merged. 
+  -->
+  <PropertyGroup>
+    <XUnitVersion>2.4.2</XUnitVersion>
+    <XUnitAnalyzersVersion>1.1.0</XUnitAnalyzersVersion>
+    <XUnitRunnerConsoleVersion>$(XUnitVersion)</XUnitRunnerConsoleVersion>
+    <XUnitRunnerVisualStudioVersion>2.4.3</XUnitRunnerVisualStudioVersion>
+  </PropertyGroup>
 </Project>

eng/cibuild_bootstrapped_msbuild.ps1

@@ -86,8 +86,8 @@ try {
   {
     $buildToolPath = Join-Path $bootstrapRoot "core\dotnet.exe"
     # The version must be consistent with BootstrapSdkVersion
-    $buildToolCommand = Join-Path $bootstrapRoot "core\sdk\8.0.302\MSBuild.dll"
-    $buildToolFramework = "net8.0"
+    $buildToolCommand = Join-Path $bootstrapRoot "core\sdk\9.0.100-preview.7.24407.12\MSBuild.dll"
+    $buildToolFramework = "net9.0"
   }
 
   # Use separate artifacts folder for stage 2

eng/cibuild_bootstrapped_msbuild.sh

@@ -60,8 +60,8 @@ if [ $host_type = "core" ]
 then
   _InitializeBuildTool="$bootstrapRoot/core/dotnet"
   # The version must be consistent with BootstrapSdkVersion
-  _InitializeBuildToolCommand="$bootstrapRoot/core/sdk/8.0.302/MSBuild.dll"
-  _InitializeBuildToolFramework="net8.0"
+  _InitializeBuildToolCommand="$bootstrapRoot/core/sdk/9.0.100-preview.7.24407.12/MSBuild.dll"
+  _InitializeBuildToolFramework="net9.0"
 else
   echo "Unsupported hostType ($host_type)"
   exit 1

eng/common/SetupNugetSources.ps1

@@ -1,17 +1,10 @@
-# This file is a temporary workaround for internal builds to be able to restore from private AzDO feeds.
-# This file should be removed as part of this issue: https://github.com/dotnet/arcade/issues/4080
+# This script adds internal feeds required to build commits that depend on internal package sources. For instance,
+# dotnet6-internal would be added automatically if dotnet6 was found in the nuget.config file. In addition also enables
+# disabled internal Maestro (darc-int*) feeds.
 #
-# What the script does is iterate over all package sources in the pointed NuGet.config and add a credential entry
-# under <packageSourceCredentials> for each Maestro managed private feed. Two additional credential 
-# entries are also added for the two private static internal feeds: dotnet3-internal and dotnet3-internal-transport.
+# Optionally, this script also adds a credential entry for each of the internal feeds if supplied.
 #
-# This script needs to be called in every job that will restore packages and which the base repo has
-# private AzDO feeds in the NuGet.config.
-#
-# See example YAML call for this script below. Note the use of the variable `$(dn-bot-dnceng-artifact-feeds-rw)`
-# from the AzureDevOps-Artifact-Feeds-Pats variable group.
-#
-# Any disabledPackageSources entries which start with "darc-int" will be re-enabled as part of this script executing
+# See example call for this script below.
 #
 #  - task: PowerShell@2
 #    displayName: Setup Private Feeds Credentials
@@ -21,11 +14,18 @@
 #      arguments: -ConfigFile $(Build.SourcesDirectory)/NuGet.config -Password $Env:Token
 #    env:
 #      Token: $(dn-bot-dnceng-artifact-feeds-rw)
+#
+# Note that the NuGetAuthenticate task should be called after SetupNugetSources.
+# This ensures that:
+# - Appropriate creds are set for the added internal feeds (if not supplied to the scrupt)
+# - The credential provider is installed.
+#
+# This logic is also abstracted into enable-internal-sources.yml.
 
 [CmdletBinding()]
 param (
     [Parameter(Mandatory = $true)][string]$ConfigFile,
-    [Parameter(Mandatory = $true)][string]$Password
+    $Password
 )
 
 $ErrorActionPreference = "Stop"
@@ -48,11 +48,17 @@ function AddPackageSource($sources, $SourceName, $SourceEndPoint, $creds, $Usern
     else {
         Write-Host "Package source $SourceName already present."
     }
+
     AddCredential -Creds $creds -Source $SourceName -Username $Username -pwd $pwd
 }
 
 # Add a credential node for the specified source
 function AddCredential($creds, $source, $username, $pwd) {
+    # If no cred supplied, don't do anything.
+    if (!$pwd) {
+        return;
+    }
+
     # Looks for credential configuration for the given SourceName. Create it if none is found.
     $sourceElement = $creds.SelectSingleNode($Source)
     if ($sourceElement -eq $null)
@@ -110,11 +116,6 @@ if (!(Test-Path $ConfigFile -PathType Leaf)) {
   ExitWithExitCode 1
 }
 
-if (!$Password) {
-    Write-PipelineTelemetryError -Category 'Build' -Message 'Eng/common/SetupNugetSources.ps1 returned a non-zero exit code. Please supply a valid PAT'
-    ExitWithExitCode 1
-}
-
 # Load NuGet.config
 $doc = New-Object System.Xml.XmlDocument
 $filename = (Get-Item $ConfigFile).FullName
@@ -127,11 +128,14 @@ if ($sources -eq $null) {
     $doc.DocumentElement.AppendChild($sources) | Out-Null
 }
 
-# Looks for a <PackageSourceCredentials> node. Create it if none is found.
-$creds = $doc.DocumentElement.SelectSingleNode("packageSourceCredentials")
-if ($creds -eq $null) {
-    $creds = $doc.CreateElement("packageSourceCredentials")
-    $doc.DocumentElement.AppendChild($creds) | Out-Null
+$creds = $null
+if ($Password) {
+    # Looks for a <PackageSourceCredentials> node. Create it if none is found.
+    $creds = $doc.DocumentElement.SelectSingleNode("packageSourceCredentials")
+    if ($creds -eq $null) {
+        $creds = $doc.CreateElement("packageSourceCredentials")
+        $doc.DocumentElement.AppendChild($creds) | Out-Null
+    }
 }
 
 # Check for disabledPackageSources; we'll enable any darc-int ones we find there
@@ -164,4 +168,4 @@ foreach ($dotnetVersion in $dotnetVersions) {
     }
 }
 
-$doc.Save($filename)
+$doc.Save($filename)

eng/common/SetupNugetSources.sh

@@ -1,28 +1,27 @@
 #!/usr/bin/env bash
 
-# This file is a temporary workaround for internal builds to be able to restore from private AzDO feeds.
-# This file should be removed as part of this issue: https://github.com/dotnet/arcade/issues/4080
+# This script adds internal feeds required to build commits that depend on internal package sources. For instance,
+# dotnet6-internal would be added automatically if dotnet6 was found in the nuget.config file. In addition also enables
+# disabled internal Maestro (darc-int*) feeds.
+# 
+# Optionally, this script also adds a credential entry for each of the internal feeds if supplied.
 #
-# What the script does is iterate over all package sources in the pointed NuGet.config and add a credential entry
-# under <packageSourceCredentials> for each Maestro's managed private feed. Two additional credential 
-# entries are also added for the two private static internal feeds: dotnet3-internal and dotnet3-internal-transport.
-#
-# This script needs to be called in every job that will restore packages and which the base repo has
-# private AzDO feeds in the NuGet.config.
-#
-# See example YAML call for this script below. Note the use of the variable `$(dn-bot-dnceng-artifact-feeds-rw)`
-# from the AzureDevOps-Artifact-Feeds-Pats variable group.
-#
-# Any disabledPackageSources entries which start with "darc-int" will be re-enabled as part of this script executing.
+# See example call for this script below.
 #
 #  - task: Bash@3
-#    displayName: Setup Private Feeds Credentials
+#    displayName: Setup Internal Feeds
 #    inputs:
 #      filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh
-#      arguments: $(Build.SourcesDirectory)/NuGet.config $Token
+#      arguments: $(Build.SourcesDirectory)/NuGet.config
 #    condition: ne(variables['Agent.OS'], 'Windows_NT')
-#    env:
-#      Token: $(dn-bot-dnceng-artifact-feeds-rw)
+#  - task: NuGetAuthenticate@1
+#
+# Note that the NuGetAuthenticate task should be called after SetupNugetSources.
+# This ensures that:
+# - Appropriate creds are set for the added internal feeds (if not supplied to the scrupt)
+# - The credential provider is installed.
+#
+# This logic is also abstracted into enable-internal-sources.yml.
 
 ConfigFile=$1
 CredToken=$2
@@ -48,11 +47,6 @@ if [ ! -f "$ConfigFile" ]; then
     ExitWithExitCode 1
 fi
 
-if [ -z "$CredToken" ]; then
-    Write-PipelineTelemetryError -category 'Build' "Error: Eng/common/SetupNugetSources.sh returned a non-zero exit code. Please supply a valid PAT"
-    ExitWithExitCode 1
-fi
-
 if [[ `uname -s` == "Darwin" ]]; then
     NL=$'\\\n'
     TB=''
@@ -140,18 +134,20 @@ PackageSources+="$IFS"
 PackageSources+=$(grep -oh '"darc-int-[^"]*"' $ConfigFile | tr -d '"')
 IFS=$PrevIFS
 
-for FeedName in ${PackageSources[@]} ; do
-    # Check if there is no existing credential for this FeedName
-    grep -i "<$FeedName>" $ConfigFile 
-    if [ "$?" != "0" ]; then
-        echo "Adding credentials for $FeedName."
+if [ "$CredToken" ]; then
+    for FeedName in ${PackageSources[@]} ; do
+        # Check if there is no existing credential for this FeedName
+        grep -i "<$FeedName>" $ConfigFile 
+        if [ "$?" != "0" ]; then
+            echo "Adding credentials for $FeedName."
 
-        PackageSourceCredentialsNodeFooter="</packageSourceCredentials>"
-        NewCredential="${TB}${TB}<$FeedName>${NL}<add key=\"Username\" value=\"dn-bot\" />${NL}<add key=\"ClearTextPassword\" value=\"$CredToken\" />${NL}</$FeedName>"
+            PackageSourceCredentialsNodeFooter="</packageSourceCredentials>"
+            NewCredential="${TB}${TB}<$FeedName>${NL}<add key=\"Username\" value=\"dn-bot\" />${NL}<add key=\"ClearTextPassword\" value=\"$CredToken\" />${NL}</$FeedName>"
 
-        sed -i.bak "s|$PackageSourceCredentialsNodeFooter|$NewCredential${NL}$PackageSourceCredentialsNodeFooter|" $ConfigFile
-    fi
-done
+            sed -i.bak "s|$PackageSourceCredentialsNodeFooter|$NewCredential${NL}$PackageSourceCredentialsNodeFooter|" $ConfigFile
+        fi
+    done
+fi
 
 # Re-enable any entries in disabledPackageSources where the feed name contains darc-int
 grep -i "<disabledPackageSources>" $ConfigFile

eng/common/build.cmd

@@ -0,0 +1,3 @@
+@echo off
+powershell -ExecutionPolicy ByPass -NoProfile -command "& """%~dp0build.ps1""" %*"
+exit /b %ErrorLevel%

eng/common/build.ps1

@@ -19,6 +19,7 @@ Param(
   [switch] $pack,
   [switch] $publish,
   [switch] $clean,
+  [switch][Alias('pb')]$productBuild,
   [switch][Alias('bl')]$binaryLog,
   [switch][Alias('nobl')]$excludeCIBinarylog,
   [switch] $ci,
@@ -58,6 +59,7 @@ function Print-Usage() {
   Write-Host "  -sign                   Sign build outputs"
   Write-Host "  -publish                Publish artifacts (e.g. symbols)"
   Write-Host "  -clean                  Clean the solution"
+  Write-Host "  -productBuild           Build the solution in the way it will be built in the full .NET product (VMR) build (short: -pb)"
   Write-Host ""
 
   Write-Host "Advanced settings:"
@@ -120,6 +122,7 @@ function Build {
     /p:Deploy=$deploy `
     /p:Test=$test `
     /p:Pack=$pack `
+    /p:DotNetBuildRepo=$productBuild `
     /p:IntegrationTest=$integrationTest `
     /p:PerformanceTest=$performanceTest `
     /p:Sign=$sign `

eng/common/build.sh

@@ -22,6 +22,9 @@ usage()
   echo "  --sourceBuild              Source-build the solution (short: -sb)"
   echo "                             Will additionally trigger the following actions: --restore, --build, --pack"
   echo "                             If --configuration is not set explicitly, will also set it to 'Release'"
+  echo "  --productBuild             Build the solution in the way it will be built in the full .NET product (VMR) build (short: -pb)"
+  echo "                             Will additionally trigger the following actions: --restore, --build, --pack"
+  echo "                             If --configuration is not set explicitly, will also set it to 'Release'"
   echo "  --rebuild                  Rebuild solution"
   echo "  --test                     Run all unit tests in the solution (short: -t)"
   echo "  --integrationTest          Run all integration tests in the solution"
@@ -59,6 +62,7 @@ scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
 restore=false
 build=false
 source_build=false
+product_build=false
 rebuild=false
 test=false
 integration_test=false
@@ -105,7 +109,7 @@ while [[ $# > 0 ]]; do
     -binarylog|-bl)
       binary_log=true
       ;;
-    -excludeCIBinarylog|-nobl)
+    -excludecibinarylog|-nobl)
       exclude_ci_binary_log=true
       ;;
     -pipelineslog|-pl)
@@ -126,6 +130,13 @@ while [[ $# > 0 ]]; do
     -sourcebuild|-sb)
       build=true
       source_build=true
+      product_build=true
+      restore=true
+      pack=true
+      ;;
+    -productBuild|-pb)
+      build=true
+      product_build=true
       restore=true
       pack=true
       ;;
@@ -219,7 +230,9 @@ function Build {
     /p:RepoRoot="$repo_root" \
     /p:Restore=$restore \
     /p:Build=$build \
+    /p:DotNetBuildRepo=$product_build \
     /p:ArcadeBuildFromSource=$source_build \
+    /p:DotNetBuildSourceOnly=$source_build \
     /p:Rebuild=$rebuild \
     /p:Test=$test \
     /p:Pack=$pack \

eng/common/core-templates/job/job.yml

@@ -0,0 +1,252 @@
+parameters:
+# Job schema parameters - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#job
+  cancelTimeoutInMinutes: ''
+  condition: ''
+  container: ''
+  continueOnError: false
+  dependsOn: ''
+  displayName: ''
+  pool: ''
+  steps: []
+  strategy: ''
+  timeoutInMinutes: ''
+  variables: []
+  workspace: ''
+  templateContext: {}
+
+# Job base template specific parameters
+  # See schema documentation - https://github.com/dotnet/arcade/blob/master/Documentation/AzureDevOps/TemplateSchema.md
+  # publishing defaults
+  artifacts: ''
+  enableMicrobuild: false
+  enablePublishBuildArtifacts: false
+  enablePublishBuildAssets: false
+  enablePublishTestResults: false
+  enablePublishUsingPipelines: false
+  enableBuildRetry: false
+  mergeTestResults: false
+  testRunTitle: ''
+  testResultsFormat: ''
+  name: ''
+  componentGovernanceSteps: []
+  preSteps: []
+  artifactPublishSteps: []
+  runAsPublic: false
+
+# Sbom related params
+  enableSbom: true
+  PackageVersion: 9.0.0
+  BuildDropPath: '$(Build.SourcesDirectory)/artifacts'
+
+# 1es specific parameters
+  is1ESPipeline: ''
+
+jobs:
+- job: ${{ parameters.name }}
+
+  ${{ if ne(parameters.cancelTimeoutInMinutes, '') }}:
+    cancelTimeoutInMinutes: ${{ parameters.cancelTimeoutInMinutes }}
+
+  ${{ if ne(parameters.condition, '') }}:
+    condition: ${{ parameters.condition }}
+
+  ${{ if ne(parameters.container, '') }}:
+    container: ${{ parameters.container }}
+
+  ${{ if ne(parameters.continueOnError, '') }}:
+    continueOnError: ${{ parameters.continueOnError }}
+
+  ${{ if ne(parameters.dependsOn, '') }}:
+    dependsOn: ${{ parameters.dependsOn }}
+
+  ${{ if ne(parameters.displayName, '') }}:
+    displayName: ${{ parameters.displayName }}
+
+  ${{ if ne(parameters.pool, '') }}:
+    pool: ${{ parameters.pool }}
+
+  ${{ if ne(parameters.strategy, '') }}:
+    strategy: ${{ parameters.strategy }}
+
+  ${{ if ne(parameters.timeoutInMinutes, '') }}:
+    timeoutInMinutes: ${{ parameters.timeoutInMinutes }}
+
+  ${{ if ne(parameters.templateContext, '') }}:
+    templateContext: ${{ parameters.templateContext }}
+
+  variables:
+  - ${{ if ne(parameters.enableTelemetry, 'false') }}:
+    - name: DOTNET_CLI_TELEMETRY_PROFILE
+      value: '$(Build.Repository.Uri)'
+  - ${{ if eq(parameters.enableRichCodeNavigation, 'true') }}:
+    - name: EnableRichCodeNavigation
+      value: 'true'
+  # Retry signature validation up to three times, waiting 2 seconds between attempts.
+  # See https://learn.microsoft.com/en-us/nuget/reference/errors-and-warnings/nu3028#retry-untrusted-root-failures
+  - name: NUGET_EXPERIMENTAL_CHAIN_BUILD_RETRY_POLICY
+    value: 3,2000
+  - ${{ each variable in parameters.variables }}:
+    # handle name-value variable syntax
+    # example:
+    # - name: [key]
+    #   value: [value]
+    - ${{ if ne(variable.name, '') }}:
+      - name: ${{ variable.name }}
+        value: ${{ variable.value }}
+
+    # handle variable groups
+    - ${{ if ne(variable.group, '') }}:
+      - group: ${{ variable.group }}
+
+    # handle template variable syntax
+    # example:
+    # - template: path/to/template.yml
+    #   parameters:
+    #     [key]: [value]
+    - ${{ if ne(variable.template, '') }}:
+      - template: ${{ variable.template }}
+        ${{ if ne(variable.parameters, '') }}:
+          parameters: ${{ variable.parameters }}
+
+    # handle key-value variable syntax.
+    # example:
+    # - [key]: [value]
+    - ${{ if and(eq(variable.name, ''), eq(variable.group, ''), eq(variable.template, '')) }}:
+      - ${{ each pair in variable }}:
+        - name: ${{ pair.key }}
+          value: ${{ pair.value }}
+
+  # DotNet-HelixApi-Access provides 'HelixApiAccessToken' for internal builds
+  - ${{ if and(eq(parameters.enableTelemetry, 'true'), eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
+    - group: DotNet-HelixApi-Access
+
+  ${{ if ne(parameters.workspace, '') }}:
+    workspace: ${{ parameters.workspace }}
+
+  steps:
+  - ${{ if eq(parameters.is1ESPipeline, '') }}:
+    - 'Illegal entry point, is1ESPipeline is not defined. Repository yaml should not directly reference templates in core-templates folder.': error
+
+  - ${{ if ne(parameters.preSteps, '') }}:
+    - ${{ each preStep in parameters.preSteps }}:
+      - ${{ preStep }}
+
+  - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
+    - ${{ if eq(parameters.enableMicrobuild, 'true') }}:
+      - task: MicroBuildSigningPlugin@4
+        displayName: Install MicroBuild plugin
+        inputs:
+          signType: $(_SignType)
+          zipSources: false
+          feedSource: https://dnceng.pkgs.visualstudio.com/_packaging/MicroBuildToolset/nuget/v3/index.json
+        env:
+          TeamName: $(_TeamName)
+          MicroBuildOutputFolderOverride: '$(Agent.TempDirectory)'
+        continueOnError: ${{ parameters.continueOnError }}
+        condition: and(succeeded(), in(variables['_SignType'], 'real', 'test'), eq(variables['Agent.Os'], 'Windows_NT'))
+
+  - ${{ if and(eq(parameters.runAsPublic, 'false'), eq(variables['System.TeamProject'], 'internal')) }}:
+    - task: NuGetAuthenticate@1
+
+  - ${{ if and(ne(parameters.artifacts.download, 'false'), ne(parameters.artifacts.download, '')) }}:
+    - task: DownloadPipelineArtifact@2
+      inputs:
+        buildType: current
+        artifactName: ${{ coalesce(parameters.artifacts.download.name, 'Artifacts_$(Agent.OS)_$(_BuildConfig)') }}
+        targetPath: ${{ coalesce(parameters.artifacts.download.path, 'artifacts') }}
+        itemPattern: ${{ coalesce(parameters.artifacts.download.pattern, '**') }}
+
+  - ${{ each step in parameters.steps }}:
+    - ${{ step }}
+
+  - ${{ if eq(parameters.enableRichCodeNavigation, true) }}:
+    - task: RichCodeNavIndexer@0
+      displayName: RichCodeNav Upload
+      inputs:
+        languages: ${{ coalesce(parameters.richCodeNavigationLanguage, 'csharp') }}
+        environment: ${{ coalesce(parameters.richCodeNavigationEnvironment, 'internal') }}
+        richNavLogOutputDirectory: $(Build.SourcesDirectory)/artifacts/bin
+        uploadRichNavArtifacts: ${{ coalesce(parameters.richCodeNavigationUploadArtifacts, false) }}
+      continueOnError: true
+
+  - ${{ each step in parameters.componentGovernanceSteps }}:
+    - ${{ step }}
+
+  - ${{ if eq(parameters.enableMicrobuild, 'true') }}:
+    - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
+      - task: MicroBuildCleanup@1
+        displayName: Execute Microbuild cleanup tasks
+        condition: and(always(), in(variables['_SignType'], 'real', 'test'), eq(variables['Agent.Os'], 'Windows_NT'))
+        continueOnError: ${{ parameters.continueOnError }}
+        env:
+          TeamName: $(_TeamName)
+
+  # Publish test results
+  - ${{ if or(and(eq(parameters.enablePublishTestResults, 'true'), eq(parameters.testResultsFormat, '')), eq(parameters.testResultsFormat, 'xunit')) }}:
+    - task: PublishTestResults@2
+      displayName: Publish XUnit Test Results
+      inputs:
+        testResultsFormat: 'xUnit'
+        testResultsFiles: '*.xml'
+        searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)'
+        testRunTitle: ${{ coalesce(parameters.testRunTitle, parameters.name, '$(System.JobName)') }}-xunit
+        mergeTestResults: ${{ parameters.mergeTestResults }}
+      continueOnError: true
+      condition: always()
+  - ${{ if or(and(eq(parameters.enablePublishTestResults, 'true'), eq(parameters.testResultsFormat, '')), eq(parameters.testResultsFormat, 'vstest')) }}:
+    - task: PublishTestResults@2
+      displayName: Publish TRX Test Results
+      inputs:
+        testResultsFormat: 'VSTest'
+        testResultsFiles: '*.trx'
+        searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)'
+        testRunTitle: ${{ coalesce(parameters.testRunTitle, parameters.name, '$(System.JobName)') }}-trx
+        mergeTestResults: ${{ parameters.mergeTestResults }}
+      continueOnError: true
+      condition: always()
+
+  # gather artifacts
+  - ${{ if ne(parameters.artifacts.publish, '') }}:
+    - ${{ if and(ne(parameters.artifacts.publish.artifacts, 'false'), ne(parameters.artifacts.publish.artifacts, '')) }}:
+      - task: CopyFiles@2
+        displayName: Gather binaries for publish to artifacts
+        inputs:
+          SourceFolder: 'artifacts/bin'
+          Contents: '**'
+          TargetFolder: '$(Build.ArtifactStagingDirectory)/artifacts/bin'
+      - task: CopyFiles@2
+        displayName: Gather packages for publish to artifacts
+        inputs:
+          SourceFolder: 'artifacts/packages'
+          Contents: '**'
+          TargetFolder: '$(Build.ArtifactStagingDirectory)/artifacts/packages'
+    - ${{ if and(ne(parameters.artifacts.publish.logs, 'false'), ne(parameters.artifacts.publish.logs, '')) }}:
+      - task: CopyFiles@2
+        displayName: Gather logs for publish to artifacts
+        inputs:
+          SourceFolder: 'artifacts/log'
+          Contents: '**'
+          TargetFolder: '$(Build.ArtifactStagingDirectory)/artifacts/log'
+        continueOnError: true
+        condition: always()
+      
+  - ${{ if eq(parameters.enablePublishBuildArtifacts, 'true') }}:
+    - task: CopyFiles@2
+      displayName: Gather logs for publish to artifacts
+      inputs:
+        SourceFolder: 'artifacts/log/$(_BuildConfig)'
+        Contents: '**'
+        TargetFolder: '$(Build.ArtifactStagingDirectory)/artifacts/log/$(_BuildConfig)'
+      continueOnError: true
+      condition: always()
+  - ${{ if eq(parameters.enableBuildRetry, 'true') }}:
+    - task: CopyFiles@2
+      displayName: Gather buildconfiguration for build retry
+      inputs:
+        SourceFolder: '$(Build.SourcesDirectory)/eng/common/BuildConfiguration'
+        Contents: '**'
+        TargetFolder: '$(Build.ArtifactStagingDirectory)/eng/common/BuildConfiguration'
+      continueOnError: true
+      condition: always()
+  - ${{ each step in parameters.artifactPublishSteps }}:
+    - ${{ step }}

eng/common/core-templates/job/onelocbuild.yml

@@ -0,0 +1,121 @@
+parameters:
+  # Optional: dependencies of the job
+  dependsOn: ''
+
+  # Optional: A defined YAML pool - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#pool
+  pool: ''
+    
+  CeapexPat: $(dn-bot-ceapex-package-r) # PAT for the loc AzDO instance https://dev.azure.com/ceapex
+  GithubPat: $(BotAccount-dotnet-bot-repo-PAT)
+
+  SourcesDirectory: $(Build.SourcesDirectory)
+  CreatePr: true
+  AutoCompletePr: false
+  ReusePr: true
+  UseLfLineEndings: true
+  UseCheckedInLocProjectJson: false
+  SkipLocProjectJsonGeneration: false
+  LanguageSet: VS_Main_Languages
+  LclSource: lclFilesInRepo
+  LclPackageId: ''
+  RepoType: gitHub
+  GitHubOrg: dotnet
+  MirrorRepo: ''
+  MirrorBranch: main
+  condition: ''
+  JobNameSuffix: ''
+  is1ESPipeline: ''
+jobs:
+- job: OneLocBuild${{ parameters.JobNameSuffix }}
+  
+  dependsOn: ${{ parameters.dependsOn }}
+
+  displayName: OneLocBuild${{ parameters.JobNameSuffix }}
+
+  variables:
+    - group: OneLocBuildVariables # Contains the CeapexPat and GithubPat
+    - name: _GenerateLocProjectArguments
+      value: -SourcesDirectory ${{ parameters.SourcesDirectory }}
+        -LanguageSet "${{ parameters.LanguageSet }}"
+        -CreateNeutralXlfs
+    - ${{ if eq(parameters.UseCheckedInLocProjectJson, 'true') }}:
+      - name: _GenerateLocProjectArguments
+        value: ${{ variables._GenerateLocProjectArguments }} -UseCheckedInLocProjectJson
+    - template: /eng/common/core-templates/variables/pool-providers.yml
+      parameters:
+        is1ESPipeline: ${{ parameters.is1ESPipeline }}
+
+  ${{ if ne(parameters.pool, '') }}:
+    pool: ${{ parameters.pool }}
+  ${{ if eq(parameters.pool, '') }}:
+    pool:
+      # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com)
+      ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}:
+        name: AzurePipelines-EO
+        image: 1ESPT-Windows2022
+        demands: Cmd
+        os: windows
+      # If it's not devdiv, it's dnceng
+      ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}:
+        name: $(DncEngInternalBuildPool)
+        image: 1es-windows-2022
+        os: windows
+
+  steps:
+    - ${{ if eq(parameters.is1ESPipeline, '') }}:
+      - 'Illegal entry point, is1ESPipeline is not defined. Repository yaml should not directly reference templates in core-templates folder.': error
+
+    - ${{ if ne(parameters.SkipLocProjectJsonGeneration, 'true') }}:
+      - task: Powershell@2
+        inputs:
+          filePath: $(Build.SourcesDirectory)/eng/common/generate-locproject.ps1
+          arguments: $(_GenerateLocProjectArguments)
+        displayName: Generate LocProject.json
+        condition: ${{ parameters.condition }}
+
+    - task: OneLocBuild@2
+      displayName: OneLocBuild
+      env:
+        SYSTEM_ACCESSTOKEN: $(System.AccessToken)
+      inputs:
+        locProj: eng/Localize/LocProject.json
+        outDir: $(Build.ArtifactStagingDirectory)
+        lclSource: ${{ parameters.LclSource }}
+        lclPackageId: ${{ parameters.LclPackageId }}
+        isCreatePrSelected: ${{ parameters.CreatePr }}
+        isAutoCompletePrSelected: ${{ parameters.AutoCompletePr }}
+        ${{ if eq(parameters.CreatePr, true) }}:
+          isUseLfLineEndingsSelected: ${{ parameters.UseLfLineEndings }}
+          ${{ if eq(parameters.RepoType, 'gitHub') }}:
+            isShouldReusePrSelected: ${{ parameters.ReusePr }}
+        packageSourceAuth: patAuth
+        patVariable: ${{ parameters.CeapexPat }}
+        ${{ if eq(parameters.RepoType, 'gitHub') }}:
+          repoType: ${{ parameters.RepoType }}
+          gitHubPatVariable: "${{ parameters.GithubPat }}"
+        ${{ if ne(parameters.MirrorRepo, '') }}:
+          isMirrorRepoSelected: true
+          gitHubOrganization: ${{ parameters.GitHubOrg }}
+          mirrorRepo: ${{ parameters.MirrorRepo }}
+          mirrorBranch: ${{ parameters.MirrorBranch }}
+      condition: ${{ parameters.condition }}
+
+    - template: /eng/common/core-templates/steps/publish-build-artifacts.yml
+      parameters:
+        is1ESPipeline: ${{ parameters.is1ESPipeline }}
+        args:
+          displayName: Publish Localization Files
+          pathToPublish: '$(Build.ArtifactStagingDirectory)/loc'
+          publishLocation: Container
+          artifactName: Loc
+          condition: ${{ parameters.condition }}
+
+    - template: /eng/common/core-templates/steps/publish-build-artifacts.yml
+      parameters:
+        is1ESPipeline: ${{ parameters.is1ESPipeline }}
+        args:
+          displayName: Publish LocProject.json
+          pathToPublish: '$(Build.SourcesDirectory)/eng/Localize/'
+          publishLocation: Container
+          artifactName: Loc
+          condition: ${{ parameters.condition }}

eng/common/core-templates/job/publish-build-assets.yml

@@ -0,0 +1,158 @@
+parameters:
+  configuration: 'Debug'
+
+  # Optional: condition for the job to run
+  condition: ''
+
+  # Optional: 'true' if future jobs should run even if this job fails
+  continueOnError: false
+
+  # Optional: dependencies of the job
+  dependsOn: ''
+
+  # Optional: Include PublishBuildArtifacts task
+  enablePublishBuildArtifacts: false
+
+  # Optional: A defined YAML pool - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#pool
+  pool: {}
+
+  # Optional: should run as a public build even in the internal project
+  #           if 'true', the build won't run any of the internal only steps, even if it is running in non-public projects.
+  runAsPublic: false
+
+  # Optional: whether the build's artifacts will be published using release pipelines or direct feed publishing
+  publishUsingPipelines: false
+
+  # Optional: whether the build's artifacts will be published using release pipelines or direct feed publishing
+  publishAssetsImmediately: false
+
+  artifactsPublishingAdditionalParameters: ''
+
+  signingValidationAdditionalParameters: ''
+
+  is1ESPipeline: ''
+
+jobs:
+- job: Asset_Registry_Publish
+
+  dependsOn: ${{ parameters.dependsOn }}
+  timeoutInMinutes: 150
+
+  ${{ if eq(parameters.publishAssetsImmediately, 'true') }}:
+    displayName: Publish Assets
+  ${{ else }}:
+    displayName: Publish to Build Asset Registry
+
+  variables:
+  - template: /eng/common/core-templates/variables/pool-providers.yml
+    parameters:
+      is1ESPipeline: ${{ parameters.is1ESPipeline }}
+  - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
+    - group: Publish-Build-Assets
+    - group: AzureDevOps-Artifact-Feeds-Pats
+    - name: runCodesignValidationInjection
+      value: false
+    # unconditional - needed for logs publishing (redactor tool version)
+    - template: /eng/common/core-templates/post-build/common-variables.yml
+
+  pool:
+    # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com)
+    ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}:
+      name: AzurePipelines-EO
+      image: 1ESPT-Windows2022
+      demands: Cmd
+      os: windows
+    # If it's not devdiv, it's dnceng
+    ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}:
+      name: NetCore1ESPool-Publishing-Internal
+      image: windows.vs2019.amd64
+      os: windows
+  steps:
+  - ${{ if eq(parameters.is1ESPipeline, '') }}:
+    - 'Illegal entry point, is1ESPipeline is not defined. Repository yaml should not directly reference templates in core-templates folder.': error
+
+  - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
+    - checkout: self
+      fetchDepth: 3
+      clean: true
+      
+    - task: DownloadBuildArtifacts@0
+      displayName: Download artifact
+      inputs:
+        artifactName: AssetManifests
+        downloadPath: '$(Build.StagingDirectory)/Download'
+        checkDownloadedFiles: true
+      condition: ${{ parameters.condition }}
+      continueOnError: ${{ parameters.continueOnError }}
+    
+    - task: NuGetAuthenticate@1
+
+    - task: AzureCLI@2
+      displayName: Publish Build Assets
+      inputs:
+        azureSubscription: "Darc: Maestro Production"
+        scriptType: ps
+        scriptLocation: scriptPath
+        scriptPath: $(Build.SourcesDirectory)/eng/common/sdk-task.ps1
+        arguments: -task PublishBuildAssets -restore -msbuildEngine dotnet
+          /p:ManifestsPath='$(Build.StagingDirectory)/Download/AssetManifests'
+          /p:MaestroApiEndpoint=https://maestro.dot.net
+          /p:PublishUsingPipelines=${{ parameters.publishUsingPipelines }}
+          /p:OfficialBuildId=$(Build.BuildNumber)
+      condition: ${{ parameters.condition }}
+      continueOnError: ${{ parameters.continueOnError }}
+    
+    - task: powershell@2
+      displayName: Create ReleaseConfigs Artifact
+      inputs:
+        targetType: inline
+        script: |
+          New-Item -Path "$(Build.StagingDirectory)/ReleaseConfigs" -ItemType Directory -Force
+          $filePath = "$(Build.StagingDirectory)/ReleaseConfigs/ReleaseConfigs.txt"
+          Add-Content -Path $filePath -Value $(BARBuildId)
+          Add-Content -Path $filePath -Value "$(DefaultChannels)"
+          Add-Content -Path $filePath -Value $(IsStableBuild)
+
+          $symbolExclusionfile = "$(Build.SourcesDirectory)/eng/SymbolPublishingExclusionsFile.txt"
+          if (Test-Path -Path $symbolExclusionfile)
+          {
+            Write-Host "SymbolExclusionFile exists"
+            Copy-Item -Path $symbolExclusionfile -Destination "$(Build.StagingDirectory)/ReleaseConfigs"
+          }
+
+    - template: /eng/common/core-templates/steps/publish-build-artifacts.yml
+      parameters:
+        is1ESPipeline: ${{ parameters.is1ESPipeline }}
+        args:
+          displayName: Publish ReleaseConfigs Artifact
+          pathToPublish: '$(Build.StagingDirectory)/ReleaseConfigs'
+          publishLocation: Container
+          artifactName: ReleaseConfigs
+
+    - ${{ if eq(parameters.publishAssetsImmediately, 'true') }}:
+      - template: /eng/common/core-templates/post-build/setup-maestro-vars.yml
+        parameters:
+          BARBuildId: ${{ parameters.BARBuildId }}
+          PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
+          is1ESPipeline: ${{ parameters.is1ESPipeline }}
+
+      - task: AzureCLI@2
+        displayName: Publish Using Darc
+        inputs:
+          azureSubscription: "Darc: Maestro Production"
+          scriptType: ps
+          scriptLocation: scriptPath
+          scriptPath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1
+          arguments: >
+            -BuildId $(BARBuildId)
+            -PublishingInfraVersion 3
+            -AzdoToken '$(System.AccessToken)'
+            -WaitPublishingFinish true
+            -ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}'
+            -SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}'
+
+    - ${{ if eq(parameters.enablePublishBuildArtifacts, 'true') }}:
+      - template: /eng/common/core-templates/steps/publish-logs.yml
+        parameters:
+          is1ESPipeline: ${{ parameters.is1ESPipeline }}
+          JobLabel: 'Publish_Artifacts_Logs'     

eng/common/core-templates/job/source-build.yml

@@ -0,0 +1,93 @@
+parameters:
+  # This template adds arcade-powered source-build to CI. The template produces a server job with a
+  # default ID 'Source_Build_Complete' to put in a dependency list if necessary.
+
+  # Specifies the prefix for source-build jobs added to pipeline. Use this if disambiguation needed.
+  jobNamePrefix: 'Source_Build'
+
+  # Defines the platform on which to run the job. By default, a linux-x64 machine, suitable for
+  # managed-only repositories. This is an object with these properties:
+  #
+  # name: ''
+  #   The name of the job. This is included in the job ID.
+  # targetRID: ''
+  #   The name of the target RID to use, instead of the one auto-detected by Arcade.
+  # nonPortable: false
+  #   Enables non-portable mode. This means a more specific RID (e.g. fedora.32-x64 rather than
+  #   linux-x64), and compiling against distro-provided packages rather than portable ones.
+  # skipPublishValidation: false
+  #   Disables publishing validation.  By default, a check is performed to ensure no packages are
+  #   published by source-build.
+  # container: ''
+  #   A container to use. Runs in docker.
+  # pool: {}
+  #   A pool to use. Runs directly on an agent.
+  # buildScript: ''
+  #   Specifies the build script to invoke to perform the build in the repo. The default
+  #   './build.sh' should work for typical Arcade repositories, but this is customizable for
+  #   difficult situations.
+  # jobProperties: {}
+  #   A list of job properties to inject at the top level, for potential extensibility beyond
+  #   container and pool.
+  platform: {}
+
+  is1ESPipeline: ''
+
+  # If set to true and running on a non-public project,
+  # Internal nuget and blob storage locations will be enabled.
+  # This is not enabled by default because many repositories do not need internal sources
+  # and do not need to have the required service connections approved in the pipeline.
+  enableInternalSources: false
+
+jobs:
+- job: ${{ parameters.jobNamePrefix }}_${{ parameters.platform.name }}
+  displayName: Source-Build (${{ parameters.platform.name }})
+
+  ${{ each property in parameters.platform.jobProperties }}:
+    ${{ property.key }}: ${{ property.value }}
+
+  ${{ if ne(parameters.platform.container, '') }}:
+    container: ${{ parameters.platform.container }}
+
+  ${{ if eq(parameters.platform.pool, '') }}:
+    # The default VM host AzDO pool. This should be capable of running Docker containers: almost all
+    # source-build builds run in Docker, including the default managed platform.
+    # /eng/common/core-templates/variables/pool-providers.yml can't be used here (some customers declare variables already), so duplicate its logic
+    ${{ if eq(parameters.is1ESPipeline, 'true') }}:
+      pool:
+        ${{ if eq(variables['System.TeamProject'], 'public') }}:
+          name: $[replace(replace(eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), True, 'NetCore-Svc-Public' ), False, 'NetCore-Public')]
+          demands: ImageOverride -equals build.ubuntu.2004.amd64
+        ${{ if eq(variables['System.TeamProject'], 'internal') }}:
+          name: $[replace(replace(eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), True, 'NetCore1ESPool-Svc-Internal'), False, 'NetCore1ESPool-Internal')]
+          image: 1es-mariner-2
+          os: linux
+    ${{ else }}:
+      pool:
+        ${{ if eq(variables['System.TeamProject'], 'public') }}:
+          name: $[replace(replace(eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), True, 'NetCore-Svc-Public' ), False, 'NetCore-Public')]
+          demands: ImageOverride -equals Build.Ubuntu.2204.Amd64.Open
+        ${{ if eq(variables['System.TeamProject'], 'internal') }}:
+          name: $[replace(replace(eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), True, 'NetCore1ESPool-Svc-Internal'), False, 'NetCore1ESPool-Internal')]
+          demands: ImageOverride -equals Build.Ubuntu.2204.Amd64
+  ${{ if ne(parameters.platform.pool, '') }}:
+    pool: ${{ parameters.platform.pool }}
+
+  workspace:
+    clean: all
+
+  steps:
+  - ${{ if eq(parameters.is1ESPipeline, '') }}:
+    - 'Illegal entry point, is1ESPipeline is not defined. Repository yaml should not directly reference templates in core-templates folder.': error
+
+  - ${{ if eq(parameters.enableInternalSources, true) }}:
+    - template: /eng/common/core-templates/steps/enable-internal-sources.yml
+      parameters:
+        is1ESPipeline: ${{ parameters.is1ESPipeline }}
+    - template: /eng/common/core-templates/steps/enable-internal-runtimes.yml
+      parameters:
+        is1ESPipeline: ${{ parameters.is1ESPipeline }}
+  - template: /eng/common/core-templates/steps/source-build.yml
+    parameters:
+      is1ESPipeline: ${{ parameters.is1ESPipeline }}
+      platform: ${{ parameters.platform }}

eng/common/core-templates/job/source-index-stage1.yml

@@ -0,0 +1,81 @@
+parameters:
+  runAsPublic: false
+  sourceIndexUploadPackageVersion: 2.0.0-20240522.1
+  sourceIndexProcessBinlogPackageVersion: 1.0.1-20240522.1
+  sourceIndexPackageSource: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json
+  sourceIndexBuildCommand: powershell -NoLogo -NoProfile -ExecutionPolicy Bypass -Command "eng/common/build.ps1 -restore -build -binarylog -ci"
+  preSteps: []
+  binlogPath: artifacts/log/Debug/Build.binlog
+  condition: ''
+  dependsOn: ''
+  pool: ''
+  is1ESPipeline: ''
+
+jobs:
+- job: SourceIndexStage1
+  dependsOn: ${{ parameters.dependsOn }}
+  condition: ${{ parameters.condition }}
+  variables:
+  - name: SourceIndexUploadPackageVersion
+    value: ${{ parameters.sourceIndexUploadPackageVersion }}
+  - name: SourceIndexProcessBinlogPackageVersion
+    value: ${{ parameters.sourceIndexProcessBinlogPackageVersion }}
+  - name: SourceIndexPackageSource
+    value: ${{ parameters.sourceIndexPackageSource }}
+  - name: BinlogPath
+    value: ${{ parameters.binlogPath }}
+  - template: /eng/common/core-templates/variables/pool-providers.yml
+    parameters:
+      is1ESPipeline: ${{ parameters.is1ESPipeline }}
+
+  ${{ if ne(parameters.pool, '') }}:
+    pool: ${{ parameters.pool }}
+  ${{ if eq(parameters.pool, '') }}:
+    pool:
+      ${{ if eq(variables['System.TeamProject'], 'public') }}:
+        name: $(DncEngPublicBuildPool)
+        image: 1es-windows-2022-open
+        os: windows
+      ${{ if eq(variables['System.TeamProject'], 'internal') }}:
+        name: $(DncEngInternalBuildPool)
+        image: 1es-windows-2022
+        os: windows
+
+  steps:
+  - ${{ if eq(parameters.is1ESPipeline, '') }}:
+    - 'Illegal entry point, is1ESPipeline is not defined. Repository yaml should not directly reference templates in core-templates folder.': error
+
+  - ${{ each preStep in parameters.preSteps }}:
+    - ${{ preStep }}
+
+  - task: UseDotNet@2
+    displayName: Use .NET 8 SDK
+    inputs:
+      packageType: sdk
+      version: 8.0.x
+      installationPath: $(Agent.TempDirectory)/dotnet
+      workingDirectory: $(Agent.TempDirectory)
+
+  - script: |
+      $(Agent.TempDirectory)/dotnet/dotnet tool install BinLogToSln --version $(sourceIndexProcessBinlogPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools
+      $(Agent.TempDirectory)/dotnet/dotnet tool install UploadIndexStage1 --version $(sourceIndexUploadPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools
+    displayName: Download Tools
+    # Set working directory to temp directory so 'dotnet' doesn't try to use global.json and use the repo's sdk.
+    workingDirectory: $(Agent.TempDirectory)
+
+  - script: ${{ parameters.sourceIndexBuildCommand }}
+    displayName: Build Repository
+
+  - script: $(Agent.TempDirectory)/.source-index/tools/BinLogToSln -i $(BinlogPath) -r $(Build.SourcesDirectory) -n $(Build.Repository.Name) -o .source-index/stage1output
+    displayName: Process Binlog into indexable sln
+
+  - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
+    - task: AzureCLI@2
+      displayName: Log in to Azure and upload stage1 artifacts to source index
+      inputs:
+        azureSubscription: 'SourceDotNet Stage1 Publish'
+        addSpnToEnvironment: true
+        scriptType: 'ps'
+        scriptLocation: 'inlineScript'
+        inlineScript: |
+          $(Agent.TempDirectory)/.source-index/tools/UploadIndexStage1 -i .source-index/stage1output -n $(Build.Repository.Name) -s netsourceindexstage1 -b stage1

eng/common/core-templates/jobs/codeql-build.yml

@@ -0,0 +1,33 @@
+parameters:
+  # See schema documentation in /Documentation/AzureDevOps/TemplateSchema.md
+  continueOnError: false
+  # Required: A collection of jobs to run - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#job
+  jobs: []
+  # Optional: if specified, restore and use this version of Guardian instead of the default.
+  overrideGuardianVersion: ''
+  is1ESPipeline: ''
+
+jobs:
+- template: /eng/common/core-templates/jobs/jobs.yml
+  parameters:
+    is1ESPipeline: ${{ parameters.is1ESPipeline }}
+    enableMicrobuild: false
+    enablePublishBuildArtifacts: false
+    enablePublishTestResults: false
+    enablePublishBuildAssets: false
+    enablePublishUsingPipelines: false
+    enableTelemetry: true
+
+    variables:
+      - group: Publish-Build-Assets
+      # The Guardian version specified in 'eng/common/sdl/packages.config'. This value must be kept in
+      # sync with the packages.config file.
+      - name: DefaultGuardianVersion
+        value: 0.109.0
+      - name: GuardianPackagesConfigFile
+        value: $(Build.SourcesDirectory)\eng\common\sdl\packages.config
+      - name: GuardianVersion
+        value: ${{ coalesce(parameters.overrideGuardianVersion, '$(DefaultGuardianVersion)') }}
+  
+    jobs: ${{ parameters.jobs }}
+        

eng/common/core-templates/jobs/jobs.yml

@@ -0,0 +1,119 @@
+parameters:
+  # See schema documentation in /Documentation/AzureDevOps/TemplateSchema.md
+  continueOnError: false
+
+  # Optional: Include PublishBuildArtifacts task
+  enablePublishBuildArtifacts: false
+
+  # Optional: Enable publishing using release pipelines
+  enablePublishUsingPipelines: false
+
+  # Optional: Enable running the source-build jobs to build repo from source
+  enableSourceBuild: false
+
+  # Optional: Parameters for source-build template.
+  #           See /eng/common/core-templates/jobs/source-build.yml for options
+  sourceBuildParameters: []
+
+  graphFileGeneration:
+    # Optional: Enable generating the graph files at the end of the build
+    enabled: false
+    # Optional: Include toolset dependencies in the generated graph files
+    includeToolset: false
+    
+  # Required: A collection of jobs to run - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#job
+  jobs: []
+
+  # Optional: Override automatically derived dependsOn value for "publish build assets" job
+  publishBuildAssetsDependsOn: ''
+
+  # Optional: Publish the assets as soon as the publish to BAR stage is complete, rather doing so in a separate stage.
+  publishAssetsImmediately: false
+
+  # Optional: If using publishAssetsImmediately and additional parameters are needed, can be used to send along additional parameters (normally sent to post-build.yml)
+  artifactsPublishingAdditionalParameters: ''
+  signingValidationAdditionalParameters: ''
+
+  # Optional: should run as a public build even in the internal project
+  #           if 'true', the build won't run any of the internal only steps, even if it is running in non-public projects.
+  runAsPublic: false
+
+  enableSourceIndex: false
+  sourceIndexParams: {}
+
+  artifacts: {}
+  is1ESPipeline: ''
+
+# Internal resources (telemetry, microbuild) can only be accessed from non-public projects,
+# and some (Microbuild) should only be applied to non-PR cases for internal builds.
+
+jobs:
+- ${{ each job in parameters.jobs }}:
+  - ${{ if eq(parameters.is1ESPipeline, 'true') }}:
+    - template: /eng/common/templates-official/job/job.yml
+      parameters: 
+        # pass along parameters
+        ${{ each parameter in parameters }}:
+          ${{ if ne(parameter.key, 'jobs') }}:
+            ${{ parameter.key }}: ${{ parameter.value }}
+
+        # pass along job properties
+        ${{ each property in job }}:
+          ${{ if ne(property.key, 'job') }}:
+            ${{ property.key }}: ${{ property.value }}
+
+        name: ${{ job.job }}
+
+  - ${{ else }}:
+    - template: /eng/common/templates/job/job.yml
+      parameters: 
+        # pass along parameters
+        ${{ each parameter in parameters }}:
+          ${{ if ne(parameter.key, 'jobs') }}:
+            ${{ parameter.key }}: ${{ parameter.value }}
+
+        # pass along job properties
+        ${{ each property in job }}:
+          ${{ if ne(property.key, 'job') }}:
+            ${{ property.key }}: ${{ property.value }}
+
+        name: ${{ job.job }}
+
+- ${{ if eq(parameters.enableSourceBuild, true) }}:
+  - template: /eng/common/core-templates/jobs/source-build.yml
+    parameters:
+      is1ESPipeline: ${{ parameters.is1ESPipeline }}
+      allCompletedJobId: Source_Build_Complete
+      ${{ each parameter in parameters.sourceBuildParameters }}:
+        ${{ parameter.key }}: ${{ parameter.value }}
+
+- ${{ if eq(parameters.enableSourceIndex, 'true') }}:
+  - template: ../job/source-index-stage1.yml
+    parameters:
+      is1ESPipeline: ${{ parameters.is1ESPipeline }}
+      runAsPublic: ${{ parameters.runAsPublic }}
+      ${{ each parameter in parameters.sourceIndexParams }}:
+        ${{ parameter.key }}: ${{ parameter.value }}
+
+- ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
+  - ${{ if or(eq(parameters.enablePublishBuildAssets, true), eq(parameters.artifacts.publish.manifests, 'true'), ne(parameters.artifacts.publish.manifests, '')) }}:
+    - template: ../job/publish-build-assets.yml
+      parameters:
+        is1ESPipeline: ${{ parameters.is1ESPipeline }}
+        continueOnError: ${{ parameters.continueOnError }}
+        dependsOn:
+        - ${{ if ne(parameters.publishBuildAssetsDependsOn, '') }}:
+          - ${{ each job in parameters.publishBuildAssetsDependsOn }}:
+            - ${{ job.job }}
+        - ${{ if eq(parameters.publishBuildAssetsDependsOn, '') }}:
+          - ${{ each job in parameters.jobs }}:
+            - ${{ job.job }}
+        - ${{ if eq(parameters.enableSourceBuild, true) }}:
+          - Source_Build_Complete
+
+        runAsPublic: ${{ parameters.runAsPublic }}
+        publishUsingPipelines: ${{ parameters.enablePublishUsingPipelines }}
+        publishAssetsImmediately: ${{ parameters.publishAssetsImmediately }}
+        enablePublishBuildArtifacts: ${{ parameters.enablePublishBuildArtifacts }}
+        artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
+        signingValidationAdditionalParameters: ${{ parameters.signingValidationAdditionalParameters }}

eng/common/core-templates/jobs/source-build.yml

@@ -0,0 +1,58 @@
+parameters:
+  # This template adds arcade-powered source-build to CI. A job is created for each platform, as
+  # well as an optional server job that completes when all platform jobs complete.
+
+  # The name of the "join" job for all source-build platforms. If set to empty string, the job is
+  # not included. Existing repo pipelines can use this job depend on all source-build jobs
+  # completing without maintaining a separate list of every single job ID: just depend on this one
+  # server job. By default, not included. Recommended name if used: 'Source_Build_Complete'.
+  allCompletedJobId: ''
+
+  # See /eng/common/core-templates/job/source-build.yml
+  jobNamePrefix: 'Source_Build'
+
+  # This is the default platform provided by Arcade, intended for use by a managed-only repo.
+  defaultManagedPlatform:
+    name: 'Managed'
+    container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream9'
+
+  # Defines the platforms on which to run build jobs. One job is created for each platform, and the
+  # object in this array is sent to the job template as 'platform'. If no platforms are specified,
+  # one job runs on 'defaultManagedPlatform'.
+  platforms: []
+
+  is1ESPipeline: ''
+
+  # If set to true and running on a non-public project,
+  # Internal nuget and blob storage locations will be enabled.
+  # This is not enabled by default because many repositories do not need internal sources
+  # and do not need to have the required service connections approved in the pipeline.
+  enableInternalSources: false
+
+jobs:
+
+- ${{ if ne(parameters.allCompletedJobId, '') }}:
+  - job: ${{ parameters.allCompletedJobId }}
+    displayName: Source-Build Complete
+    pool: server
+    dependsOn:
+    - ${{ each platform in parameters.platforms }}:
+      - ${{ parameters.jobNamePrefix }}_${{ platform.name }}
+    - ${{ if eq(length(parameters.platforms), 0) }}:
+      - ${{ parameters.jobNamePrefix }}_${{ parameters.defaultManagedPlatform.name }}
+
+- ${{ each platform in parameters.platforms }}:
+  - template: /eng/common/core-templates/job/source-build.yml
+    parameters:
+      is1ESPipeline: ${{ parameters.is1ESPipeline }}
+      jobNamePrefix: ${{ parameters.jobNamePrefix }}
+      platform: ${{ platform }}
+      enableInternalSources: ${{ parameters.enableInternalSources }}
+
+- ${{ if eq(length(parameters.platforms), 0) }}:
+  - template: /eng/common/core-templates/job/source-build.yml
+    parameters:
+      is1ESPipeline: ${{ parameters.is1ESPipeline }}
+      jobNamePrefix: ${{ parameters.jobNamePrefix }}
+      platform: ${{ parameters.defaultManagedPlatform }}
+      enableInternalSources: ${{ parameters.enableInternalSources }}

eng/common/core-templates/post-build/common-variables.yml

@@ -0,0 +1,22 @@
+variables:
+  - group: Publish-Build-Assets
+
+  # Whether the build is internal or not
+  - name: IsInternalBuild
+    value: ${{ and(ne(variables['System.TeamProject'], 'public'), contains(variables['Build.SourceBranch'], 'internal')) }}
+
+  # Default Maestro++ API Endpoint and API Version
+  - name: MaestroApiEndPoint
+    value: "https://maestro.dot.net"
+  - name: MaestroApiVersion
+    value: "2020-02-20"
+
+  - name: SourceLinkCLIVersion
+    value: 3.0.0
+  - name: SymbolToolVersion
+    value: 1.0.1
+  - name: BinlogToolVersion
+    value: 1.0.11
+
+  - name: runCodesignValidationInjection
+    value: false

eng/common/core-templates/post-build/post-build.yml

@@ -0,0 +1,316 @@
+parameters:
+  # Which publishing infra should be used. THIS SHOULD MATCH THE VERSION ON THE BUILD MANIFEST.
+  # Publishing V1 is no longer supported
+  # Publishing V2 is no longer supported
+  # Publishing V3 is the default
+  - name: publishingInfraVersion
+    displayName: Which version of publishing should be used to promote the build definition?
+    type: number
+    default: 3
+    values:
+    - 3
+
+  - name: BARBuildId
+    displayName: BAR Build Id
+    type: number
+    default: 0
+
+  - name: PromoteToChannelIds
+    displayName: Channel to promote BARBuildId to
+    type: string
+    default: ''
+
+  - name: enableSourceLinkValidation
+    displayName: Enable SourceLink validation
+    type: boolean
+    default: false
+
+  - name: enableSigningValidation
+    displayName: Enable signing validation
+    type: boolean
+    default: true
+
+  - name: enableSymbolValidation
+    displayName: Enable symbol validation
+    type: boolean
+    default: false
+
+  - name: enableNugetValidation
+    displayName: Enable NuGet validation
+    type: boolean
+    default: true
+    
+  - name: publishInstallersAndChecksums
+    displayName: Publish installers and checksums
+    type: boolean
+    default: true
+
+  - name: SDLValidationParameters
+    type: object
+    default:
+      enable: false
+      publishGdn: false
+      continueOnError: false
+      params: ''
+      artifactNames: ''
+      downloadArtifacts: true
+
+  # These parameters let the user customize the call to sdk-task.ps1 for publishing
+  # symbols & general artifacts as well as for signing validation
+  - name: symbolPublishingAdditionalParameters
+    displayName: Symbol publishing additional parameters
+    type: string
+    default: ''
+
+  - name: artifactsPublishingAdditionalParameters
+    displayName: Artifact publishing additional parameters
+    type: string
+    default: ''
+
+  - name: signingValidationAdditionalParameters
+    displayName: Signing validation additional parameters
+    type: string
+    default: ''
+
+  # Which stages should finish execution before post-build stages start
+  - name: validateDependsOn
+    type: object
+    default:
+    - build
+
+  - name: publishDependsOn
+    type: object
+    default:
+    - Validate
+
+  # Optional: Call asset publishing rather than running in a separate stage
+  - name: publishAssetsImmediately
+    type: boolean
+    default: false
+
+  - name: is1ESPipeline
+    type: boolean
+    default: false
+
+stages:
+- ${{ if or(eq( parameters.enableNugetValidation, 'true'), eq(parameters.enableSigningValidation, 'true'), eq(parameters.enableSourceLinkValidation, 'true'), eq(parameters.SDLValidationParameters.enable, 'true')) }}:
+  - stage: Validate
+    dependsOn: ${{ parameters.validateDependsOn }}
+    displayName: Validate Build Assets
+    variables:
+      - template: /eng/common/core-templates/post-build/common-variables.yml
+      - template: /eng/common/core-templates/variables/pool-providers.yml
+        parameters:
+          is1ESPipeline: ${{ parameters.is1ESPipeline }}
+    jobs:
+    - job:
+      displayName: NuGet Validation
+      condition: and(succeededOrFailed(), eq( ${{ parameters.enableNugetValidation }}, 'true'))
+      pool:
+        # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com)
+        ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}:
+          name: AzurePipelines-EO
+          image: 1ESPT-Windows2022
+          demands: Cmd
+          os: windows
+        # If it's not devdiv, it's dnceng
+        ${{ else }}:
+          ${{ if eq(parameters.is1ESPipeline, true) }}:
+            name: $(DncEngInternalBuildPool)
+            image: windows.vs2022.amd64
+            os: windows
+          ${{ else }}:
+            name: $(DncEngInternalBuildPool)
+            demands: ImageOverride -equals windows.vs2022.amd64
+
+      steps:
+        - template: /eng/common/core-templates/post-build/setup-maestro-vars.yml
+          parameters:
+            BARBuildId: ${{ parameters.BARBuildId }}
+            PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
+            is1ESPipeline: ${{ parameters.is1ESPipeline }}
+
+        - task: DownloadBuildArtifacts@0
+          displayName: Download Package Artifacts
+          inputs:
+            buildType: specific
+            buildVersionToDownload: specific
+            project: $(AzDOProjectName)
+            pipeline: $(AzDOPipelineId)
+            buildId: $(AzDOBuildId)
+            artifactName: PackageArtifacts
+            checkDownloadedFiles: true
+
+        - task: PowerShell@2
+          displayName: Validate
+          inputs:
+            filePath: $(Build.SourcesDirectory)/eng/common/post-build/nuget-validation.ps1
+            arguments: -PackagesPath $(Build.ArtifactStagingDirectory)/PackageArtifacts/
+
+    - job:
+      displayName: Signing Validation
+      condition: and( eq( ${{ parameters.enableSigningValidation }}, 'true'), ne( variables['PostBuildSign'], 'true'))
+      pool:
+        # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com)
+        ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}:
+          name: AzurePipelines-EO
+          image: 1ESPT-Windows2022
+          demands: Cmd
+          os: windows
+        # If it's not devdiv, it's dnceng
+        ${{ else }}:
+          ${{ if eq(parameters.is1ESPipeline, true) }}:        
+            name: $(DncEngInternalBuildPool)
+            image: 1es-windows-2022
+            os: windows
+          ${{ else }}:
+            name: $(DncEngInternalBuildPool)
+            demands: ImageOverride -equals windows.vs2022.amd64          
+      steps:
+        - template: /eng/common/core-templates/post-build/setup-maestro-vars.yml
+          parameters:
+            BARBuildId: ${{ parameters.BARBuildId }}
+            PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
+            is1ESPipeline: ${{ parameters.is1ESPipeline }}
+
+        - task: DownloadBuildArtifacts@0
+          displayName: Download Package Artifacts
+          inputs:
+            buildType: specific
+            buildVersionToDownload: specific
+            project: $(AzDOProjectName)
+            pipeline: $(AzDOPipelineId)
+            buildId: $(AzDOBuildId)
+            artifactName: PackageArtifacts
+            checkDownloadedFiles: true
+            itemPattern: |
+              **
+              !**/Microsoft.SourceBuild.Intermediate.*.nupkg
+
+        # This is necessary whenever we want to publish/restore to an AzDO private feed
+        # Since sdk-task.ps1 tries to restore packages we need to do this authentication here
+        # otherwise it'll complain about accessing a private feed.
+        - task: NuGetAuthenticate@1
+          displayName: 'Authenticate to AzDO Feeds'
+
+        # Signing validation will optionally work with the buildmanifest file which is downloaded from
+        # Azure DevOps above.
+        - task: PowerShell@2
+          displayName: Validate
+          inputs:
+            filePath: eng\common\sdk-task.ps1
+            arguments: -task SigningValidation -restore -msbuildEngine vs
+              /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts'
+              /p:SignCheckExclusionsFile='$(Build.SourcesDirectory)/eng/SignCheckExclusionsFile.txt'
+              ${{ parameters.signingValidationAdditionalParameters }}
+
+        - template: /eng/common/core-templates/steps/publish-logs.yml
+          parameters:
+            is1ESPipeline: ${{ parameters.is1ESPipeline }}
+            StageLabel: 'Validation'
+            JobLabel: 'Signing'
+            BinlogToolVersion: $(BinlogToolVersion)
+
+    - job:
+      displayName: SourceLink Validation
+      condition: eq( ${{ parameters.enableSourceLinkValidation }}, 'true')
+      pool:
+        # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com)
+        ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}:
+          name: AzurePipelines-EO
+          image: 1ESPT-Windows2022
+          demands: Cmd
+          os: windows
+        # If it's not devdiv, it's dnceng
+        ${{ else }}:
+          ${{ if eq(parameters.is1ESPipeline, true) }}:          
+            name: $(DncEngInternalBuildPool)
+            image: 1es-windows-2022
+            os: windows
+          ${{ else }}:
+            name: $(DncEngInternalBuildPool)
+            demands: ImageOverride -equals windows.vs2022.amd64          
+      steps:
+        - template: /eng/common/core-templates/post-build/setup-maestro-vars.yml
+          parameters:
+            BARBuildId: ${{ parameters.BARBuildId }}
+            PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
+            is1ESPipeline: ${{ parameters.is1ESPipeline }}
+
+        - task: DownloadBuildArtifacts@0
+          displayName: Download Blob Artifacts
+          inputs:
+            buildType: specific
+            buildVersionToDownload: specific
+            project: $(AzDOProjectName)
+            pipeline: $(AzDOPipelineId)
+            buildId: $(AzDOBuildId)
+            artifactName: BlobArtifacts
+            checkDownloadedFiles: true
+
+        - task: PowerShell@2
+          displayName: Validate
+          inputs:
+            filePath: $(Build.SourcesDirectory)/eng/common/post-build/sourcelink-validation.ps1
+            arguments: -InputPath $(Build.ArtifactStagingDirectory)/BlobArtifacts/ 
+              -ExtractPath $(Agent.BuildDirectory)/Extract/ 
+              -GHRepoName $(Build.Repository.Name) 
+              -GHCommit $(Build.SourceVersion)
+              -SourcelinkCliVersion $(SourceLinkCLIVersion)
+          continueOnError: true
+
+- ${{ if ne(parameters.publishAssetsImmediately, 'true') }}:
+  - stage: publish_using_darc
+    ${{ if or(eq(parameters.enableNugetValidation, 'true'), eq(parameters.enableSigningValidation, 'true'), eq(parameters.enableSourceLinkValidation, 'true'), eq(parameters.SDLValidationParameters.enable, 'true')) }}:
+      dependsOn: ${{ parameters.publishDependsOn }}
+    ${{ else }}:
+      dependsOn: ${{ parameters.validateDependsOn }}
+    displayName: Publish using Darc
+    variables:
+      - template: /eng/common/core-templates/post-build/common-variables.yml
+      - template: /eng/common/core-templates/variables/pool-providers.yml
+        parameters:
+          is1ESPipeline: ${{ parameters.is1ESPipeline }}
+    jobs:
+    - job:
+      displayName: Publish Using Darc
+      timeoutInMinutes: 120
+      pool:
+        # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com)
+        ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}:
+          name: AzurePipelines-EO
+          image: 1ESPT-Windows2022
+          demands: Cmd
+          os: windows
+        # If it's not devdiv, it's dnceng
+        ${{ else }}:
+          ${{ if eq(parameters.is1ESPipeline, true) }}:          
+            name: NetCore1ESPool-Publishing-Internal
+            image: windows.vs2019.amd64
+            os: windows
+          ${{ else }}:
+            name: NetCore1ESPool-Publishing-Internal
+            demands: ImageOverride -equals windows.vs2019.amd64          
+      steps:
+        - template: /eng/common/core-templates/post-build/setup-maestro-vars.yml
+          parameters:
+            BARBuildId: ${{ parameters.BARBuildId }}
+            PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
+            is1ESPipeline: ${{ parameters.is1ESPipeline }}
+
+        - task: NuGetAuthenticate@1
+
+        - task: AzureCLI@2
+          displayName: Publish Using Darc
+          inputs:
+            azureSubscription: "Darc: Maestro Production"
+            scriptType: ps
+            scriptLocation: scriptPath
+            scriptPath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1
+            arguments: >
+              -BuildId $(BARBuildId)
+              -PublishingInfraVersion ${{ parameters.publishingInfraVersion }}
+              -AzdoToken '$(System.AccessToken)'
+              -WaitPublishingFinish true
+              -ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}'
+              -SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}'

eng/common/core-templates/post-build/setup-maestro-vars.yml

@@ -0,0 +1,74 @@
+parameters:
+  BARBuildId: ''
+  PromoteToChannelIds: ''
+  is1ESPipeline: ''
+
+steps:
+  - ${{ if eq(parameters.is1ESPipeline, '') }}:
+    - 'Illegal entry point, is1ESPipeline is not defined. Repository yaml should not directly reference templates in core-templates folder.': error
+
+  - ${{ if eq(coalesce(parameters.PromoteToChannelIds, 0), 0) }}:
+    - task: DownloadBuildArtifacts@0
+      displayName: Download Release Configs
+      inputs:
+        buildType: current
+        artifactName: ReleaseConfigs
+        checkDownloadedFiles: true
+
+  - task: AzureCLI@2
+    name: setReleaseVars
+    displayName: Set Release Configs Vars
+    inputs:
+      azureSubscription: "Darc: Maestro Production"
+      scriptType: pscore
+      scriptLocation: inlineScript
+      inlineScript: |
+        try {
+          if (!$Env:PromoteToMaestroChannels -or $Env:PromoteToMaestroChannels.Trim() -eq '') {
+            $Content = Get-Content $(Build.StagingDirectory)/ReleaseConfigs/ReleaseConfigs.txt
+
+            $BarId = $Content | Select -Index 0
+            $Channels = $Content | Select -Index 1
+            $IsStableBuild = $Content | Select -Index 2
+
+            $AzureDevOpsProject = $Env:System_TeamProject
+            $AzureDevOpsBuildDefinitionId = $Env:System_DefinitionId
+            $AzureDevOpsBuildId = $Env:Build_BuildId
+          }
+          else {
+            . $(Build.SourcesDirectory)\eng\common\tools.ps1
+            $darc = Get-Darc
+            $buildInfo = & $darc get-build `
+              --id ${{ parameters.BARBuildId }} `
+              --extended `
+              --output-format json `
+              --ci `
+              | convertFrom-Json
+
+            $BarId = ${{ parameters.BARBuildId }}
+            $Channels = $Env:PromoteToMaestroChannels -split ","
+            $Channels = $Channels -join "]["
+            $Channels = "[$Channels]"
+
+            $IsStableBuild = $buildInfo.stable
+            $AzureDevOpsProject = $buildInfo.azureDevOpsProject
+            $AzureDevOpsBuildDefinitionId = $buildInfo.azureDevOpsBuildDefinitionId
+            $AzureDevOpsBuildId = $buildInfo.azureDevOpsBuildId
+          }
+
+          Write-Host "##vso[task.setvariable variable=BARBuildId]$BarId"
+          Write-Host "##vso[task.setvariable variable=TargetChannels]$Channels"
+          Write-Host "##vso[task.setvariable variable=IsStableBuild]$IsStableBuild"
+
+          Write-Host "##vso[task.setvariable variable=AzDOProjectName]$AzureDevOpsProject"
+          Write-Host "##vso[task.setvariable variable=AzDOPipelineId]$AzureDevOpsBuildDefinitionId"
+          Write-Host "##vso[task.setvariable variable=AzDOBuildId]$AzureDevOpsBuildId"
+        }
+        catch {
+          Write-Host $_
+          Write-Host $_.Exception
+          Write-Host $_.ScriptStackTrace
+          exit 1
+        }
+    env:
+      PromoteToMaestroChannels: ${{ parameters.PromoteToChannelIds }}

eng/common/core-templates/steps/component-governance.yml

@@ -0,0 +1,16 @@
+parameters:
+  disableComponentGovernance: false
+  componentGovernanceIgnoreDirectories: ''
+  is1ESPipeline: false
+  displayName: 'Component Detection'
+
+steps:
+- ${{ if eq(parameters.disableComponentGovernance, 'true') }}:
+  - script: echo "##vso[task.setvariable variable=skipComponentGovernanceDetection]true"
+    displayName: Set skipComponentGovernanceDetection variable
+- ${{ if ne(parameters.disableComponentGovernance, 'true') }}:
+  - task: ComponentGovernanceComponentDetection@0
+    continueOnError: true
+    displayName: ${{ parameters.displayName }}
+    inputs:
+      ignoreDirectories: ${{ parameters.componentGovernanceIgnoreDirectories }}

eng/common/core-templates/steps/enable-internal-runtimes.yml

@@ -0,0 +1,32 @@
+# Obtains internal runtime download credentials and populates the 'dotnetbuilds-internal-container-read-token-base64'
+# variable with the base64-encoded SAS token, by default
+
+parameters:
+- name: federatedServiceConnection
+  type: string
+  default: 'dotnetbuilds-internal-read'
+- name: outputVariableName
+  type: string
+  default: 'dotnetbuilds-internal-container-read-token-base64'
+- name: expiryInHours
+  type: number
+  default: 1
+- name: base64Encode
+  type: boolean
+  default: true
+- name: is1ESPipeline
+  type: boolean
+  default: false
+
+steps:
+- ${{ if ne(variables['System.TeamProject'], 'public') }}:
+  - template: /eng/common/core-templates/steps/get-delegation-sas.yml
+    parameters:
+      federatedServiceConnection: ${{ parameters.federatedServiceConnection }}
+      outputVariableName: ${{ parameters.outputVariableName }}
+      expiryInHours: ${{ parameters.expiryInHours }}
+      base64Encode: ${{ parameters.base64Encode }}
+      storageAccount: dotnetbuilds
+      container: internal
+      permissions: rl
+      is1ESPipeline: ${{ parameters.is1ESPipeline }}

eng/common/core-templates/steps/enable-internal-sources.yml

@@ -0,0 +1,47 @@
+parameters:
+# This is the Azure federated service connection that we log into to get an access token.
+- name: nugetFederatedServiceConnection
+  type: string
+  default: 'dnceng-artifacts-feeds-read'
+- name: is1ESPipeline
+  type: boolean
+  default: false
+# Legacy parameters to allow for PAT usage
+- name: legacyCredential
+  type: string
+  default: ''
+
+steps:
+- ${{ if ne(variables['System.TeamProject'], 'public') }}:
+  - ${{ if ne(parameters.legacyCredential, '') }}:
+    - task: PowerShell@2
+      displayName: Setup Internal Feeds
+      inputs:
+        filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.ps1
+        arguments: -ConfigFile $(Build.SourcesDirectory)/NuGet.config -Password $Env:Token
+      env:
+        Token: ${{ parameters.legacyCredential }}
+  # If running on dnceng (internal project), just use the default behavior for NuGetAuthenticate.
+  # If running on DevDiv, NuGetAuthenticate is not really an option. It's scoped to a single feed, and we have many feeds that
+  # may be added. Instead, we'll use the traditional approach (add cred to nuget.config), but use an account token.
+  - ${{ else }}:
+    - ${{ if eq(variables['System.TeamProject'], 'internal') }}:
+      - task: PowerShell@2
+        displayName: Setup Internal Feeds
+        inputs:
+          filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.ps1
+          arguments: -ConfigFile $(Build.SourcesDirectory)/NuGet.config
+    - ${{ else }}:
+      - template: /eng/common/templates/steps/get-federated-access-token.yml
+        parameters:
+          federatedServiceConnection: ${{ parameters.nugetFederatedServiceConnection }}
+          outputVariableName: 'dnceng-artifacts-feeds-read-access-token'
+      - task: PowerShell@2
+        displayName: Setup Internal Feeds
+        inputs:
+          filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.ps1
+          arguments: -ConfigFile $(Build.SourcesDirectory)/NuGet.config -Password $(dnceng-artifacts-feeds-read-access-token)
+  # This is required in certain scenarios to install the ADO credential provider.
+  # It installed by default in some msbuild invocations (e.g. VS msbuild), but needs to be installed for others
+  # (e.g. dotnet msbuild).
+  - task: NuGetAuthenticate@1

eng/common/core-templates/steps/generate-sbom.yml

@@ -0,0 +1,54 @@
+# BuildDropPath - The root folder of the drop directory for which the manifest file will be generated.
+# PackageName - The name of the package this SBOM represents.
+# PackageVersion - The version of the package this SBOM represents. 
+# ManifestDirPath - The path of the directory where the generated manifest files will be placed
+# IgnoreDirectories - Directories to ignore for SBOM generation. This will be passed through to the CG component detector.
+
+parameters:
+  PackageVersion: 9.0.0
+  BuildDropPath: '$(Build.SourcesDirectory)/artifacts'
+  PackageName: '.NET'
+  ManifestDirPath: $(Build.ArtifactStagingDirectory)/sbom
+  IgnoreDirectories: ''
+  sbomContinueOnError: true
+  is1ESPipeline: false
+  # disable publishArtifacts if some other step is publishing the artifacts (like job.yml).
+  publishArtifacts: true
+
+steps:
+- task: PowerShell@2 
+  displayName: Prep for SBOM generation in (Non-linux)
+  condition: or(eq(variables['Agent.Os'], 'Windows_NT'), eq(variables['Agent.Os'], 'Darwin'))
+  inputs: 
+    filePath: ./eng/common/generate-sbom-prep.ps1
+    arguments: ${{parameters.manifestDirPath}}
+
+# Chmodding is a workaround for https://github.com/dotnet/arcade/issues/8461
+- script: |
+    chmod +x ./eng/common/generate-sbom-prep.sh
+    ./eng/common/generate-sbom-prep.sh ${{parameters.manifestDirPath}}
+  displayName: Prep for SBOM generation in (Linux)
+  condition: eq(variables['Agent.Os'], 'Linux')
+  continueOnError: ${{ parameters.sbomContinueOnError }}
+
+- task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0
+  displayName: 'Generate SBOM manifest'
+  continueOnError: ${{ parameters.sbomContinueOnError }}
+  inputs:
+      PackageName: ${{ parameters.packageName }}
+      BuildDropPath: ${{ parameters.buildDropPath }}
+      PackageVersion: ${{ parameters.packageVersion }}
+      ManifestDirPath: ${{ parameters.manifestDirPath }}
+      ${{ if ne(parameters.IgnoreDirectories, '') }}:
+        AdditionalComponentDetectorArgs: '--IgnoreDirectories ${{ parameters.IgnoreDirectories }}'
+
+- ${{ if eq(parameters.publishArtifacts, 'true')}}:
+  - template: /eng/common/core-templates/steps/publish-pipeline-artifacts.yml
+    parameters:
+      is1ESPipeline: ${{ parameters.is1ESPipeline }}
+      args:
+        displayName: Publish SBOM manifest
+        continueOnError: ${{parameters.sbomContinueOnError}}
+        targetPath: '${{ parameters.manifestDirPath }}'
+        artifactName: $(ARTIFACT_NAME)
+

eng/common/core-templates/steps/get-delegation-sas.yml

@@ -0,0 +1,46 @@
+parameters:
+- name: federatedServiceConnection
+  type: string
+- name: outputVariableName
+  type: string
+- name: expiryInHours
+  type: number
+  default: 1
+- name: base64Encode
+  type: boolean
+  default: false
+- name: storageAccount
+  type: string
+- name: container
+  type: string
+- name: permissions
+  type: string
+  default: 'rl'
+- name: is1ESPipeline
+  type: boolean
+  default: false
+
+steps:
+- task: AzureCLI@2
+  displayName: 'Generate delegation SAS Token for ${{ parameters.storageAccount }}/${{ parameters.container }}'
+  inputs:
+    azureSubscription: ${{ parameters.federatedServiceConnection }}
+    scriptType: 'pscore'
+    scriptLocation: 'inlineScript'
+    inlineScript: |
+      # Calculate the expiration of the SAS token and convert to UTC
+      $expiry = (Get-Date).AddHours(${{ parameters.expiryInHours }}).ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ssZ")
+
+      $sas = az storage container generate-sas --account-name ${{ parameters.storageAccount }} --name ${{ parameters.container }} --permissions ${{ parameters.permissions }} --expiry $expiry --auth-mode login --as-user -o tsv
+
+      if ($LASTEXITCODE -ne 0) {
+        Write-Error "Failed to generate SAS token."
+        exit 1
+      }
+
+      if ('${{ parameters.base64Encode }}' -eq 'true') {
+        $sas = [Convert]::ToBase64String([System.Text.Encoding]::UTF8.GetBytes($sas))
+      }
+
+      Write-Host "Setting '${{ parameters.outputVariableName }}' with the access token value"
+      Write-Host "##vso[task.setvariable variable=${{ parameters.outputVariableName }};issecret=true]$sas"

eng/common/core-templates/steps/get-federated-access-token.yml

@@ -0,0 +1,42 @@
+parameters:
+- name: federatedServiceConnection
+  type: string
+- name: outputVariableName
+  type: string
+- name: is1ESPipeline
+  type: boolean
+- name: stepName
+  type: string
+  default: 'getFederatedAccessToken'
+- name: condition
+  type: string
+  default: ''
+# Resource to get a token for. Common values include:
+# - '499b84ac-1321-427f-aa17-267ca6975798' for Azure DevOps
+# - 'https://storage.azure.com/' for storage
+# Defaults to Azure DevOps
+- name: resource
+  type: string
+  default: '499b84ac-1321-427f-aa17-267ca6975798'
+- name: isStepOutputVariable
+  type: boolean
+  default: false
+
+steps:
+- task: AzureCLI@2
+  displayName: 'Getting federated access token for feeds'
+  name: ${{ parameters.stepName }}
+  ${{ if ne(parameters.condition, '') }}:
+    condition: ${{ parameters.condition }}
+  inputs:
+    azureSubscription: ${{ parameters.federatedServiceConnection }}
+    scriptType: 'pscore'
+    scriptLocation: 'inlineScript'
+    inlineScript: |
+      $accessToken = az account get-access-token --query accessToken --resource ${{ parameters.resource }} --output tsv
+      if ($LASTEXITCODE -ne 0) {
+        Write-Error "Failed to get access token for resource '${{ parameters.resource }}'"
+        exit 1
+      }
+      Write-Host "Setting '${{ parameters.outputVariableName }}' with the access token value"
+      Write-Host "##vso[task.setvariable variable=${{ parameters.outputVariableName }};issecret=true;isOutput=${{ parameters.isStepOutputVariable }}]$accessToken"

eng/common/core-templates/steps/publish-build-artifacts.yml

@@ -0,0 +1,20 @@
+parameters:
+- name: is1ESPipeline
+  type: boolean
+  default: false
+- name: args
+  type: object
+  default: {}
+steps:
+- ${{ if ne(parameters.is1ESPipeline, true) }}:
+  - template: /eng/common/templates/steps/publish-build-artifacts.yml
+    parameters:
+      is1ESPipeline: ${{ parameters.is1ESPipeline }}
+      ${{ each parameter in parameters.args }}:
+        ${{ parameter.key }}: ${{ parameter.value }}
+- ${{ else }}:
+  - template: /eng/common/templates-official/steps/publish-build-artifacts.yml
+    parameters:
+      is1ESPipeline: ${{ parameters.is1ESPipeline }}
+      ${{ each parameter in parameters.args }}:
+        ${{ parameter.key }}: ${{ parameter.value }}

eng/common/core-templates/steps/publish-logs.yml

@@ -0,0 +1,58 @@
+parameters:
+  StageLabel: ''
+  JobLabel: ''
+  CustomSensitiveDataList: ''
+  # A default - in case value from eng/common/core-templates/post-build/common-variables.yml is not passed
+  BinlogToolVersion: '1.0.11'
+  is1ESPipeline: false
+
+steps:
+- task: Powershell@2
+  displayName: Prepare Binlogs to Upload
+  inputs:
+    targetType: inline
+    script: |
+      New-Item -ItemType Directory $(Build.SourcesDirectory)/PostBuildLogs/${{parameters.StageLabel}}/${{parameters.JobLabel}}/
+      Move-Item -Path $(Build.SourcesDirectory)/artifacts/log/Debug/* $(Build.SourcesDirectory)/PostBuildLogs/${{parameters.StageLabel}}/${{parameters.JobLabel}}/
+  continueOnError: true
+  condition: always()
+    
+- task: PowerShell@2
+  displayName: Redact Logs
+  inputs:
+    filePath: $(Build.SourcesDirectory)/eng/common/post-build/redact-logs.ps1
+    # For now this needs to have explicit list of all sensitive data. Taken from eng/publishing/v3/publish.yml
+    # Sensitive data can as well be added to $(Build.SourcesDirectory)/eng/BinlogSecretsRedactionFile.txt'
+    #  If the file exists - sensitive data for redaction will be sourced from it
+    #  (single entry per line, lines starting with '# ' are considered comments and skipped)
+    arguments: -InputPath '$(Build.SourcesDirectory)/PostBuildLogs' 
+      -BinlogToolVersion ${{parameters.BinlogToolVersion}}
+      -TokensFilePath '$(Build.SourcesDirectory)/eng/BinlogSecretsRedactionFile.txt'
+      '$(publishing-dnceng-devdiv-code-r-build-re)'
+      '$(MaestroAccessToken)'
+      '$(dn-bot-all-orgs-artifact-feeds-rw)'
+      '$(akams-client-id)'
+      '$(microsoft-symbol-server-pat)'
+      '$(symweb-symbol-server-pat)'
+      '$(dn-bot-all-orgs-build-rw-code-rw)'
+      ${{parameters.CustomSensitiveDataList}}
+  continueOnError: true
+  condition: always()
+
+- task: CopyFiles@2
+  displayName: Gather post build logs
+  inputs:
+    SourceFolder: '$(Build.SourcesDirectory)/PostBuildLogs'
+    Contents: '**'
+    TargetFolder: '$(Build.ArtifactStagingDirectory)/PostBuildLogs'
+
+- template: /eng/common/core-templates/steps/publish-build-artifacts.yml
+  parameters:
+    is1ESPipeline: ${{ parameters.is1ESPipeline }}
+    args:
+      displayName: Publish Logs
+      pathToPublish: '$(Build.ArtifactStagingDirectory)/PostBuildLogs'
+      publishLocation: Container
+      artifactName: PostBuildLogs
+      continueOnError: true
+      condition: always()

eng/common/core-templates/steps/publish-pipeline-artifacts.yml

@@ -0,0 +1,20 @@
+parameters:
+- name: is1ESPipeline
+  type: boolean
+  default: false
+
+- name: args
+  type: object
+  default: {}  
+
+steps:
+- ${{ if ne(parameters.is1ESPipeline, true) }}:
+  - template: /eng/common/templates/steps/publish-pipeline-artifacts.yml
+    parameters:
+      ${{ each parameter in parameters }}:
+        ${{ parameter.key }}: ${{ parameter.value }}
+- ${{ else }}:
+  - template: /eng/common/templates-official/steps/publish-pipeline-artifacts.yml
+    parameters:
+      ${{ each parameter in parameters }}:
+        ${{ parameter.key }}: ${{ parameter.value }}

eng/common/core-templates/steps/retain-build.yml

@@ -0,0 +1,28 @@
+parameters:
+  # Optional azure devops PAT with build execute permissions for the build's organization,
+  # only needed if the build that should be retained ran on a different organization than 
+  # the pipeline where this template is executing from
+  Token: ''
+  # Optional BuildId to retain, defaults to the current running build
+  BuildId: ''
+  # Azure devops Organization URI for the build in the https://dev.azure.com/<organization> format.
+  # Defaults to the organization the current pipeline is running on
+  AzdoOrgUri: '$(System.CollectionUri)'
+  # Azure devops project for the build. Defaults to the project the current pipeline is running on
+  AzdoProject: '$(System.TeamProject)'
+
+steps:
+  - task: powershell@2
+    inputs:
+      targetType: 'filePath'
+      filePath: eng/common/retain-build.ps1
+      pwsh: true
+      arguments: >
+        -AzdoOrgUri: ${{parameters.AzdoOrgUri}}
+        -AzdoProject ${{parameters.AzdoProject}}
+        -Token ${{coalesce(parameters.Token, '$env:SYSTEM_ACCESSTOKEN') }}
+        -BuildId ${{coalesce(parameters.BuildId, '$env:BUILD_ID')}}
+    displayName: Enable permanent build retention
+    env:
+      SYSTEM_ACCESSTOKEN: $(System.AccessToken)
+      BUILD_ID: $(Build.BuildId)

eng/common/core-templates/steps/send-to-helix.yml

@@ -0,0 +1,93 @@
+# Please remember to update the documentation if you make changes to these parameters!
+parameters:
+  HelixSource: 'pr/default'              # required -- sources must start with pr/, official/, prodcon/, or agent/
+  HelixType: 'tests/default/'            # required -- Helix telemetry which identifies what type of data this is; should include "test" for clarity and must end in '/'
+  HelixBuild: $(Build.BuildNumber)       # required -- the build number Helix will use to identify this -- automatically set to the AzDO build number
+  HelixTargetQueues: ''                  # required -- semicolon-delimited list of Helix queues to test on; see https://helix.dot.net/ for a list of queues
+  HelixAccessToken: ''                   # required -- access token to make Helix API requests; should be provided by the appropriate variable group
+  HelixProjectPath: 'eng/common/helixpublish.proj'  # optional -- path to the project file to build relative to BUILD_SOURCESDIRECTORY
+  HelixProjectArguments: ''              # optional -- arguments passed to the build command
+  HelixConfiguration: ''                 # optional -- additional property attached to a job
+  HelixPreCommands: ''                   # optional -- commands to run before Helix work item execution
+  HelixPostCommands: ''                  # optional -- commands to run after Helix work item execution
+  WorkItemDirectory: ''                  # optional -- a payload directory to zip up and send to Helix; requires WorkItemCommand; incompatible with XUnitProjects
+  WorkItemCommand: ''                    # optional -- a command to execute on the payload; requires WorkItemDirectory; incompatible with XUnitProjects
+  WorkItemTimeout: ''                    # optional -- a timeout in TimeSpan.Parse-ready value (e.g. 00:02:00) for the work item command; requires WorkItemDirectory; incompatible with XUnitProjects
+  CorrelationPayloadDirectory: ''        # optional -- a directory to zip up and send to Helix as a correlation payload
+  XUnitProjects: ''                      # optional -- semicolon-delimited list of XUnitProjects to parse and send to Helix; requires XUnitRuntimeTargetFramework, XUnitPublishTargetFramework, XUnitRunnerVersion, and IncludeDotNetCli=true
+  XUnitWorkItemTimeout: ''               # optional -- the workitem timeout in seconds for all workitems created from the xUnit projects specified by XUnitProjects
+  XUnitPublishTargetFramework: ''        # optional -- framework to use to publish your xUnit projects
+  XUnitRuntimeTargetFramework: ''        # optional -- framework to use for the xUnit console runner
+  XUnitRunnerVersion: ''                 # optional -- version of the xUnit nuget package you wish to use on Helix; required for XUnitProjects
+  IncludeDotNetCli: false                # optional -- true will download a version of the .NET CLI onto the Helix machine as a correlation payload; requires DotNetCliPackageType and DotNetCliVersion
+  DotNetCliPackageType: ''               # optional -- either 'sdk', 'runtime' or 'aspnetcore-runtime'; determines whether the sdk or runtime will be sent to Helix; see https://raw.githubusercontent.com/dotnet/core/main/release-notes/releases-index.json
+  DotNetCliVersion: ''                   # optional -- version of the CLI to send to Helix; based on this: https://raw.githubusercontent.com/dotnet/core/main/release-notes/releases-index.json
+  WaitForWorkItemCompletion: true        # optional -- true will make the task wait until work items have been completed and fail the build if work items fail. False is "fire and forget."
+  IsExternal: false                      # [DEPRECATED] -- doesn't do anything, jobs are external if HelixAccessToken is empty and Creator is set
+  HelixBaseUri: 'https://helix.dot.net/' # optional -- sets the Helix API base URI (allows targeting https://helix.int-dot.net )
+  Creator: ''                            # optional -- if the build is external, use this to specify who is sending the job
+  DisplayNamePrefix: 'Run Tests'         # optional -- rename the beginning of the displayName of the steps in AzDO 
+  condition: succeeded()                 # optional -- condition for step to execute; defaults to succeeded()
+  continueOnError: false                 # optional -- determines whether to continue the build if the step errors; defaults to false
+
+steps:
+  - powershell: 'powershell "$env:BUILD_SOURCESDIRECTORY\eng\common\msbuild.ps1 $env:BUILD_SOURCESDIRECTORY/${{ parameters.HelixProjectPath }} /restore /p:TreatWarningsAsErrors=false ${{ parameters.HelixProjectArguments }} /t:Test /bl:$env:BUILD_SOURCESDIRECTORY\artifacts\log\$env:BuildConfig\SendToHelix.binlog"'
+    displayName: ${{ parameters.DisplayNamePrefix }} (Windows)
+    env:
+      BuildConfig: $(_BuildConfig)
+      HelixSource: ${{ parameters.HelixSource }}
+      HelixType: ${{ parameters.HelixType }}
+      HelixBuild: ${{ parameters.HelixBuild }}
+      HelixConfiguration:  ${{ parameters.HelixConfiguration }}
+      HelixTargetQueues: ${{ parameters.HelixTargetQueues }}
+      HelixAccessToken: ${{ parameters.HelixAccessToken }}
+      HelixPreCommands: ${{ parameters.HelixPreCommands }}
+      HelixPostCommands: ${{ parameters.HelixPostCommands }}
+      WorkItemDirectory: ${{ parameters.WorkItemDirectory }}
+      WorkItemCommand: ${{ parameters.WorkItemCommand }}
+      WorkItemTimeout: ${{ parameters.WorkItemTimeout }}
+      CorrelationPayloadDirectory: ${{ parameters.CorrelationPayloadDirectory }}
+      XUnitProjects: ${{ parameters.XUnitProjects }}
+      XUnitWorkItemTimeout: ${{ parameters.XUnitWorkItemTimeout }}
+      XUnitPublishTargetFramework: ${{ parameters.XUnitPublishTargetFramework }}
+      XUnitRuntimeTargetFramework: ${{ parameters.XUnitRuntimeTargetFramework }}
+      XUnitRunnerVersion: ${{ parameters.XUnitRunnerVersion }}
+      IncludeDotNetCli: ${{ parameters.IncludeDotNetCli }}
+      DotNetCliPackageType: ${{ parameters.DotNetCliPackageType }}
+      DotNetCliVersion: ${{ parameters.DotNetCliVersion }}
+      WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }}
+      HelixBaseUri: ${{ parameters.HelixBaseUri }}
+      Creator: ${{ parameters.Creator }}
+      SYSTEM_ACCESSTOKEN: $(System.AccessToken)
+    condition: and(${{ parameters.condition }}, eq(variables['Agent.Os'], 'Windows_NT'))
+    continueOnError: ${{ parameters.continueOnError }}
+  - script: $BUILD_SOURCESDIRECTORY/eng/common/msbuild.sh $BUILD_SOURCESDIRECTORY/${{ parameters.HelixProjectPath }} /restore /p:TreatWarningsAsErrors=false ${{ parameters.HelixProjectArguments }} /t:Test /bl:$BUILD_SOURCESDIRECTORY/artifacts/log/$BuildConfig/SendToHelix.binlog
+    displayName: ${{ parameters.DisplayNamePrefix }} (Unix)
+    env:
+      BuildConfig: $(_BuildConfig)
+      HelixSource: ${{ parameters.HelixSource }}
+      HelixType: ${{ parameters.HelixType }}
+      HelixBuild: ${{ parameters.HelixBuild }}
+      HelixConfiguration:  ${{ parameters.HelixConfiguration }}
+      HelixTargetQueues: ${{ parameters.HelixTargetQueues }}
+      HelixAccessToken: ${{ parameters.HelixAccessToken }}
+      HelixPreCommands: ${{ parameters.HelixPreCommands }}
+      HelixPostCommands: ${{ parameters.HelixPostCommands }}
+      WorkItemDirectory: ${{ parameters.WorkItemDirectory }}
+      WorkItemCommand: ${{ parameters.WorkItemCommand }}
+      WorkItemTimeout: ${{ parameters.WorkItemTimeout }}
+      CorrelationPayloadDirectory: ${{ parameters.CorrelationPayloadDirectory }}
+      XUnitProjects: ${{ parameters.XUnitProjects }}
+      XUnitWorkItemTimeout: ${{ parameters.XUnitWorkItemTimeout }}
+      XUnitPublishTargetFramework: ${{ parameters.XUnitPublishTargetFramework }}
+      XUnitRuntimeTargetFramework: ${{ parameters.XUnitRuntimeTargetFramework }}
+      XUnitRunnerVersion: ${{ parameters.XUnitRunnerVersion }}
+      IncludeDotNetCli: ${{ parameters.IncludeDotNetCli }}
+      DotNetCliPackageType: ${{ parameters.DotNetCliPackageType }}
+      DotNetCliVersion: ${{ parameters.DotNetCliVersion }}
+      WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }}
+      HelixBaseUri: ${{ parameters.HelixBaseUri }}
+      Creator: ${{ parameters.Creator }}
+      SYSTEM_ACCESSTOKEN: $(System.AccessToken)
+    condition: and(${{ parameters.condition }}, ne(variables['Agent.Os'], 'Windows_NT'))
+    continueOnError: ${{ parameters.continueOnError }}

eng/common/core-templates/steps/source-build.yml

@@ -0,0 +1,129 @@
+parameters:
+  # This template adds arcade-powered source-build to CI.
+
+  # This is a 'steps' template, and is intended for advanced scenarios where the existing build
+  # infra has a careful build methodology that must be followed. For example, a repo
+  # (dotnet/runtime) might choose to clone the GitHub repo only once and store it as a pipeline
+  # artifact for all subsequent jobs to use, to reduce dependence on a strong network connection to
+  # GitHub. Using this steps template leaves room for that infra to be included.
+
+  # Defines the platform on which to run the steps. See 'eng/common/core-templates/job/source-build.yml'
+  # for details. The entire object is described in the 'job' template for simplicity, even though
+  # the usage of the properties on this object is split between the 'job' and 'steps' templates.
+  platform: {}
+  is1ESPipeline: false
+
+steps:
+# Build. Keep it self-contained for simple reusability. (No source-build-specific job variables.)
+- script: |
+    set -x
+    df -h
+
+    # If file changes are detected, set CopyWipIntoInnerSourceBuildRepo to copy the WIP changes into the inner source build repo.
+    internalRestoreArgs=
+    if ! git diff --quiet; then
+      internalRestoreArgs='/p:CopyWipIntoInnerSourceBuildRepo=true'
+      # The 'Copy WIP' feature of source build uses git stash to apply changes from the original repo.
+      # This only works if there is a username/email configured, which won't be the case in most CI runs.
+      git config --get user.email
+      if [ $? -ne 0 ]; then
+        git config user.email dn-bot@microsoft.com
+        git config user.name dn-bot
+      fi
+    fi
+
+    # If building on the internal project, the internal storage variable may be available (usually only if needed)
+    # In that case, add variables to allow the download of internal runtimes if the specified versions are not found
+    # in the default public locations.
+    internalRuntimeDownloadArgs=
+    if [ '$(dotnetbuilds-internal-container-read-token-base64)' != '$''(dotnetbuilds-internal-container-read-token-base64)' ]; then
+      internalRuntimeDownloadArgs='/p:DotNetRuntimeSourceFeed=https://dotnetbuilds.blob.core.windows.net/internal /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64) --runtimesourcefeed https://dotnetbuilds.blob.core.windows.net/internal --runtimesourcefeedkey $(dotnetbuilds-internal-container-read-token-base64)'
+    fi
+
+    buildConfig=Release
+    # Check if AzDO substitutes in a build config from a variable, and use it if so.
+    if [ '$(_BuildConfig)' != '$''(_BuildConfig)' ]; then
+      buildConfig='$(_BuildConfig)'
+    fi
+
+    officialBuildArgs=
+    if [ '${{ and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}' = 'True' ]; then
+      officialBuildArgs='/p:DotNetPublishUsingPipelines=true /p:OfficialBuildId=$(BUILD.BUILDNUMBER)'
+    fi
+
+    targetRidArgs=
+    if [ '${{ parameters.platform.targetRID }}' != '' ]; then
+      targetRidArgs='/p:TargetRid=${{ parameters.platform.targetRID }}'
+    fi
+
+    runtimeOsArgs=
+    if [ '${{ parameters.platform.runtimeOS }}' != '' ]; then
+      runtimeOsArgs='/p:RuntimeOS=${{ parameters.platform.runtimeOS }}'
+    fi
+
+    baseOsArgs=
+    if [ '${{ parameters.platform.baseOS }}' != '' ]; then
+      baseOsArgs='/p:BaseOS=${{ parameters.platform.baseOS }}'
+    fi
+
+    publishArgs=
+    if [ '${{ parameters.platform.skipPublishValidation }}' != 'true' ]; then
+      publishArgs='--publish'
+    fi
+
+    assetManifestFileName=SourceBuild_RidSpecific.xml
+    if [ '${{ parameters.platform.name }}' != '' ]; then
+      assetManifestFileName=SourceBuild_${{ parameters.platform.name }}.xml
+    fi
+
+    ${{ coalesce(parameters.platform.buildScript, './build.sh') }} --ci \
+      --configuration $buildConfig \
+      --restore --build --pack $publishArgs -bl \
+      $officialBuildArgs \
+      $internalRuntimeDownloadArgs \
+      $internalRestoreArgs \
+      $targetRidArgs \
+      $runtimeOsArgs \
+      $baseOsArgs \
+      /p:SourceBuildNonPortable=${{ parameters.platform.nonPortable }} \
+      /p:ArcadeBuildFromSource=true \
+      /p:DotNetBuildSourceOnly=true \
+      /p:DotNetBuildRepo=true \
+      /p:AssetManifestFileName=$assetManifestFileName
+  displayName: Build
+
+# Upload build logs for diagnosis.
+- task: CopyFiles@2
+  displayName: Prepare BuildLogs staging directory
+  inputs:
+    SourceFolder: '$(Build.SourcesDirectory)'
+    Contents: |
+      **/*.log
+      **/*.binlog
+      artifacts/sb/prebuilt-report/**
+    TargetFolder: '$(Build.StagingDirectory)/BuildLogs'
+    CleanTargetFolder: true
+  continueOnError: true
+  condition: succeededOrFailed()
+
+- template: /eng/common/core-templates/steps/publish-pipeline-artifacts.yml
+  parameters:
+    is1ESPipeline: ${{ parameters.is1ESPipeline }}
+    args:
+      displayName: Publish BuildLogs
+      targetPath: '$(Build.StagingDirectory)/BuildLogs'
+      artifactName: BuildLogs_SourceBuild_${{ parameters.platform.name }}_Attempt$(System.JobAttempt)
+      continueOnError: true
+      condition: succeededOrFailed()
+      sbomEnabled: false  # we don't need SBOM for logs
+
+# Manually inject component detection so that we can ignore the source build upstream cache, which contains
+# a nupkg cache of input packages (a local feed).
+# This path must match the upstream cache path in property 'CurrentRepoSourceBuiltNupkgCacheDir'
+# in src\Microsoft.DotNet.Arcade.Sdk\tools\SourceBuild\SourceBuildArcade.targets
+- template: /eng/common/core-templates/steps/component-governance.yml
+  parameters:
+    displayName: Component Detection (Exclude upstream cache)
+    is1ESPipeline: ${{ parameters.is1ESPipeline }}
+    componentGovernanceIgnoreDirectories: '$(Build.SourcesDirectory)/artifacts/sb/src/artifacts/obj/source-built-upstream-cache'
+    disableComponentGovernance: ${{ eq(variables['System.TeamProject'], 'public') }}

eng/common/core-templates/variables/pool-providers.yml

@@ -0,0 +1,8 @@
+parameters:
+  is1ESPipeline: false
+
+variables:
+  - ${{ if eq(parameters.is1ESPipeline, 'true') }}:
+    - template: /eng/common/templates-official/variables/pool-providers.yml
+  - ${{ else }}:
+    - template: /eng/common/templates/variables/pool-providers.yml

eng/common/cross/arm/sources.list.bionic

@@ -1,11 +0,0 @@
-deb http://ports.ubuntu.com/ubuntu-ports/ bionic main restricted universe
-deb-src http://ports.ubuntu.com/ubuntu-ports/ bionic main restricted universe
-
-deb http://ports.ubuntu.com/ubuntu-ports/ bionic-updates main restricted universe
-deb-src http://ports.ubuntu.com/ubuntu-ports/ bionic-updates main restricted universe
-
-deb http://ports.ubuntu.com/ubuntu-ports/ bionic-backports main restricted
-deb-src http://ports.ubuntu.com/ubuntu-ports/ bionic-backports main restricted
-
-deb http://ports.ubuntu.com/ubuntu-ports/ bionic-security main restricted universe multiverse
-deb-src http://ports.ubuntu.com/ubuntu-ports/ bionic-security main restricted universe multiverse

eng/common/cross/arm/sources.list.focal

@@ -1,11 +0,0 @@
-deb http://ports.ubuntu.com/ubuntu-ports/ focal main restricted universe
-deb-src http://ports.ubuntu.com/ubuntu-ports/ focal main restricted universe
-
-deb http://ports.ubuntu.com/ubuntu-ports/ focal-updates main restricted universe
-deb-src http://ports.ubuntu.com/ubuntu-ports/ focal-updates main restricted universe
-
-deb http://ports.ubuntu.com/ubuntu-ports/ focal-backports main restricted
-deb-src http://ports.ubuntu.com/ubuntu-ports/ focal-backports main restricted
-
-deb http://ports.ubuntu.com/ubuntu-ports/ focal-security main restricted universe multiverse
-deb-src http://ports.ubuntu.com/ubuntu-ports/ focal-security main restricted universe multiverse

eng/common/cross/arm/sources.list.jammy

@@ -1,11 +0,0 @@
-deb http://ports.ubuntu.com/ubuntu-ports/ jammy main restricted universe
-deb-src http://ports.ubuntu.com/ubuntu-ports/ jammy main restricted universe
-
-deb http://ports.ubuntu.com/ubuntu-ports/ jammy-updates main restricted universe
-deb-src http://ports.ubuntu.com/ubuntu-ports/ jammy-updates main restricted universe
-
-deb http://ports.ubuntu.com/ubuntu-ports/ jammy-backports main restricted
-deb-src http://ports.ubuntu.com/ubuntu-ports/ jammy-backports main restricted
-
-deb http://ports.ubuntu.com/ubuntu-ports/ jammy-security main restricted universe multiverse
-deb-src http://ports.ubuntu.com/ubuntu-ports/ jammy-security main restricted universe multiverse

eng/common/cross/arm/sources.list.jessie

@@ -1,3 +0,0 @@
-# Debian (sid)   # UNSTABLE
-deb http://ftp.debian.org/debian/ sid main contrib non-free
-deb-src http://ftp.debian.org/debian/ sid main contrib non-free

eng/common/cross/arm/sources.list.xenial

@@ -1,11 +0,0 @@
-deb http://ports.ubuntu.com/ubuntu-ports/ xenial main restricted universe
-deb-src http://ports.ubuntu.com/ubuntu-ports/ xenial main restricted universe
-
-deb http://ports.ubuntu.com/ubuntu-ports/ xenial-updates main restricted universe
-deb-src http://ports.ubuntu.com/ubuntu-ports/ xenial-updates main restricted universe
-
-deb http://ports.ubuntu.com/ubuntu-ports/ xenial-backports main restricted
-deb-src http://ports.ubuntu.com/ubuntu-ports/ xenial-backports main restricted
-
-deb http://ports.ubuntu.com/ubuntu-ports/ xenial-security main restricted universe multiverse
-deb-src http://ports.ubuntu.com/ubuntu-ports/ xenial-security main restricted universe multiverse

eng/common/cross/arm/sources.list.zesty

@@ -1,11 +0,0 @@
-deb http://ports.ubuntu.com/ubuntu-ports/ zesty main restricted universe
-deb-src http://ports.ubuntu.com/ubuntu-ports/ zesty main restricted universe
-
-deb http://ports.ubuntu.com/ubuntu-ports/ zesty-updates main restricted universe
-deb-src http://ports.ubuntu.com/ubuntu-ports/ zesty-updates main restricted universe
-
-deb http://ports.ubuntu.com/ubuntu-ports/ zesty-backports main restricted
-deb-src http://ports.ubuntu.com/ubuntu-ports/ zesty-backports main restricted
-
-deb http://ports.ubuntu.com/ubuntu-ports/ zesty-security main restricted universe multiverse
-deb-src http://ports.ubuntu.com/ubuntu-ports/ zesty-security main restricted universe multiverse

eng/common/cross/arm64/sources.list.bionic

@@ -1,11 +0,0 @@
-deb http://ports.ubuntu.com/ubuntu-ports/ bionic main restricted universe
-deb-src http://ports.ubuntu.com/ubuntu-ports/ bionic main restricted universe
-
-deb http://ports.ubuntu.com/ubuntu-ports/ bionic-updates main restricted universe
-deb-src http://ports.ubuntu.com/ubuntu-ports/ bionic-updates main restricted universe
-
-deb http://ports.ubuntu.com/ubuntu-ports/ bionic-backports main restricted
-deb-src http://ports.ubuntu.com/ubuntu-ports/ bionic-backports main restricted
-
-deb http://ports.ubuntu.com/ubuntu-ports/ bionic-security main restricted universe multiverse
-deb-src http://ports.ubuntu.com/ubuntu-ports/ bionic-security main restricted universe multiverse

eng/common/cross/arm64/sources.list.buster

@@ -1,11 +0,0 @@
-deb http://deb.debian.org/debian buster main
-deb-src http://deb.debian.org/debian buster main
-
-deb http://deb.debian.org/debian-security/ buster/updates main
-deb-src http://deb.debian.org/debian-security/ buster/updates main
-
-deb http://deb.debian.org/debian buster-updates main
-deb-src http://deb.debian.org/debian buster-updates main
-
-deb http://deb.debian.org/debian buster-backports main contrib non-free
-deb-src http://deb.debian.org/debian buster-backports main contrib non-free

eng/common/cross/arm64/sources.list.focal

@@ -1,11 +0,0 @@
-deb http://ports.ubuntu.com/ubuntu-ports/ focal main restricted universe
-deb-src http://ports.ubuntu.com/ubuntu-ports/ focal main restricted universe
-
-deb http://ports.ubuntu.com/ubuntu-ports/ focal-updates main restricted universe
-deb-src http://ports.ubuntu.com/ubuntu-ports/ focal-updates main restricted universe
-
-deb http://ports.ubuntu.com/ubuntu-ports/ focal-backports main restricted
-deb-src http://ports.ubuntu.com/ubuntu-ports/ focal-backports main restricted
-
-deb http://ports.ubuntu.com/ubuntu-ports/ focal-security main restricted universe multiverse
-deb-src http://ports.ubuntu.com/ubuntu-ports/ focal-security main restricted universe multiverse

eng/common/cross/arm64/sources.list.jammy

@@ -1,11 +0,0 @@
-deb http://ports.ubuntu.com/ubuntu-ports/ jammy main restricted universe
-deb-src http://ports.ubuntu.com/ubuntu-ports/ jammy main restricted universe
-
-deb http://ports.ubuntu.com/ubuntu-ports/ jammy-updates main restricted universe
-deb-src http://ports.ubuntu.com/ubuntu-ports/ jammy-updates main restricted universe
-
-deb http://ports.ubuntu.com/ubuntu-ports/ jammy-backports main restricted
-deb-src http://ports.ubuntu.com/ubuntu-ports/ jammy-backports main restricted
-
-deb http://ports.ubuntu.com/ubuntu-ports/ jammy-security main restricted universe multiverse
-deb-src http://ports.ubuntu.com/ubuntu-ports/ jammy-security main restricted universe multiverse

eng/common/cross/arm64/sources.list.stretch

@@ -1,12 +0,0 @@
-deb http://deb.debian.org/debian stretch main
-deb-src http://deb.debian.org/debian stretch main
-
-deb http://deb.debian.org/debian-security/ stretch/updates main
-deb-src http://deb.debian.org/debian-security/ stretch/updates main
-
-deb http://deb.debian.org/debian stretch-updates main
-deb-src http://deb.debian.org/debian stretch-updates main
-
-deb http://deb.debian.org/debian stretch-backports main contrib non-free
-deb-src http://deb.debian.org/debian stretch-backports main contrib non-free
-

eng/common/cross/arm64/sources.list.xenial

@@ -1,11 +0,0 @@
-deb http://ports.ubuntu.com/ubuntu-ports/ xenial main restricted universe
-deb-src http://ports.ubuntu.com/ubuntu-ports/ xenial main restricted universe
-
-deb http://ports.ubuntu.com/ubuntu-ports/ xenial-updates main restricted universe
-deb-src http://ports.ubuntu.com/ubuntu-ports/ xenial-updates main restricted universe
-
-deb http://ports.ubuntu.com/ubuntu-ports/ xenial-backports main restricted
-deb-src http://ports.ubuntu.com/ubuntu-ports/ xenial-backports main restricted
-
-deb http://ports.ubuntu.com/ubuntu-ports/ xenial-security main restricted universe multiverse
-deb-src http://ports.ubuntu.com/ubuntu-ports/ xenial-security main restricted universe multiverse

eng/common/cross/arm64/sources.list.zesty

@@ -1,11 +0,0 @@
-deb http://ports.ubuntu.com/ubuntu-ports/ zesty main restricted universe
-deb-src http://ports.ubuntu.com/ubuntu-ports/ zesty main restricted universe
-
-deb http://ports.ubuntu.com/ubuntu-ports/ zesty-updates main restricted universe
-deb-src http://ports.ubuntu.com/ubuntu-ports/ zesty-updates main restricted universe
-
-deb http://ports.ubuntu.com/ubuntu-ports/ zesty-backports main restricted
-deb-src http://ports.ubuntu.com/ubuntu-ports/ zesty-backports main restricted
-
-deb http://ports.ubuntu.com/ubuntu-ports/ zesty-security main restricted universe multiverse
-deb-src http://ports.ubuntu.com/ubuntu-ports/ zesty-security main restricted universe multiverse

eng/common/cross/armel/sources.list.jessie

@@ -1,3 +0,0 @@
-# Debian (jessie)   # Stable
-deb http://ftp.debian.org/debian/ jessie main contrib non-free
-deb-src http://ftp.debian.org/debian/ jessie main contrib non-free

eng/common/cross/armv6/sources.list.buster

@@ -1,2 +0,0 @@
-deb http://raspbian.raspberrypi.org/raspbian/ buster main contrib non-free rpi
-deb-src http://raspbian.raspberrypi.org/raspbian/ buster main contrib non-free rpi

eng/common/cross/build-android-rootfs.sh

@@ -5,15 +5,15 @@ __NDK_Version=r21
 usage()
 {
     echo "Creates a toolchain and sysroot used for cross-compiling for Android."
-    echo.
+    echo
     echo "Usage: $0 [BuildArch] [ApiLevel]"
-    echo.
+    echo
     echo "BuildArch is the target architecture of Android. Currently only arm64 is supported."
     echo "ApiLevel is the target Android API level. API levels usually match to Android releases. See https://source.android.com/source/build-numbers.html"
-    echo.
+    echo
     echo "By default, the toolchain and sysroot will be generated in cross/android-rootfs/toolchain/[BuildArch]. You can change this behavior"
     echo "by setting the TOOLCHAIN_DIR environment variable"
-    echo.
+    echo
     echo "By default, the NDK will be downloaded into the cross/android-rootfs/android-ndk-$__NDK_Version directory. If you already have an NDK installation,"
     echo "you can set the NDK_DIR environment variable to have this script use that installation of the NDK."
     echo "By default, this script will generate a file, android_platform, in the root of the ROOTFS_DIR directory that contains the RID for the supported and tested Android build: android.28-arm64. This file is to replace '/etc/os-release', which is not available for Android."

eng/common/cross/build-rootfs.sh

@@ -8,7 +8,7 @@ usage()
     echo "BuildArch can be: arm(default), arm64, armel, armv6, ppc64le, riscv64, s390x, x64, x86"
     echo "CodeName - optional, Code name for Linux, can be: xenial(default), zesty, bionic, alpine"
     echo "                               for alpine can be specified with version: alpineX.YY or alpineedge"
-    echo "                               for FreeBSD can be: freebsd12, freebsd13"
+    echo "                               for FreeBSD can be: freebsd13, freebsd14"
     echo "                               for illumos can be: illumos"
     echo "                               for Haiku can be: haiku."
     echo "lldbx.y - optional, LLDB version, can be: lldb3.9(default), lldb4.0, lldb5.0, lldb6.0 no-lldb. Ignored for alpine and FreeBSD"
@@ -30,7 +30,8 @@ __IllumosArch=arm7
 __HaikuArch=arm
 __QEMUArch=arm
 __UbuntuArch=armhf
-__UbuntuRepo="http://ports.ubuntu.com/"
+__UbuntuRepo=
+__UbuntuSuites="updates security backports"
 __LLDB_Package="liblldb-3.9-dev"
 __SkipUnmount=0
 
@@ -71,9 +72,9 @@ __AlpinePackages+=" krb5-dev"
 __AlpinePackages+=" openssl-dev"
 __AlpinePackages+=" zlib-dev"
 
-__FreeBSDBase="12.4-RELEASE"
+__FreeBSDBase="13.3-RELEASE"
 __FreeBSDPkg="1.17.0"
-__FreeBSDABI="12"
+__FreeBSDABI="13"
 __FreeBSDPackages="libunwind"
 __FreeBSDPackages+=" icu"
 __FreeBSDPackages+=" libinotify"
@@ -129,6 +130,7 @@ __AlpineKeys='
 616db30d:MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAnpUpyWDWjlUk3smlWeA0\nlIMW+oJ38t92CRLHH3IqRhyECBRW0d0aRGtq7TY8PmxjjvBZrxTNDpJT6KUk4LRm\na6A6IuAI7QnNK8SJqM0DLzlpygd7GJf8ZL9SoHSH+gFsYF67Cpooz/YDqWrlN7Vw\ntO00s0B+eXy+PCXYU7VSfuWFGK8TGEv6HfGMALLjhqMManyvfp8hz3ubN1rK3c8C\nUS/ilRh1qckdbtPvoDPhSbTDmfU1g/EfRSIEXBrIMLg9ka/XB9PvWRrekrppnQzP\nhP9YE3x/wbFc5QqQWiRCYyQl/rgIMOXvIxhkfe8H5n1Et4VAorkpEAXdsfN8KSVv\nLSMazVlLp9GYq5SUpqYX3KnxdWBgN7BJoZ4sltsTpHQ/34SXWfu3UmyUveWj7wp0\nx9hwsPirVI00EEea9AbP7NM2rAyu6ukcm4m6ATd2DZJIViq2es6m60AE6SMCmrQF\nwmk4H/kdQgeAELVfGOm2VyJ3z69fQuywz7xu27S6zTKi05Qlnohxol4wVb6OB7qG\nLPRtK9ObgzRo/OPumyXqlzAi/Yvyd1ZQk8labZps3e16bQp8+pVPiumWioMFJDWV\nGZjCmyMSU8V6MB6njbgLHoyg2LCukCAeSjbPGGGYhnKLm1AKSoJh3IpZuqcKCk5C\n8CM1S15HxV78s9dFntEqIokCAwEAAQ==
 '
 __Keyring=
+__KeyringFile="/usr/share/keyrings/ubuntu-archive-keyring.gpg"
 __SkipSigCheck=0
 __UseMirror=0
 
@@ -142,7 +144,6 @@ while :; do
     case $lowerI in
         -\?|-h|--help)
             usage
-            exit 1
             ;;
         arm)
             __BuildArch=arm
@@ -163,6 +164,7 @@ while :; do
             __UbuntuArch=armel
             __UbuntuRepo="http://ftp.debian.org/debian/"
             __CodeName=jessie
+            __KeyringFile="/usr/share/keyrings/debian-archive-keyring.gpg"
             ;;
         armv6)
             __BuildArch=armv6
@@ -170,10 +172,12 @@ while :; do
             __QEMUArch=arm
             __UbuntuRepo="http://raspbian.raspberrypi.org/raspbian/"
             __CodeName=buster
+            __KeyringFile="/usr/share/keyrings/raspbian-archive-keyring.gpg"
             __LLDB_Package="liblldb-6.0-dev"
+            __UbuntuSuites=
 
-            if [[ -e "/usr/share/keyrings/raspbian-archive-keyring.gpg" ]]; then
-                __Keyring="--keyring /usr/share/keyrings/raspbian-archive-keyring.gpg"
+            if [[ -e "$__KeyringFile" ]]; then
+                __Keyring="--keyring $__KeyringFile"
             fi
             ;;
         riscv64)
@@ -182,13 +186,8 @@ while :; do
             __AlpinePackages="${__AlpinePackages// lldb-dev/}"
             __QEMUArch=riscv64
             __UbuntuArch=riscv64
-            __UbuntuRepo="http://deb.debian.org/debian-ports"
             __UbuntuPackages="${__UbuntuPackages// libunwind8-dev/}"
             unset __LLDB_Package
-
-            if [[ -e "/usr/share/keyrings/debian-ports-archive-keyring.gpg" ]]; then
-                __Keyring="--keyring /usr/share/keyrings/debian-ports-archive-keyring.gpg --include=debian-ports-archive-keyring"
-            fi
             ;;
         ppc64le)
             __BuildArch=ppc64le
@@ -229,12 +228,19 @@ while :; do
             __UbuntuRepo="http://archive.ubuntu.com/ubuntu/"
             ;;
         lldb*)
-            version="${lowerI/lldb/}"
-            parts=(${version//./ })
+            version="$(echo "$lowerI" | tr -d '[:alpha:]-=')"
+            majorVersion="${version%%.*}"
+
+            [ -z "${version##*.*}" ] && minorVersion="${version#*.}"
+            if [ -z "$minorVersion" ]; then
+                minorVersion=0
+            fi
 
             # for versions > 6.0, lldb has dropped the minor version
-            if [[ "${parts[0]}" -gt 6 ]]; then
-                version="${parts[0]}"
+            if [ "$majorVersion" -le 6 ]; then
+                version="$majorVersion.$minorVersion"
+            else
+                version="$majorVersion"
             fi
 
             __LLDB_Package="liblldb-${version}-dev"
@@ -243,15 +249,19 @@ while :; do
             unset __LLDB_Package
             ;;
         llvm*)
-            version="${lowerI/llvm/}"
-            parts=(${version//./ })
-            __LLVM_MajorVersion="${parts[0]}"
-            __LLVM_MinorVersion="${parts[1]}"
+            version="$(echo "$lowerI" | tr -d '[:alpha:]-=')"
+            __LLVM_MajorVersion="${version%%.*}"
 
-            # for versions > 6.0, llvm has dropped the minor version
-            if [[ -z "$__LLVM_MinorVersion" && "$__LLVM_MajorVersion" -le 6 ]]; then
-                __LLVM_MinorVersion=0;
+            [ -z "${version##*.*}" ] && __LLVM_MinorVersion="${version#*.}"
+            if [ -z "$__LLVM_MinorVersion" ]; then
+                __LLVM_MinorVersion=0
             fi
+
+            # for versions > 6.0, lldb has dropped the minor version
+            if [ "$__LLVM_MajorVersion" -gt 6 ]; then
+                __LLVM_MinorVersion=
+            fi
+
             ;;
         xenial) # Ubuntu 16.04
             if [[ "$__CodeName" != "jessie" ]]; then
@@ -278,8 +288,17 @@ while :; do
                 __CodeName=jammy
             fi
             ;;
+        noble) # Ubuntu 24.04
+            if [[ "$__CodeName" != "jessie" ]]; then
+                __CodeName=noble
+            fi
+            if [[ -n "$__LLDB_Package" ]]; then
+                __LLDB_Package="liblldb-18-dev"
+            fi
+            ;;
         jessie) # Debian 8
             __CodeName=jessie
+            __KeyringFile="/usr/share/keyrings/debian-archive-keyring.gpg"
 
             if [[ -z "$__UbuntuRepo" ]]; then
                 __UbuntuRepo="http://ftp.debian.org/debian/"
@@ -288,6 +307,7 @@ while :; do
         stretch) # Debian 9
             __CodeName=stretch
             __LLDB_Package="liblldb-6.0-dev"
+            __KeyringFile="/usr/share/keyrings/debian-archive-keyring.gpg"
 
             if [[ -z "$__UbuntuRepo" ]]; then
                 __UbuntuRepo="http://ftp.debian.org/debian/"
@@ -296,6 +316,7 @@ while :; do
         buster) # Debian 10
             __CodeName=buster
             __LLDB_Package="liblldb-6.0-dev"
+            __KeyringFile="/usr/share/keyrings/debian-archive-keyring.gpg"
 
             if [[ -z "$__UbuntuRepo" ]]; then
                 __UbuntuRepo="http://ftp.debian.org/debian/"
@@ -303,6 +324,15 @@ while :; do
             ;;
         bullseye) # Debian 11
             __CodeName=bullseye
+            __KeyringFile="/usr/share/keyrings/debian-archive-keyring.gpg"
+
+            if [[ -z "$__UbuntuRepo" ]]; then
+                __UbuntuRepo="http://ftp.debian.org/debian/"
+            fi
+            ;;
+        bookworm) # Debian 12
+            __CodeName=bookworm
+            __KeyringFile="/usr/share/keyrings/debian-archive-keyring.gpg"
 
             if [[ -z "$__UbuntuRepo" ]]; then
                 __UbuntuRepo="http://ftp.debian.org/debian/"
@@ -310,6 +340,7 @@ while :; do
             ;;
         sid) # Debian sid
             __CodeName=sid
+            __KeyringFile="/usr/share/keyrings/debian-archive-keyring.gpg"
 
             if [[ -z "$__UbuntuRepo" ]]; then
                 __UbuntuRepo="http://ftp.debian.org/debian/"
@@ -323,25 +354,24 @@ while :; do
         alpine*)
             __CodeName=alpine
             __UbuntuRepo=
-            version="${lowerI/alpine/}"
 
-            if [[ "$version" == "edge" ]]; then
+            if [[ "$lowerI" == "alpineedge" ]]; then
                 __AlpineVersion=edge
             else
-                parts=(${version//./ })
-                __AlpineMajorVersion="${parts[0]}"
-                __AlpineMinoVersion="${parts[1]}"
-                __AlpineVersion="$__AlpineMajorVersion.$__AlpineMinoVersion"
+                version="$(echo "$lowerI" | tr -d '[:alpha:]-=')"
+                __AlpineMajorVersion="${version%%.*}"
+                __AlpineMinorVersion="${version#*.}"
+                __AlpineVersion="$__AlpineMajorVersion.$__AlpineMinorVersion"
             fi
             ;;
-        freebsd12)
-            __CodeName=freebsd
-            __SkipUnmount=1
-            ;;
         freebsd13)
             __CodeName=freebsd
-            __FreeBSDBase="13.2-RELEASE"
-            __FreeBSDABI="13"
+            __SkipUnmount=1
+            ;;
+        freebsd14)
+            __CodeName=freebsd
+            __FreeBSDBase="14.0-RELEASE"
+            __FreeBSDABI="14"
             __SkipUnmount=1
             ;;
         illumos)
@@ -420,6 +450,10 @@ fi
 
 __UbuntuPackages+=" ${__LLDB_Package:-}"
 
+if [[ -z "$__UbuntuRepo" ]]; then
+    __UbuntuRepo="http://ports.ubuntu.com/"
+fi
+
 if [[ -n "$__LLVM_MajorVersion" ]]; then
     __UbuntuPackages+=" libclang-common-${__LLVM_MajorVersion}${__LLVM_MinorVersion:+.$__LLVM_MinorVersion}-dev"
 fi
@@ -442,13 +476,39 @@ fi
 mkdir -p "$__RootfsDir"
 __RootfsDir="$( cd "$__RootfsDir" && pwd )"
 
+__hasWget=
+ensureDownloadTool()
+{
+    if command -v wget &> /dev/null; then
+        __hasWget=1
+    elif command -v curl &> /dev/null; then
+        __hasWget=0
+    else
+        >&2 echo "ERROR: either wget or curl is required by this script."
+        exit 1
+    fi
+}
+
 if [[ "$__CodeName" == "alpine" ]]; then
     __ApkToolsVersion=2.12.11
-    __ApkToolsSHA512SUM=53e57b49230da07ef44ee0765b9592580308c407a8d4da7125550957bb72cb59638e04f8892a18b584451c8d841d1c7cb0f0ab680cc323a3015776affaa3be33
     __ApkToolsDir="$(mktemp -d)"
     __ApkKeysDir="$(mktemp -d)"
+    arch="$(uname -m)"
 
-    wget "https://gitlab.alpinelinux.org/api/v4/projects/5/packages/generic//v$__ApkToolsVersion/x86_64/apk.static" -P "$__ApkToolsDir"
+    ensureDownloadTool
+    
+    if [[ "$__hasWget" == 1 ]]; then
+        wget -P "$__ApkToolsDir" "https://gitlab.alpinelinux.org/api/v4/projects/5/packages/generic/v$__ApkToolsVersion/$arch/apk.static"
+    else
+        curl -SLO --create-dirs --output-dir "$__ApkToolsDir" "https://gitlab.alpinelinux.org/api/v4/projects/5/packages/generic/v$__ApkToolsVersion/$arch/apk.static"
+    fi
+    if [[ "$arch" == "x86_64" ]]; then
+      __ApkToolsSHA512SUM="53e57b49230da07ef44ee0765b9592580308c407a8d4da7125550957bb72cb59638e04f8892a18b584451c8d841d1c7cb0f0ab680cc323a3015776affaa3be33"
+    elif [[ "$arch" == "aarch64" ]]; then
+      __ApkToolsSHA512SUM="9e2b37ecb2b56c05dad23d379be84fd494c14bd730b620d0d576bda760588e1f2f59a7fcb2f2080577e0085f23a0ca8eadd993b4e61c2ab29549fdb71969afd0"
+    else
+      echo "WARNING: add missing hash for your host architecture. To find the value, use: 'find /tmp -name apk.static -exec sha512sum {} \;'"
+    fi
     echo "$__ApkToolsSHA512SUM $__ApkToolsDir/apk.static" | sha512sum -c
     chmod +x "$__ApkToolsDir/apk.static"
 
@@ -477,20 +537,23 @@ if [[ "$__CodeName" == "alpine" ]]; then
     fi
 
     # initialize DB
+    # shellcheck disable=SC2086
     "$__ApkToolsDir/apk.static" \
         -X "http://dl-cdn.alpinelinux.org/alpine/$version/main" \
         -X "http://dl-cdn.alpinelinux.org/alpine/$version/community" \
         -U $__ApkSignatureArg --root "$__RootfsDir" --arch "$__AlpineArch" --initdb add
 
     if [[ "$__AlpineLlvmLibsLookup" == 1 ]]; then
+        # shellcheck disable=SC2086
         __AlpinePackages+=" $("$__ApkToolsDir/apk.static" \
             -X "http://dl-cdn.alpinelinux.org/alpine/$version/main" \
             -X "http://dl-cdn.alpinelinux.org/alpine/$version/community" \
             -U $__ApkSignatureArg --root "$__RootfsDir" --arch "$__AlpineArch" \
-            search 'llvm*-libs' | sort | tail -1 | sed 's/-[^-]*//2g')"
+            search 'llvm*-libs' | grep -E '^llvm' | sort | tail -1 | sed 's/-[^-]*//2g')"
     fi
 
     # install all packages in one go
+    # shellcheck disable=SC2086
     "$__ApkToolsDir/apk.static" \
         -X "http://dl-cdn.alpinelinux.org/alpine/$version/main" \
         -X "http://dl-cdn.alpinelinux.org/alpine/$version/community" \
@@ -501,12 +564,23 @@ if [[ "$__CodeName" == "alpine" ]]; then
 elif [[ "$__CodeName" == "freebsd" ]]; then
     mkdir -p "$__RootfsDir"/usr/local/etc
     JOBS=${MAXJOBS:="$(getconf _NPROCESSORS_ONLN)"}
-    wget -O - "https://download.freebsd.org/ftp/releases/${__FreeBSDArch}/${__FreeBSDMachineArch}/${__FreeBSDBase}/base.txz" | tar -C "$__RootfsDir" -Jxf - ./lib ./usr/lib ./usr/libdata ./usr/include ./usr/share/keys ./etc ./bin/freebsd-version
+
+    ensureDownloadTool
+
+    if [[ "$__hasWget" == 1 ]]; then
+        wget -O- "https://download.freebsd.org/ftp/releases/${__FreeBSDArch}/${__FreeBSDMachineArch}/${__FreeBSDBase}/base.txz" | tar -C "$__RootfsDir" -Jxf - ./lib ./usr/lib ./usr/libdata ./usr/include ./usr/share/keys ./etc ./bin/freebsd-version
+    else
+        curl -SL "https://download.freebsd.org/ftp/releases/${__FreeBSDArch}/${__FreeBSDMachineArch}/${__FreeBSDBase}/base.txz" | tar -C "$__RootfsDir" -Jxf - ./lib ./usr/lib ./usr/libdata ./usr/include ./usr/share/keys ./etc ./bin/freebsd-version
+    fi
     echo "ABI = \"FreeBSD:${__FreeBSDABI}:${__FreeBSDMachineArch}\"; FINGERPRINTS = \"${__RootfsDir}/usr/share/keys\"; REPOS_DIR = [\"${__RootfsDir}/etc/pkg\"]; REPO_AUTOUPDATE = NO; RUN_SCRIPTS = NO;" > "${__RootfsDir}"/usr/local/etc/pkg.conf
     echo "FreeBSD: { url: \"pkg+http://pkg.FreeBSD.org/\${ABI}/quarterly\", mirror_type: \"srv\", signature_type: \"fingerprints\", fingerprints: \"${__RootfsDir}/usr/share/keys/pkg\", enabled: yes }" > "${__RootfsDir}"/etc/pkg/FreeBSD.conf
     mkdir -p "$__RootfsDir"/tmp
     # get and build package manager
-    wget -O - "https://github.com/freebsd/pkg/archive/${__FreeBSDPkg}.tar.gz" | tar -C "$__RootfsDir"/tmp -zxf -
+    if [[ "$__hasWget" == 1 ]]; then
+        wget -O- "https://github.com/freebsd/pkg/archive/${__FreeBSDPkg}.tar.gz" | tar -C "$__RootfsDir"/tmp -zxf -
+    else
+        curl -SL "https://github.com/freebsd/pkg/archive/${__FreeBSDPkg}.tar.gz" | tar -C "$__RootfsDir"/tmp -zxf -
+    fi
     cd "$__RootfsDir/tmp/pkg-${__FreeBSDPkg}"
     # needed for install to succeed
     mkdir -p "$__RootfsDir"/host/etc
@@ -514,27 +588,43 @@ elif [[ "$__CodeName" == "freebsd" ]]; then
     rm -rf "$__RootfsDir/tmp/pkg-${__FreeBSDPkg}"
     # install packages we need.
     INSTALL_AS_USER=$(whoami) "$__RootfsDir"/host/sbin/pkg -r "$__RootfsDir" -C "$__RootfsDir"/usr/local/etc/pkg.conf update
+    # shellcheck disable=SC2086
     INSTALL_AS_USER=$(whoami) "$__RootfsDir"/host/sbin/pkg -r "$__RootfsDir" -C "$__RootfsDir"/usr/local/etc/pkg.conf install --yes $__FreeBSDPackages
 elif [[ "$__CodeName" == "illumos" ]]; then
     mkdir "$__RootfsDir/tmp"
     pushd "$__RootfsDir/tmp"
     JOBS=${MAXJOBS:="$(getconf _NPROCESSORS_ONLN)"}
+
+    ensureDownloadTool
+
     echo "Downloading sysroot."
-    wget -O - https://github.com/illumos/sysroot/releases/download/20181213-de6af22ae73b-v1/illumos-sysroot-i386-20181213-de6af22ae73b-v1.tar.gz | tar -C "$__RootfsDir" -xzf -
+    if [[ "$__hasWget" == 1 ]]; then
+        wget -O- https://github.com/illumos/sysroot/releases/download/20181213-de6af22ae73b-v1/illumos-sysroot-i386-20181213-de6af22ae73b-v1.tar.gz | tar -C "$__RootfsDir" -xzf -
+    else
+        curl -SL https://github.com/illumos/sysroot/releases/download/20181213-de6af22ae73b-v1/illumos-sysroot-i386-20181213-de6af22ae73b-v1.tar.gz | tar -C "$__RootfsDir" -xzf -
+    fi
     echo "Building binutils. Please wait.."
-    wget -O - https://ftp.gnu.org/gnu/binutils/binutils-2.33.1.tar.bz2 | tar -xjf -
+    if [[ "$__hasWget" == 1 ]]; then
+        wget -O- https://ftp.gnu.org/gnu/binutils/binutils-2.42.tar.xz | tar -xJf -
+    else
+        curl -SL https://ftp.gnu.org/gnu/binutils/binutils-2.42.tar.xz | tar -xJf -
+    fi
     mkdir build-binutils && cd build-binutils
-    ../binutils-2.33.1/configure --prefix="$__RootfsDir" --target="${__illumosArch}-sun-solaris2.10" --program-prefix="${__illumosArch}-illumos-" --with-sysroot="$__RootfsDir"
+    ../binutils-2.42/configure --prefix="$__RootfsDir" --target="${__illumosArch}-sun-solaris2.11" --program-prefix="${__illumosArch}-illumos-" --with-sysroot="$__RootfsDir"
     make -j "$JOBS" && make install && cd ..
     echo "Building gcc. Please wait.."
-    wget -O - https://ftp.gnu.org/gnu/gcc/gcc-8.4.0/gcc-8.4.0.tar.xz | tar -xJf -
+    if [[ "$__hasWget" == 1 ]]; then
+        wget -O- https://ftp.gnu.org/gnu/gcc/gcc-13.3.0/gcc-13.3.0.tar.xz | tar -xJf -
+    else
+        curl -SL https://ftp.gnu.org/gnu/gcc/gcc-13.3.0/gcc-13.3.0.tar.xz | tar -xJf -
+    fi
     CFLAGS="-fPIC"
     CXXFLAGS="-fPIC"
     CXXFLAGS_FOR_TARGET="-fPIC"
     CFLAGS_FOR_TARGET="-fPIC"
     export CFLAGS CXXFLAGS CXXFLAGS_FOR_TARGET CFLAGS_FOR_TARGET
     mkdir build-gcc && cd build-gcc
-    ../gcc-8.4.0/configure --prefix="$__RootfsDir" --target="${__illumosArch}-sun-solaris2.10" --program-prefix="${__illumosArch}-illumos-" --with-sysroot="$__RootfsDir" --with-gnu-as       \
+    ../gcc-13.3.0/configure --prefix="$__RootfsDir" --target="${__illumosArch}-sun-solaris2.11" --program-prefix="${__illumosArch}-illumos-" --with-sysroot="$__RootfsDir" --with-gnu-as       \
         --with-gnu-ld --disable-nls --disable-libgomp --disable-libquadmath --disable-libssp --disable-libvtv --disable-libcilkrts --disable-libada --disable-libsanitizer \
         --disable-libquadmath-support --disable-shared --enable-tls
     make -j "$JOBS" && make install && cd ..
@@ -542,9 +632,13 @@ elif [[ "$__CodeName" == "illumos" ]]; then
     if [[ "$__UseMirror" == 1 ]]; then
         BaseUrl=https://pkgsrc.smartos.skylime.net
     fi
-    BaseUrl="$BaseUrl/packages/SmartOS/trunk/${__illumosArch}/All"
+    BaseUrl="$BaseUrl/packages/SmartOS/2019Q4/${__illumosArch}/All"
     echo "Downloading manifest"
-    wget "$BaseUrl"
+    if [[ "$__hasWget" == 1 ]]; then
+        wget "$BaseUrl"
+    else
+        curl -SLO "$BaseUrl"
+    fi
     echo "Downloading dependencies."
     read -ra array <<<"$__IllumosPackages"
     for package in "${array[@]}"; do
@@ -552,7 +646,11 @@ elif [[ "$__CodeName" == "illumos" ]]; then
         # find last occurrence of package in listing and extract its name
         package="$(sed -En '/.*href="('"$package"'-[0-9].*).tgz".*/h;$!d;g;s//\1/p' All)"
         echo "Resolved name '$package'"
-        wget "$BaseUrl"/"$package".tgz
+        if [[ "$__hasWget" == 1 ]]; then
+            wget "$BaseUrl"/"$package".tgz
+        else
+            curl -SLO "$BaseUrl"/"$package".tgz
+        fi
         ar -x "$package".tgz
         tar --skip-old-files -xzf "$package".tmp.tg* -C "$__RootfsDir" 2>/dev/null
     done
@@ -561,10 +659,17 @@ elif [[ "$__CodeName" == "illumos" ]]; then
     rm -rf "$__RootfsDir"/{tmp,+*}
     mkdir -p "$__RootfsDir"/usr/include/net
     mkdir -p "$__RootfsDir"/usr/include/netpacket
-    wget -P "$__RootfsDir"/usr/include/net https://raw.githubusercontent.com/illumos/illumos-gate/master/usr/src/uts/common/io/bpf/net/bpf.h
-    wget -P "$__RootfsDir"/usr/include/net https://raw.githubusercontent.com/illumos/illumos-gate/master/usr/src/uts/common/io/bpf/net/dlt.h
-    wget -P "$__RootfsDir"/usr/include/netpacket https://raw.githubusercontent.com/illumos/illumos-gate/master/usr/src/uts/common/inet/sockmods/netpacket/packet.h
-    wget -P "$__RootfsDir"/usr/include/sys https://raw.githubusercontent.com/illumos/illumos-gate/master/usr/src/uts/common/sys/sdt.h
+    if [[ "$__hasWget" == 1 ]]; then
+        wget -P "$__RootfsDir"/usr/include/net https://raw.githubusercontent.com/illumos/illumos-gate/master/usr/src/uts/common/io/bpf/net/bpf.h
+        wget -P "$__RootfsDir"/usr/include/net https://raw.githubusercontent.com/illumos/illumos-gate/master/usr/src/uts/common/io/bpf/net/dlt.h
+        wget -P "$__RootfsDir"/usr/include/netpacket https://raw.githubusercontent.com/illumos/illumos-gate/master/usr/src/uts/common/inet/sockmods/netpacket/packet.h
+        wget -P "$__RootfsDir"/usr/include/sys https://raw.githubusercontent.com/illumos/illumos-gate/master/usr/src/uts/common/sys/sdt.h
+    else
+        curl -SLO --create-dirs --output-dir "$__RootfsDir"/usr/include/net https://raw.githubusercontent.com/illumos/illumos-gate/master/usr/src/uts/common/io/bpf/net/bpf.h
+        curl -SLO --create-dirs --output-dir "$__RootfsDir"/usr/include/net https://raw.githubusercontent.com/illumos/illumos-gate/master/usr/src/uts/common/io/bpf/net/dlt.h
+        curl -SLO --create-dirs --output-dir "$__RootfsDir"/usr/include/netpacket https://raw.githubusercontent.com/illumos/illumos-gate/master/usr/src/uts/common/inet/sockmods/netpacket/packet.h
+        curl -SLO --create-dirs --output-dir "$__RootfsDir"/usr/include/sys https://raw.githubusercontent.com/illumos/illumos-gate/master/usr/src/uts/common/sys/sdt.h
+    fi
 elif [[ "$__CodeName" == "haiku" ]]; then
     JOBS=${MAXJOBS:="$(getconf _NPROCESSORS_ONLN)"}
 
@@ -574,9 +679,16 @@ elif [[ "$__CodeName" == "haiku" ]]; then
 
     mkdir "$__RootfsDir/tmp/download"
 
+    ensureDownloadTool
+
     echo "Downloading Haiku package tool"
-    git clone https://github.com/haiku/haiku-toolchains-ubuntu --depth 1 $__RootfsDir/tmp/script
-    wget -O "$__RootfsDir/tmp/download/hosttools.zip" $($__RootfsDir/tmp/script/fetch.sh --hosttools)
+    git clone https://github.com/haiku/haiku-toolchains-ubuntu --depth 1 "$__RootfsDir/tmp/script"
+    if [[ "$__hasWget" == 1 ]]; then
+        wget -O "$__RootfsDir/tmp/download/hosttools.zip" "$("$__RootfsDir/tmp/script/fetch.sh" --hosttools)"
+    else
+        curl -SLo "$__RootfsDir/tmp/download/hosttools.zip" "$("$__RootfsDir/tmp/script/fetch.sh" --hosttools)"
+    fi
+
     unzip -o "$__RootfsDir/tmp/download/hosttools.zip" -d "$__RootfsDir/tmp/bin"
 
     DepotBaseUrl="https://depot.haiku-os.org/__api/v2/pkg/get-pkg"
@@ -589,14 +701,25 @@ elif [[ "$__CodeName" == "haiku" ]]; then
         echo "Downloading $package..."
         # API documented here: https://github.com/haiku/haikudepotserver/blob/master/haikudepotserver-api2/src/main/resources/api2/pkg.yaml#L60
         # The schema here: https://github.com/haiku/haikudepotserver/blob/master/haikudepotserver-api2/src/main/resources/api2/pkg.yaml#L598
-        hpkgDownloadUrl="$(wget -qO- --post-data='{"name":"'"$package"'","repositorySourceCode":"haikuports_'$__HaikuArch'","versionType":"LATEST","naturalLanguageCode":"en"}' \
-            --header='Content-Type:application/json' "$DepotBaseUrl" | jq -r '.result.versions[].hpkgDownloadURL')"
-        wget -P "$__RootfsDir/tmp/download" "$hpkgDownloadUrl"
+        if [[ "$__hasWget" == 1 ]]; then
+            hpkgDownloadUrl="$(wget -qO- --post-data '{"name":"'"$package"'","repositorySourceCode":"haikuports_'$__HaikuArch'","versionType":"LATEST","naturalLanguageCode":"en"}' \
+                --header 'Content-Type:application/json' "$DepotBaseUrl" | jq -r '.result.versions[].hpkgDownloadURL')"
+            wget -P "$__RootfsDir/tmp/download" "$hpkgDownloadUrl"
+        else
+            hpkgDownloadUrl="$(curl -sSL -XPOST --data '{"name":"'"$package"'","repositorySourceCode":"haikuports_'$__HaikuArch'","versionType":"LATEST","naturalLanguageCode":"en"}' \
+                --header 'Content-Type:application/json' "$DepotBaseUrl" | jq -r '.result.versions[].hpkgDownloadURL')"
+            curl -SLO --create-dirs --output-dir "$__RootfsDir/tmp/download" "$hpkgDownloadUrl"
+        fi
     done
     for package in haiku haiku_devel; do
         echo "Downloading $package..."
-        hpkgVersion="$(wget -qO- $HpkgBaseUrl | sed -n 's/^.*version: "\([^"]*\)".*$/\1/p')"
-        wget -P "$__RootfsDir/tmp/download" "$HpkgBaseUrl/packages/$package-$hpkgVersion-1-$__HaikuArch.hpkg"
+        if [[ "$__hasWget" == 1 ]]; then
+            hpkgVersion="$(wget -qO- "$HpkgBaseUrl" | sed -n 's/^.*version: "\([^"]*\)".*$/\1/p')"
+            wget -P "$__RootfsDir/tmp/download" "$HpkgBaseUrl/packages/$package-$hpkgVersion-1-$__HaikuArch.hpkg"
+        else
+            hpkgVersion="$(curl -sSL "$HpkgBaseUrl" | sed -n 's/^.*version: "\([^"]*\)".*$/\1/p')"
+            curl -SLO --create-dirs --output-dir "$__RootfsDir/tmp/download" "$HpkgBaseUrl/packages/$package-$hpkgVersion-1-$__HaikuArch.hpkg"
+        fi
     done
 
     # Set up the sysroot
@@ -609,7 +732,11 @@ elif [[ "$__CodeName" == "haiku" ]]; then
 
     # Download buildtools
     echo "Downloading Haiku buildtools"
-    wget -O "$__RootfsDir/tmp/download/buildtools.zip" $($__RootfsDir/tmp/script/fetch.sh --buildtools --arch=$__HaikuArch)
+    if [[ "$__hasWget" == 1 ]]; then
+        wget -O "$__RootfsDir/tmp/download/buildtools.zip" "$("$__RootfsDir/tmp/script/fetch.sh" --buildtools --arch=$__HaikuArch)"
+    else
+        curl -SLo "$__RootfsDir/tmp/download/buildtools.zip" "$("$__RootfsDir/tmp/script/fetch.sh" --buildtools --arch=$__HaikuArch)"
+    fi
     unzip -o "$__RootfsDir/tmp/download/buildtools.zip" -d "$__RootfsDir"
 
     # Cleaning up temporary files
@@ -622,10 +749,22 @@ elif [[ -n "$__CodeName" ]]; then
         __Keyring="$__Keyring --force-check-gpg"
     fi
 
+    # shellcheck disable=SC2086
+    echo running debootstrap "--variant=minbase" $__Keyring --arch "$__UbuntuArch" "$__CodeName" "$__RootfsDir" "$__UbuntuRepo"
     debootstrap "--variant=minbase" $__Keyring --arch "$__UbuntuArch" "$__CodeName" "$__RootfsDir" "$__UbuntuRepo"
-    cp "$__CrossDir/$__BuildArch/sources.list.$__CodeName" "$__RootfsDir/etc/apt/sources.list"
+
+    mkdir -p "$__RootfsDir/etc/apt/sources.list.d/"
+    cat > "$__RootfsDir/etc/apt/sources.list.d/$__CodeName.sources" <<EOF
+Types: deb
+URIs: $__UbuntuRepo
+Suites: $__CodeName $(echo $__UbuntuSuites | xargs -n 1 | xargs -I {} echo -n "$__CodeName-{} ")
+Components: main universe
+Signed-By: $__KeyringFile
+EOF
+
     chroot "$__RootfsDir" apt-get update
     chroot "$__RootfsDir" apt-get -f -y install
+    # shellcheck disable=SC2086
     chroot "$__RootfsDir" apt-get -y install $__UbuntuPackages
     chroot "$__RootfsDir" symlinks -cr /usr
     chroot "$__RootfsDir" apt-get clean
@@ -643,6 +782,5 @@ elif [[ "$__Tizen" == "tizen" ]]; then
     ROOTFS_DIR="$__RootfsDir" "$__CrossDir/tizen-build-rootfs.sh" "$__BuildArch"
 else
     echo "Unsupported target platform."
-    usage;
-    exit 1
+    usage
 fi

eng/common/cross/ppc64le/sources.list.bionic

@@ -1,11 +0,0 @@
-deb http://ports.ubuntu.com/ubuntu-ports/ bionic main restricted universe
-deb-src http://ports.ubuntu.com/ubuntu-ports/ bionic main restricted universe
-
-deb http://ports.ubuntu.com/ubuntu-ports/ bionic-updates main restricted universe
-deb-src http://ports.ubuntu.com/ubuntu-ports/ bionic-updates main restricted universe
-
-deb http://ports.ubuntu.com/ubuntu-ports/ bionic-backports main restricted
-deb-src http://ports.ubuntu.com/ubuntu-ports/ bionic-backports main restricted
-
-deb http://ports.ubuntu.com/ubuntu-ports/ bionic-security main restricted universe multiverse
-deb-src http://ports.ubuntu.com/ubuntu-ports/ bionic-security main restricted universe multiverse

eng/common/cross/riscv64/sources.list.sid

@@ -1 +0,0 @@
-deb http://deb.debian.org/debian-ports sid main

eng/common/cross/riscv64/tizen/tizen.patch

@@ -0,0 +1,9 @@
+diff -u -r a/usr/lib/libc.so b/usr/lib/libc.so
+--- a/usr/lib64/libc.so	2016-12-30 23:00:08.284951863 +0900
++++ b/usr/lib64/libc.so	2016-12-30 23:00:32.140951815 +0900
+@@ -2,4 +2,4 @@
+    Use the shared library, but some functions are only in
+    the static library, so try that secondarily.  */
+ OUTPUT_FORMAT(elf64-littleriscv)
+-GROUP ( /lib64/libc.so.6 /usr/lib64/libc_nonshared.a  AS_NEEDED ( /lib64/ld-linux-riscv64-lp64d.so.1 ) )
++GROUP ( libc.so.6 libc_nonshared.a  AS_NEEDED ( ld-linux-riscv64-lp64d.so.1 ) )

eng/common/cross/s390x/sources.list.bionic

@@ -1,11 +0,0 @@
-deb http://ports.ubuntu.com/ubuntu-ports/ bionic main restricted universe
-deb-src http://ports.ubuntu.com/ubuntu-ports/ bionic main restricted universe
-
-deb http://ports.ubuntu.com/ubuntu-ports/ bionic-updates main restricted universe
-deb-src http://ports.ubuntu.com/ubuntu-ports/ bionic-updates main restricted universe
-
-deb http://ports.ubuntu.com/ubuntu-ports/ bionic-backports main restricted
-deb-src http://ports.ubuntu.com/ubuntu-ports/ bionic-backports main restricted
-
-deb http://ports.ubuntu.com/ubuntu-ports/ bionic-security main restricted universe multiverse
-deb-src http://ports.ubuntu.com/ubuntu-ports/ bionic-security main restricted universe multiverse

eng/common/cross/tizen-build-rootfs.sh

@@ -22,6 +22,10 @@ case "$ARCH" in
         TIZEN_ARCH="x86_64"
         LINK_ARCH="x86"
         ;;
+    riscv64)
+        TIZEN_ARCH="riscv64"
+        LINK_ARCH="riscv"
+        ;;
     *)
         echo "Unsupported architecture for tizen: $ARCH"
         exit 1
@@ -58,4 +62,21 @@ rm -rf $TIZEN_TMP_DIR
 echo ">>Start configuring Tizen rootfs"
 ln -sfn asm-${LINK_ARCH} ./usr/include/asm
 patch -p1 < $__TIZEN_CROSSDIR/tizen.patch
+if [[ "$TIZEN_ARCH" == "riscv64" ]]; then
+    echo "Fixing broken symlinks in $PWD"
+    rm ./usr/lib64/libresolv.so
+    ln -s ../../lib64/libresolv.so.2 ./usr/lib64/libresolv.so
+    rm ./usr/lib64/libpthread.so
+    ln -s ../../lib64/libpthread.so.0 ./usr/lib64/libpthread.so
+    rm ./usr/lib64/libdl.so
+    ln -s ../../lib64/libdl.so.2 ./usr/lib64/libdl.so
+    rm ./usr/lib64/libutil.so
+    ln -s ../../lib64/libutil.so.1 ./usr/lib64/libutil.so
+    rm ./usr/lib64/libm.so
+    ln -s ../../lib64/libm.so.6 ./usr/lib64/libm.so
+    rm ./usr/lib64/librt.so
+    ln -s ../../lib64/librt.so.1 ./usr/lib64/librt.so
+    rm ./lib/ld-linux-riscv64-lp64d.so.1
+    ln -s ../lib64/ld-linux-riscv64-lp64d.so.1 ./lib/ld-linux-riscv64-lp64d.so.1
+fi
 echo "<<Finish configuring Tizen rootfs"

eng/common/cross/tizen-fetch.sh

@@ -7,7 +7,7 @@ fi
 
 Log()
 {
-    if [ $VERBOSE -ge $1 ]; then
+    if [ $VERBOSE -ge 1 ]; then
         echo ${@:2}
     fi
 }
@@ -156,17 +156,28 @@ fetch_tizen_pkgs()
     done
 }
 
+if [ "$TIZEN_ARCH" == "riscv64" ]; then
+    BASE="Tizen-Base-RISCV"
+    UNIFIED="Tizen-Unified-RISCV"
+else
+    BASE="Tizen-Base"
+    UNIFIED="Tizen-Unified"
+fi
+
 Inform "Initialize ${TIZEN_ARCH} base"
-fetch_tizen_pkgs_init standard Tizen-Base
+fetch_tizen_pkgs_init standard $BASE
 Inform "fetch common packages"
 fetch_tizen_pkgs ${TIZEN_ARCH} gcc gcc-devel-static glibc glibc-devel libicu libicu-devel libatomic linux-glibc-devel keyutils keyutils-devel libkeyutils
 Inform "fetch coreclr packages"
-fetch_tizen_pkgs ${TIZEN_ARCH} lldb lldb-devel libgcc libstdc++ libstdc++-devel libunwind libunwind-devel lttng-ust-devel lttng-ust userspace-rcu-devel userspace-rcu
+fetch_tizen_pkgs ${TIZEN_ARCH} libgcc libstdc++ libstdc++-devel libunwind libunwind-devel lttng-ust-devel lttng-ust userspace-rcu-devel userspace-rcu
+if [ "$TIZEN_ARCH" != "riscv64" ]; then
+    fetch_tizen_pkgs ${TIZEN_ARCH} lldb lldb-devel
+fi
 Inform "fetch corefx packages"
 fetch_tizen_pkgs ${TIZEN_ARCH} libcom_err libcom_err-devel zlib zlib-devel libopenssl11 libopenssl1.1-devel krb5 krb5-devel
 
 Inform "Initialize standard unified"
-fetch_tizen_pkgs_init standard Tizen-Unified
+fetch_tizen_pkgs_init standard $UNIFIED
 Inform "fetch corefx packages"
 fetch_tizen_pkgs ${TIZEN_ARCH} gssdp gssdp-devel tizen-release
 

eng/common/cross/toolchain.cmake

@@ -80,6 +80,9 @@ elseif(TARGET_ARCH_NAME STREQUAL "riscv64")
     set(TOOLCHAIN "riscv64-alpine-linux-musl")
   else()
     set(TOOLCHAIN "riscv64-linux-gnu")
+    if(TIZEN)
+      set(TIZEN_TOOLCHAIN "riscv64-tizen-linux-gnu/13.1.0")
+    endif()
   endif()
 elseif(TARGET_ARCH_NAME STREQUAL "s390x")
   set(CMAKE_SYSTEM_PROCESSOR s390x)
@@ -144,6 +147,10 @@ if(TIZEN)
     include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}/include/c++/)
     include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}/include/c++/x86_64-tizen-linux-gnu)
   endif()
+  if(TARGET_ARCH_NAME STREQUAL "riscv64")
+    include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}/include/c++/)
+    include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}/include/c++/riscv64-tizen-linux-gnu)
+  endif()
 endif()
 
 if(ANDROID)
@@ -270,7 +277,7 @@ if(TARGET_ARCH_NAME MATCHES "^(arm|armel)$")
     add_toolchain_linker_flag("-L${CROSS_ROOTFS}/usr/lib")
     add_toolchain_linker_flag("-L${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}")
   endif()
-elseif(TARGET_ARCH_NAME MATCHES "^(arm64|x64)$")
+elseif(TARGET_ARCH_NAME MATCHES "^(arm64|x64|riscv64)$")
   if(TIZEN)
     add_toolchain_linker_flag("-B${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}")
     add_toolchain_linker_flag("-L${CROSS_ROOTFS}/lib64")
@@ -281,6 +288,8 @@ elseif(TARGET_ARCH_NAME MATCHES "^(arm64|x64)$")
     add_toolchain_linker_flag("-Wl,--rpath-link=${CROSS_ROOTFS}/usr/lib64")
     add_toolchain_linker_flag("-Wl,--rpath-link=${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}")
   endif()
+elseif(TARGET_ARCH_NAME STREQUAL "s390x")
+  add_toolchain_linker_flag("--target=${TOOLCHAIN}")
 elseif(TARGET_ARCH_NAME STREQUAL "x86")
   if(EXISTS ${CROSS_ROOTFS}/usr/lib/gcc/i586-alpine-linux-musl)
     add_toolchain_linker_flag("--target=${TOOLCHAIN}")
@@ -328,6 +337,8 @@ if(TARGET_ARCH_NAME MATCHES "^(arm|armel)$")
   if(TARGET_ARCH_NAME STREQUAL "armel")
     add_compile_options(-mfloat-abi=softfp)
   endif()
+elseif(TARGET_ARCH_NAME STREQUAL "s390x")
+  add_compile_options("--target=${TOOLCHAIN}")
 elseif(TARGET_ARCH_NAME STREQUAL "x86")
   if(EXISTS ${CROSS_ROOTFS}/usr/lib/gcc/i586-alpine-linux-musl)
     add_compile_options(--target=${TOOLCHAIN})
@@ -371,6 +382,26 @@ if(TARGET_ARCH_NAME MATCHES "^(arm|armel|x86)$")
   endif()
 endif()
 
+# Set C++ standard library options if specified
+set(CLR_CMAKE_CXX_STANDARD_LIBRARY "" CACHE STRING "Standard library flavor to link against. Only supported with the Clang compiler.")
+if (CLR_CMAKE_CXX_STANDARD_LIBRARY)
+  add_compile_options($<$<COMPILE_LANG_AND_ID:CXX,Clang>:--stdlib=${CLR_CMAKE_CXX_STANDARD_LIBRARY}>)
+  add_link_options($<$<LINK_LANG_AND_ID:CXX,Clang>:--stdlib=${CLR_CMAKE_CXX_STANDARD_LIBRARY}>)
+endif()
+
+option(CLR_CMAKE_CXX_STANDARD_LIBRARY_STATIC "Statically link against the C++ standard library" OFF)
+if(CLR_CMAKE_CXX_STANDARD_LIBRARY_STATIC)
+  add_link_options($<$<LINK_LANGUAGE:CXX>:-static-libstdc++>)
+endif()
+
+set(CLR_CMAKE_CXX_ABI_LIBRARY "" CACHE STRING "C++ ABI implementation library to link against. Only supported with the Clang compiler.")
+if (CLR_CMAKE_CXX_ABI_LIBRARY)
+  # The user may specify the ABI library with the 'lib' prefix, like 'libstdc++'. Strip the prefix here so the linker finds the right library.
+  string(REGEX REPLACE "^lib(.+)" "\\1" CLR_CMAKE_CXX_ABI_LIBRARY ${CLR_CMAKE_CXX_ABI_LIBRARY})
+  # We need to specify this as a linker-backend option as Clang will filter this option out when linking to libc++.
+  add_link_options("LINKER:-l${CLR_CMAKE_CXX_ABI_LIBRARY}")
+endif()
+
 set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER)
 set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY)
 set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY)

eng/common/cross/x64/sources.list.bionic

@@ -1,11 +0,0 @@
-deb http://archive.ubuntu.com/ubuntu/ bionic main restricted universe
-deb-src http://archive.ubuntu.com/ubuntu/ bionic main restricted universe
-
-deb http://archive.ubuntu.com/ubuntu/ bionic-updates main restricted universe
-deb-src http://archive.ubuntu.com/ubuntu/ bionic-updates main restricted universe
-
-deb http://archive.ubuntu.com/ubuntu/ bionic-backports main restricted
-deb-src http://archive.ubuntu.com/ubuntu/ bionic-backports main restricted
-
-deb http://archive.ubuntu.com/ubuntu/ bionic-security main restricted universe multiverse
-deb-src http://archive.ubuntu.com/ubuntu/ bionic-security main restricted universe multiverse

eng/common/cross/x64/sources.list.xenial

@@ -1,11 +0,0 @@
-deb http://archive.ubuntu.com/ubuntu/ xenial main restricted universe
-deb-src http://archive.ubuntu.com/ubuntu/ xenial main restricted universe
-
-deb http://archive.ubuntu.com/ubuntu/ xenial-updates main restricted universe
-deb-src http://archive.ubuntu.com/ubuntu/ xenial-updates main restricted universe
-
-deb http://archive.ubuntu.com/ubuntu/ xenial-backports main restricted
-deb-src http://archive.ubuntu.com/ubuntu/ xenial-backports main restricted
-
-deb http://archive.ubuntu.com/ubuntu/ xenial-security main restricted universe multiverse
-deb-src http://archive.ubuntu.com/ubuntu/ xenial-security main restricted universe multiverse

eng/common/cross/x86/sources.list.bionic

@@ -1,11 +0,0 @@
-deb http://archive.ubuntu.com/ubuntu/ bionic main restricted universe
-deb-src http://archive.ubuntu.com/ubuntu/ bionic main restricted universe
-
-deb http://archive.ubuntu.com/ubuntu/ bionic-updates main restricted universe
-deb-src http://archive.ubuntu.com/ubuntu/ bionic-updates main restricted universe
-
-deb http://archive.ubuntu.com/ubuntu/ bionic-backports main restricted
-deb-src http://archive.ubuntu.com/ubuntu/ bionic-backports main restricted
-
-deb http://archive.ubuntu.com/ubuntu/ bionic-security main restricted universe multiverse
-deb-src http://archive.ubuntu.com/ubuntu/ bionic-security main restricted universe multiverse

eng/common/cross/x86/sources.list.focal

@@ -1,11 +0,0 @@
-deb http://archive.ubuntu.com/ubuntu/ focal main restricted universe
-deb-src http://archive.ubuntu.com/ubuntu/ focal main restricted universe
-
-deb http://archive.ubuntu.com/ubuntu/ focal-updates main restricted universe
-deb-src http://archive.ubuntu.com/ubuntu/ focal-updates main restricted universe
-
-deb http://archive.ubuntu.com/ubuntu/ focal-backports main restricted
-deb-src http://archive.ubuntu.com/ubuntu/ focal-backports main restricted
-
-deb http://archive.ubuntu.com/ubuntu/ focal-security main restricted universe multiverse
-deb-src http://archive.ubuntu.com/ubuntu/ focal-security main restricted universe multiverse

eng/common/cross/x86/sources.list.jammy

@@ -1,11 +0,0 @@
-deb http://archive.ubuntu.com/ubuntu/ jammy main restricted universe
-deb-src http://archive.ubuntu.com/ubuntu/ jammy main restricted universe
-
-deb http://archive.ubuntu.com/ubuntu/ jammy-updates main restricted universe
-deb-src http://archive.ubuntu.com/ubuntu/ jammy-updates main restricted universe
-
-deb http://archive.ubuntu.com/ubuntu/ jammy-backports main restricted
-deb-src http://archive.ubuntu.com/ubuntu/ jammy-backports main restricted
-
-deb http://archive.ubuntu.com/ubuntu/ jammy-security main restricted universe multiverse
-deb-src http://archive.ubuntu.com/ubuntu/ jammy-security main restricted universe multiverse

eng/common/cross/x86/sources.list.xenial

@@ -1,11 +0,0 @@
-deb http://archive.ubuntu.com/ubuntu/ xenial main restricted universe
-deb-src http://archive.ubuntu.com/ubuntu/ xenial main restricted universe
-
-deb http://archive.ubuntu.com/ubuntu/ xenial-updates main restricted universe
-deb-src http://archive.ubuntu.com/ubuntu/ xenial-updates main restricted universe
-
-deb http://archive.ubuntu.com/ubuntu/ xenial-backports main restricted
-deb-src http://archive.ubuntu.com/ubuntu/ xenial-backports main restricted
-
-deb http://archive.ubuntu.com/ubuntu/ xenial-security main restricted universe multiverse
-deb-src http://archive.ubuntu.com/ubuntu/ xenial-security main restricted universe multiverse

eng/common/dotnet-install.sh

@@ -71,6 +71,9 @@ case $cpuname in
   i[3-6]86)
     buildarch=x86
     ;;
+  riscv64)
+    buildarch=riscv64
+    ;;
   *)
     echo "Unknown CPU $cpuname detected, treating it as x64"
     buildarch=x64
@@ -82,7 +85,7 @@ if [[ $architecture != "" ]] && [[ $architecture != $buildarch ]]; then
   dotnetRoot="$dotnetRoot/$architecture"
 fi
 
-InstallDotNet $dotnetRoot $version "$architecture" $runtime true $runtimeSourceFeed $runtimeSourceFeedKey || {
+InstallDotNet "$dotnetRoot" $version "$architecture" $runtime true $runtimeSourceFeed $runtimeSourceFeedKey || {
   local exit_code=$?
   Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "dotnet-install.sh failed (exit code '$exit_code')." >&2
   ExitWithExitCode $exit_code

eng/common/helixpublish.proj

@@ -1,3 +1,4 @@
+<!-- Licensed to the .NET Foundation under one or more agreements. The .NET Foundation licenses this file to you under the MIT license. -->
 <Project Sdk="Microsoft.DotNet.Helix.Sdk" DefaultTargets="Test">
 
   <PropertyGroup>

eng/common/internal/Directory.Build.props

@@ -1,4 +1,11 @@
 <!-- Licensed to the .NET Foundation under one or more agreements. The .NET Foundation licenses this file to you under the MIT license. -->
 <Project>
+
+  <PropertyGroup>
+    <ImportDirectoryBuildTargets>false</ImportDirectoryBuildTargets>
+    <ImportDirectoryPackagesProps>false</ImportDirectoryPackagesProps>
+  </PropertyGroup>
+
   <Import Project="Sdk.props" Sdk="Microsoft.DotNet.Arcade.Sdk" />
+
 </Project>

eng/common/internal/Tools.csproj

@@ -1,8 +1,8 @@
 <!-- Licensed to the .NET Foundation under one or more agreements. The .NET Foundation licenses this file to you under the MIT license. -->
 <Project Sdk="Microsoft.NET.Sdk">
+
   <PropertyGroup>
     <TargetFramework>net472</TargetFramework>
-    <ImportDirectoryBuildTargets>false</ImportDirectoryBuildTargets>
     <AutomaticallyUseReferenceAssemblyPackages>false</AutomaticallyUseReferenceAssemblyPackages>
   </PropertyGroup>
   <ItemGroup>
@@ -27,4 +27,5 @@
 
   <!-- Repository extensibility point -->
   <Import Project="$(RepositoryEngineeringDir)InternalTools.props" Condition="Exists('$(RepositoryEngineeringDir)InternalTools.props')" />
+
 </Project>

eng/common/native/CommonLibrary.psm1

@@ -277,7 +277,8 @@ function Get-MachineArchitecture {
   if (($ProcessorArchitecture -Eq "AMD64") -Or
       ($ProcessorArchitecture -Eq "IA64") -Or
       ($ProcessorArchitecture -Eq "ARM64") -Or
-      ($ProcessorArchitecture -Eq "LOONGARCH64")) {
+      ($ProcessorArchitecture -Eq "LOONGARCH64") -Or
+      ($ProcessorArchitecture -Eq "RISCV64")) {
     return "x64"
   }
   return "x86"

eng/common/native/init-compiler.sh

@@ -2,7 +2,9 @@
 #
 # This file detects the C/C++ compiler and exports it to the CC/CXX environment variables
 #
-# NOTE: some scripts source this file and rely on stdout being empty, make sure to not output anything here!
+# NOTE: some scripts source this file and rely on stdout being empty, make sure
+# to not output *anything* here, unless it is an error message that fails the
+# build.
 
 if [ -z "$build_arch" ] || [ -z "$compiler" ]; then
   echo "Usage..."
@@ -17,11 +19,9 @@ case "$compiler" in
         # clangx.y or clang-x.y
         version="$(echo "$compiler" | tr -d '[:alpha:]-=')"
         majorVersion="${version%%.*}"
-        [ -z "${version##*.*}" ] && minorVersion="${version#*.}"
 
-        if [ -z "$minorVersion" ] && [ -n "$majorVersion" ] && [ "$majorVersion" -le 6 ]; then
-            minorVersion=0;
-        fi
+        # LLVM based on v18 released in early 2024, with two releases per year
+        maxVersion="$((18 + ((($(date +%Y) - 2024) * 12 + $(date +%-m) - 3) / 6)))"
         compiler=clang
         ;;
 
@@ -29,7 +29,9 @@ case "$compiler" in
         # gccx.y or gcc-x.y
         version="$(echo "$compiler" | tr -d '[:alpha:]-=')"
         majorVersion="${version%%.*}"
-        [ -z "${version##*.*}" ] && minorVersion="${version#*.}"
+
+        # GCC based on v14 released in early 2024, with one release per year
+        maxVersion="$((14 + ((($(date +%Y) - 2024) * 12 + $(date +%-m) - 3) / 12)))"
         compiler=gcc
         ;;
 esac
@@ -47,91 +49,98 @@ check_version_exists() {
     desired_version=-1
 
     # Set up the environment to be used for building with the desired compiler.
-    if command -v "$compiler-$1.$2" > /dev/null; then
-        desired_version="-$1.$2"
-    elif command -v "$compiler$1$2" > /dev/null; then
-        desired_version="$1$2"
-    elif command -v "$compiler-$1$2" > /dev/null; then
-        desired_version="-$1$2"
+    if command -v "$compiler-$1" > /dev/null; then
+        desired_version="-$1"
+    elif command -v "$compiler$1" > /dev/null; then
+        desired_version="$1"
     fi
 
     echo "$desired_version"
 }
 
+__baseOS="$(uname)"
+set_compiler_version_from_CC() {
+    if [ "$__baseOS" = "Darwin" ]; then
+        # On Darwin, the versions from -version/-dumpversion refer to Xcode
+        # versions, not llvm versions, so we can't rely on them.
+        return
+    fi
+
+    version="$("$CC" -dumpversion)"
+    if [ -z "$version" ]; then
+        echo "Error: $CC -dumpversion didn't provide a version"
+        exit 1
+    fi
+
+    # gcc and clang often display 3 part versions. However, gcc can show only 1 part in some environments.
+    IFS=. read -r majorVersion _ <<EOF
+$version
+EOF
+}
+
 if [ -z "$CLR_CC" ]; then
 
     # Set default versions
     if [ -z "$majorVersion" ]; then
-        # note: gcc (all versions) and clang versions higher than 6 do not have minor version in file name, if it is zero.
-        if [ "$compiler" = "clang" ]; then versions="18 17 16 15 14 13 12 11 10 9 8 7 6.0 5.0 4.0 3.9 3.8 3.7 3.6 3.5"
-        elif [ "$compiler" = "gcc" ]; then versions="13 12 11 10 9 8 7 6 5 4.9"; fi
-
-        for version in $versions; do
-            _major="${version%%.*}"
-            [ -z "${version##*.*}" ] && _minor="${version#*.}"
-            desired_version="$(check_version_exists "$_major" "$_minor")"
-            if [ "$desired_version" != "-1" ]; then majorVersion="$_major"; break; fi
+        minVersion=8
+        maxVersion="$((maxVersion + 1))" # +1 for headspace
+        i="$maxVersion"
+        while [ "$i" -ge $minVersion ]; do
+            desired_version="$(check_version_exists "$i")"
+            if [ "$desired_version" != "-1" ]; then majorVersion="$i"; break; fi
+            i=$((i - 1))
         done
 
         if [ -z "$majorVersion" ]; then
-            if command -v "$compiler" > /dev/null; then
-                if [ "$(uname)" != "Darwin" ]; then
-                    echo "Warning: Specific version of $compiler not found, falling back to use the one in PATH."
-                fi
-                CC="$(command -v "$compiler")"
-                CXX="$(command -v "$cxxCompiler")"
-            else
-                echo "No usable version of $compiler found."
+            if ! command -v "$compiler" > /dev/null; then
+                echo "Error: No compatible version of $compiler was found within the range of $minVersion to $maxVersion. Please upgrade your toolchain or specify the compiler explicitly using CLR_CC and CLR_CXX environment variables."
                 exit 1
             fi
-        else
-            if [ "$compiler" = "clang" ] && [ "$majorVersion" -lt 5 ]; then
-                if [ "$build_arch" = "arm" ] || [ "$build_arch" = "armel" ]; then
-                    if command -v "$compiler" > /dev/null; then
-                        echo "Warning: Found clang version $majorVersion which is not supported on arm/armel architectures, falling back to use clang from PATH."
-                        CC="$(command -v "$compiler")"
-                        CXX="$(command -v "$cxxCompiler")"
-                    else
-                        echo "Found clang version $majorVersion which is not supported on arm/armel architectures, and there is no clang in PATH."
-                        exit 1
-                    fi
-                fi
-            fi
+
+            CC="$(command -v "$compiler" 2> /dev/null)"
+            CXX="$(command -v "$cxxCompiler" 2> /dev/null)"
+            set_compiler_version_from_CC
         fi
     else
-        desired_version="$(check_version_exists "$majorVersion" "$minorVersion")"
+        desired_version="$(check_version_exists "$majorVersion")"
         if [ "$desired_version" = "-1" ]; then
-            echo "Could not find specific version of $compiler: $majorVersion $minorVersion."
+            echo "Error: Could not find specific version of $compiler: $majorVersion."
             exit 1
         fi
     fi
 
     if [ -z "$CC" ]; then
-        CC="$(command -v "$compiler$desired_version")"
-        CXX="$(command -v "$cxxCompiler$desired_version")"
-        if [ -z "$CXX" ]; then CXX="$(command -v "$cxxCompiler")"; fi
+        CC="$(command -v "$compiler$desired_version" 2> /dev/null)"
+        CXX="$(command -v "$cxxCompiler$desired_version" 2> /dev/null)"
+        if [ -z "$CXX" ]; then CXX="$(command -v "$cxxCompiler" 2> /dev/null)"; fi
+        set_compiler_version_from_CC
     fi
 else
     if [ ! -f "$CLR_CC" ]; then
-        echo "CLR_CC is set but path '$CLR_CC' does not exist"
+        echo "Error: CLR_CC is set but path '$CLR_CC' does not exist"
         exit 1
     fi
     CC="$CLR_CC"
     CXX="$CLR_CXX"
+    set_compiler_version_from_CC
 fi
 
 if [ -z "$CC" ]; then
-    echo "Unable to find $compiler."
+    echo "Error: Unable to find $compiler."
     exit 1
 fi
 
-# Only lld version >= 9 can be considered stable. lld doesn't support s390x.
-if [ "$compiler" = "clang" ] && [ -n "$majorVersion" ] && [ "$majorVersion" -ge 9 ] && [ "$build_arch" != "s390x" ]; then
-    if "$CC" -fuse-ld=lld -Wl,--version >/dev/null 2>&1; then
-        LDFLAGS="-fuse-ld=lld"
+if [ "$__baseOS" != "Darwin" ]; then
+    # On Darwin, we always want to use the Apple linker.
+
+    # Only lld version >= 9 can be considered stable. lld supports s390x starting from 18.0.
+    if [ "$compiler" = "clang" ] && [ -n "$majorVersion" ] && [ "$majorVersion" -ge 9 ] && { [ "$build_arch" != "s390x" ] || [ "$majorVersion" -ge 18 ]; }; then
+        if "$CC" -fuse-ld=lld -Wl,--version >/dev/null 2>&1; then
+            LDFLAGS="-fuse-ld=lld"
+        fi
     fi
 fi
 
-SCAN_BUILD_COMMAND="$(command -v "scan-build$desired_version")"
+SCAN_BUILD_COMMAND="$(command -v "scan-build$desired_version" 2> /dev/null)"
 
 export CC CXX LDFLAGS SCAN_BUILD_COMMAND

eng/common/native/init-distro-rid.sh

@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/sh
 
 # getNonPortableDistroRid
 #
@@ -11,21 +11,16 @@
 #   non-portable rid
 getNonPortableDistroRid()
 {
-    local targetOs="$1"
-    local targetArch="$2"
-    local rootfsDir="$3"
-    local nonPortableRid=""
+    targetOs="$1"
+    targetArch="$2"
+    rootfsDir="$3"
+    nonPortableRid=""
 
     if [ "$targetOs" = "linux" ]; then
+        # shellcheck disable=SC1091
         if [ -e "${rootfsDir}/etc/os-release" ]; then
-            source "${rootfsDir}/etc/os-release"
-
-            if [[ "${ID}" == "rhel" || "${ID}" == "rocky" || "${ID}" == "alpine" ]]; then
-                # remove the last version digit
-                VERSION_ID="${VERSION_ID%.*}"
-            fi
-
-            if [[ "${VERSION_ID:-}" =~ ^([[:digit:]]|\.)+$ ]]; then
+            . "${rootfsDir}/etc/os-release"
+            if echo "${VERSION_ID:-}" | grep -qE '^([[:digit:]]|\.)+$'; then
                 nonPortableRid="${ID}.${VERSION_ID}-${targetArch}"
             else
                 # Rolling release distros either do not set VERSION_ID, set it as blank or
@@ -33,45 +28,33 @@ getNonPortableDistroRid()
                 # so omit it here to be consistent with everything else.
                 nonPortableRid="${ID}-${targetArch}"
             fi
-
         elif [ -e "${rootfsDir}/android_platform" ]; then
-            source "$rootfsDir"/android_platform
+            # shellcheck disable=SC1091
+            . "${rootfsDir}/android_platform"
             nonPortableRid="$RID"
         fi
     fi
 
     if [ "$targetOs" = "freebsd" ]; then
-        # $rootfsDir can be empty. freebsd-version is shell script and it should always work.
-        __freebsd_major_version=$($rootfsDir/bin/freebsd-version | { read v; echo "${v%%.*}"; })
+        # $rootfsDir can be empty. freebsd-version is a shell script and should always work.
+        __freebsd_major_version=$("$rootfsDir"/bin/freebsd-version | cut -d'.' -f1)
         nonPortableRid="freebsd.$__freebsd_major_version-${targetArch}"
-    elif command -v getprop && getprop ro.product.system.model 2>&1 | grep -qi android; then
+    elif command -v getprop >/dev/null && getprop ro.product.system.model | grep -qi android; then
         __android_sdk_version=$(getprop ro.build.version.sdk)
         nonPortableRid="android.$__android_sdk_version-${targetArch}"
     elif [ "$targetOs" = "illumos" ]; then
         __uname_version=$(uname -v)
-        case "$__uname_version" in
-            omnios-*)
-                __omnios_major_version=$(echo "${__uname_version:8:2}")
-                nonPortableRid=omnios."$__omnios_major_version"-"$targetArch"
-            ;;
-            joyent_*)
-                __smartos_major_version=$(echo "${__uname_version:7:4}")
-                nonPortableRid=smartos."$__smartos_major_version"-"$targetArch"
-            ;;
-            illumos_*)
-                nonPortableRid=openindiana-"$targetArch"
-            ;;
-        esac
+        nonPortableRid="illumos-${targetArch}"
     elif [ "$targetOs" = "solaris" ]; then
         __uname_version=$(uname -v)
-        __solaris_major_version=$(echo "${__uname_version%.*}")
-        nonPortableRid=solaris."$__solaris_major_version"-"$targetArch"
+        __solaris_major_version=$(echo "$__uname_version" | cut -d'.' -f1)
+        nonPortableRid="solaris.$__solaris_major_version-${targetArch}"
     elif [ "$targetOs" = "haiku" ]; then
-        __uname_release=$(uname -r)
+        __uname_release="$(uname -r)"
         nonPortableRid=haiku.r"$__uname_release"-"$targetArch"
     fi
 
-    echo "$(echo $nonPortableRid | tr '[:upper:]' '[:lower:]')"
+    echo "$nonPortableRid" | tr '[:upper:]' '[:lower:]'
 }
 
 # initDistroRidGlobal
@@ -85,26 +68,23 @@ getNonPortableDistroRid()
 #   None
 #
 # Notes:
-#
-# It is important to note that the function does not return anything, but it
-# exports the following variables on success:
-#
-#   __DistroRid   : Non-portable rid of the target platform.
-#   __PortableTargetOS  : OS-part of the portable rid that corresponds to the target platform.
-#
+#   It is important to note that the function does not return anything, but it
+#   exports the following variables on success:
+#     __DistroRid   : Non-portable rid of the target platform.
+#     __PortableTargetOS  : OS-part of the portable rid that corresponds to the target platform.
 initDistroRidGlobal()
 {
-    local targetOs="$1"
-    local targetArch="$2"
-    local rootfsDir=""
-    if [ "$#" -ge 3 ]; then
+    targetOs="$1"
+    targetArch="$2"
+    rootfsDir=""
+    if [ $# -ge 3 ]; then
         rootfsDir="$3"
     fi
 
     if [ -n "${rootfsDir}" ]; then
         # We may have a cross build. Check for the existence of the rootfsDir
         if [ ! -e "${rootfsDir}" ]; then
-            echo "Error rootfsDir has been passed, but the location is not valid."
+            echo "Error: rootfsDir has been passed, but the location is not valid."
             exit 1
         fi
     fi
@@ -119,7 +99,7 @@ initDistroRidGlobal()
             STRINGS="$(command -v llvm-strings || true)"
         fi
 
-        # Check for musl-based distros (e.g Alpine Linux, Void Linux).
+        # Check for musl-based distros (e.g. Alpine Linux, Void Linux).
         if "${rootfsDir}/usr/bin/ldd" --version 2>&1 | grep -q musl ||
                 ( [ -n "$STRINGS" ] && "$STRINGS" "${rootfsDir}/usr/bin/ldd" 2>&1 | grep -q musl ); then
             __PortableTargetOS="linux-musl"

eng/common/native/init-os-and-arch.sh

@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/sh
 
 # Use uname to determine what the OS is.
 OSName=$(uname -s | tr '[:upper:]' '[:lower:]')
@@ -35,6 +35,10 @@ fi
 case "$CPUName" in
     arm64|aarch64)
         arch=arm64
+        if [ "$(getconf LONG_BIT)" -lt 64 ]; then
+            # This is 32-bit OS running on 64-bit CPU (for example Raspberry Pi OS)
+            arch=arm
+        fi
         ;;
 
     loongarch64)
@@ -50,6 +54,7 @@ case "$CPUName" in
         ;;
 
     armv7l|armv8l)
+        # shellcheck disable=SC1091
         if (NAME=""; . /etc/os-release; test "$NAME" = "Tizen"); then
             arch=armel
         else

eng/common/post-build/add-build-to-channel.ps1

@@ -1,48 +0,0 @@
-param(
-  [Parameter(Mandatory=$true)][int] $BuildId,
-  [Parameter(Mandatory=$true)][int] $ChannelId,
-  [Parameter(Mandatory=$true)][string] $MaestroApiAccessToken,
-  [Parameter(Mandatory=$false)][string] $MaestroApiEndPoint = 'https://maestro.dot.net',
-  [Parameter(Mandatory=$false)][string] $MaestroApiVersion = '2019-01-16'
-)
-
-try {
-  . $PSScriptRoot\post-build-utils.ps1
-
-  # Check that the channel we are going to promote the build to exist
-  $channelInfo = Get-MaestroChannel -ChannelId $ChannelId
-
-  if (!$channelInfo) {
-    Write-PipelineTelemetryCategory -Category 'PromoteBuild' -Message "Channel with BAR ID $ChannelId was not found in BAR!"
-    ExitWithExitCode 1
-  }
-
-  # Get info about which channel(s) the build has already been promoted to
-  $buildInfo = Get-MaestroBuild -BuildId $BuildId
-  
-  if (!$buildInfo) {
-    Write-PipelineTelemetryError -Category 'PromoteBuild' -Message "Build with BAR ID $BuildId was not found in BAR!"
-    ExitWithExitCode 1
-  }
-
-  # Find whether the build is already assigned to the channel or not
-  if ($buildInfo.channels) {
-    foreach ($channel in $buildInfo.channels) {
-      if ($channel.Id -eq $ChannelId) {
-        Write-Host "The build with BAR ID $BuildId is already on channel $ChannelId!"
-        ExitWithExitCode 0
-      }
-    }
-  }
-
-  Write-Host "Promoting build '$BuildId' to channel '$ChannelId'."
-
-  Assign-BuildToChannel -BuildId $BuildId -ChannelId $ChannelId
-
-  Write-Host 'done.'
-} 
-catch {
-  Write-Host $_
-  Write-PipelineTelemetryError -Category 'PromoteBuild' -Message "There was an error while trying to promote build '$BuildId' to channel '$ChannelId'"
-  ExitWithExitCode 1
-}

eng/common/post-build/check-channel-consistency.ps1

@@ -4,10 +4,18 @@ param(
 )
 
 try {
-  . $PSScriptRoot\post-build-utils.ps1
+  $ErrorActionPreference = 'Stop'
+  Set-StrictMode -Version 2.0
+
+  # `tools.ps1` checks $ci to perform some actions. Since the post-build
+  # scripts don't necessarily execute in the same agent that run the
+  # build.ps1/sh script this variable isn't automatically set.
+  $ci = $true
+  $disableConfigureToolsetImport = $true
+  . $PSScriptRoot\..\tools.ps1
 
   if ($PromoteToChannels -eq "") {
-    Write-PipelineTaskError -Type 'warning' -Message "This build won't publish assets as it's not configured to any Maestro channel. If that wasn't intended use Darc to configure a default channel using add-default-channel for this branch or to promote it to a channel using add-build-to-channel. See https://github.com/dotnet/arcade/blob/master/Documentation/Darc.md#assigning-an-individual-build-to-a-channel for more info."
+    Write-PipelineTaskError -Type 'warning' -Message "This build won't publish assets as it's not configured to any Maestro channel. If that wasn't intended use Darc to configure a default channel using add-default-channel for this branch or to promote it to a channel using add-build-to-channel. See https://github.com/dotnet/arcade/blob/main/Documentation/Darc.md#assigning-an-individual-build-to-a-channel for more info."
     ExitWithExitCode 0
   }
 

eng/common/post-build/nuget-validation.ps1

@@ -2,20 +2,18 @@
 # tool: https://github.com/NuGet/NuGetGallery/tree/jver-verify/src/VerifyMicrosoftPackage
 
 param(
-  [Parameter(Mandatory=$true)][string] $PackagesPath,           # Path to where the packages to be validated are
-  [Parameter(Mandatory=$true)][string] $ToolDestinationPath     # Where the validation tool should be downloaded to
+  [Parameter(Mandatory=$true)][string] $PackagesPath # Path to where the packages to be validated are
 )
 
+# `tools.ps1` checks $ci to perform some actions. Since the post-build
+# scripts don't necessarily execute in the same agent that run the
+# build.ps1/sh script this variable isn't automatically set.
+$ci = $true
+$disableConfigureToolsetImport = $true
+. $PSScriptRoot\..\tools.ps1
+
 try {
-  . $PSScriptRoot\post-build-utils.ps1
-
-  $url = 'https://raw.githubusercontent.com/NuGet/NuGetGallery/3e25ad135146676bcab0050a516939d9958bfa5d/src/VerifyMicrosoftPackage/verify.ps1'
-
-  New-Item -ItemType 'directory' -Path ${ToolDestinationPath} -Force
-
-  Invoke-WebRequest $url -OutFile ${ToolDestinationPath}\verify.ps1 
-
-  & ${ToolDestinationPath}\verify.ps1 ${PackagesPath}\*.nupkg
+  & $PSScriptRoot\nuget-verification.ps1 ${PackagesPath}\*.nupkg
 } 
 catch {
   Write-Host $_.ScriptStackTrace

eng/common/post-build/nuget-verification.ps1

@@ -0,0 +1,121 @@
+<#
+.SYNOPSIS
+    Verifies that Microsoft NuGet packages have proper metadata.
+.DESCRIPTION
+    Downloads a verification tool and runs metadata validation on the provided NuGet packages. This script writes an
+    error if any of the provided packages fail validation. All arguments provided to this PowerShell script that do not
+    match PowerShell parameters are passed on to the verification tool downloaded during the execution of this script.
+.PARAMETER NuGetExePath
+    The path to the nuget.exe binary to use. If not provided, nuget.exe will be downloaded into the -DownloadPath
+    directory.
+.PARAMETER PackageSource
+    The package source to use to download the verification tool. If not provided, nuget.org will be used.
+.PARAMETER DownloadPath
+    The directory path to download the verification tool and nuget.exe to. If not provided,
+    %TEMP%\NuGet.VerifyNuGetPackage will be used.
+.PARAMETER args
+    Arguments that will be passed to the verification tool.
+.EXAMPLE
+    PS> .\verify.ps1 *.nupkg
+    Verifies the metadata of all .nupkg files in the currect working directory.
+.EXAMPLE
+    PS> .\verify.ps1 --help
+    Displays the help text of the downloaded verifiction tool.
+.LINK
+    https://github.com/NuGet/NuGetGallery/blob/master/src/VerifyMicrosoftPackage/README.md
+#>
+
+# This script was copied from https://github.com/NuGet/NuGetGallery/blob/3e25ad135146676bcab0050a516939d9958bfa5d/src/VerifyMicrosoftPackage/verify.ps1
+
+[CmdletBinding(PositionalBinding = $false)]
+param(
+   [string]$NuGetExePath,
+   [string]$PackageSource = "https://api.nuget.org/v3/index.json",
+   [string]$DownloadPath,
+   [Parameter(ValueFromRemainingArguments = $true)]
+   [string[]]$args
+)
+
+# The URL to download nuget.exe.
+$nugetExeUrl = "https://dist.nuget.org/win-x86-commandline/v4.9.4/nuget.exe"
+
+# The package ID of the verification tool.
+$packageId = "NuGet.VerifyMicrosoftPackage"
+
+# The location that nuget.exe and the verification tool will be downloaded to.
+if (!$DownloadPath) {
+    $DownloadPath = (Join-Path $env:TEMP "NuGet.VerifyMicrosoftPackage")
+}
+
+$fence = New-Object -TypeName string -ArgumentList '=', 80
+
+# Create the download directory, if it doesn't already exist.
+if (!(Test-Path $DownloadPath)) {
+    New-Item -ItemType Directory $DownloadPath | Out-Null
+}
+Write-Host "Using download path: $DownloadPath"
+
+if ($NuGetExePath) {
+    $nuget = $NuGetExePath
+} else {
+    $downloadedNuGetExe = Join-Path $DownloadPath "nuget.exe"
+    
+    # Download nuget.exe, if it doesn't already exist.
+    if (!(Test-Path $downloadedNuGetExe)) {
+        Write-Host "Downloading nuget.exe from $nugetExeUrl..."
+        $ProgressPreference = 'SilentlyContinue'
+        try {
+            Invoke-WebRequest $nugetExeUrl -OutFile $downloadedNuGetExe
+            $ProgressPreference = 'Continue'
+        } catch {
+            $ProgressPreference = 'Continue'
+            Write-Error $_
+            Write-Error "nuget.exe failed to download."
+            exit
+        }
+    }
+
+    $nuget = $downloadedNuGetExe
+}
+
+Write-Host "Using nuget.exe path: $nuget"
+Write-Host " "
+
+# Download the latest version of the verification tool.
+Write-Host "Downloading the latest version of $packageId from $packageSource..."
+Write-Host $fence
+& $nuget install $packageId `
+    -Prerelease `
+    -OutputDirectory $DownloadPath `
+    -Source $PackageSource
+Write-Host $fence
+Write-Host " "
+
+if ($LASTEXITCODE -ne 0) {
+    Write-Error "nuget.exe failed to fetch the verify tool."
+    exit
+}
+
+# Find the most recently downloaded tool
+Write-Host "Finding the most recently downloaded verification tool."
+$verifyProbePath = Join-Path $DownloadPath "$packageId.*"
+$verifyPath = Get-ChildItem -Path $verifyProbePath -Directory `
+    | Sort-Object -Property LastWriteTime -Descending `
+    | Select-Object -First 1
+$verify = Join-Path $verifyPath "tools\NuGet.VerifyMicrosoftPackage.exe"
+Write-Host "Using verification tool: $verify"
+Write-Host " "
+
+# Execute the verification tool.
+Write-Host "Executing the verify tool..."
+Write-Host $fence
+& $verify $args
+Write-Host $fence
+Write-Host " "
+
+# Respond to the exit code.
+if ($LASTEXITCODE -ne 0) {
+    Write-Error "The verify tool found some problems."
+} else {
+    Write-Output "The verify tool succeeded."
+}

eng/common/post-build/post-build-utils.ps1

@@ -1,91 +0,0 @@
-# Most of the functions in this file require the variables `MaestroApiEndPoint`, 
-# `MaestroApiVersion` and `MaestroApiAccessToken` to be globally available.
-
-$ErrorActionPreference = 'Stop'
-Set-StrictMode -Version 2.0
-
-# `tools.ps1` checks $ci to perform some actions. Since the post-build
-# scripts don't necessarily execute in the same agent that run the
-# build.ps1/sh script this variable isn't automatically set.
-$ci = $true
-$disableConfigureToolsetImport = $true
-. $PSScriptRoot\..\tools.ps1
-
-function Create-MaestroApiRequestHeaders([string]$ContentType = 'application/json') {
-  Validate-MaestroVars
-
-  $headers = New-Object 'System.Collections.Generic.Dictionary[[String],[String]]'
-  $headers.Add('Accept', $ContentType)
-  $headers.Add('Authorization',"Bearer $MaestroApiAccessToken")
-  return $headers
-}
-
-function Get-MaestroChannel([int]$ChannelId) {
-  Validate-MaestroVars
-
-  $apiHeaders = Create-MaestroApiRequestHeaders
-  $apiEndpoint = "$MaestroApiEndPoint/api/channels/${ChannelId}?api-version=$MaestroApiVersion"
-  
-  $result = try { Invoke-WebRequest -Method Get -Uri $apiEndpoint -Headers $apiHeaders | ConvertFrom-Json } catch { Write-Host "Error: $_" }
-  return $result
-}
-
-function Get-MaestroBuild([int]$BuildId) {
-  Validate-MaestroVars
-
-  $apiHeaders = Create-MaestroApiRequestHeaders -AuthToken $MaestroApiAccessToken
-  $apiEndpoint = "$MaestroApiEndPoint/api/builds/${BuildId}?api-version=$MaestroApiVersion"
-
-  $result = try { return Invoke-WebRequest -Method Get -Uri $apiEndpoint -Headers $apiHeaders | ConvertFrom-Json } catch { Write-Host "Error: $_" }
-  return $result
-}
-
-function Get-MaestroSubscriptions([string]$SourceRepository, [int]$ChannelId) {
-  Validate-MaestroVars
-
-  $SourceRepository = [System.Web.HttpUtility]::UrlEncode($SourceRepository) 
-  $apiHeaders = Create-MaestroApiRequestHeaders -AuthToken $MaestroApiAccessToken
-  $apiEndpoint = "$MaestroApiEndPoint/api/subscriptions?sourceRepository=$SourceRepository&channelId=$ChannelId&api-version=$MaestroApiVersion"
-
-  $result = try { Invoke-WebRequest -Method Get -Uri $apiEndpoint -Headers $apiHeaders | ConvertFrom-Json } catch { Write-Host "Error: $_" }
-  return $result
-}
-
-function Assign-BuildToChannel([int]$BuildId, [int]$ChannelId) {
-  Validate-MaestroVars
-
-  $apiHeaders = Create-MaestroApiRequestHeaders -AuthToken $MaestroApiAccessToken
-  $apiEndpoint = "$MaestroApiEndPoint/api/channels/${ChannelId}/builds/${BuildId}?api-version=$MaestroApiVersion"
-  Invoke-WebRequest -Method Post -Uri $apiEndpoint -Headers $apiHeaders | Out-Null
-}
-
-function Trigger-Subscription([string]$SubscriptionId) {
-  Validate-MaestroVars
-
-  $apiHeaders = Create-MaestroApiRequestHeaders -AuthToken $MaestroApiAccessToken
-  $apiEndpoint = "$MaestroApiEndPoint/api/subscriptions/$SubscriptionId/trigger?api-version=$MaestroApiVersion"
-  Invoke-WebRequest -Uri $apiEndpoint -Headers $apiHeaders -Method Post | Out-Null
-}
-
-function Validate-MaestroVars {
-  try {
-    Get-Variable MaestroApiEndPoint | Out-Null
-    Get-Variable MaestroApiVersion | Out-Null
-    Get-Variable MaestroApiAccessToken | Out-Null
-
-    if (!($MaestroApiEndPoint -Match '^http[s]?://maestro-(int|prod).westus2.cloudapp.azure.com$')) {
-      Write-PipelineTelemetryError -Category 'MaestroVars' -Message "MaestroApiEndPoint is not a valid Maestro URL. '$MaestroApiEndPoint'"
-      ExitWithExitCode 1  
-    }
-
-    if (!($MaestroApiVersion -Match '^[0-9]{4}-[0-9]{2}-[0-9]{2}$')) {
-      Write-PipelineTelemetryError -Category 'MaestroVars' -Message "MaestroApiVersion does not match a version string in the format yyyy-MM-DD. '$MaestroApiVersion'"
-      ExitWithExitCode 1
-    }
-  }
-  catch {
-    Write-PipelineTelemetryError -Category 'MaestroVars' -Message 'Error: Variables `MaestroApiEndPoint`, `MaestroApiVersion` and `MaestroApiAccessToken` are required while using this script.'
-    Write-Host $_
-    ExitWithExitCode 1
-  }
-}

eng/common/post-build/publish-using-darc.ps1

@@ -9,7 +9,12 @@ param(
 )
 
 try {
-  . $PSScriptRoot\post-build-utils.ps1
+  # `tools.ps1` checks $ci to perform some actions. Since the post-build
+  # scripts don't necessarily execute in the same agent that run the
+  # build.ps1/sh script this variable isn't automatically set.
+  $ci = $true
+  $disableConfigureToolsetImport = $true
+  . $PSScriptRoot\..\tools.ps1
 
   $darc = Get-Darc
 
@@ -37,6 +42,7 @@ try {
     --azdev-pat "$AzdoToken" `
     --bar-uri "$MaestroApiEndPoint" `
     --ci `
+    --verbose `
 	@optionalParams
 
   if ($LastExitCode -ne 0) {

eng/common/post-build/redact-logs.ps1

@@ -0,0 +1,89 @@
+[CmdletBinding(PositionalBinding=$False)]
+param(
+  [Parameter(Mandatory=$true, Position=0)][string] $InputPath,
+  [Parameter(Mandatory=$true)][string] $BinlogToolVersion,
+  [Parameter(Mandatory=$false)][string] $DotnetPath,
+  [Parameter(Mandatory=$false)][string] $PackageFeed = 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public/nuget/v3/index.json',
+  # File with strings to redact - separated by newlines.
+  #  For comments start the line with '# ' - such lines are ignored 
+  [Parameter(Mandatory=$false)][string] $TokensFilePath,
+  [Parameter(ValueFromRemainingArguments=$true)][String[]]$TokensToRedact
+)
+
+try {
+  $ErrorActionPreference = 'Stop'
+  Set-StrictMode -Version 2.0
+
+  # `tools.ps1` checks $ci to perform some actions. Since the post-build
+  # scripts don't necessarily execute in the same agent that run the
+  # build.ps1/sh script this variable isn't automatically set.
+  $ci = $true
+  $disableConfigureToolsetImport = $true
+  . $PSScriptRoot\..\tools.ps1
+
+  $packageName = 'binlogtool'
+
+  $dotnet = $DotnetPath
+
+  if (!$dotnet) {
+    $dotnetRoot = InitializeDotNetCli -install:$true
+    $dotnet = "$dotnetRoot\dotnet.exe"
+  }
+  
+  $toolList = & "$dotnet" tool list -g
+
+  if ($toolList -like "*$packageName*") {
+    & "$dotnet" tool uninstall $packageName -g
+  }
+
+  $toolPath  = "$PSScriptRoot\..\..\..\.tools"
+  $verbosity = 'minimal'
+  
+  New-Item -ItemType Directory -Force -Path $toolPath
+  
+  Push-Location -Path $toolPath
+
+  try {
+    Write-Host "Installing Binlog redactor CLI..."
+    Write-Host "'$dotnet' new tool-manifest"
+    & "$dotnet" new tool-manifest
+    Write-Host "'$dotnet' tool install $packageName --local --add-source '$PackageFeed' -v $verbosity --version $BinlogToolVersion"
+    & "$dotnet" tool install $packageName --local --add-source "$PackageFeed" -v $verbosity --version $BinlogToolVersion
+
+    if (Test-Path $TokensFilePath) {
+        Write-Host "Adding additional sensitive data for redaction from file: " $TokensFilePath
+        $TokensToRedact += Get-Content -Path $TokensFilePath | Foreach {$_.Trim()} | Where { $_ -notmatch "^# " }
+    }
+
+    $optionalParams = [System.Collections.ArrayList]::new()
+  
+    Foreach ($p in $TokensToRedact)
+    {
+      if($p -match '^\$\(.*\)$')
+      {
+        Write-Host ("Ignoring token {0} as it is probably unexpanded AzDO variable"  -f $p)
+      }          
+      elseif($p)
+      {
+        $optionalParams.Add("-p:" + $p) | Out-Null
+      }
+    }
+
+    & $dotnet binlogtool redact --input:$InputPath --recurse --in-place `
+      @optionalParams
+
+    if ($LastExitCode -ne 0) {
+      Write-PipelineTelemetryError -Category 'Redactor' -Type 'warning' -Message "Problems using Redactor tool (exit code: $LastExitCode). But ignoring them now."
+    }
+  }
+  finally {
+    Pop-Location
+  }
+
+  Write-Host 'done.'
+} 
+catch {
+  Write-Host $_
+  Write-PipelineTelemetryError -Category 'Redactor' -Message "There was an error while trying to redact logs. Error: $_"
+  ExitWithExitCode 1
+}

eng/common/post-build/sourcelink-validation.ps1

@@ -6,7 +6,15 @@ param(
   [Parameter(Mandatory=$true)][string] $SourcelinkCliVersion    # Version of SourceLink CLI to use
 )
 
-. $PSScriptRoot\post-build-utils.ps1
+$ErrorActionPreference = 'Stop'
+Set-StrictMode -Version 2.0
+
+# `tools.ps1` checks $ci to perform some actions. Since the post-build
+# scripts don't necessarily execute in the same agent that run the
+# build.ps1/sh script this variable isn't automatically set.
+$ci = $true
+$disableConfigureToolsetImport = $true
+. $PSScriptRoot\..\tools.ps1
 
 # Cache/HashMap (File -> Exist flag) used to consult whether a file exist 
 # in the repository at a specific commit point. This is populated by inserting

eng/common/post-build/symbols-validation.ps1

@@ -322,8 +322,6 @@ function InstallDotnetSymbol {
 }
 
 try {
-  . $PSScriptRoot\post-build-utils.ps1
-  
   InstallDotnetSymbol
 
   foreach ($Job in @(Get-Job)) {

eng/common/post-build/trigger-subscriptions.ps1

@@ -1,64 +0,0 @@
-param(
-  [Parameter(Mandatory=$true)][string] $SourceRepo,
-  [Parameter(Mandatory=$true)][int] $ChannelId,
-  [Parameter(Mandatory=$true)][string] $MaestroApiAccessToken,
-  [Parameter(Mandatory=$false)][string] $MaestroApiEndPoint = 'https://maestro.dot.net',
-  [Parameter(Mandatory=$false)][string] $MaestroApiVersion = '2019-01-16'
-)
-
-try {
-  . $PSScriptRoot\post-build-utils.ps1
-
-  # Get all the $SourceRepo subscriptions
-  $normalizedSourceRepo = $SourceRepo.Replace('dnceng@', '')
-  $subscriptions = Get-MaestroSubscriptions -SourceRepository $normalizedSourceRepo -ChannelId $ChannelId
-
-  if (!$subscriptions) {
-    Write-PipelineTelemetryError -Category 'TriggerSubscriptions' -Message "No subscriptions found for source repo '$normalizedSourceRepo' in channel '$ChannelId'"
-    ExitWithExitCode 0
-  }
-
-  $subscriptionsToTrigger = New-Object System.Collections.Generic.List[string]
-  $failedTriggeredSubscription = $false
-
-  # Get all enabled subscriptions that need dependency flow on 'everyBuild'
-  foreach ($subscription in $subscriptions) {
-    if ($subscription.enabled -and $subscription.policy.updateFrequency -like 'everyBuild' -and $subscription.channel.id -eq $ChannelId) {
-      Write-Host "Should trigger this subscription: ${$subscription.id}"
-      [void]$subscriptionsToTrigger.Add($subscription.id)
-    }
-  }
-
-  foreach ($subscriptionToTrigger in $subscriptionsToTrigger) {
-    try {
-      Write-Host "Triggering subscription '$subscriptionToTrigger'."
-
-      Trigger-Subscription -SubscriptionId $subscriptionToTrigger
-    
-      Write-Host 'done.'
-    } 
-    catch
-    {
-      Write-Host "There was an error while triggering subscription '$subscriptionToTrigger'"
-      Write-Host $_
-      Write-Host $_.ScriptStackTrace
-      $failedTriggeredSubscription = $true
-    }
-  }
-
-  if ($subscriptionsToTrigger.Count -eq 0) {
-    Write-Host "No subscription matched source repo '$normalizedSourceRepo' and channel ID '$ChannelId'."
-  }
-  elseif ($failedTriggeredSubscription) {
-    Write-PipelineTelemetryError -Category 'TriggerSubscriptions' -Message 'At least one subscription failed to be triggered...'
-    ExitWithExitCode 1
-  }
-  else {
-    Write-Host 'All subscriptions were triggered successfully!'
-  }
-}
-catch {
-  Write-Host $_.ScriptStackTrace
-  Write-PipelineTelemetryError -Category 'TriggerSubscriptions' -Message $_
-  ExitWithExitCode 1
-}

eng/common/sdk-task.ps1

@@ -64,7 +64,7 @@ try {
       $GlobalJson.tools | Add-Member -Name "vs" -Value (ConvertFrom-Json "{ `"version`": `"16.5`" }") -MemberType NoteProperty
     }
     if( -not ($GlobalJson.tools.PSObject.Properties.Name -match "xcopy-msbuild" )) {
-      $GlobalJson.tools | Add-Member -Name "xcopy-msbuild" -Value "17.8.1-2" -MemberType NoteProperty
+      $GlobalJson.tools | Add-Member -Name "xcopy-msbuild" -Value "17.10.0-pre.4.0" -MemberType NoteProperty
     }
     if ($GlobalJson.tools."xcopy-msbuild".Trim() -ine "none") {
         $xcopyMSBuildToolsFolder = InitializeXCopyMSBuild $GlobalJson.tools."xcopy-msbuild" -install $true

eng/common/sdl/NuGet.config

@@ -5,11 +5,11 @@
   </solution>
   <packageSources>
     <clear />
-    <add key="guardian" value="https://pkgs.dev.azure.com/dnceng/_packaging/Guardian1ESPTUpstreamOrgFeed/nuget/v3/index.json" />
+    <add key="guardian" value="https://securitytools.pkgs.visualstudio.com/_packaging/Guardian/nuget/v3/index.json" />
   </packageSources>
   <packageSourceMapping>
     <packageSource key="guardian">
-      <package pattern="Microsoft.Guardian.Cli.win-x64" />
+      <package pattern="microsoft.guardian.cli" />
     </packageSource>
   </packageSourceMapping>
   <disabledPackageSources>

eng/common/sdl/execute-all-sdl-tools.ps1

@@ -6,6 +6,7 @@ Param(
   [string] $BranchName=$env:BUILD_SOURCEBRANCH,                                                  # Optional: name of branch or version of gdn settings; defaults to master
   [string] $SourceDirectory=$env:BUILD_SOURCESDIRECTORY,                                         # Required: the directory where source files are located
   [string] $ArtifactsDirectory = (Join-Path $env:BUILD_ARTIFACTSTAGINGDIRECTORY ('artifacts')),  # Required: the directory where build artifacts are located
+  [string] $AzureDevOpsAccessToken,                                                              # Required: access token for dnceng; should be provided via KeyVault
 
   # Optional: list of SDL tools to run on source code. See 'configure-sdl-tool.ps1' for tools list
   # format.
@@ -74,7 +75,7 @@ try {
   }
 
   Exec-BlockVerbosely {
-    & $(Join-Path $PSScriptRoot 'init-sdl.ps1') -GuardianCliLocation $guardianCliLocation -Repository $RepoName -BranchName $BranchName -WorkingDirectory $workingDirectory -GuardianLoggerLevel $GuardianLoggerLevel
+    & $(Join-Path $PSScriptRoot 'init-sdl.ps1') -GuardianCliLocation $guardianCliLocation -Repository $RepoName -BranchName $BranchName -WorkingDirectory $workingDirectory -AzureDevOpsAccessToken $AzureDevOpsAccessToken -GuardianLoggerLevel $GuardianLoggerLevel
   }
   $gdnFolder = Join-Path $workingDirectory '.gdn'
 
@@ -103,6 +104,7 @@ try {
           -TargetDirectory $targetDirectory `
           -GdnFolder $gdnFolder `
           -ToolsList $tools `
+          -AzureDevOpsAccessToken $AzureDevOpsAccessToken `
           -GuardianLoggerLevel $GuardianLoggerLevel `
           -CrScanAdditionalRunConfigParams $CrScanAdditionalRunConfigParams `
           -PoliCheckAdditionalRunConfigParams $PoliCheckAdditionalRunConfigParams `

eng/common/sdl/init-sdl.ps1

@@ -3,6 +3,7 @@ Param(
   [string] $Repository,
   [string] $BranchName='master',
   [string] $WorkingDirectory,
+  [string] $AzureDevOpsAccessToken,
   [string] $GuardianLoggerLevel='Standard'
 )
 
@@ -20,7 +21,14 @@ $ci = $true
 # Don't display the console progress UI - it's a huge perf hit
 $ProgressPreference = 'SilentlyContinue'
 
+# Construct basic auth from AzDO access token; construct URI to the repository's gdn folder stored in that repository; construct location of zip file
+$encodedPat = [Convert]::ToBase64String([System.Text.Encoding]::ASCII.GetBytes(":$AzureDevOpsAccessToken"))
+$escapedRepository = [Uri]::EscapeDataString("/$Repository/$BranchName/.gdn")
+$uri = "https://dev.azure.com/dnceng/internal/_apis/git/repositories/sdl-tool-cfg/Items?path=$escapedRepository&versionDescriptor[versionOptions]=0&`$format=zip&api-version=5.0"
+$zipFile = "$WorkingDirectory/gdn.zip"
+
 Add-Type -AssemblyName System.IO.Compression.FileSystem
+$gdnFolder = (Join-Path $WorkingDirectory '.gdn')
 
 try {
   # if the folder does not exist, we'll do a guardian init and push it to the remote repository

eng/common/sdl/sdl.ps1

@@ -4,8 +4,6 @@ function Install-Gdn {
         [Parameter(Mandatory=$true)]
         [string]$Path,
 
-        [string]$Source = "https://pkgs.dev.azure.com/dnceng/_packaging/Guardian1ESPTUpstreamOrgFeed/nuget/v3/index.json",
-
         # If omitted, install the latest version of Guardian, otherwise install that specific version.
         [string]$Version
     )
@@ -21,7 +19,7 @@ function Install-Gdn {
     $ci = $true
     . $PSScriptRoot\..\tools.ps1
 
-    $argumentList = @("install", "Microsoft.Guardian.Cli.win-x64", "-Source $Source", "-OutputDirectory $Path", "-NonInteractive", "-NoCache")
+    $argumentList = @("install", "Microsoft.Guardian.Cli", "-Source https://securitytools.pkgs.visualstudio.com/_packaging/Guardian/nuget/v3/index.json", "-OutputDirectory $Path", "-NonInteractive", "-NoCache")
 
     if ($Version) {
         $argumentList += "-Version $Version"

eng/common/sdl/trim-assets-version.ps1

@@ -72,4 +72,4 @@ catch {
   Write-Host $_
   Write-PipelineTelemetryError -Force -Category 'Sdl' -Message $_
   ExitWithExitCode 1
-}
+}

eng/common/template-guidance.md

@@ -0,0 +1,133 @@
+# Overview
+
+Arcade provides templates for public (`/templates`) and 1ES pipeline templates (`/templates-official`) scenarios.  Pipelines which are required to be managed by 1ES pipeline templates should reference `/templates-offical`, all other pipelines may reference `/templates`.
+
+## How to use
+
+Basic guidance is:
+
+- 1ES Pipeline Template or 1ES Microbuild template runs should reference `eng/common/templates-official`. Any internal production-graded pipeline should use these templates.
+
+- All other runs should reference `eng/common/templates`.
+
+See [azure-pipelines.yml](../../azure-pipelines.yml) (templates-official example) or [azure-pipelines-pr.yml](../../azure-pipelines-pr.yml) (templates example) for examples.
+
+#### The `templateIs1ESManaged` parameter
+
+The `templateIs1ESManaged` is available on most templates and affects which of the variants is used for nested templates. See [Development Notes](#development-notes) below for more information on the `templateIs1ESManaged1 parameter.
+
+- For templates under `job/`, `jobs/`, `steps`, or `post-build/`, this parameter must be explicitly set.
+
+## Multiple outputs
+
+1ES pipeline templates impose a policy where every publish artifact execution results in additional security scans being injected into your pipeline.  When using `templates-official/jobs/jobs.yml`, Arcade reduces the number of additional security injections by gathering all publishing outputs into the [Build.ArtifactStagingDirectory](https://learn.microsoft.com/en-us/azure/devops/pipelines/build/variables?view=azure-devops&tabs=yaml#build-variables-devops-services), and utilizing the [outputParentDirectory](https://eng.ms/docs/cloud-ai-platform/devdiv/one-engineering-system-1es/1es-docs/1es-pipeline-templates/features/outputs#multiple-outputs) feature of 1ES pipeline templates.  When implementing your pipeline, if you ensure publish artifacts are located in the `$(Build.ArtifactStagingDirectory)`, and utilize the 1ES provided template context, then you can reduce the number of security scans for your pipeline.
+
+Example:
+``` yaml
+# azure-pipelines.yml
+extends:
+  template: azure-pipelines/MicroBuild.1ES.Official.yml@MicroBuildTemplate
+  parameters:
+    stages:
+    - stage: build
+      jobs:
+      - template: /eng/common/templates-official/jobs/jobs.yml@self
+        parameters:
+          # 1ES makes use of outputs to reduce security task injection overhead
+          templateContext:
+            outputs:
+            - output: pipelineArtifact
+              displayName: 'Publish logs from source'
+              continueOnError: true
+              condition: always()
+              targetPath: $(Build.ArtifactStagingDirectory)/artifacts/log
+              artifactName: Logs
+          jobs:
+          - job: Windows
+            steps:
+            - script: echo "friendly neighborhood" > artifacts/marvel/spiderman.txt
+          # copy build outputs to artifact staging directory for publishing
+          - task: CopyFiles@2
+              displayName: Gather build output
+              inputs:
+                SourceFolder: '$(Build.SourcesDirectory)/artifacts/marvel'
+                Contents: '**'
+                TargetFolder: '$(Build.ArtifactStagingDirectory)/artifacts/marvel'
+```
+
+Note: Multiple outputs are ONLY applicable to 1ES PT publishing (only usable when referencing `templates-official`).
+
+# Development notes
+
+**Folder / file structure**
+
+``` text
+eng\common\
+    [templates || templates-official]\
+        job\
+            job.yml                          (shim + artifact publishing logic)
+            onelocbuild.yml                  (shim)
+            publish-build-assets.yml         (shim)
+            source-build.yml                 (shim)
+            source-index-stage1.yml          (shim)
+        jobs\
+            codeql-build.yml                 (shim)
+            jobs.yml                         (shim)
+            source-build.yml                 (shim)
+        post-build\
+            post-build.yml                   (shim)
+            common-variabls.yml              (shim)
+            setup-maestro-vars.yml           (shim)
+        steps\
+            publish-build-artifacts.yml      (logic)
+            publish-pipeline-artifacts.yml   (logic)
+            component-governance.yml         (shim)
+            generate-sbom.yml                (shim)
+            publish-logs.yml                 (shim)
+            retain-build.yml                 (shim)
+            send-to-helix.yml                (shim)
+            source-build.yml                 (shim)
+        variables\
+            pool-providers.yml               (logic + redirect) # templates/variables/pool-providers.yml will redirect to templates-official/variables/pool-providers.yml if you are running in the internal project
+            sdl-variables.yml                (logic)
+    core-templates\
+        job\
+            job.yml                          (logic)
+            onelocbuild.yml                  (logic)
+            publish-build-assets.yml         (logic)
+            source-build.yml                 (logic)
+            source-index-stage1.yml          (logic)
+        jobs\
+            codeql-build.yml                 (logic)
+            jobs.yml                         (logic)
+            source-build.yml                 (logic)
+        post-build\
+            common-variabls.yml              (logic)
+            post-build.yml                   (logic)
+            setup-maestro-vars.yml           (logic)
+        steps\
+            component-governance.yml         (logic)
+            generate-sbom.yml                (logic)
+            publish-build-artifacts.yml      (redirect)
+            publish-logs.yml                 (logic)
+            publish-pipeline-artifacts.yml   (redirect)
+            retain-build.yml                 (logic)
+            send-to-helix.yml                (logic)
+            source-build.yml                 (logic)
+        variables\
+            pool-providers.yml               (redirect)
+```
+
+In the table above, a file is designated as "shim", "logic", or "redirect".
+
+- shim - represents a yaml file which is an intermediate step between pipeline logic and .Net Core Engineering's templates (`core-templates`) and defines the `is1ESPipeline` parameter value.
+
+- logic - represents actual base template logic.
+
+- redirect- represents a file in `core-templates` which redirects to the "logic" file in either `templates` or `templates-official`.
+
+Logic for Arcade's templates live **primarily** in the `core-templates` folder.  The exceptions to the location of the logic files are around artifact publishing, which is handled differently between 1es pipeline templates and standard templates.  `templates` and `templates-official` provide shim entry points which redirect to `core-templates` while also defining the `is1ESPipeline` parameter.  If a shim is referenced in `templates`, then `is1ESPipeline` is set to `false`.  If a shim is referenced in `templates-official`, then `is1ESPipeline` is set to `true`.
+
+Within `templates` and `templates-official`, the templates at the "stages", and "jobs" / "job" level have been replaced with shims.  Templates at the "steps" and "variables" level are typically too granular to be replaced with shims and instead persist logic which is directly applicable to either scenario.
+
+Within `core-templates`, there are a handful of places where logic is dependent on which shim entry point was used.  In those places, we redirect back to the respective logic file in `templates` or `templates-official`.

eng/common/templates-official/job/job.yml

@@ -1,264 +1,65 @@
-# Internal resources (telemetry, microbuild) can only be accessed from non-public projects,
-# and some (Microbuild) should only be applied to non-PR cases for internal builds.
-
-parameters:
-# Job schema parameters - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#job
-  cancelTimeoutInMinutes: ''
-  condition: ''
-  container: ''
-  continueOnError: false
-  dependsOn: ''
-  displayName: ''
-  pool: ''
-  steps: []
-  strategy: ''
-  timeoutInMinutes: ''
-  variables: []
-  workspace: ''
-  templateContext: ''
-
-# Job base template specific parameters
-  # See schema documentation - https://github.com/dotnet/arcade/blob/master/Documentation/AzureDevOps/TemplateSchema.md
-  artifacts: ''
-  enableMicrobuild: false
-  enablePublishBuildArtifacts: false
-  enablePublishBuildAssets: false
-  enablePublishTestResults: false
-  enablePublishUsingPipelines: false
-  enableBuildRetry: false
-  disableComponentGovernance: ''
-  componentGovernanceIgnoreDirectories: ''
-  mergeTestResults: false
-  testRunTitle: ''
-  testResultsFormat: ''
-  name: ''
-  preSteps: []
-  runAsPublic: false
-# Sbom related params
-  enableSbom: true
-  PackageVersion: 7.0.0
-  BuildDropPath: '$(Build.SourcesDirectory)/artifacts'
-
 jobs:
-- job: ${{ parameters.name }}
+- template: /eng/common/core-templates/job/job.yml
+  parameters:
+    is1ESPipeline: true
 
-  ${{ if ne(parameters.cancelTimeoutInMinutes, '') }}:
-    cancelTimeoutInMinutes: ${{ parameters.cancelTimeoutInMinutes }}
+    # publish artifacts
+    # for 1ES managed templates, use the templateContext.output to handle multiple outputs.
+    templateContext:
+      outputParentDirectory: $(Build.ArtifactStagingDirectory)
+      outputs:
+      - ${{ if ne(parameters.artifacts.publish, '') }}:
+        - ${{ if and(ne(parameters.artifacts.publish.artifacts, 'false'), ne(parameters.artifacts.publish.artifacts, '')) }}:
+          - output: buildArtifacts
+            displayName: Publish pipeline artifacts
+            PathtoPublish: '$(Build.ArtifactStagingDirectory)/artifacts'
+            ArtifactName: ${{ coalesce(parameters.artifacts.publish.artifacts.name , 'Artifacts_$(Agent.Os)_$(_BuildConfig)') }}
+            condition: always()
+            continueOnError: true
+        - ${{ if and(ne(parameters.artifacts.publish.logs, 'false'), ne(parameters.artifacts.publish.logs, '')) }}:
+          - output: pipelineArtifact
+            targetPath: '$(Build.ArtifactStagingDirectory)/artifacts/log'
+            artifactName: ${{ coalesce(parameters.artifacts.publish.logs.name, 'Logs_Build_$(Agent.Os)_$(_BuildConfig)_Attempt$(System.JobAttempt)') }}
+            displayName: 'Publish logs'
+            continueOnError: true
+            condition: always()
+            sbomEnabled: false  # we don't need SBOM for logs
 
-  ${{ if ne(parameters.condition, '') }}:
-    condition: ${{ parameters.condition }}
+      - ${{ if eq(parameters.enablePublishBuildArtifacts, true) }}:
+        - output: buildArtifacts
+          displayName: Publish Logs
+          PathtoPublish: '$(Build.ArtifactStagingDirectory)/artifacts/log/$(_BuildConfig)'
+          publishLocation: Container
+          ArtifactName: ${{ coalesce(parameters.enablePublishBuildArtifacts.artifactName, '$(Agent.Os)_$(Agent.JobName)' ) }}
+          continueOnError: true
+          condition: always()
+          sbomEnabled: false  # we don't need SBOM for logs
 
-  ${{ if ne(parameters.container, '') }}:
-    container: ${{ parameters.container }}
+      - ${{ if eq(parameters.enableBuildRetry, 'true') }}:
+        - output: pipelineArtifact
+          targetPath: '$(Build.ArtifactStagingDirectory)/artifacts/eng/common/BuildConfiguration'
+          artifactName: 'BuildConfiguration'
+          displayName: 'Publish build retry configuration'
+          continueOnError: true
+          sbomEnabled: false  # we don't need SBOM for BuildConfiguration
 
-  ${{ if ne(parameters.continueOnError, '') }}:
-    continueOnError: ${{ parameters.continueOnError }}
+      - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(parameters.enableSbom, 'true')) }}:
+        - output: pipelineArtifact
+          displayName: Publish SBOM manifest
+          continueOnError: true
+          targetPath: $(Build.ArtifactStagingDirectory)/sbom
+          artifactName: $(ARTIFACT_NAME)
 
-  ${{ if ne(parameters.dependsOn, '') }}:
-    dependsOn: ${{ parameters.dependsOn }}
+      # add any outputs provided via root yaml
+      - ${{ if ne(parameters.templateContext.outputs, '') }}:
+        - ${{ each output in parameters.templateContext.outputs }}:
+          - ${{ output }}
+      
+      # add any remaining templateContext properties
+      ${{ each context in parameters.templateContext }}:
+        ${{ if and(ne(context.key, 'outputParentDirectory'), ne(context.key, 'outputs')) }}:
+          ${{ context.key }}: ${{ context.value }}
 
-  ${{ if ne(parameters.displayName, '') }}:
-    displayName: ${{ parameters.displayName }}
-
-  ${{ if ne(parameters.pool, '') }}:
-    pool: ${{ parameters.pool }}
-
-  ${{ if ne(parameters.strategy, '') }}:
-    strategy: ${{ parameters.strategy }}
-
-  ${{ if ne(parameters.timeoutInMinutes, '') }}:
-    timeoutInMinutes: ${{ parameters.timeoutInMinutes }}
-
-  ${{ if ne(parameters.templateContext, '') }}:
-    templateContext: ${{ parameters.templateContext }}
-
-  variables:
-  - ${{ if ne(parameters.enableTelemetry, 'false') }}:
-    - name: DOTNET_CLI_TELEMETRY_PROFILE
-      value: '$(Build.Repository.Uri)'
-  - ${{ if eq(parameters.enableRichCodeNavigation, 'true') }}:
-    - name: EnableRichCodeNavigation
-      value: 'true'
-  # Retry signature validation up to three times, waiting 2 seconds between attempts.
-  # See https://learn.microsoft.com/en-us/nuget/reference/errors-and-warnings/nu3028#retry-untrusted-root-failures
-  - name: NUGET_EXPERIMENTAL_CHAIN_BUILD_RETRY_POLICY
-    value: 3,2000
-  - ${{ each variable in parameters.variables }}:
-    # handle name-value variable syntax
-    # example:
-    # - name: [key]
-    #   value: [value]
-    - ${{ if ne(variable.name, '') }}:
-      - name: ${{ variable.name }}
-        value: ${{ variable.value }}
-
-    # handle variable groups
-    - ${{ if ne(variable.group, '') }}:
-      - group: ${{ variable.group }}
-
-    # handle template variable syntax
-    # example:
-    # - template: path/to/template.yml
-    #   parameters:
-    #     [key]: [value]
-    - ${{ if ne(variable.template, '') }}:
-      - template: ${{ variable.template }}
-        ${{ if ne(variable.parameters, '') }}:
-          parameters: ${{ variable.parameters }}
-
-    # handle key-value variable syntax.
-    # example:
-    # - [key]: [value]
-    - ${{ if and(eq(variable.name, ''), eq(variable.group, ''), eq(variable.template, '')) }}:
-      - ${{ each pair in variable }}:
-        - name: ${{ pair.key }}
-          value: ${{ pair.value }}
-
-  # DotNet-HelixApi-Access provides 'HelixApiAccessToken' for internal builds
-  - ${{ if and(eq(parameters.enableTelemetry, 'true'), eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
-    - group: DotNet-HelixApi-Access
-
-  ${{ if ne(parameters.workspace, '') }}:
-    workspace: ${{ parameters.workspace }}
-
-  steps:
-  - ${{ if ne(parameters.preSteps, '') }}:
-    - ${{ each preStep in parameters.preSteps }}:
-      - ${{ preStep }}
-
-  - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
-    - ${{ if eq(parameters.enableMicrobuild, 'true') }}:
-      - task: MicroBuildSigningPlugin@4
-        displayName: Install MicroBuild plugin
-        inputs:
-          signType: $(_SignType)
-          zipSources: false
-          feedSource: https://dnceng.pkgs.visualstudio.com/_packaging/MicroBuildToolset/nuget/v3/index.json
-        env:
-          TeamName: $(_TeamName)
-          MicroBuildOutputFolderOverride: '$(Agent.TempDirectory)'
-        continueOnError: ${{ parameters.continueOnError }}
-        condition: and(succeeded(), in(variables['_SignType'], 'real', 'test'), eq(variables['Agent.Os'], 'Windows_NT'))
-
-  - ${{ if and(eq(parameters.runAsPublic, 'false'), eq(variables['System.TeamProject'], 'internal')) }}:
-    - task: NuGetAuthenticate@1
-
-  - ${{ if and(ne(parameters.artifacts.download, 'false'), ne(parameters.artifacts.download, '')) }}:
-    - task: DownloadPipelineArtifact@2
-      inputs:
-        buildType: current
-        artifactName: ${{ coalesce(parameters.artifacts.download.name, 'Artifacts_$(Agent.OS)_$(_BuildConfig)') }}
-        targetPath: ${{ coalesce(parameters.artifacts.download.path, 'artifacts') }}
-        itemPattern: ${{ coalesce(parameters.artifacts.download.pattern, '**') }}
-
-  - ${{ each step in parameters.steps }}:
-    - ${{ step }}
-
-  - ${{ if eq(parameters.enableRichCodeNavigation, true) }}:
-    - task: RichCodeNavIndexer@0
-      displayName: RichCodeNav Upload
-      inputs:
-        languages: ${{ coalesce(parameters.richCodeNavigationLanguage, 'csharp') }}
-        environment: ${{ coalesce(parameters.richCodeNavigationEnvironment, 'production') }}
-        richNavLogOutputDirectory: $(Build.SourcesDirectory)/artifacts/bin
-        uploadRichNavArtifacts: ${{ coalesce(parameters.richCodeNavigationUploadArtifacts, false) }}
-      continueOnError: true
-
-  - template: /eng/common/templates-official/steps/component-governance.yml
-    parameters:
-      ${{ if eq(parameters.disableComponentGovernance, '') }}:
-        ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(parameters.runAsPublic, 'false'), or(startsWith(variables['Build.SourceBranch'], 'refs/heads/release/'), startsWith(variables['Build.SourceBranch'], 'refs/heads/dotnet/'), startsWith(variables['Build.SourceBranch'], 'refs/heads/microsoft/'), eq(variables['Build.SourceBranch'], 'refs/heads/main'))) }}:
-          disableComponentGovernance: false
-        ${{ else }}:
-          disableComponentGovernance: true
-      ${{ else }}:
-        disableComponentGovernance: ${{ parameters.disableComponentGovernance }}
-      componentGovernanceIgnoreDirectories: ${{ parameters.componentGovernanceIgnoreDirectories }}
-
-  - ${{ if eq(parameters.enableMicrobuild, 'true') }}:
-    - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
-      - task: MicroBuildCleanup@1
-        displayName: Execute Microbuild cleanup tasks
-        condition: and(always(), in(variables['_SignType'], 'real', 'test'), eq(variables['Agent.Os'], 'Windows_NT'))
-        continueOnError: ${{ parameters.continueOnError }}
-        env:
-          TeamName: $(_TeamName)
-
-  - ${{ if ne(parameters.artifacts.publish, '') }}:
-    - ${{ if and(ne(parameters.artifacts.publish.artifacts, 'false'), ne(parameters.artifacts.publish.artifacts, '')) }}:
-      - task: CopyFiles@2
-        displayName: Gather binaries for publish to artifacts
-        inputs:
-          SourceFolder: 'artifacts/bin'
-          Contents: '**'
-          TargetFolder: '$(Build.ArtifactStagingDirectory)/artifacts/bin'
-      - task: CopyFiles@2
-        displayName: Gather packages for publish to artifacts
-        inputs:
-          SourceFolder: 'artifacts/packages'
-          Contents: '**'
-          TargetFolder: '$(Build.ArtifactStagingDirectory)/artifacts/packages'
-      - task: 1ES.PublishBuildArtifacts@1
-        displayName: Publish pipeline artifacts
-        inputs:
-          PathtoPublish: '$(Build.ArtifactStagingDirectory)/artifacts'
-          PublishLocation: Container
-          ArtifactName: ${{ coalesce(parameters.artifacts.publish.artifacts.name , 'Artifacts_$(Agent.Os)_$(_BuildConfig)') }}
-        continueOnError: true
-        condition: always()
-    - ${{ if and(ne(parameters.artifacts.publish.logs, 'false'), ne(parameters.artifacts.publish.logs, '')) }}:
-      - task: 1ES.PublishPipelineArtifact@1
-        inputs:
-          targetPath: 'artifacts/log'
-          artifactName: ${{ coalesce(parameters.artifacts.publish.logs.name, 'Logs_Build_$(Agent.Os)_$(_BuildConfig)') }}
-        displayName: 'Publish logs'
-        continueOnError: true
-        condition: always()
-
-  - ${{ if ne(parameters.enablePublishBuildArtifacts, 'false') }}:
-    - task: 1ES.PublishBuildArtifacts@1
-      displayName: Publish Logs
-      inputs:
-        PathtoPublish: '$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)'
-        PublishLocation: Container
-        ArtifactName: ${{ coalesce(parameters.enablePublishBuildArtifacts.artifactName, '$(Agent.Os)_$(Agent.JobName)' ) }}
-      continueOnError: true
-      condition: always()
-
-  - ${{ if or(and(eq(parameters.enablePublishTestResults, 'true'), eq(parameters.testResultsFormat, '')), eq(parameters.testResultsFormat, 'xunit')) }}:
-    - task: PublishTestResults@2
-      displayName: Publish XUnit Test Results
-      inputs:
-        testResultsFormat: 'xUnit'
-        testResultsFiles: '*.xml'
-        searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)'
-        testRunTitle: ${{ coalesce(parameters.testRunTitle, parameters.name, '$(System.JobName)') }}-xunit
-        mergeTestResults: ${{ parameters.mergeTestResults }}
-      continueOnError: true
-      condition: always()
-  - ${{ if or(and(eq(parameters.enablePublishTestResults, 'true'), eq(parameters.testResultsFormat, '')), eq(parameters.testResultsFormat, 'vstest')) }}:
-    - task: PublishTestResults@2
-      displayName: Publish TRX Test Results
-      inputs:
-        testResultsFormat: 'VSTest'
-        testResultsFiles: '*.trx'
-        searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)'
-        testRunTitle: ${{ coalesce(parameters.testRunTitle, parameters.name, '$(System.JobName)') }}-trx
-        mergeTestResults: ${{ parameters.mergeTestResults }}
-      continueOnError: true
-      condition: always()
-
-  - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(parameters.enableSbom, 'true')) }}:
-    - template: /eng/common/templates-official/steps/generate-sbom.yml
-      parameters:
-        PackageVersion: ${{ parameters.packageVersion}}
-        BuildDropPath: ${{ parameters.buildDropPath }}
-        IgnoreDirectories: ${{ parameters.componentGovernanceIgnoreDirectories }}
-
-  - ${{ if eq(parameters.enableBuildRetry, 'true') }}:
-    - task: 1ES.PublishPipelineArtifact@1
-      inputs:
-        targetPath: '$(Build.SourcesDirectory)\eng\common\BuildConfiguration'
-        artifactName: 'BuildConfiguration'
-      displayName: 'Publish build retry configuration'
-      continueOnError: true
+    ${{ each parameter in parameters }}:
+      ${{ if and(ne(parameter.key, 'templateContext'), ne(parameter.key, 'is1ESPipeline')) }}:
+        ${{ parameter.key }}: ${{ parameter.value }}

eng/common/templates-official/job/onelocbuild.yml

@@ -1,112 +1,7 @@
-parameters:
-  # Optional: dependencies of the job
-  dependsOn: ''
-
-  # Optional: A defined YAML pool - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#pool
-  pool: ''
-    
-  CeapexPat: $(dn-bot-ceapex-package-r) # PAT for the loc AzDO instance https://dev.azure.com/ceapex
-  GithubPat: $(BotAccount-dotnet-bot-repo-PAT)
-
-  SourcesDirectory: $(Build.SourcesDirectory)
-  CreatePr: true
-  AutoCompletePr: false
-  ReusePr: true
-  UseLfLineEndings: true
-  UseCheckedInLocProjectJson: false
-  SkipLocProjectJsonGeneration: false
-  LanguageSet: VS_Main_Languages
-  LclSource: lclFilesInRepo
-  LclPackageId: ''
-  RepoType: gitHub
-  GitHubOrg: dotnet
-  MirrorRepo: ''
-  MirrorBranch: main
-  condition: ''
-  JobNameSuffix: ''
-
 jobs:
-- job: OneLocBuild${{ parameters.JobNameSuffix }}
-  
-  dependsOn: ${{ parameters.dependsOn }}
+- template: /eng/common/core-templates/job/onelocbuild.yml
+  parameters:
+    is1ESPipeline: true
 
-  displayName: OneLocBuild${{ parameters.JobNameSuffix }}
-
-  variables:
-    - group: OneLocBuildVariables # Contains the CeapexPat and GithubPat
-    - name: _GenerateLocProjectArguments
-      value: -SourcesDirectory ${{ parameters.SourcesDirectory }}
-        -LanguageSet "${{ parameters.LanguageSet }}"
-        -CreateNeutralXlfs
-    - ${{ if eq(parameters.UseCheckedInLocProjectJson, 'true') }}:
-      - name: _GenerateLocProjectArguments
-        value: ${{ variables._GenerateLocProjectArguments }} -UseCheckedInLocProjectJson
-    - template: /eng/common/templates-official/variables/pool-providers.yml
-
-  ${{ if ne(parameters.pool, '') }}:
-    pool: ${{ parameters.pool }}
-  ${{ if eq(parameters.pool, '') }}:
-    pool:
-      # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com)
-      ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}:
-        name: AzurePipelines-EO
-        image: 1ESPT-Windows2022
-        demands: Cmd
-        os: windows
-      # If it's not devdiv, it's dnceng
-      ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}:
-        name: $(DncEngInternalBuildPool)
-        image: 1es-windows-2022
-        os: windows
-
-  steps:
-    - ${{ if ne(parameters.SkipLocProjectJsonGeneration, 'true') }}:
-      - task: Powershell@2
-        inputs:
-          filePath: $(Build.SourcesDirectory)/eng/common/generate-locproject.ps1
-          arguments: $(_GenerateLocProjectArguments)
-        displayName: Generate LocProject.json
-        condition: ${{ parameters.condition }}
-
-    - task: OneLocBuild@2
-      displayName: OneLocBuild
-      env:
-        SYSTEM_ACCESSTOKEN: $(System.AccessToken)
-      inputs:
-        locProj: eng/Localize/LocProject.json
-        outDir: $(Build.ArtifactStagingDirectory)
-        lclSource: ${{ parameters.LclSource }}
-        lclPackageId: ${{ parameters.LclPackageId }}
-        isCreatePrSelected: ${{ parameters.CreatePr }}
-        isAutoCompletePrSelected: ${{ parameters.AutoCompletePr }}
-        ${{ if eq(parameters.CreatePr, true) }}:
-          isUseLfLineEndingsSelected: ${{ parameters.UseLfLineEndings }}
-          ${{ if eq(parameters.RepoType, 'gitHub') }}:
-            isShouldReusePrSelected: ${{ parameters.ReusePr }}
-        packageSourceAuth: patAuth
-        patVariable: ${{ parameters.CeapexPat }}
-        ${{ if eq(parameters.RepoType, 'gitHub') }}:
-          repoType: ${{ parameters.RepoType }}
-          gitHubPatVariable: "${{ parameters.GithubPat }}"
-        ${{ if ne(parameters.MirrorRepo, '') }}:
-          isMirrorRepoSelected: true
-          gitHubOrganization: ${{ parameters.GitHubOrg }}
-          mirrorRepo: ${{ parameters.MirrorRepo }}
-          mirrorBranch: ${{ parameters.MirrorBranch }}
-      condition: ${{ parameters.condition }}
-
-    - task: 1ES.PublishBuildArtifacts@1
-      displayName: Publish Localization Files
-      inputs:
-        PathtoPublish: '$(Build.ArtifactStagingDirectory)/loc'
-        PublishLocation: Container
-        ArtifactName: Loc
-      condition: ${{ parameters.condition }}
-
-    - task: 1ES.PublishBuildArtifacts@1
-      displayName: Publish LocProject.json
-      inputs:
-        PathtoPublish: '$(Build.SourcesDirectory)/eng/Localize/'
-        PublishLocation: Container
-        ArtifactName: Loc
-      condition: ${{ parameters.condition }}
+    ${{ each parameter in parameters }}:
+      ${{ parameter.key }}: ${{ parameter.value }}

eng/common/templates-official/job/publish-build-assets.yml

@@ -1,160 +1,7 @@
-parameters:
-  configuration: 'Debug'
-
-  # Optional: condition for the job to run
-  condition: ''
-
-  # Optional: 'true' if future jobs should run even if this job fails
-  continueOnError: false
-
-  # Optional: dependencies of the job
-  dependsOn: ''
-
-  # Optional: Include PublishBuildArtifacts task
-  enablePublishBuildArtifacts: false
-
-  # Optional: A defined YAML pool - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#pool
-  pool: {}
-
-  # Optional: should run as a public build even in the internal project
-  #           if 'true', the build won't run any of the internal only steps, even if it is running in non-public projects.
-  runAsPublic: false
-
-  # Optional: whether the build's artifacts will be published using release pipelines or direct feed publishing
-  publishUsingPipelines: false
-
-  # Optional: whether the build's artifacts will be published using release pipelines or direct feed publishing
-  publishAssetsImmediately: false
-
-  artifactsPublishingAdditionalParameters: ''
-
-  signingValidationAdditionalParameters: ''
-
 jobs:
-- job: Asset_Registry_Publish
+- template: /eng/common/core-templates/job/publish-build-assets.yml
+  parameters:
+    is1ESPipeline: true
 
-  dependsOn: ${{ parameters.dependsOn }}
-  timeoutInMinutes: 150
-
-  ${{ if eq(parameters.publishAssetsImmediately, 'true') }}:
-    displayName: Publish Assets
-  ${{ else }}:
-    displayName: Publish to Build Asset Registry
-
-  variables:
-  - template: /eng/common/templates-official/variables/pool-providers.yml
-  - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
-    - group: Publish-Build-Assets
-    - group: AzureDevOps-Artifact-Feeds-Pats
-    - name: runCodesignValidationInjection
-      value: false
-    - ${{ if eq(parameters.publishAssetsImmediately, 'true') }}:
-      - template: /eng/common/templates-official/post-build/common-variables.yml
-
-  pool:
-    # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com)
-    ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}:
-      name: AzurePipelines-EO
-      image: 1ESPT-Windows2022
-      demands: Cmd
-      os: windows
-    # If it's not devdiv, it's dnceng
-    ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}:
-      name: NetCore1ESPool-Publishing-Internal
-      image: windows.vs2019.amd64
-      os: windows
-  steps:
-  - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
-    - task: DownloadBuildArtifacts@0
-      displayName: Download artifact
-      inputs:
-        artifactName: AssetManifests
-        downloadPath: '$(Build.StagingDirectory)/Download'
-        checkDownloadedFiles: true
-      condition: ${{ parameters.condition }}
-      continueOnError: ${{ parameters.continueOnError }}
-    
-    - task: NuGetAuthenticate@1
-
-    - task: AzureCLI@2
-      displayName: Publish Build Assets
-      inputs:
-        azureSubscription: "Darc: Maestro Production"
-        scriptType: ps
-        scriptLocation: scriptPath
-        scriptPath: $(Build.SourcesDirectory)/eng/common/sdk-task.ps1
-        arguments: >
-          -task PublishBuildAssets -restore -msbuildEngine dotnet
-          /p:ManifestsPath='$(Build.StagingDirectory)/Download/AssetManifests'
-          /p:MaestroApiEndpoint=https://maestro-prod.westus2.cloudapp.azure.com
-          /p:PublishUsingPipelines=${{ parameters.publishUsingPipelines }}
-          /p:OfficialBuildId=$(Build.BuildNumber)
-      condition: ${{ parameters.condition }}
-      continueOnError: ${{ parameters.continueOnError }}
-    
-    - task: powershell@2
-      displayName: Create ReleaseConfigs Artifact
-      inputs:
-        targetType: inline
-        script: |
-          New-Item -Path "$(Build.StagingDirectory)/ReleaseConfigs" -ItemType Directory -Force
-          $filePath = "$(Build.StagingDirectory)/ReleaseConfigs/ReleaseConfigs.txt"
-          Add-Content -Path $filePath -Value $(BARBuildId)
-          Add-Content -Path $filePath -Value "$(DefaultChannels)"
-          Add-Content -Path $filePath -Value $(IsStableBuild)
-    
-    - task: 1ES.PublishBuildArtifacts@1
-      displayName: Publish ReleaseConfigs Artifact
-      inputs:
-        PathtoPublish: '$(Build.StagingDirectory)/ReleaseConfigs'
-        PublishLocation: Container
-        ArtifactName: ReleaseConfigs
-
-    - task: powershell@2
-      displayName: Check if SymbolPublishingExclusionsFile.txt exists
-      inputs:
-        targetType: inline
-        script: |
-          $symbolExclusionfile = "$(Build.SourcesDirectory)/eng/SymbolPublishingExclusionsFile.txt"
-          if(Test-Path -Path $symbolExclusionfile)
-          {
-            Write-Host "SymbolExclusionFile exists"
-            Write-Host "##vso[task.setvariable variable=SymbolExclusionFile]true"
-          }
-          else{
-           Write-Host "Symbols Exclusion file does not exists"
-           Write-Host "##vso[task.setvariable variable=SymbolExclusionFile]false"
-          }
-
-    - task: 1ES.PublishBuildArtifacts@1
-      displayName: Publish SymbolPublishingExclusionsFile Artifact
-      condition: eq(variables['SymbolExclusionFile'], 'true') 
-      inputs:
-        PathtoPublish: '$(Build.SourcesDirectory)/eng/SymbolPublishingExclusionsFile.txt'
-        PublishLocation: Container
-        ArtifactName: ReleaseConfigs
-
-    - ${{ if eq(parameters.publishAssetsImmediately, 'true') }}:
-      - template: /eng/common/templates-official/post-build/setup-maestro-vars.yml
-        parameters:
-          BARBuildId: ${{ parameters.BARBuildId }}
-          PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
-
-      - task: AzureCLI@2
-        displayName: Publish Using Darc
-        inputs:
-          azureSubscription: "Darc: Maestro Production"
-          scriptType: ps
-          scriptLocation: scriptPath
-          scriptPath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1
-          arguments: -BuildId $(BARBuildId)
-            -PublishingInfraVersion 3
-            -AzdoToken '$(System.AccessToken)'
-            -WaitPublishingFinish true
-            -ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}'
-            -SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}'
-
-    - ${{ if eq(parameters.enablePublishBuildArtifacts, 'true') }}:
-      - template: /eng/common/templates-official/steps/publish-logs.yml
-        parameters:
-          JobLabel: 'Publish_Artifacts_Logs'     
+    ${{ each parameter in parameters }}:
+      ${{ parameter.key }}: ${{ parameter.value }}

eng/common/templates-official/job/source-build.yml

@@ -1,75 +1,7 @@
-parameters:
-  # This template adds arcade-powered source-build to CI. The template produces a server job with a
-  # default ID 'Source_Build_Complete' to put in a dependency list if necessary.
-
-  # Specifies the prefix for source-build jobs added to pipeline. Use this if disambiguation needed.
-  jobNamePrefix: 'Source_Build'
-
-  # Defines the platform on which to run the job. By default, a linux-x64 machine, suitable for
-  # managed-only repositories. This is an object with these properties:
-  #
-  # name: ''
-  #   The name of the job. This is included in the job ID.
-  # targetRID: ''
-  #   The name of the target RID to use, instead of the one auto-detected by Arcade.
-  # nonPortable: false
-  #   Enables non-portable mode. This means a more specific RID (e.g. fedora.32-x64 rather than
-  #   linux-x64), and compiling against distro-provided packages rather than portable ones.
-  # skipPublishValidation: false
-  #   Disables publishing validation.  By default, a check is performed to ensure no packages are
-  #   published by source-build.
-  # container: ''
-  #   A container to use. Runs in docker.
-  # pool: {}
-  #   A pool to use. Runs directly on an agent.
-  # buildScript: ''
-  #   Specifies the build script to invoke to perform the build in the repo. The default
-  #   './build.sh' should work for typical Arcade repositories, but this is customizable for
-  #   difficult situations.
-  # jobProperties: {}
-  #   A list of job properties to inject at the top level, for potential extensibility beyond
-  #   container and pool.
-  platform: {}
-
-  # If set to true and running on a non-public project,
-  # Internal blob storage locations will be enabled.
-  # This is not enabled by default because many repositories do not need internal sources
-  # and do not need to have the required service connections approved in the pipeline.
-  enableInternalSources: false
-
 jobs:
-- job: ${{ parameters.jobNamePrefix }}_${{ parameters.platform.name }}
-  displayName: Source-Build (${{ parameters.platform.name }})
+- template: /eng/common/core-templates/job/source-build.yml
+  parameters:
+    is1ESPipeline: true
 
-  ${{ each property in parameters.platform.jobProperties }}:
-    ${{ property.key }}: ${{ property.value }}
-
-  ${{ if ne(parameters.platform.container, '') }}:
-    container: ${{ parameters.platform.container }}
-
-  ${{ if eq(parameters.platform.pool, '') }}:
-    # The default VM host AzDO pool. This should be capable of running Docker containers: almost all
-    # source-build builds run in Docker, including the default managed platform.
-    # /eng/common/templates-official/variables/pool-providers.yml can't be used here (some customers declare variables already), so duplicate its logic
-    pool:
-      ${{ if eq(variables['System.TeamProject'], 'public') }}:
-        name: $[replace(replace(eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), True, 'NetCore-Svc-Public' ), False, 'NetCore-Public')]
-        demands: ImageOverride -equals Build.Ubuntu.1804.Amd64.Open
-
-      ${{ if eq(variables['System.TeamProject'], 'internal') }}:
-        name: $[replace(replace(eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), True, 'NetCore1ESPool-Svc-Internal'), False, 'NetCore1ESPool-Internal')]
-        image: 1es-mariner-2
-        os: linux
-
-  ${{ if ne(parameters.platform.pool, '') }}:
-    pool: ${{ parameters.platform.pool }}
-
-  workspace:
-    clean: all
-
-  steps:
-  - ${{ if eq(parameters.enableInternalSources, true) }}:
-    - template: /eng/common/templates-official/steps/enable-internal-runtimes.yml
-  - template: /eng/common/templates-official/steps/source-build.yml
-    parameters:
-      platform: ${{ parameters.platform }}
+    ${{ each parameter in parameters }}:
+      ${{ parameter.key }}: ${{ parameter.value }}

eng/common/templates-official/job/source-index-stage1.yml

@@ -1,83 +1,7 @@
-parameters:
-  runAsPublic: false
-  sourceIndexUploadPackageVersion: 2.0.0-20240502.12
-  sourceIndexProcessBinlogPackageVersion: 1.0.1-20240129.2
-  sourceIndexPackageSource: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json
-  sourceIndexBuildCommand: powershell -NoLogo -NoProfile -ExecutionPolicy Bypass -Command "eng/common/build.ps1 -restore -build -binarylog -ci"
-  preSteps: []
-  binlogPath: artifacts/log/Debug/Build.binlog
-  condition: ''
-  dependsOn: ''
-  pool: ''
-
 jobs:
-- job: SourceIndexStage1
-  dependsOn: ${{ parameters.dependsOn }}
-  condition: ${{ parameters.condition }}
-  variables:
-  - name: SourceIndexUploadPackageVersion
-    value: ${{ parameters.sourceIndexUploadPackageVersion }}
-  - name: SourceIndexProcessBinlogPackageVersion
-    value: ${{ parameters.sourceIndexProcessBinlogPackageVersion }}
-  - name: SourceIndexPackageSource
-    value: ${{ parameters.sourceIndexPackageSource }}
-  - name: BinlogPath
-    value: ${{ parameters.binlogPath }}
-  - template: /eng/common/templates-official/variables/pool-providers.yml
+- template: /eng/common/core-templates/job/source-index-stage1.yml
+  parameters:
+    is1ESPipeline: true
 
-  ${{ if ne(parameters.pool, '') }}:
-    pool: ${{ parameters.pool }}
-  ${{ if eq(parameters.pool, '') }}:
-    pool:
-      ${{ if eq(variables['System.TeamProject'], 'public') }}:
-        name: $(DncEngPublicBuildPool)
-        demands: ImageOverride -equals windows.vs2019.amd64.open
-      ${{ if eq(variables['System.TeamProject'], 'internal') }}:
-        name: $(DncEngInternalBuildPool)
-        image: windows.vs2022.amd64
-        os: windows
-
-  steps:
-  - ${{ each preStep in parameters.preSteps }}:
-    - ${{ preStep }}
-
-  - task: UseDotNet@2
-    displayName: Use .NET 8 SDK
-    inputs:
-      packageType: sdk
-      version: 8.0.x
-      installationPath: $(Agent.TempDirectory)/dotnet
-      workingDirectory: $(Agent.TempDirectory)
-
-  - script: |
-      $(Agent.TempDirectory)/dotnet/dotnet tool install BinLogToSln --version $(sourceIndexProcessBinlogPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools
-      $(Agent.TempDirectory)/dotnet/dotnet tool install UploadIndexStage1 --version $(sourceIndexUploadPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools
-    displayName: Download Tools
-    # Set working directory to temp directory so 'dotnet' doesn't try to use global.json and use the repo's sdk.
-    workingDirectory: $(Agent.TempDirectory)
-
-  - script: ${{ parameters.sourceIndexBuildCommand }}
-    displayName: Build Repository
-
-  - script: $(Agent.TempDirectory)/.source-index/tools/BinLogToSln -i $(BinlogPath) -r $(Build.SourcesDirectory) -n $(Build.Repository.Name) -o .source-index/stage1output
-    displayName: Process Binlog into indexable sln
-
-  - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
-    - task: AzureCLI@2
-      displayName: Get stage 1 auth token
-      inputs:
-        azureSubscription: 'SourceDotNet Stage1 Publish'
-        addSpnToEnvironment: true
-        scriptType: 'ps'
-        scriptLocation: 'inlineScript'
-        inlineScript: |
-          echo "##vso[task.setvariable variable=ARM_CLIENT_ID;issecret=true]$env:servicePrincipalId"
-          echo "##vso[task.setvariable variable=ARM_ID_TOKEN;issecret=true]$env:idToken"
-          echo "##vso[task.setvariable variable=ARM_TENANT_ID;issecret=true]$env:tenantId"
-
-    - script: |
-        az login --service-principal -u $(ARM_CLIENT_ID) --tenant $(ARM_TENANT_ID) --allow-no-subscriptions --federated-token $(ARM_ID_TOKEN)
-      displayName: "Login to Azure"
-
-    - script: $(Agent.TempDirectory)/.source-index/tools/UploadIndexStage1 -i .source-index/stage1output -n $(Build.Repository.Name) -s netsourceindexstage1 -b stage1
-      displayName: Upload stage1 artifacts to source index
+    ${{ each parameter in parameters }}:
+      ${{ parameter.key }}: ${{ parameter.value }}

eng/common/templates-official/jobs/codeql-build.yml

@@ -1,31 +1,7 @@
-parameters:
-  # See schema documentation in /Documentation/AzureDevOps/TemplateSchema.md
-  continueOnError: false
-  # Required: A collection of jobs to run - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#job
-  jobs: []
-  # Optional: if specified, restore and use this version of Guardian instead of the default.
-  overrideGuardianVersion: ''
-
 jobs:
-- template: /eng/common/templates-official/jobs/jobs.yml
+- template: /eng/common/core-templates/jobs/codeql-build.yml
   parameters:
-    enableMicrobuild: false
-    enablePublishBuildArtifacts: false
-    enablePublishTestResults: false
-    enablePublishBuildAssets: false
-    enablePublishUsingPipelines: false
-    enableTelemetry: true
+    is1ESPipeline: true
 
-    variables:
-      - group: Publish-Build-Assets
-      # The Guardian version specified in 'eng/common/sdl/packages.config'. This value must be kept in
-      # sync with the packages.config file.
-      - name: DefaultGuardianVersion
-        value: 0.109.0
-      - name: GuardianPackagesConfigFile
-        value: $(Build.SourcesDirectory)\eng\common\sdl\packages.config
-      - name: GuardianVersion
-        value: ${{ coalesce(parameters.overrideGuardianVersion, '$(DefaultGuardianVersion)') }}
-  
-    jobs: ${{ parameters.jobs }}
-        
+    ${{ each parameter in parameters }}:
+      ${{ parameter.key }}: ${{ parameter.value }}

eng/common/templates-official/jobs/jobs.yml

@@ -1,97 +1,7 @@
-parameters:
-  # See schema documentation in /Documentation/AzureDevOps/TemplateSchema.md
-  continueOnError: false
-
-  # Optional: Include PublishBuildArtifacts task
-  enablePublishBuildArtifacts: false
-
-  # Optional: Enable publishing using release pipelines
-  enablePublishUsingPipelines: false
-
-  # Optional: Enable running the source-build jobs to build repo from source
-  enableSourceBuild: false
-
-  # Optional: Parameters for source-build template.
-  #           See /eng/common/templates-official/jobs/source-build.yml for options
-  sourceBuildParameters: []
-
-  graphFileGeneration:
-    # Optional: Enable generating the graph files at the end of the build
-    enabled: false
-    # Optional: Include toolset dependencies in the generated graph files
-    includeToolset: false
-    
-  # Required: A collection of jobs to run - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#job
-  jobs: []
-
-  # Optional: Override automatically derived dependsOn value for "publish build assets" job
-  publishBuildAssetsDependsOn: ''
-
-  # Optional: Publish the assets as soon as the publish to BAR stage is complete, rather doing so in a separate stage.
-  publishAssetsImmediately: false
-
-  # Optional: If using publishAssetsImmediately and additional parameters are needed, can be used to send along additional parameters (normally sent to post-build.yml)
-  artifactsPublishingAdditionalParameters: ''
-  signingValidationAdditionalParameters: ''
-
-  # Optional: should run as a public build even in the internal project
-  #           if 'true', the build won't run any of the internal only steps, even if it is running in non-public projects.
-  runAsPublic: false
-
-  enableSourceIndex: false
-  sourceIndexParams: {}
-
-# Internal resources (telemetry, microbuild) can only be accessed from non-public projects,
-# and some (Microbuild) should only be applied to non-PR cases for internal builds.
-
 jobs:
-- ${{ each job in parameters.jobs }}:
-  - template: ../job/job.yml
-    parameters: 
-      # pass along parameters
-      ${{ each parameter in parameters }}:
-        ${{ if ne(parameter.key, 'jobs') }}:
-          ${{ parameter.key }}: ${{ parameter.value }}
+- template: /eng/common/core-templates/jobs/jobs.yml
+  parameters:
+    is1ESPipeline: true
 
-      # pass along job properties
-      ${{ each property in job }}:
-        ${{ if ne(property.key, 'job') }}:
-          ${{ property.key }}: ${{ property.value }}
-
-      name: ${{ job.job }}
-
-- ${{ if eq(parameters.enableSourceBuild, true) }}:
-  - template: /eng/common/templates-official/jobs/source-build.yml
-    parameters:
-      allCompletedJobId: Source_Build_Complete
-      ${{ each parameter in parameters.sourceBuildParameters }}:
-        ${{ parameter.key }}: ${{ parameter.value }}
-
-- ${{ if eq(parameters.enableSourceIndex, 'true') }}:
-  - template: ../job/source-index-stage1.yml
-    parameters:
-      runAsPublic: ${{ parameters.runAsPublic }}
-      ${{ each parameter in parameters.sourceIndexParams }}:
-        ${{ parameter.key }}: ${{ parameter.value }}
-
-- ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
-  - ${{ if or(eq(parameters.enablePublishBuildAssets, true), eq(parameters.artifacts.publish.manifests, 'true'), ne(parameters.artifacts.publish.manifests, '')) }}:
-    - template: ../job/publish-build-assets.yml
-      parameters:
-        continueOnError: ${{ parameters.continueOnError }}
-        dependsOn:
-        - ${{ if ne(parameters.publishBuildAssetsDependsOn, '') }}:
-          - ${{ each job in parameters.publishBuildAssetsDependsOn }}:
-            - ${{ job.job }}
-        - ${{ if eq(parameters.publishBuildAssetsDependsOn, '') }}:
-          - ${{ each job in parameters.jobs }}:
-            - ${{ job.job }}
-        - ${{ if eq(parameters.enableSourceBuild, true) }}:
-          - Source_Build_Complete
-
-        runAsPublic: ${{ parameters.runAsPublic }}
-        publishUsingPipelines: ${{ parameters.enablePublishUsingPipelines }}
-        publishAssetsImmediately: ${{ parameters.publishAssetsImmediately }}
-        enablePublishBuildArtifacts: ${{ parameters.enablePublishBuildArtifacts }}
-        artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
-        signingValidationAdditionalParameters: ${{ parameters.signingValidationAdditionalParameters }}
+    ${{ each parameter in parameters }}:
+      ${{ parameter.key }}: ${{ parameter.value }}

eng/common/templates-official/jobs/source-build.yml

@@ -1,54 +1,7 @@
-parameters:
-  # This template adds arcade-powered source-build to CI. A job is created for each platform, as
-  # well as an optional server job that completes when all platform jobs complete.
-
-  # The name of the "join" job for all source-build platforms. If set to empty string, the job is
-  # not included. Existing repo pipelines can use this job depend on all source-build jobs
-  # completing without maintaining a separate list of every single job ID: just depend on this one
-  # server job. By default, not included. Recommended name if used: 'Source_Build_Complete'.
-  allCompletedJobId: ''
-
-  # See /eng/common/templates-official/job/source-build.yml
-  jobNamePrefix: 'Source_Build'
-
-  # This is the default platform provided by Arcade, intended for use by a managed-only repo.
-  defaultManagedPlatform:
-    name: 'Managed'
-    container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream8'
-
-  # Defines the platforms on which to run build jobs. One job is created for each platform, and the
-  # object in this array is sent to the job template as 'platform'. If no platforms are specified,
-  # one job runs on 'defaultManagedPlatform'.
-  platforms: []
-
-  # If set to true and running on a non-public project,
-  # Internal nuget and blob storage locations will be enabled.
-  # This is not enabled by default because many repositories do not need internal sources
-  # and do not need to have the required service connections approved in the pipeline.
-  enableInternalSources: false
-
 jobs:
+- template: /eng/common/core-templates/jobs/source-build.yml
+  parameters:
+    is1ESPipeline: true
 
-- ${{ if ne(parameters.allCompletedJobId, '') }}:
-  - job: ${{ parameters.allCompletedJobId }}
-    displayName: Source-Build Complete
-    pool: server
-    dependsOn:
-    - ${{ each platform in parameters.platforms }}:
-      - ${{ parameters.jobNamePrefix }}_${{ platform.name }}
-    - ${{ if eq(length(parameters.platforms), 0) }}:
-      - ${{ parameters.jobNamePrefix }}_${{ parameters.defaultManagedPlatform.name }}
-
-- ${{ each platform in parameters.platforms }}:
-  - template: /eng/common/templates-official/job/source-build.yml
-    parameters:
-      jobNamePrefix: ${{ parameters.jobNamePrefix }}
-      platform: ${{ platform }}
-      enableInternalSources: ${{ parameters.enableInternalSources }}
-
-- ${{ if eq(length(parameters.platforms), 0) }}:
-  - template: /eng/common/templates-official/job/source-build.yml
-    parameters:
-      jobNamePrefix: ${{ parameters.jobNamePrefix }}
-      platform: ${{ parameters.defaultManagedPlatform }}
-      enableInternalSources: ${{ parameters.enableInternalSources }}
+    ${{ each parameter in parameters }}:
+      ${{ parameter.key }}: ${{ parameter.value }}