Upgrade Arcade to 6.0.0-beta.21256.3.

NuGet.config

@@ -2,6 +2,6 @@
 <configuration>
   <!-- Only specify feed for Arcade SDK (see https://github.com/Microsoft/msbuild/issues/2982) -->->
   <packageSources>
-    <add key="dotnet-core" value="https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json" />
+    <add key="dotnet-eng" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json" />
   </packageSources>
 </configuration>

eng/Versions.props

@@ -16,7 +16,7 @@
   <!-- Toolset Dependencies -->
   <PropertyGroup>
     <DotNetCliProjectJsonVersion>1.0.0-preview2-1-003177</DotNetCliProjectJsonVersion>
-    <VSWhereVersion>2.2.7</VSWhereVersion>
+    <VSWhereVersion>2.6.7</VSWhereVersion>
   </PropertyGroup>
   <!-- Test Dependencies -->
   <PropertyGroup>

eng/common/PSScriptAnalyzerSettings.psd1

@@ -0,0 +1,11 @@
+@{
+    IncludeRules=@('PSAvoidUsingCmdletAliases',
+                   'PSAvoidUsingWMICmdlet',
+                   'PSAvoidUsingPositionalParameters',
+                   'PSAvoidUsingInvokeExpression',
+                   'PSUseDeclaredVarsMoreThanAssignments',
+                   'PSUseCmdletCorrectly',
+                   'PSStandardDSCFunctionsInResource',
+                   'PSUseIdenticalMandatoryParametersForDSC',
+                   'PSUseIdenticalParametersForDSC')
+}

eng/common/PublishToPackageFeed.proj

@@ -1,81 +0,0 @@
-<!-- Licensed to the .NET Foundation under one or more agreements. The .NET Foundation licenses this file to you under the MIT license. See the LICENSE file in the project root for more information. -->
-<Project Sdk="Microsoft.NET.Sdk">
-  <!--
-    This MSBuild file is intended to be used as the body of the default 
-    publishing release pipeline. The release pipeline will use this file
-    to invoke the PushToStaticFeed task that will read the build asset
-    manifest and publish the assets described in the manifest to
-    informed target feeds.
-  -->
-
-  <PropertyGroup>
-    <TargetFramework>netcoreapp2.1</TargetFramework>
-  </PropertyGroup>
-
-  <Import Project="$(MSBuildThisFileDirectory)DefaultVersions.props" Condition="Exists('$(MSBuildThisFileDirectory)DefaultVersions.props')" />
-
-  <!--
-    This won't be necessary once we solve this issue:
-    https://github.com/dotnet/arcade/issues/2266
-  -->
-  <Import Project="$(MSBuildThisFileDirectory)ArtifactsCategory.props" Condition="Exists('$(MSBuildThisFileDirectory)ArtifactsCategory.props')" />
-  
-  <Import Project="$(NuGetPackageRoot)microsoft.dotnet.build.tasks.feed\$(MicrosoftDotNetBuildTasksFeedVersion)\build\Microsoft.DotNet.Build.Tasks.Feed.targets" />
-
-  <Target Name="PublishToFeed">
-    <Error Condition="'$(ArtifactsCategory)' == ''" Text="ArtifactsCategory: The artifacts' category produced by the build wasn't provided." />
-    <Error Condition="'$(AccountKeyToStaticFeed)' == ''" Text="AccountKeyToStaticFeed: Account key for target feed wasn't provided." />
-    <Error Condition="'$(ManifestsBasePath)' == ''" Text="Full path to asset manifests directory wasn't provided." />
-    <Error Condition="'$(BlobBasePath)' == '' AND '$(PackageBasePath)' == ''" Text="A valid full path to BlobBasePath of PackageBasePath is required." />
-
-    <ItemGroup>
-      <!-- Include all manifests found in the manifest folder. -->
-      <ManifestFiles Include="$(ManifestsBasePath)*.xml" />
-    </ItemGroup>
-
-    <Error Condition="'@(ManifestFiles)' == ''" Text="No manifest file was found in the provided path: $(ManifestsBasePath)" />
-
-    <!--
-      For now the type of packages being published will be informed for the whole build.
-      Eventually this will be specified on a per package basis:
-      TODO: https://github.com/dotnet/arcade/issues/2266
-    -->
-    <PropertyGroup>
-      <TargetStaticFeed Condition="'$(ArtifactsCategory.ToUpper())' == '.NETCORE'">https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json</TargetStaticFeed>
-      <TargetStaticFeed Condition="'$(ArtifactsCategory.ToUpper())' == '.NETCOREVALIDATION'">https://dotnetfeed.blob.core.windows.net/arcade-validation/index.json</TargetStaticFeed>
-      <TargetStaticFeed Condition="'$(ArtifactsCategory.ToUpper())' == 'ASPNETCORE'">https://dotnetfeed.blob.core.windows.net/aspnet-aspnetcore/index.json</TargetStaticFeed>
-      <TargetStaticFeed Condition="'$(ArtifactsCategory.ToUpper())' == 'ASPNETCORETOOLING'">https://dotnetfeed.blob.core.windows.net/aspnet-aspnetcore-tooling/index.json</TargetStaticFeed>
-      <TargetStaticFeed Condition="'$(ArtifactsCategory.ToUpper())' == 'ENTITYFRAMEWORKCORE'">https://dotnetfeed.blob.core.windows.net/aspnet-entityframeworkcore/index.json</TargetStaticFeed>
-      <TargetStaticFeed Condition="'$(ArtifactsCategory.ToUpper())' == 'ASPNETEXTENSIONS'">https://dotnetfeed.blob.core.windows.net/aspnet-extensions/index.json</TargetStaticFeed>
-      <TargetStaticFeed Condition="'$(ArtifactsCategory.ToUpper())' == 'CORECLR'">https://dotnetfeed.blob.core.windows.net/dotnet-coreclr/index.json</TargetStaticFeed>
-      <TargetStaticFeed Condition="'$(ArtifactsCategory.ToUpper())' == 'CORESDK'">https://dotnetfeed.blob.core.windows.net/dotnet-sdk/index.json</TargetStaticFeed>
-      <TargetStaticFeed Condition="'$(ArtifactsCategory.ToUpper())' == 'TOOLSINTERNAL'">https://dotnetfeed.blob.core.windows.net/dotnet-tools-internal/index.json</TargetStaticFeed>
-      <TargetStaticFeed Condition="'$(ArtifactsCategory.ToUpper())' == 'TOOLSET'">https://dotnetfeed.blob.core.windows.net/dotnet-toolset/index.json</TargetStaticFeed>
-      <TargetStaticFeed Condition="'$(ArtifactsCategory.ToUpper())' == 'WINDOWSDESKTOP'">https://dotnetfeed.blob.core.windows.net/dotnet-windowsdesktop/index.json</TargetStaticFeed>
-      <TargetStaticFeed Condition="'$(ArtifactsCategory.ToUpper())' == 'NUGETCLIENT'">https://dotnetfeed.blob.core.windows.net/nuget-nugetclient/index.json</TargetStaticFeed>
-    </PropertyGroup>
-
-    <Error 
-      Condition="'$(TargetStaticFeed)' == ''" 
-      Text="'$(ArtifactsCategory)' wasn't recognized as a valid artifact category. Valid categories are: '.NetCore' and '.NetCoreValidation'" />
-
-    <!-- Iterate publishing assets from each manifest file. -->
-    <PushArtifactsInManifestToFeed
-      ExpectedFeedUrl="$(TargetStaticFeed)"
-      AccountKey="$(AccountKeyToStaticFeed)"
-      BARBuildId="$(BARBuildId)"
-      MaestroApiEndpoint="$(MaestroApiEndpoint)"
-      BuildAssetRegistryToken="$(BuildAssetRegistryToken)"
-      Overwrite="$(OverrideAssetsWithSameName)"
-      PassIfExistingItemIdentical="$(PassIfExistingItemIdentical)"
-      MaxClients="$(MaxParallelUploads)"
-      UploadTimeoutInMinutes="$(MaxUploadTimeoutInMinutes)"
-      AssetManifestPath="%(ManifestFiles.Identity)"
-      BlobAssetsBasePath="$(BlobBasePath)"
-      PackageAssetsBasePath="$(PackageBasePath)"/>
-  </Target>
-
-  <ItemGroup>
-    <PackageReference Include="Microsoft.DotNet.Build.Tasks.Feed" Version="$(MicrosoftDotNetBuildTasksFeedVersion)" />
-  </ItemGroup>
-</Project>

eng/common/PublishToSymbolServers.proj

@@ -1,74 +0,0 @@
-<!-- Licensed to the .NET Foundation under one or more agreements. The .NET Foundation licenses this file to you under the MIT license. See the LICENSE file in the project root for more information. -->
-<Project Sdk="Microsoft.NET.Sdk">
-  <!--
-    This MSBuild file is intended to be used as the body of the default 
-    publishing release pipeline. The release pipeline will use this file
-    to invoke the PublishSymbols tasks to publish symbols to MSDL and SymWeb.
-  
-    Parameters:
-  
-      - PDBArtifactsDirectory   : Full path to directory containing PDB files to be published.
-      - BlobBasePath            : Full path containing *.symbols.nupkg packages to be published.
-      - DotNetSymbolServerTokenMsdl   : PAT to access MSDL.
-      - DotNetSymbolServerTokenSymWeb : PAT to access SymWeb.
-      - DotNetSymbolExpirationInDays  : Expiration days for published packages. Default is 3650.
-  -->
-
-  <PropertyGroup>
-    <TargetFramework>netcoreapp2.1</TargetFramework>
-  </PropertyGroup>
-
-  <Import Project="$(NuGetPackageRoot)microsoft.symboluploader.build.task\$(SymbolUploaderVersion)\build\PublishSymbols.targets" />
-    
-  <Target Name="PublishSymbols">
-    <ItemGroup>
-      <FilesToPublishToSymbolServer Include="$(PDBArtifactsDirectory)\*.pdb"/>
-      <PackagesToPublishToSymbolServer Include="$(BlobBasePath)\*.symbols.nupkg"/>
-    </ItemGroup>
-
-    <PropertyGroup>
-      <DotNetSymbolExpirationInDays Condition="'$(DotNetSymbolExpirationInDays)' == ''">3650</DotNetSymbolExpirationInDays>
-      <PublishToSymbolServer>true</PublishToSymbolServer>
-      <PublishToSymbolServer Condition="'@(FilesToPublishToSymbolServer)' == '' and '@(PackagesToPublishToSymbolServer)' == ''">false</PublishToSymbolServer>
-    </PropertyGroup>
-
-    <Message
-      Importance="High"
-      Text="No symbol package(s) were found to publish." 
-      Condition="$(PublishToSymbolServer) == false" />
-    
-    <!-- Symbol Uploader: MSDL -->
-    <Message Importance="High" Text="Publishing symbol packages to MSDL ..." Condition="$(PublishToSymbolServer)" />
-    <PublishSymbols PackagesToPublish="@(PackagesToPublishToSymbolServer)"
-                    FilesToPublish="@(FilesToPublishToSymbolServer)"
-                    PersonalAccessToken="$(DotNetSymbolServerTokenMsdl)"
-                    SymbolServerPath="https://microsoftpublicsymbols.artifacts.visualstudio.com/DefaultCollection"
-                    ExpirationInDays="$(DotNetSymbolExpirationInDays)"
-                    VerboseLogging="true"
-                    DryRun="false"
-                    ConvertPortablePdbsToWindowsPdbs="false"
-                    PdbConversionTreatAsWarning=""
-                    Condition="$(PublishToSymbolServer)"/>
-
-    <!-- 
-      Symbol Uploader: SymWeb 
-      Watson, VS insertion testings and the typical internal dev usage require SymWeb.
-      Currently we need to call the task twice (https://github.com/dotnet/core-eng/issues/3489).
-    -->
-    <Message Importance="High" Text="Publishing symbol packages to SymWeb ..." Condition="$(PublishToSymbolServer)" />
-    <PublishSymbols PackagesToPublish="@(PackagesToPublishToSymbolServer)"
-                    FilesToPublish="@(FilesToPublishToSymbolServer)"
-                    PersonalAccessToken="$(DotNetSymbolServerTokenSymWeb)"
-                    SymbolServerPath="https://microsoft.artifacts.visualstudio.com/DefaultCollection"
-                    ExpirationInDays="$(DotNetSymbolExpirationInDays)"
-                    VerboseLogging="true"
-                    DryRun="false"
-                    ConvertPortablePdbsToWindowsPdbs="false"
-                    PdbConversionTreatAsWarning=""
-                    Condition="$(PublishToSymbolServer)"/>
-  </Target>
-
-  <ItemGroup>
-    <PackageReference Include="Microsoft.SymbolUploader.Build.Task" Version="$(SymbolUploaderVersion)" />
-  </ItemGroup>
-</Project>

eng/common/SetupNugetSources.ps1

@@ -0,0 +1,161 @@
+# This file is a temporary workaround for internal builds to be able to restore from private AzDO feeds.
+# This file should be removed as part of this issue: https://github.com/dotnet/arcade/issues/4080
+#
+# What the script does is iterate over all package sources in the pointed NuGet.config and add a credential entry
+# under <packageSourceCredentials> for each Maestro managed private feed. Two additional credential 
+# entries are also added for the two private static internal feeds: dotnet3-internal and dotnet3-internal-transport.
+#
+# This script needs to be called in every job that will restore packages and which the base repo has
+# private AzDO feeds in the NuGet.config.
+#
+# See example YAML call for this script below. Note the use of the variable `$(dn-bot-dnceng-artifact-feeds-rw)`
+# from the AzureDevOps-Artifact-Feeds-Pats variable group.
+#
+# Any disabledPackageSources entries which start with "darc-int" will be re-enabled as part of this script executing
+#
+#  - task: PowerShell@2
+#    displayName: Setup Private Feeds Credentials
+#    condition: eq(variables['Agent.OS'], 'Windows_NT')
+#    inputs:
+#      filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.ps1
+#      arguments: -ConfigFile $(Build.SourcesDirectory)/NuGet.config -Password $Env:Token
+#    env:
+#      Token: $(dn-bot-dnceng-artifact-feeds-rw)
+
+[CmdletBinding()]
+param (
+    [Parameter(Mandatory = $true)][string]$ConfigFile,
+    [Parameter(Mandatory = $true)][string]$Password
+)
+
+$ErrorActionPreference = "Stop"
+Set-StrictMode -Version 2.0
+[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
+
+. $PSScriptRoot\tools.ps1
+
+# Add source entry to PackageSources
+function AddPackageSource($sources, $SourceName, $SourceEndPoint, $creds, $Username, $Password) {
+    $packageSource = $sources.SelectSingleNode("add[@key='$SourceName']")
+    
+    if ($packageSource -eq $null)
+    {
+        $packageSource = $doc.CreateElement("add")
+        $packageSource.SetAttribute("key", $SourceName)
+        $packageSource.SetAttribute("value", $SourceEndPoint)
+        $sources.AppendChild($packageSource) | Out-Null
+    }
+    else {
+        Write-Host "Package source $SourceName already present."
+    }
+    
+    AddCredential -Creds $creds -Source $SourceName -Username $Username -Password $Password
+}
+
+# Add a credential node for the specified source
+function AddCredential($creds, $source, $username, $password) {
+    # Looks for credential configuration for the given SourceName. Create it if none is found.
+    $sourceElement = $creds.SelectSingleNode($Source)
+    if ($sourceElement -eq $null)
+    {
+        $sourceElement = $doc.CreateElement($Source)
+        $creds.AppendChild($sourceElement) | Out-Null
+    }
+
+    # Add the <Username> node to the credential if none is found.
+    $usernameElement = $sourceElement.SelectSingleNode("add[@key='Username']")
+    if ($usernameElement -eq $null)
+    {
+        $usernameElement = $doc.CreateElement("add")
+        $usernameElement.SetAttribute("key", "Username")
+        $sourceElement.AppendChild($usernameElement) | Out-Null
+    }
+    $usernameElement.SetAttribute("value", $Username)
+
+    # Add the <ClearTextPassword> to the credential if none is found.
+    # Add it as a clear text because there is no support for encrypted ones in non-windows .Net SDKs.
+    #   -> https://github.com/NuGet/Home/issues/5526
+    $passwordElement = $sourceElement.SelectSingleNode("add[@key='ClearTextPassword']")
+    if ($passwordElement -eq $null)
+    {
+        $passwordElement = $doc.CreateElement("add")
+        $passwordElement.SetAttribute("key", "ClearTextPassword")
+        $sourceElement.AppendChild($passwordElement) | Out-Null
+    }
+    $passwordElement.SetAttribute("value", $Password)
+}
+
+function InsertMaestroPrivateFeedCredentials($Sources, $Creds, $Username, $Password) {
+    $maestroPrivateSources = $Sources.SelectNodes("add[contains(@key,'darc-int')]")
+
+    Write-Host "Inserting credentials for $($maestroPrivateSources.Count) Maestro's private feeds."
+    
+    ForEach ($PackageSource in $maestroPrivateSources) {
+        Write-Host "`tInserting credential for Maestro's feed:" $PackageSource.Key
+        AddCredential -Creds $creds -Source $PackageSource.Key -Username $Username -Password $Password
+    }
+}
+
+function EnablePrivatePackageSources($DisabledPackageSources) {
+    $maestroPrivateSources = $DisabledPackageSources.SelectNodes("add[contains(@key,'darc-int')]")
+    ForEach ($DisabledPackageSource in $maestroPrivateSources) {
+        Write-Host "`tEnsuring private source '$($DisabledPackageSource.key)' is enabled by deleting it from disabledPackageSource"
+        # Due to https://github.com/NuGet/Home/issues/10291, we must actually remove the disabled entries
+        $DisabledPackageSources.RemoveChild($DisabledPackageSource)
+    }
+}
+
+if (!(Test-Path $ConfigFile -PathType Leaf)) {
+  Write-PipelineTelemetryError -Category 'Build' -Message "Eng/common/SetupNugetSources.ps1 returned a non-zero exit code. Couldn't find the NuGet config file: $ConfigFile"
+  ExitWithExitCode 1
+}
+
+if (!$Password) {
+    Write-PipelineTelemetryError -Category 'Build' -Message 'Eng/common/SetupNugetSources.ps1 returned a non-zero exit code. Please supply a valid PAT'
+    ExitWithExitCode 1
+}
+
+# Load NuGet.config
+$doc = New-Object System.Xml.XmlDocument
+$filename = (Get-Item $ConfigFile).FullName
+$doc.Load($filename)
+
+# Get reference to <PackageSources> or create one if none exist already
+$sources = $doc.DocumentElement.SelectSingleNode("packageSources")
+if ($sources -eq $null) {
+    $sources = $doc.CreateElement("packageSources")
+    $doc.DocumentElement.AppendChild($sources) | Out-Null
+}
+
+# Looks for a <PackageSourceCredentials> node. Create it if none is found.
+$creds = $doc.DocumentElement.SelectSingleNode("packageSourceCredentials")
+if ($creds -eq $null) {
+    $creds = $doc.CreateElement("packageSourceCredentials")
+    $doc.DocumentElement.AppendChild($creds) | Out-Null
+}
+
+# Check for disabledPackageSources; we'll enable any darc-int ones we find there
+$disabledSources = $doc.DocumentElement.SelectSingleNode("disabledPackageSources")
+if ($disabledSources -ne $null) {
+    Write-Host "Checking for any darc-int disabled package sources in the disabledPackageSources node"
+    EnablePrivatePackageSources -DisabledPackageSources $disabledSources
+}
+
+$userName = "dn-bot"
+
+# Insert credential nodes for Maestro's private feeds
+InsertMaestroPrivateFeedCredentials -Sources $sources -Creds $creds -Username $userName -Password $Password
+
+$dotnet31Source = $sources.SelectSingleNode("add[@key='dotnet3.1']")
+if ($dotnet31Source -ne $null) {
+    AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal/nuget/v2" -Creds $creds -Username $userName -Password $Password
+    AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-transport/nuget/v2" -Creds $creds -Username $userName -Password $Password
+}
+
+$dotnet5Source = $sources.SelectSingleNode("add[@key='dotnet5']")
+if ($dotnet5Source -ne $null) {
+    AddPackageSource -Sources $sources -SourceName "dotnet5-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet5-internal/nuget/v2" -Creds $creds -Username $userName -Password $Password
+    AddPackageSource -Sources $sources -SourceName "dotnet5-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet5-internal-transport/nuget/v2" -Creds $creds -Username $userName -Password $Password
+}
+
+$doc.Save($filename)

eng/common/SetupNugetSources.sh

@@ -0,0 +1,167 @@
+#!/usr/bin/env bash
+
+# This file is a temporary workaround for internal builds to be able to restore from private AzDO feeds.
+# This file should be removed as part of this issue: https://github.com/dotnet/arcade/issues/4080
+#
+# What the script does is iterate over all package sources in the pointed NuGet.config and add a credential entry
+# under <packageSourceCredentials> for each Maestro's managed private feed. Two additional credential 
+# entries are also added for the two private static internal feeds: dotnet3-internal and dotnet3-internal-transport.
+#
+# This script needs to be called in every job that will restore packages and which the base repo has
+# private AzDO feeds in the NuGet.config.
+#
+# See example YAML call for this script below. Note the use of the variable `$(dn-bot-dnceng-artifact-feeds-rw)`
+# from the AzureDevOps-Artifact-Feeds-Pats variable group.
+#
+# Any disabledPackageSources entries which start with "darc-int" will be re-enabled as part of this script executing.
+#
+#  - task: Bash@3
+#    displayName: Setup Private Feeds Credentials
+#    inputs:
+#      filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh
+#      arguments: $(Build.SourcesDirectory)/NuGet.config $Token
+#    condition: ne(variables['Agent.OS'], 'Windows_NT')
+#    env:
+#      Token: $(dn-bot-dnceng-artifact-feeds-rw)
+
+ConfigFile=$1
+CredToken=$2
+NL='\n'
+TB='    '
+
+source="${BASH_SOURCE[0]}"
+
+# resolve $source until the file is no longer a symlink
+while [[ -h "$source" ]]; do
+  scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
+  source="$(readlink "$source")"
+  # if $source was a relative symlink, we need to resolve it relative to the path where the
+  # symlink file was located
+  [[ $source != /* ]] && source="$scriptroot/$source"
+done
+scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
+
+. "$scriptroot/tools.sh"
+
+if [ ! -f "$ConfigFile" ]; then
+    Write-PipelineTelemetryError -Category 'Build' "Error: Eng/common/SetupNugetSources.sh returned a non-zero exit code. Couldn't find the NuGet config file: $ConfigFile"
+    ExitWithExitCode 1
+fi
+
+if [ -z "$CredToken" ]; then
+    Write-PipelineTelemetryError -category 'Build' "Error: Eng/common/SetupNugetSources.sh returned a non-zero exit code. Please supply a valid PAT"
+    ExitWithExitCode 1
+fi
+
+if [[ `uname -s` == "Darwin" ]]; then
+    NL=$'\\\n'
+    TB=''
+fi
+
+# Ensure there is a <packageSources>...</packageSources> section.
+grep -i "<packageSources>" $ConfigFile
+if [ "$?" != "0" ]; then
+    echo "Adding <packageSources>...</packageSources> section."
+    ConfigNodeHeader="<configuration>"
+    PackageSourcesTemplate="${TB}<packageSources>${NL}${TB}</packageSources>"
+
+    sed -i.bak "s|$ConfigNodeHeader|$ConfigNodeHeader${NL}$PackageSourcesTemplate|" $ConfigFile
+fi
+
+# Ensure there is a <packageSourceCredentials>...</packageSourceCredentials> section. 
+grep -i "<packageSourceCredentials>" $ConfigFile
+if [ "$?" != "0" ]; then
+    echo "Adding <packageSourceCredentials>...</packageSourceCredentials> section."
+
+    PackageSourcesNodeFooter="</packageSources>"
+    PackageSourceCredentialsTemplate="${TB}<packageSourceCredentials>${NL}${TB}</packageSourceCredentials>"
+
+    sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourcesNodeFooter${NL}$PackageSourceCredentialsTemplate|" $ConfigFile
+fi
+
+PackageSources=()
+
+# Ensure dotnet3.1-internal and dotnet3.1-internal-transport are in the packageSources if the public dotnet3.1 feeds are present
+grep -i "<add key=\"dotnet3.1\"" $ConfigFile
+if [ "$?" == "0" ]; then
+    grep -i "<add key=\"dotnet3.1-internal\"" $ConfigFile
+    if [ "$?" != "0" ]; then
+        echo "Adding dotnet3.1-internal to the packageSources."
+        PackageSourcesNodeFooter="</packageSources>"
+        PackageSourceTemplate="${TB}<add key=\"dotnet3.1-internal\" value=\"https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal/nuget/v2\" />"
+
+        sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" $ConfigFile
+    fi
+    PackageSources+=('dotnet3.1-internal')
+
+    grep -i "<add key=\"dotnet3.1-internal-transport\">" $ConfigFile
+    if [ "$?" != "0" ]; then
+        echo "Adding dotnet3.1-internal-transport to the packageSources."
+        PackageSourcesNodeFooter="</packageSources>"
+        PackageSourceTemplate="${TB}<add key=\"dotnet3.1-internal-transport\" value=\"https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-transport/nuget/v2\" />"
+
+        sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" $ConfigFile
+    fi
+    PackageSources+=('dotnet3.1-internal-transport')
+fi
+
+# Ensure dotnet5-internal and dotnet5-internal-transport are in the packageSources if the public dotnet5 feeds are present
+grep -i "<add key=\"dotnet5\"" $ConfigFile
+if [ "$?" == "0" ]; then
+    grep -i "<add key=\"dotnet5-internal\"" $ConfigFile
+    if [ "$?" != "0" ]; then
+        echo "Adding dotnet5-internal to the packageSources."
+        PackageSourcesNodeFooter="</packageSources>"
+        PackageSourceTemplate="${TB}<add key=\"dotnet5-internal\" value=\"https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet5-internal/nuget/v2\" />"
+
+        sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" $ConfigFile
+    fi
+    PackageSources+=('dotnet5-internal')
+
+    grep -i "<add key=\"dotnet5-internal-transport\">" $ConfigFile
+    if [ "$?" != "0" ]; then
+        echo "Adding dotnet5-internal-transport to the packageSources."
+        PackageSourcesNodeFooter="</packageSources>"
+        PackageSourceTemplate="${TB}<add key=\"dotnet5-internal-transport\" value=\"https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet5-internal-transport/nuget/v2\" />"
+
+        sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" $ConfigFile
+    fi
+    PackageSources+=('dotnet5-internal-transport')
+fi
+
+# I want things split line by line
+PrevIFS=$IFS
+IFS=$'\n'
+PackageSources+="$IFS"
+PackageSources+=$(grep -oh '"darc-int-[^"]*"' $ConfigFile | tr -d '"')
+IFS=$PrevIFS
+
+for FeedName in ${PackageSources[@]} ; do
+    # Check if there is no existing credential for this FeedName
+    grep -i "<$FeedName>" $ConfigFile 
+    if [ "$?" != "0" ]; then
+        echo "Adding credentials for $FeedName."
+
+        PackageSourceCredentialsNodeFooter="</packageSourceCredentials>"
+        NewCredential="${TB}${TB}<$FeedName>${NL}<add key=\"Username\" value=\"dn-bot\" />${NL}<add key=\"ClearTextPassword\" value=\"$CredToken\" />${NL}</$FeedName>"
+
+        sed -i.bak "s|$PackageSourceCredentialsNodeFooter|$NewCredential${NL}$PackageSourceCredentialsNodeFooter|" $ConfigFile
+    fi
+done
+
+# Re-enable any entries in disabledPackageSources where the feed name contains darc-int
+grep -i "<disabledPackageSources>" $ConfigFile
+if [ "$?" == "0" ]; then
+    DisabledDarcIntSources=()
+    echo "Re-enabling any disabled \"darc-int\" package sources in $ConfigFile"
+    DisabledDarcIntSources+=$(grep -oh '"darc-int-[^"]*" value="true"' $ConfigFile  | tr -d '"')
+    for DisabledSourceName in ${DisabledDarcIntSources[@]} ; do
+        if [[ $DisabledSourceName == darc-int* ]]
+            then
+                OldDisableValue="<add key=\"$DisabledSourceName\" value=\"true\" />"
+                NewDisableValue="<!-- Reenabled for build : $DisabledSourceName -->"
+                sed -i.bak "s|$OldDisableValue|$NewDisableValue|" $ConfigFile
+                echo "Neutralized disablePackageSources entry for '$DisabledSourceName'"
+        fi
+    done
+fi

eng/common/SigningValidation.proj

@@ -1,83 +0,0 @@
-<!-- Licensed to the .NET Foundation under one or more agreements. The .NET Foundation licenses this file to you under the MIT license. See the LICENSE file in the project root for more information. -->
-<Project Sdk="Microsoft.NET.Sdk">
-  <!--
-    This MSBuild file is intended to be used as the body of the default 
-    publishing release pipeline. The release pipeline will use this file
-    to invoke the the SignCheck tool to validate that packages about to
-    be published are correctly signed.
-  
-    Parameters:
-  
-      - PackageBasePath   : Directory containing all files that need to be validated.
-      - SignCheckVersion  : Version of SignCheck package to be used.
-      - SignValidationExclusionList   : ItemGroup containing exclusion list to be forwarded to SignCheck.
-      - EnableJarSigningCheck    : Whether .jar files should be validated.
-      - EnableStrongNameCheck    : Whether strong name check should be performed.
-  -->
-
-  <PropertyGroup>
-    <TargetFramework>netcoreapp2.1</TargetFramework>
-  </PropertyGroup>
-
-  <!--
-    From 'Signing.props' we import $(SignValidationExclusionList)
-  -->
-  <Import Project="$(MSBuildThisFileDirectory)Signing.props" Condition="Exists('$(MSBuildThisFileDirectory)Signing.props')" />
-
-  <Target Name="ValidateSigning">
-    <PropertyGroup>
-      <SignCheckToolPath>$(NuGetPackageRoot)Microsoft.DotNet.SignCheck\$(SignCheckVersion)\tools\Microsoft.DotNet.SignCheck.exe</SignCheckToolPath>
-
-      <SignCheckInputDir>$(PackageBasePath)</SignCheckInputDir>
-      <SignCheckLog>signcheck.log</SignCheckLog>
-      <SignCheckErrorLog>signcheck.errors.log</SignCheckErrorLog>
-      <SignCheckExclusionsFile>signcheck.exclusions.txt</SignCheckExclusionsFile>
-    </PropertyGroup>
-    
-    <ItemGroup>
-      <!--
-        Documentation for these arguments is available here:
-        https://github.com/dotnet/arcade/tree/master/src/SignCheck
-      -->
-      <SignCheckArgs Include="--recursive" />
-      <SignCheckArgs Include="--traverse-subfolders" />
-      <SignCheckArgs Include="--file-status AllFiles" />
-      <SignCheckArgs Include="--log-file $(SignCheckLog)" />
-      <SignCheckArgs Include="--error-log-file $(SignCheckErrorLog)" />
-      <SignCheckArgs Include="--input-files $(SignCheckInputDir)" />
-      
-      <SignCheckArgs Include="--exclusions-file $(SignCheckExclusionsFile)" Condition="'@(SignValidationExclusionList)' != ''" />
-      <SignCheckArgs Include="--verify-jar" Condition="'$(EnableJarSigningCheck)' == 'true'" />
-      <SignCheckArgs Include="--verify-strongname" Condition="'$(EnableStrongNameCheck)' == 'true'" />
-    </ItemGroup>
-   
-    <WriteLinesToFile 
-      File="$(SignCheckExclusionsFile)"
-      Lines="@(SignValidationExclusionList)"
-      Condition="'@(SignValidationExclusionList)' != ''"
-      Overwrite="true"
-      Encoding="Unicode"/>
-    
-    <!--
-      IgnoreExitCode='true' because the tool doesn't return '0' on success.
-    -->
-    <Exec 
-      Command="&quot;$(SignCheckToolPath)&quot; @(SignCheckArgs, ' ')"
-      IgnoreExitCode='true' 
-      ConsoleToMsBuild="false" 
-      StandardErrorImportance="high" />
-
-    <Error 
-      Text="Signing validation failed. Check $(SignCheckErrorLog) for more information." 
-      Condition="Exists($(SignCheckErrorLog)) and '$([System.IO.File]::ReadAllText($(SignCheckErrorLog)))' != ''" />
-
-    <Message
-      Text="##vso[artifact.upload containerfolder=LogFiles;artifactname=LogFiles]{SignCheckErrorLog}"
-      Condition="Exists($(SignCheckErrorLog)) and '$([System.IO.File]::ReadAllText($(SignCheckErrorLog)))' != ''" />
-    
-  </Target>
-
-  <ItemGroup>
-    <PackageReference Include="Microsoft.DotNet.SignCheck" Version="$(SignCheckVersion)" />
-  </ItemGroup>
-</Project>

eng/common/build.ps1

@@ -1,6 +1,7 @@
 [CmdletBinding(PositionalBinding=$false)]
 Param(
   [string][Alias('c')]$configuration = "Debug",
+  [string]$platform = $null,
   [string] $projects,
   [string][Alias('v')]$verbosity = "minimal",
   [string] $msbuildEngine = $null,
@@ -17,55 +18,67 @@ Param(
   [switch] $sign,
   [switch] $pack,
   [switch] $publish,
+  [switch] $clean,
   [switch][Alias('bl')]$binaryLog,
+  [switch][Alias('nobl')]$excludeCIBinarylog,
   [switch] $ci,
   [switch] $prepareMachine,
+  [string] $runtimeSourceFeed = '',
+  [string] $runtimeSourceFeedKey = '',
   [switch] $help,
   [Parameter(ValueFromRemainingArguments=$true)][String[]]$properties
 )
 
-. $PSScriptRoot\tools.ps1
-
-function Print-Usage() {
-    Write-Host "Common settings:"
-    Write-Host "  -configuration <value>  Build configuration: 'Debug' or 'Release' (short: -c)"
-    Write-Host "  -verbosity <value>      Msbuild verbosity: q[uiet], m[inimal], n[ormal], d[etailed], and diag[nostic] (short: -v)"
-    Write-Host "  -binaryLog              Output binary log (short: -bl)"
-    Write-Host "  -help                   Print help and exit"
-    Write-Host ""
-
-    Write-Host "Actions:"
-    Write-Host "  -restore                Restore dependencies (short: -r)"
-    Write-Host "  -build                  Build solution (short: -b)"
-    Write-Host "  -rebuild                Rebuild solution"
-    Write-Host "  -deploy                 Deploy built VSIXes"
-    Write-Host "  -deployDeps             Deploy dependencies (e.g. VSIXes for integration tests)"
-    Write-Host "  -test                   Run all unit tests in the solution (short: -t)"
-    Write-Host "  -integrationTest        Run all integration tests in the solution"
-    Write-Host "  -performanceTest        Run all performance tests in the solution"
-    Write-Host "  -pack                   Package build outputs into NuGet packages and Willow components"
-    Write-Host "  -sign                   Sign build outputs"
-    Write-Host "  -publish                Publish artifacts (e.g. symbols)"
-    Write-Host ""
-
-    Write-Host "Advanced settings:"
-    Write-Host "  -projects <value>       Semi-colon delimited list of sln/proj's to build. Globbing is supported (*.sln)"
-    Write-Host "  -ci                     Set when running on CI server"
-    Write-Host "  -prepareMachine         Prepare machine for CI run, clean up processes after build"
-    Write-Host "  -warnAsError <value>    Sets warnaserror msbuild parameter ('true' or 'false')"
-    Write-Host "  -msbuildEngine <value>  Msbuild engine to use to run build ('dotnet', 'vs', or unspecified)."
-    Write-Host ""
-
-    Write-Host "Command line arguments not listed above are passed thru to msbuild."
-    Write-Host "The above arguments can be shortened as much as to be unambiguous (e.g. -co for configuration, -t for test, etc.)."
+# Unset 'Platform' environment variable to avoid unwanted collision in InstallDotNetCore.targets file
+# some computer has this env var defined (e.g. Some HP)
+if($env:Platform) {
+  $env:Platform=""  
 }
+function Print-Usage() {
+  Write-Host "Common settings:"
+  Write-Host "  -configuration <value>  Build configuration: 'Debug' or 'Release' (short: -c)"
+  Write-Host "  -platform <value>       Platform configuration: 'x86', 'x64' or any valid Platform value to pass to msbuild"
+  Write-Host "  -verbosity <value>      Msbuild verbosity: q[uiet], m[inimal], n[ormal], d[etailed], and diag[nostic] (short: -v)"
+  Write-Host "  -binaryLog              Output binary log (short: -bl)"
+  Write-Host "  -help                   Print help and exit"
+  Write-Host ""
+
+  Write-Host "Actions:"
+  Write-Host "  -restore                Restore dependencies (short: -r)"
+  Write-Host "  -build                  Build solution (short: -b)"
+  Write-Host "  -rebuild                Rebuild solution"
+  Write-Host "  -deploy                 Deploy built VSIXes"
+  Write-Host "  -deployDeps             Deploy dependencies (e.g. VSIXes for integration tests)"
+  Write-Host "  -test                   Run all unit tests in the solution (short: -t)"
+  Write-Host "  -integrationTest        Run all integration tests in the solution"
+  Write-Host "  -performanceTest        Run all performance tests in the solution"
+  Write-Host "  -pack                   Package build outputs into NuGet packages and Willow components"
+  Write-Host "  -sign                   Sign build outputs"
+  Write-Host "  -publish                Publish artifacts (e.g. symbols)"
+  Write-Host "  -clean                  Clean the solution"
+  Write-Host ""
+
+  Write-Host "Advanced settings:"
+  Write-Host "  -projects <value>       Semi-colon delimited list of sln/proj's to build. Globbing is supported (*.sln)"
+  Write-Host "  -ci                     Set when running on CI server"
+  Write-Host "  -excludeCIBinarylog     Don't output binary log (short: -nobl)"
+  Write-Host "  -prepareMachine         Prepare machine for CI run, clean up processes after build"
+  Write-Host "  -warnAsError <value>    Sets warnaserror msbuild parameter ('true' or 'false')"
+  Write-Host "  -msbuildEngine <value>  Msbuild engine to use to run build ('dotnet', 'vs', or unspecified)."
+  Write-Host ""
+
+  Write-Host "Command line arguments not listed above are passed thru to msbuild."
+  Write-Host "The above arguments can be shortened as much as to be unambiguous (e.g. -co for configuration, -t for test, etc.)."
+}
+
+. $PSScriptRoot\tools.ps1
 
 function InitializeCustomToolset {
   if (-not $restore) {
     return
   }
 
-  $script = Join-Path $EngRoot "restore-toolset.ps1"
+  $script = Join-Path $EngRoot 'restore-toolset.ps1'
 
   if (Test-Path $script) {
     . $script
@@ -76,18 +89,24 @@ function Build {
   $toolsetBuildProj = InitializeToolset
   InitializeCustomToolset
 
-  $bl = if ($binaryLog) { "/bl:" + (Join-Path $LogDir "Build.binlog") } else { "" }
+  $bl = if ($binaryLog) { '/bl:' + (Join-Path $LogDir 'Build.binlog') } else { '' }
+  $platformArg = if ($platform) { "/p:Platform=$platform" } else { '' }
 
   if ($projects) {
     # Re-assign properties to a new variable because PowerShell doesn't let us append properties directly for unclear reasons.
     # Explicitly set the type as string[] because otherwise PowerShell would make this char[] if $properties is empty.
     [string[]] $msbuildArgs = $properties
+    
+    # Resolve relative project paths into full paths 
+    $projects = ($projects.Split(';').ForEach({Resolve-Path $_}) -join ';')
+    
     $msbuildArgs += "/p:Projects=$projects"
     $properties = $msbuildArgs
   }
 
   MSBuild $toolsetBuildProj `
     $bl `
+    $platformArg `
     /p:Configuration=$configuration `
     /p:RepoRoot=$RepoRoot `
     /p:Restore=$restore `
@@ -105,29 +124,35 @@ function Build {
 }
 
 try {
-  if ($help -or (($null -ne $properties) -and ($properties.Contains("/help") -or $properties.Contains("/?")))) {
+  if ($clean) {
+    if (Test-Path $ArtifactsDir) {
+      Remove-Item -Recurse -Force $ArtifactsDir
+      Write-Host 'Artifacts directory deleted.'
+    }
+    exit 0
+  }
+
+  if ($help -or (($null -ne $properties) -and ($properties.Contains('/help') -or $properties.Contains('/?')))) {
     Print-Usage
     exit 0
   }
 
   if ($ci) {
-    $binaryLog = $true
+    if (-not $excludeCIBinarylog) {
+      $binaryLog = $true
+    }
     $nodeReuse = $false
   }
 
-  # Import custom tools configuration, if present in the repo.
-  # Note: Import in global scope so that the script set top-level variables without qualification.
-  $configureToolsetScript = Join-Path $EngRoot "configure-toolset.ps1"
-  if (Test-Path $configureToolsetScript) {
-    . $configureToolsetScript
+  if ($restore) {
+    InitializeNativeTools
   }
 
   Build
 }
 catch {
-  Write-Host $_
-  Write-Host $_.Exception
   Write-Host $_.ScriptStackTrace
+  Write-PipelineTelemetryError -Category 'InitializeToolset' -Message $_
   ExitWithExitCode 1
 }
 

eng/common/build.sh

@@ -26,16 +26,18 @@ usage()
   echo "  --pack                     Package build outputs into NuGet packages and Willow components"
   echo "  --sign                     Sign build outputs"
   echo "  --publish                  Publish artifacts (e.g. symbols)"
+  echo "  --clean                    Clean the solution"
   echo ""
 
   echo "Advanced settings:"
   echo "  --projects <value>       Project or solution file(s) to build"
   echo "  --ci                     Set when running on CI server"
+  echo "  --excludeCIBinarylog     Don't output binary log (short: -nobl)"
   echo "  --prepareMachine         Prepare machine for CI run, clean up processes after build"
   echo "  --nodeReuse <value>      Sets nodereuse msbuild parameter ('true' or 'false')"
   echo "  --warnAsError <value>    Sets warnaserror msbuild parameter ('true' or 'false')"
   echo ""
-  echo "Command line arguments starting with '/p:' are passed through to MSBuild."
+  echo "Command line arguments not listed above are passed thru to msbuild."
   echo "Arguments can also be passed in with a single hyphen."
 }
 
@@ -62,25 +64,32 @@ publish=false
 sign=false
 public=false
 ci=false
+clean=false
 
 warn_as_error=true
 node_reuse=true
 binary_log=false
+exclude_ci_binary_log=false
+pipelines_log=false
 
 projects=''
 configuration='Debug'
 prepare_machine=false
 verbosity='minimal'
+runtime_source_feed=''
+runtime_source_feed_key=''
 
 properties=''
-
 while [[ $# > 0 ]]; do
-  opt="$(echo "${1/#--/-}" | awk '{print tolower($0)}')"
+  opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")"
   case "$opt" in
     -help|-h)
       usage
       exit 0
       ;;
+    -clean)
+      clean=true
+      ;;
     -configuration|-c)
       configuration=$2
       shift
@@ -92,6 +101,12 @@ while [[ $# > 0 ]]; do
     -binarylog|-bl)
       binary_log=true
       ;;
+    -excludeCIBinarylog|-nobl)
+      exclude_ci_binary_log=true
+      ;;
+    -pipelineslog|-pl)
+      pipelines_log=true
+      ;;
     -restore|-r)
       restore=true
       ;;
@@ -137,22 +152,16 @@ while [[ $# > 0 ]]; do
       node_reuse=$2
       shift
       ;;
-    -p:*|/p:*)
-      properties="$properties $1"
+    -runtimesourcefeed)
+      runtime_source_feed=$2
+      shift
       ;;
-    -m:*|/m:*)
-      properties="$properties $1"
-      ;;
-    -bl:*|/bl:*)
-      properties="$properties $1"
-      ;;
-    -dl:*|/dl:*)
-      properties="$properties $1"
+     -runtimesourcefeedkey)
+      runtime_source_feed_key=$2
+      shift
       ;;
     *)
-      echo "Invalid argument: $1"
-      usage
-      exit 1
+      properties="$properties $1"
       ;;
   esac
 
@@ -160,8 +169,11 @@ while [[ $# > 0 ]]; do
 done
 
 if [[ "$ci" == true ]]; then
-  binary_log=true
+  pipelines_log=true
   node_reuse=false
+  if [[ "$exclude_ci_binary_log" == false ]]; then
+    binary_log=true
+  fi
 fi
 
 . "$scriptroot/tools.sh"
@@ -205,17 +217,16 @@ function Build {
   ExitWithExitCode 0
 }
 
-# Import custom tools configuration, if present in the repo.
-configure_toolset_script="$eng_root/configure-toolset.sh"
-if [[ -a "$configure_toolset_script" ]]; then
-  . "$configure_toolset_script"
+if [[ "$clean" == true ]]; then
+  if [ -d "$artifacts_dir" ]; then
+    rm -rf $artifacts_dir
+    echo "Artifacts directory deleted."
+  fi
+  exit 0
 fi
 
-# TODO: https://github.com/dotnet/arcade/issues/1468
-# Temporary workaround to avoid breaking change.
-# Remove once repos are updated.
-if [[ -n "${useInstalledDotNetCli:-}" ]]; then
-  use_installed_dotnet_cli="$useInstalledDotNetCli"
+if [[ "$restore" == true ]]; then
+  InitializeNativeTools
 fi
 
 Build

eng/common/cross/android/arm/toolchain.cmake

@@ -1,41 +0,0 @@
-set(CROSS_NDK_TOOLCHAIN $ENV{ROOTFS_DIR}/../)
-set(CROSS_ROOTFS ${CROSS_NDK_TOOLCHAIN}/sysroot)
-set(CLR_CMAKE_PLATFORM_ANDROID "Android")
-
-set(CMAKE_SYSTEM_NAME Linux)
-set(CMAKE_SYSTEM_VERSION 1)
-set(CMAKE_SYSTEM_PROCESSOR arm)
-
-## Specify the toolchain
-set(TOOLCHAIN "arm-linux-androideabi")
-set(CMAKE_PREFIX_PATH ${CROSS_NDK_TOOLCHAIN})
-set(TOOLCHAIN_PREFIX ${TOOLCHAIN}-)
-
-find_program(CMAKE_C_COMPILER ${TOOLCHAIN_PREFIX}clang)
-find_program(CMAKE_CXX_COMPILER ${TOOLCHAIN_PREFIX}clang++)
-find_program(CMAKE_ASM_COMPILER ${TOOLCHAIN_PREFIX}clang)
-find_program(CMAKE_AR ${TOOLCHAIN_PREFIX}ar)
-find_program(CMAKE_LD ${TOOLCHAIN_PREFIX}ar)
-find_program(CMAKE_OBJCOPY ${TOOLCHAIN_PREFIX}objcopy)
-find_program(CMAKE_OBJDUMP ${TOOLCHAIN_PREFIX}objdump)
-
-add_compile_options(--sysroot=${CROSS_ROOTFS})
-add_compile_options(-fPIE)
-add_compile_options(-mfloat-abi=soft)
-include_directories(SYSTEM ${CROSS_NDK_TOOLCHAIN}/include/c++/4.9.x/)
-include_directories(SYSTEM ${CROSS_NDK_TOOLCHAIN}/include/c++/4.9.x/arm-linux-androideabi/)
-
-set(CROSS_LINK_FLAGS "${CROSS_LINK_FLAGS} -B ${CROSS_ROOTFS}/usr/lib/gcc/${TOOLCHAIN}")
-set(CROSS_LINK_FLAGS "${CROSS_LINK_FLAGS} -L${CROSS_ROOTFS}/lib/${TOOLCHAIN}")
-set(CROSS_LINK_FLAGS "${CROSS_LINK_FLAGS} --sysroot=${CROSS_ROOTFS}")
-set(CROSS_LINK_FLAGS "${CROSS_LINK_FLAGS} -fPIE -pie")
-
-set(CMAKE_EXE_LINKER_FLAGS    "${CMAKE_EXE_LINKER_FLAGS}    ${CROSS_LINK_FLAGS}" CACHE STRING "" FORCE)
-set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} ${CROSS_LINK_FLAGS}" CACHE STRING "" FORCE)
-set(CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} ${CROSS_LINK_FLAGS}" CACHE STRING "" FORCE)
-
-set(CMAKE_FIND_ROOT_PATH "${CROSS_ROOTFS}")
-set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER)
-set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY)
-set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY)
-set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY)

eng/common/cross/android/arm64/toolchain.cmake

@@ -1,42 +0,0 @@
-set(CROSS_NDK_TOOLCHAIN $ENV{ROOTFS_DIR}/../)
-set(CROSS_ROOTFS ${CROSS_NDK_TOOLCHAIN}/sysroot)
-set(CLR_CMAKE_PLATFORM_ANDROID "Android")
-
-set(CMAKE_SYSTEM_NAME Linux)
-set(CMAKE_SYSTEM_VERSION 1)
-set(CMAKE_SYSTEM_PROCESSOR aarch64)
-
-## Specify the toolchain
-set(TOOLCHAIN "aarch64-linux-android")
-set(CMAKE_PREFIX_PATH ${CROSS_NDK_TOOLCHAIN})
-set(TOOLCHAIN_PREFIX ${TOOLCHAIN}-)
-
-find_program(CMAKE_C_COMPILER ${TOOLCHAIN_PREFIX}clang)
-find_program(CMAKE_CXX_COMPILER ${TOOLCHAIN_PREFIX}clang++)
-find_program(CMAKE_ASM_COMPILER ${TOOLCHAIN_PREFIX}clang)
-find_program(CMAKE_AR ${TOOLCHAIN_PREFIX}ar)
-find_program(CMAKE_LD ${TOOLCHAIN_PREFIX}ar)
-find_program(CMAKE_OBJCOPY ${TOOLCHAIN_PREFIX}objcopy)
-find_program(CMAKE_OBJDUMP ${TOOLCHAIN_PREFIX}objdump)
-
-add_compile_options(--sysroot=${CROSS_ROOTFS})
-add_compile_options(-fPIE)
-
-## Needed for Android or bionic specific conditionals
-add_compile_options(-D__ANDROID__)
-add_compile_options(-D__BIONIC__)
-
-set(CROSS_LINK_FLAGS "${CROSS_LINK_FLAGS} -B ${CROSS_ROOTFS}/usr/lib/gcc/${TOOLCHAIN}")
-set(CROSS_LINK_FLAGS "${CROSS_LINK_FLAGS} -L${CROSS_ROOTFS}/lib/${TOOLCHAIN}")
-set(CROSS_LINK_FLAGS "${CROSS_LINK_FLAGS} --sysroot=${CROSS_ROOTFS}")
-set(CROSS_LINK_FLAGS "${CROSS_LINK_FLAGS} -fPIE -pie")
-
-set(CMAKE_EXE_LINKER_FLAGS    "${CMAKE_EXE_LINKER_FLAGS}    ${CROSS_LINK_FLAGS}" CACHE STRING "" FORCE)
-set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} ${CROSS_LINK_FLAGS}" CACHE STRING "" FORCE)
-set(CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} ${CROSS_LINK_FLAGS}" CACHE STRING "" FORCE)
-
-set(CMAKE_FIND_ROOT_PATH "${CROSS_ROOTFS}")
-set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER)
-set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY)
-set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY)
-set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY)

eng/common/cross/arm/sources.list.vivid

@@ -1,11 +0,0 @@
-deb http://ports.ubuntu.com/ubuntu-ports/ vivid main restricted universe
-deb-src http://ports.ubuntu.com/ubuntu-ports/ vivid main restricted universe
-
-deb http://ports.ubuntu.com/ubuntu-ports/ vivid-updates main restricted universe
-deb-src http://ports.ubuntu.com/ubuntu-ports/ vivid-updates main restricted universe
-
-deb http://ports.ubuntu.com/ubuntu-ports/ vivid-backports main restricted
-deb-src http://ports.ubuntu.com/ubuntu-ports/ vivid-backports main restricted
-
-deb http://ports.ubuntu.com/ubuntu-ports/ vivid-security main restricted universe multiverse
-deb-src http://ports.ubuntu.com/ubuntu-ports/ vivid-security main restricted universe multiverse

eng/common/cross/arm/sources.list.wily

@@ -1,11 +0,0 @@
-deb http://ports.ubuntu.com/ubuntu-ports/ wily main restricted universe
-deb-src http://ports.ubuntu.com/ubuntu-ports/ wily main restricted universe
-
-deb http://ports.ubuntu.com/ubuntu-ports/ wily-updates main restricted universe
-deb-src http://ports.ubuntu.com/ubuntu-ports/ wily-updates main restricted universe
-
-deb http://ports.ubuntu.com/ubuntu-ports/ wily-backports main restricted
-deb-src http://ports.ubuntu.com/ubuntu-ports/ wily-backports main restricted
-
-deb http://ports.ubuntu.com/ubuntu-ports/ wily-security main restricted universe multiverse
-deb-src http://ports.ubuntu.com/ubuntu-ports/ wily-security main restricted universe multiverse

eng/common/cross/arm64/sources.list.buster

@@ -0,0 +1,11 @@
+deb http://deb.debian.org/debian buster main
+deb-src http://deb.debian.org/debian buster main
+
+deb http://deb.debian.org/debian-security/ buster/updates main
+deb-src http://deb.debian.org/debian-security/ buster/updates main
+
+deb http://deb.debian.org/debian buster-updates main
+deb-src http://deb.debian.org/debian buster-updates main
+
+deb http://deb.debian.org/debian buster-backports main contrib non-free
+deb-src http://deb.debian.org/debian buster-backports main contrib non-free

eng/common/cross/arm64/sources.list.stretch

@@ -0,0 +1,12 @@
+deb http://deb.debian.org/debian stretch main
+deb-src http://deb.debian.org/debian stretch main
+
+deb http://deb.debian.org/debian-security/ stretch/updates main
+deb-src http://deb.debian.org/debian-security/ stretch/updates main
+
+deb http://deb.debian.org/debian stretch-updates main
+deb-src http://deb.debian.org/debian stretch-updates main
+
+deb http://deb.debian.org/debian stretch-backports main contrib non-free
+deb-src http://deb.debian.org/debian stretch-backports main contrib non-free
+

eng/common/cross/arm64/sources.list.vivid

@@ -1,11 +0,0 @@
-deb http://ports.ubuntu.com/ubuntu-ports/ vivid main restricted universe
-deb-src http://ports.ubuntu.com/ubuntu-ports/ vivid main restricted universe
-
-deb http://ports.ubuntu.com/ubuntu-ports/ vivid-updates main restricted universe
-deb-src http://ports.ubuntu.com/ubuntu-ports/ vivid-updates main restricted universe
-
-deb http://ports.ubuntu.com/ubuntu-ports/ vivid-backports main restricted
-deb-src http://ports.ubuntu.com/ubuntu-ports/ vivid-backports main restricted
-
-deb http://ports.ubuntu.com/ubuntu-ports/ vivid-security main restricted universe multiverse
-deb-src http://ports.ubuntu.com/ubuntu-ports/ vivid-security main restricted universe multiverse

eng/common/cross/arm64/sources.list.wily

@@ -1,11 +0,0 @@
-deb http://ports.ubuntu.com/ubuntu-ports/ wily main restricted universe
-deb-src http://ports.ubuntu.com/ubuntu-ports/ wily main restricted universe
-
-deb http://ports.ubuntu.com/ubuntu-ports/ wily-updates main restricted universe
-deb-src http://ports.ubuntu.com/ubuntu-ports/ wily-updates main restricted universe
-
-deb http://ports.ubuntu.com/ubuntu-ports/ wily-backports main restricted
-deb-src http://ports.ubuntu.com/ubuntu-ports/ wily-backports main restricted
-
-deb http://ports.ubuntu.com/ubuntu-ports/ wily-security main restricted universe multiverse
-deb-src http://ports.ubuntu.com/ubuntu-ports/ wily-security main restricted universe multiverse

eng/common/cross/arm64/tizen-build-rootfs.sh

@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+set -e
+
+__CrossDir=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
+__TIZEN_CROSSDIR="$__CrossDir/tizen"
+
+if [[ -z "$ROOTFS_DIR" ]]; then
+    echo "ROOTFS_DIR is not defined."
+    exit 1;
+fi
+
+TIZEN_TMP_DIR=$ROOTFS_DIR/tizen_tmp
+mkdir -p $TIZEN_TMP_DIR
+
+# Download files
+echo ">>Start downloading files"
+VERBOSE=1 $__CrossDir/tizen-fetch.sh $TIZEN_TMP_DIR
+echo "<<Finish downloading files"
+
+echo ">>Start constructing Tizen rootfs"
+TIZEN_RPM_FILES=`ls $TIZEN_TMP_DIR/*.rpm`
+cd $ROOTFS_DIR
+for f in $TIZEN_RPM_FILES; do
+    rpm2cpio $f  | cpio -idm --quiet
+done
+echo "<<Finish constructing Tizen rootfs"
+
+# Cleanup tmp
+rm -rf $TIZEN_TMP_DIR
+
+# Configure Tizen rootfs
+echo ">>Start configuring Tizen rootfs"
+ln -sfn asm-arm64 ./usr/include/asm
+patch -p1 < $__TIZEN_CROSSDIR/tizen.patch
+echo "<<Finish configuring Tizen rootfs"

eng/common/cross/arm64/tizen-fetch.sh

@@ -0,0 +1,170 @@
+#!/usr/bin/env bash
+set -e
+
+if [[ -z "${VERBOSE// }" ]] || [ "$VERBOSE" -ne "$VERBOSE" ] 2>/dev/null; then
+	VERBOSE=0
+fi
+
+Log()
+{
+	if [ $VERBOSE -ge $1 ]; then
+		echo ${@:2}
+	fi
+}
+
+Inform()
+{
+	Log 1 -e "\x1B[0;34m$@\x1B[m"
+}
+
+Debug()
+{
+	Log 2 -e "\x1B[0;32m$@\x1B[m"
+}
+
+Error()
+{
+	>&2 Log 0 -e "\x1B[0;31m$@\x1B[m"
+}
+
+Fetch()
+{
+	URL=$1
+	FILE=$2
+	PROGRESS=$3
+	if [ $VERBOSE -ge 1 ] && [ $PROGRESS ]; then
+		CURL_OPT="--progress-bar"
+	else
+		CURL_OPT="--silent"
+	fi
+	curl $CURL_OPT $URL > $FILE
+}
+
+hash curl 2> /dev/null || { Error "Require 'curl' Aborting."; exit 1; }
+hash xmllint 2> /dev/null || { Error "Require 'xmllint' Aborting."; exit 1; }
+hash sha256sum 2> /dev/null || { Error "Require 'sha256sum' Aborting."; exit 1; }
+
+TMPDIR=$1
+if [ ! -d $TMPDIR ]; then
+	TMPDIR=./tizen_tmp
+	Debug "Create temporary directory : $TMPDIR"
+	mkdir -p $TMPDIR
+fi
+
+TIZEN_URL=http://download.tizen.org/snapshots/tizen/
+BUILD_XML=build.xml
+REPOMD_XML=repomd.xml
+PRIMARY_XML=primary.xml
+TARGET_URL="http://__not_initialized"
+
+Xpath_get()
+{
+	XPATH_RESULT=''
+	XPATH=$1
+	XML_FILE=$2
+	RESULT=$(xmllint --xpath $XPATH $XML_FILE)
+	if [[ -z ${RESULT// } ]]; then
+		Error "Can not find target from $XML_FILE"
+		Debug "Xpath = $XPATH"
+		exit 1
+	fi
+	XPATH_RESULT=$RESULT
+}
+
+fetch_tizen_pkgs_init()
+{
+	TARGET=$1
+	PROFILE=$2
+	Debug "Initialize TARGET=$TARGET, PROFILE=$PROFILE"
+
+	TMP_PKG_DIR=$TMPDIR/tizen_${PROFILE}_pkgs
+	if [ -d $TMP_PKG_DIR ]; then rm -rf $TMP_PKG_DIR; fi
+	mkdir -p $TMP_PKG_DIR
+
+	PKG_URL=$TIZEN_URL/$PROFILE/latest
+
+	BUILD_XML_URL=$PKG_URL/$BUILD_XML
+	TMP_BUILD=$TMP_PKG_DIR/$BUILD_XML
+	TMP_REPOMD=$TMP_PKG_DIR/$REPOMD_XML
+	TMP_PRIMARY=$TMP_PKG_DIR/$PRIMARY_XML
+	TMP_PRIMARYGZ=${TMP_PRIMARY}.gz
+
+	Fetch $BUILD_XML_URL $TMP_BUILD
+
+	Debug "fetch $BUILD_XML_URL to $TMP_BUILD"
+
+	TARGET_XPATH="//build/buildtargets/buildtarget[@name=\"$TARGET\"]/repo[@type=\"binary\"]/text()"
+	Xpath_get $TARGET_XPATH $TMP_BUILD
+	TARGET_PATH=$XPATH_RESULT
+	TARGET_URL=$PKG_URL/$TARGET_PATH
+
+	REPOMD_URL=$TARGET_URL/repodata/repomd.xml
+	PRIMARY_XPATH='string(//*[local-name()="data"][@type="primary"]/*[local-name()="location"]/@href)'
+
+	Fetch $REPOMD_URL $TMP_REPOMD
+
+	Debug "fetch $REPOMD_URL to $TMP_REPOMD"
+
+	Xpath_get $PRIMARY_XPATH $TMP_REPOMD
+	PRIMARY_XML_PATH=$XPATH_RESULT
+	PRIMARY_URL=$TARGET_URL/$PRIMARY_XML_PATH
+
+	Fetch $PRIMARY_URL $TMP_PRIMARYGZ
+
+	Debug "fetch $PRIMARY_URL to $TMP_PRIMARYGZ"
+
+	gunzip $TMP_PRIMARYGZ
+
+	Debug "unzip $TMP_PRIMARYGZ to $TMP_PRIMARY"
+}
+
+fetch_tizen_pkgs()
+{
+	ARCH=$1
+	PACKAGE_XPATH_TPL='string(//*[local-name()="metadata"]/*[local-name()="package"][*[local-name()="name"][text()="_PKG_"]][*[local-name()="arch"][text()="_ARCH_"]]/*[local-name()="location"]/@href)'
+
+	PACKAGE_CHECKSUM_XPATH_TPL='string(//*[local-name()="metadata"]/*[local-name()="package"][*[local-name()="name"][text()="_PKG_"]][*[local-name()="arch"][text()="_ARCH_"]]/*[local-name()="checksum"]/text())'
+
+	for pkg in ${@:2}
+	do
+		Inform "Fetching... $pkg"
+		XPATH=${PACKAGE_XPATH_TPL/_PKG_/$pkg}
+		XPATH=${XPATH/_ARCH_/$ARCH}
+		Xpath_get $XPATH $TMP_PRIMARY
+		PKG_PATH=$XPATH_RESULT
+
+		XPATH=${PACKAGE_CHECKSUM_XPATH_TPL/_PKG_/$pkg}
+		XPATH=${XPATH/_ARCH_/$ARCH}
+		Xpath_get $XPATH $TMP_PRIMARY
+		CHECKSUM=$XPATH_RESULT
+
+		PKG_URL=$TARGET_URL/$PKG_PATH
+		PKG_FILE=$(basename $PKG_PATH)
+		PKG_PATH=$TMPDIR/$PKG_FILE
+
+		Debug "Download $PKG_URL to $PKG_PATH"
+		Fetch $PKG_URL $PKG_PATH true
+
+		echo "$CHECKSUM $PKG_PATH" | sha256sum -c - > /dev/null
+		if [ $? -ne 0 ]; then
+			Error "Fail to fetch $PKG_URL to $PKG_PATH"
+			Debug "Checksum = $CHECKSUM"
+			exit 1
+		fi
+	done
+}
+
+Inform "Initialize arm base"
+fetch_tizen_pkgs_init standard base
+Inform "fetch common packages"
+fetch_tizen_pkgs aarch64 gcc glibc glibc-devel libicu libicu-devel libatomic linux-glibc-devel keyutils keyutils-devel libkeyutils
+Inform "fetch coreclr packages"
+fetch_tizen_pkgs aarch64 lldb lldb-devel libgcc libstdc++ libstdc++-devel libunwind libunwind-devel lttng-ust-devel lttng-ust userspace-rcu-devel userspace-rcu
+Inform "fetch corefx packages"
+fetch_tizen_pkgs aarch64 libcom_err libcom_err-devel zlib zlib-devel libopenssl11 libopenssl1.1-devel krb5 krb5-devel
+
+Inform "Initialize standard unified"
+fetch_tizen_pkgs_init standard unified
+Inform "fetch corefx packages"
+fetch_tizen_pkgs aarch64 gssdp gssdp-devel tizen-release
+

eng/common/cross/arm64/tizen/tizen.patch

@@ -0,0 +1,9 @@
+diff -u -r a/usr/lib/libc.so b/usr/lib/libc.so
+--- a/usr/lib64/libc.so	2016-12-30 23:00:08.284951863 +0900
++++ b/usr/lib64/libc.so	2016-12-30 23:00:32.140951815 +0900
+@@ -2,4 +2,4 @@
+    Use the shared library, but some functions are only in
+    the static library, so try that secondarily.  */
+ OUTPUT_FORMAT(elf64-littleaarch64)
+-GROUP ( /lib64/libc.so.6 /usr/lib64/libc_nonshared.a  AS_NEEDED ( /lib/ld-linux-aarch64.so.1 ) )
++GROUP ( libc.so.6 libc_nonshared.a  AS_NEEDED ( ld-linux-aarch64.so.1 ) )

eng/common/cross/armel/armel.jessie.patch

@@ -0,0 +1,43 @@
+diff -u -r a/usr/include/urcu/uatomic/generic.h b/usr/include/urcu/uatomic/generic.h
+--- a/usr/include/urcu/uatomic/generic.h	2014-10-22 15:00:58.000000000 -0700
++++ b/usr/include/urcu/uatomic/generic.h	2020-10-30 21:38:28.550000000 -0700
+@@ -69,10 +69,10 @@
+ #endif
+ #ifdef UATOMIC_HAS_ATOMIC_SHORT
+ 	case 2:
+-		return __sync_val_compare_and_swap_2(addr, old, _new);
++		return __sync_val_compare_and_swap_2((uint16_t*) addr, old, _new);
+ #endif
+ 	case 4:
+-		return __sync_val_compare_and_swap_4(addr, old, _new);
++		return __sync_val_compare_and_swap_4((uint32_t*) addr, old, _new);
+ #if (CAA_BITS_PER_LONG == 64)
+ 	case 8:
+ 		return __sync_val_compare_and_swap_8(addr, old, _new);
+@@ -109,7 +109,7 @@
+ 		return;
+ #endif
+ 	case 4:
+-		__sync_and_and_fetch_4(addr, val);
++		__sync_and_and_fetch_4((uint32_t*) addr, val);
+ 		return;
+ #if (CAA_BITS_PER_LONG == 64)
+ 	case 8:
+@@ -148,7 +148,7 @@
+ 		return;
+ #endif
+ 	case 4:
+-		__sync_or_and_fetch_4(addr, val);
++		__sync_or_and_fetch_4((uint32_t*) addr, val);
+ 		return;
+ #if (CAA_BITS_PER_LONG == 64)
+ 	case 8:
+@@ -187,7 +187,7 @@
+ 		return __sync_add_and_fetch_2(addr, val);
+ #endif
+ 	case 4:
+-		return __sync_add_and_fetch_4(addr, val);
++		return __sync_add_and_fetch_4((uint32_t*) addr, val);
+ #if (CAA_BITS_PER_LONG == 64)
+ 	case 8:
+ 		return __sync_add_and_fetch_8(addr, val);

eng/common/cross/armel/tizen-build-rootfs.sh

@@ -9,13 +9,6 @@ if [[ -z "$ROOTFS_DIR" ]]; then
     exit 1;
 fi
 
-# Clean-up (TODO-Cleanup: We may already delete  $ROOTFS_DIR at ./cross/build-rootfs.sh.)
-# hk0110
-if [ -d "$ROOTFS_DIR" ]; then
-    umount $ROOTFS_DIR/*
-    rm -rf $ROOTFS_DIR
-fi
-
 TIZEN_TMP_DIR=$ROOTFS_DIR/tizen_tmp
 mkdir -p $TIZEN_TMP_DIR
 
@@ -37,8 +30,6 @@ rm -rf $TIZEN_TMP_DIR
 
 # Configure Tizen rootfs
 echo ">>Start configuring Tizen rootfs"
-rm ./usr/lib/libunwind.so
-ln -s libunwind.so.8 ./usr/lib/libunwind.so
 ln -sfn asm-arm ./usr/include/asm
 patch -p1 < $__TIZEN_CROSSDIR/tizen.patch
 echo "<<Finish configuring Tizen rootfs"

eng/common/cross/armel/tizen-fetch.sh

@@ -51,7 +51,7 @@ if [ ! -d $TMPDIR ]; then
 	mkdir -p $TMPDIR 
 fi
 
-TIZEN_URL=http://download.tizen.org/releases/milestone/tizen
+TIZEN_URL=http://download.tizen.org/snapshots/tizen
 BUILD_XML=build.xml
 REPOMD_XML=repomd.xml
 PRIMARY_XML=primary.xml
@@ -157,15 +157,14 @@ fetch_tizen_pkgs()
 Inform "Initialize arm base"
 fetch_tizen_pkgs_init standard base
 Inform "fetch common packages"
-fetch_tizen_pkgs armv7l gcc glibc glibc-devel libicu libicu-devel
-fetch_tizen_pkgs noarch linux-glibc-devel
+fetch_tizen_pkgs armv7l gcc gcc-devel-static glibc glibc-devel libicu libicu-devel libatomic linux-glibc-devel keyutils keyutils-devel libkeyutils
 Inform "fetch coreclr packages"
-fetch_tizen_pkgs armv7l lldb lldb-devel libgcc libstdc++ libstdc++-devel libunwind libunwind-devel tizen-release lttng-ust-devel lttng-ust userspace-rcu-devel userspace-rcu
+fetch_tizen_pkgs armv7l lldb lldb-devel libgcc libstdc++ libstdc++-devel libunwind libunwind-devel lttng-ust-devel lttng-ust userspace-rcu-devel userspace-rcu
 Inform "fetch corefx packages"
-fetch_tizen_pkgs armv7l libcom_err libcom_err-devel zlib zlib-devel libopenssl libopenssl-devel krb5 krb5-devel libcurl libcurl-devel
+fetch_tizen_pkgs armv7l libcom_err libcom_err-devel zlib zlib-devel libopenssl11 libopenssl1.1-devel krb5 krb5-devel
 
 Inform "Initialize standard unified"
 fetch_tizen_pkgs_init standard unified
 Inform "fetch corefx packages"
-fetch_tizen_pkgs armv7l gssdp gssdp-devel
+fetch_tizen_pkgs armv7l gssdp gssdp-devel tizen-release
 

eng/common/cross/armel/tizen/tizen.patch

@@ -7,12 +7,3 @@ diff -u -r a/usr/lib/libc.so b/usr/lib/libc.so
  OUTPUT_FORMAT(elf32-littlearm)
 -GROUP ( /lib/libc.so.6 /usr/lib/libc_nonshared.a  AS_NEEDED ( /lib/ld-linux.so.3 ) )
 +GROUP ( libc.so.6 libc_nonshared.a  AS_NEEDED ( ld-linux.so.3 ) )
-diff -u -r a/usr/lib/libpthread.so b/usr/lib/libpthread.so
---- a/usr/lib/libpthread.so	2016-12-30 23:00:19.408951841 +0900
-+++ b/usr/lib/libpthread.so	2016-12-30 23:00:39.068951801 +0900
-@@ -2,4 +2,4 @@
-    Use the shared library, but some functions are only in
-    the static library, so try that secondarily.  */
- OUTPUT_FORMAT(elf32-littlearm)
--GROUP ( /lib/libpthread.so.0 /usr/lib/libpthread_nonshared.a )
-+GROUP ( libpthread.so.0 libpthread_nonshared.a )

eng/common/cross/build-android-rootfs.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 set -e
-__NDK_Version=r14
+__NDK_Version=r21
 
 usage()
 {
@@ -16,18 +16,18 @@ usage()
     echo.
     echo "By default, the NDK will be downloaded into the cross/android-rootfs/android-ndk-$__NDK_Version directory. If you already have an NDK installation,"
     echo "you can set the NDK_DIR environment variable to have this script use that installation of the NDK."
-    echo "By default, this script will generate a file, android_platform, in the root of the ROOTFS_DIR directory that contains the RID for the supported and tested Android build: android.21-arm64. This file is to replace '/etc/os-release', which is not available for Android."
+    echo "By default, this script will generate a file, android_platform, in the root of the ROOTFS_DIR directory that contains the RID for the supported and tested Android build: android.28-arm64. This file is to replace '/etc/os-release', which is not available for Android."
     exit 1
 }
 
-__ApiLevel=21 # The minimum platform for arm64 is API level 21
+__ApiLevel=28 # The minimum platform for arm64 is API level 21 but the minimum version that support glob(3) is 28. See $ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/include/glob.h
 __BuildArch=arm64
 __AndroidArch=aarch64
 __AndroidToolchain=aarch64-linux-android
 
 for i in "$@"
     do
-        lowerI="$(echo $i | awk '{print tolower($0)}')"
+        lowerI="$(echo $i | tr "[:upper:]" "[:lower:]")"
         case $lowerI in
         -?|-h|--help)
             usage
@@ -53,13 +53,20 @@ for i in "$@"
 done
 
 # Obtain the location of the bash script to figure out where the root of the repo is.
-__CrossDir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+__ScriptBaseDir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 
-__Android_Cross_Dir="$__CrossDir/android-rootfs"
-__NDK_Dir="$__Android_Cross_Dir/android-ndk-$__NDK_Version"
-__libunwind_Dir="$__Android_Cross_Dir/libunwind"
-__lldb_Dir="$__Android_Cross_Dir/lldb"
-__ToolchainDir="$__Android_Cross_Dir/toolchain/$__BuildArch"
+__CrossDir="$__ScriptBaseDir/../../../.tools/android-rootfs"
+
+if [[ ! -f "$__CrossDir" ]]; then
+    mkdir -p "$__CrossDir"
+fi
+
+# Resolve absolute path to avoid `../` in build logs
+__CrossDir="$( cd "$__CrossDir" && pwd )"
+
+__NDK_Dir="$__CrossDir/android-ndk-$__NDK_Version"
+__lldb_Dir="$__CrossDir/lldb"
+__ToolchainDir="$__CrossDir/android-ndk-$__NDK_Version"
 
 if [[ -n "$TOOLCHAIN_DIR" ]]; then
     __ToolchainDir=$TOOLCHAIN_DIR
@@ -78,60 +85,48 @@ echo "Target Toolchain location: $__ToolchainDir"
 if [ ! -d $__NDK_Dir ]; then
     echo Downloading the NDK into $__NDK_Dir
     mkdir -p $__NDK_Dir
-    wget -nv -nc --show-progress https://dl.google.com/android/repository/android-ndk-$__NDK_Version-linux-x86_64.zip -O $__Android_Cross_Dir/android-ndk-$__NDK_Version-linux-x86_64.zip
-    unzip -q $__Android_Cross_Dir/android-ndk-$__NDK_Version-linux-x86_64.zip -d $__Android_Cross_Dir
+    wget -q --progress=bar:force:noscroll --show-progress https://dl.google.com/android/repository/android-ndk-$__NDK_Version-linux-x86_64.zip -O $__CrossDir/android-ndk-$__NDK_Version-linux-x86_64.zip
+    unzip -q $__CrossDir/android-ndk-$__NDK_Version-linux-x86_64.zip -d $__CrossDir
 fi
 
 if [ ! -d $__lldb_Dir ]; then
     mkdir -p $__lldb_Dir
     echo Downloading LLDB into $__lldb_Dir
-    wget -nv -nc --show-progress https://dl.google.com/android/repository/lldb-2.3.3614996-linux-x86_64.zip -O $__Android_Cross_Dir/lldb-2.3.3614996-linux-x86_64.zip
-    unzip -q $__Android_Cross_Dir/lldb-2.3.3614996-linux-x86_64.zip -d $__lldb_Dir
+    wget -q --progress=bar:force:noscroll --show-progress https://dl.google.com/android/repository/lldb-2.3.3614996-linux-x86_64.zip -O $__CrossDir/lldb-2.3.3614996-linux-x86_64.zip
+    unzip -q $__CrossDir/lldb-2.3.3614996-linux-x86_64.zip -d $__lldb_Dir
 fi
 
-# Create the RootFS for both arm64 as well as aarch
-rm -rf $__Android_Cross_Dir/toolchain
+echo "Download dependencies..."
+__TmpDir=$__CrossDir/tmp/$__BuildArch/
+mkdir -p "$__TmpDir"
 
-echo Generating the $__BuildArch toolchain
-$__NDK_Dir/build/tools/make_standalone_toolchain.py --arch $__BuildArch --api $__ApiLevel --install-dir $__ToolchainDir
+# combined dependencies for coreclr, installer and libraries
+__AndroidPackages="libicu"
+__AndroidPackages+=" libandroid-glob"
+__AndroidPackages+=" liblzma"
+__AndroidPackages+=" krb5"
+__AndroidPackages+=" openssl"
+__AndroidPackages+=" openldap"
 
-# Install the required packages into the toolchain
-# TODO: Add logic to get latest pkg version instead of specific version number
-rm -rf $__Android_Cross_Dir/deb/
-rm -rf $__Android_Cross_Dir/tmp
+for path in $(wget -qO- http://termux.net/dists/stable/main/binary-$__AndroidArch/Packages |\
+    grep -A15 "Package: \(${__AndroidPackages// /\\|}\)" | grep -v "static\|tool" | grep Filename); do
 
-mkdir -p $__Android_Cross_Dir/deb/
-mkdir -p $__Android_Cross_Dir/tmp/$arch/
-wget -nv -nc http://termux.net/dists/stable/main/binary-$__AndroidArch/libicu_60.2_$__AndroidArch.deb -O $__Android_Cross_Dir/deb/libicu_60.2_$__AndroidArch.deb
-wget -nv -nc http://termux.net/dists/stable/main/binary-$__AndroidArch/libicu-dev_60.2_$__AndroidArch.deb -O $__Android_Cross_Dir/deb/libicu-dev_60.2_$__AndroidArch.deb
+    if [[ "$path" != "Filename:" ]]; then
+        echo "Working on: $path"
+        wget -qO- http://termux.net/$path | dpkg -x - "$__TmpDir"
+    fi
+done
 
-wget -nv -nc http://termux.net/dists/stable/main/binary-$__AndroidArch/libandroid-glob-dev_0.4_$__AndroidArch.deb -O $__Android_Cross_Dir/deb/libandroid-glob-dev_0.4_$__AndroidArch.deb
-wget -nv -nc http://termux.net/dists/stable/main/binary-$__AndroidArch/libandroid-glob_0.4_$__AndroidArch.deb -O $__Android_Cross_Dir/deb/libandroid-glob_0.4_$__AndroidArch.deb
-wget -nv -nc http://termux.net/dists/stable/main/binary-$__AndroidArch/libandroid-support-dev_22_$__AndroidArch.deb -O $__Android_Cross_Dir/deb/libandroid-support-dev_22_$__AndroidArch.deb
-wget -nv -nc http://termux.net/dists/stable/main/binary-$__AndroidArch/libandroid-support_22_$__AndroidArch.deb -O $__Android_Cross_Dir/deb/libandroid-support_22_$__AndroidArch.deb
-wget -nv -nc http://termux.net/dists/stable/main/binary-$__AndroidArch/liblzma-dev_5.2.3_$__AndroidArch.deb  -O $__Android_Cross_Dir/deb/liblzma-dev_5.2.3_$__AndroidArch.deb
-wget -nv -nc http://termux.net/dists/stable/main/binary-$__AndroidArch/liblzma_5.2.3_$__AndroidArch.deb -O $__Android_Cross_Dir/deb/liblzma_5.2.3_$__AndroidArch.deb
-wget -nv -nc http://termux.net/dists/stable/main/binary-$__AndroidArch/libunwind-dev_1.2.20170304_$__AndroidArch.deb  -O $__Android_Cross_Dir/deb/libunwind-dev_1.2.20170304_$__AndroidArch.deb
-wget -nv -nc http://termux.net/dists/stable/main/binary-$__AndroidArch/libunwind_1.2.20170304_$__AndroidArch.deb -O $__Android_Cross_Dir/deb/libunwind_1.2.20170304_$__AndroidArch.deb
-
-echo Unpacking Termux packages
-dpkg -x $__Android_Cross_Dir/deb/libicu_60.2_$__AndroidArch.deb $__Android_Cross_Dir/tmp/$__AndroidArch/
-dpkg -x $__Android_Cross_Dir/deb/libicu-dev_60.2_$__AndroidArch.deb $__Android_Cross_Dir/tmp/$__AndroidArch/
-dpkg -x $__Android_Cross_Dir/deb/libandroid-glob-dev_0.4_$__AndroidArch.deb $__Android_Cross_Dir/tmp/$__AndroidArch/
-dpkg -x $__Android_Cross_Dir/deb/libandroid-glob_0.4_$__AndroidArch.deb $__Android_Cross_Dir/tmp/$__AndroidArch/
-dpkg -x $__Android_Cross_Dir/deb/libandroid-support-dev_22_$__AndroidArch.deb $__Android_Cross_Dir/tmp/$__AndroidArch/
-dpkg -x $__Android_Cross_Dir/deb/libandroid-support_22_$__AndroidArch.deb $__Android_Cross_Dir/tmp/$__AndroidArch/
-dpkg -x $__Android_Cross_Dir/deb/liblzma-dev_5.2.3_$__AndroidArch.deb $__Android_Cross_Dir/tmp/$__AndroidArch/
-dpkg -x $__Android_Cross_Dir/deb/liblzma_5.2.3_$__AndroidArch.deb $__Android_Cross_Dir/tmp/$__AndroidArch/
-dpkg -x $__Android_Cross_Dir/deb/libunwind-dev_1.2.20170304_$__AndroidArch.deb $__Android_Cross_Dir/tmp/$__AndroidArch/
-dpkg -x $__Android_Cross_Dir/deb/libunwind_1.2.20170304_$__AndroidArch.deb $__Android_Cross_Dir/tmp/$__AndroidArch/
-
-cp -R $__Android_Cross_Dir/tmp/$__AndroidArch/data/data/com.termux/files/usr/* $__ToolchainDir/sysroot/usr/
+cp -R "$__TmpDir/data/data/com.termux/files/usr/"* "$__ToolchainDir/sysroot/usr/"
 
 # Generate platform file for build.sh script to assign to __DistroRid
 echo "Generating platform file..."
+echo "RID=android.${__ApiLevel}-${__BuildArch}" > $__ToolchainDir/sysroot/android_platform
 
-echo "RID=android.21-arm64" > $__ToolchainDir/sysroot/android_platform
-echo Now run:
-echo CONFIG_DIR=\`realpath cross/android/$__BuildArch\` ROOTFS_DIR=\`realpath $__ToolchainDir/sysroot\` ./build.sh cross $__BuildArch skipgenerateversion skipnuget cmakeargs -DENABLE_LLDBPLUGIN=0
-
+echo "Now to build coreclr, libraries and installers; run:"
+echo ROOTFS_DIR=\$\(realpath $__ToolchainDir/sysroot\) ./build.sh --cross --arch $__BuildArch \
+    --subsetCategory coreclr
+echo ROOTFS_DIR=\$\(realpath $__ToolchainDir/sysroot\) ./build.sh --cross --arch $__BuildArch \
+    --subsetCategory libraries
+echo ROOTFS_DIR=\$\(realpath $__ToolchainDir/sysroot\) ./build.sh --cross --arch $__BuildArch \
+    --subsetCategory installer

eng/common/cross/build-rootfs.sh

@@ -1,22 +1,29 @@
 #!/usr/bin/env bash
 
+set -e
+
 usage()
 {
-    echo "Usage: $0 [BuildArch] [LinuxCodeName] [lldbx.y] [--skipunmount]"
+    echo "Usage: $0 [BuildArch] [CodeName] [lldbx.y] [--skipunmount] --rootfsdir <directory>]"
     echo "BuildArch can be: arm(default), armel, arm64, x86"
-    echo "LinuxCodeName - optional, Code name for Linux, can be: trusty(default), vivid, wily, xenial, zesty, bionic, alpine. If BuildArch is armel, LinuxCodeName is jessie(default) or tizen."
-    echo "lldbx.y - optional, LLDB version, can be: lldb3.6(default), lldb3.8, lldb3.9, lldb4.0, no-lldb. Ignored for alpine"
+    echo "CodeName - optional, Code name for Linux, can be: trusty, xenial(default), zesty, bionic, alpine, alpine3.9 or alpine3.13. If BuildArch is armel, LinuxCodeName is jessie(default) or tizen."
+    echo "                              for FreeBSD can be: freebsd11 or freebsd12."
+    echo "                              for illumos can be: illumos."
+    echo "lldbx.y - optional, LLDB version, can be: lldb3.9(default), lldb4.0, lldb5.0, lldb6.0 no-lldb. Ignored for alpine and FReeBSD"
     echo "--skipunmount - optional, will skip the unmount of rootfs folder."
+    echo "--use-mirror - optional, use mirror URL to fetch resources, when available."
     exit 1
 }
 
-__LinuxCodeName=trusty
+__CodeName=xenial
 __CrossDir=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
 __InitialDir=$PWD
 __BuildArch=arm
+__AlpineArch=armv7
+__QEMUArch=arm
 __UbuntuArch=armhf
 __UbuntuRepo="http://ports.ubuntu.com/"
-__LLDB_Package="lldb-3.6-dev"
+__LLDB_Package="liblldb-3.9-dev"
 __SkipUnmount=0
 
 # base development support
@@ -25,8 +32,10 @@ __UbuntuPackages="build-essential"
 __AlpinePackages="alpine-base"
 __AlpinePackages+=" build-base"
 __AlpinePackages+=" linux-headers"
-__AlpinePackages+=" lldb-dev"
-__AlpinePackages+=" llvm-dev"
+__AlpinePackagesEdgeCommunity=" lldb-dev"
+__AlpinePackagesEdgeMain=" llvm10-libs"
+__AlpinePackagesEdgeMain+=" python3"
+__AlpinePackagesEdgeMain+=" libedit"
 
 # symlinks fixer
 __UbuntuPackages+=" symlinks"
@@ -46,15 +55,42 @@ __UbuntuPackages+=" libcurl4-openssl-dev"
 __UbuntuPackages+=" libkrb5-dev"
 __UbuntuPackages+=" libssl-dev"
 __UbuntuPackages+=" zlib1g-dev"
+__UbuntuPackages+=" libldap2-dev"
 
 __AlpinePackages+=" curl-dev"
 __AlpinePackages+=" krb5-dev"
 __AlpinePackages+=" openssl-dev"
 __AlpinePackages+=" zlib-dev"
+__AlpinePackages+=" openldap-dev"
+
+__FreeBSDBase="12.1-RELEASE"
+__FreeBSDPkg="1.12.0"
+__FreeBSDPackages="libunwind"
+__FreeBSDPackages+=" icu"
+__FreeBSDPackages+=" libinotify"
+__FreeBSDPackages+=" lttng-ust"
+__FreeBSDPackages+=" krb5"
+__FreeBSDPackages+=" libslapi-2.4"
+
+__IllumosPackages="icu-64.2nb2"
+__IllumosPackages+=" mit-krb5-1.16.2nb4"
+__IllumosPackages+=" openssl-1.1.1e"
+__IllumosPackages+=" zlib-1.2.11"
+__IllumosPackages+=" openldap-client-2.4.49"
+
+# ML.NET dependencies
+__UbuntuPackages+=" libomp5"
+__UbuntuPackages+=" libomp-dev"
+
+__UseMirror=0
 
 __UnprocessedBuildArgs=
-for i in "$@" ; do
-    lowerI="$(echo $i | awk '{print tolower($0)}')"
+while :; do
+    if [ $# -le 0 ]; then
+        break
+    fi
+
+    lowerI="$(echo $1 | tr "[:upper:]" "[:lower:]")"
     case $lowerI in
         -?|-h|--help)
             usage
@@ -63,7 +99,7 @@ for i in "$@" ; do
         arm)
             __BuildArch=arm
             __UbuntuArch=armhf
-            __AlpineArch=armhf
+            __AlpineArch=armv7
             __QEMUArch=arm
             ;;
         arm64)
@@ -76,7 +112,14 @@ for i in "$@" ; do
             __BuildArch=armel
             __UbuntuArch=armel
             __UbuntuRepo="http://ftp.debian.org/debian/"
-            __LinuxCodeName=jessie
+            __CodeName=jessie
+            ;;
+        s390x)
+            __BuildArch=s390x
+            __UbuntuArch=s390x
+            __UbuntuRepo="http://ports.ubuntu.com/ubuntu-ports/"
+            __UbuntuPackages=$(echo ${__UbuntuPackages} | sed 's/ libunwind8-dev//')
+            unset __LLDB_Package
             ;;
         x86)
             __BuildArch=x86
@@ -95,59 +138,103 @@ for i in "$@" ; do
         lldb4.0)
             __LLDB_Package="liblldb-4.0-dev"
             ;;
+        lldb5.0)
+            __LLDB_Package="liblldb-5.0-dev"
+            ;;
+        lldb6.0)
+            __LLDB_Package="liblldb-6.0-dev"
+            ;;
         no-lldb)
             unset __LLDB_Package
             ;;
-        vivid)
-            if [ "$__LinuxCodeName" != "jessie" ]; then
-                __LinuxCodeName=vivid
+        trusty) # Ubuntu 14.04
+            if [ "$__CodeName" != "jessie" ]; then
+                __CodeName=trusty
             fi
             ;;
-        wily)
-            if [ "$__LinuxCodeName" != "jessie" ]; then
-                __LinuxCodeName=wily
+        xenial) # Ubuntu 16.04
+            if [ "$__CodeName" != "jessie" ]; then
+                __CodeName=xenial
             fi
             ;;
-        xenial)
-            if [ "$__LinuxCodeName" != "jessie" ]; then
-                __LinuxCodeName=xenial
+        zesty) # Ubuntu 17.04
+            if [ "$__CodeName" != "jessie" ]; then
+                __CodeName=zesty
             fi
             ;;
-        zesty)
-            if [ "$__LinuxCodeName" != "jessie" ]; then
-                __LinuxCodeName=zesty
+        bionic) # Ubuntu 18.04
+            if [ "$__CodeName" != "jessie" ]; then
+                __CodeName=bionic
             fi
             ;;
-        bionic)
-            if [ "$__LinuxCodeName" != "jessie" ]; then
-                __LinuxCodeName=bionic
-            fi
-            ;;
-        jessie)
-            __LinuxCodeName=jessie
+        jessie) # Debian 8
+            __CodeName=jessie
             __UbuntuRepo="http://ftp.debian.org/debian/"
             ;;
+        stretch) # Debian 9
+            __CodeName=stretch
+            __UbuntuRepo="http://ftp.debian.org/debian/"
+            __LLDB_Package="liblldb-6.0-dev"
+            ;;
+        buster) # Debian 10
+            __CodeName=buster
+            __UbuntuRepo="http://ftp.debian.org/debian/"
+            __LLDB_Package="liblldb-6.0-dev"
+            ;;
         tizen)
-            if [ "$__BuildArch" != "armel" ]; then
-                echo "Tizen is available only for armel."
+            if [ "$__BuildArch" != "armel" ] && [ "$__BuildArch" != "arm64" ]; then
+                echo "Tizen is available only for armel and arm64."
                 usage;
                 exit 1;
             fi
-            __LinuxCodeName=
+            __CodeName=
             __UbuntuRepo=
             __Tizen=tizen
             ;;
-        alpine)
-            __LinuxCodeName=alpine
+        alpine|alpine3.9)
+            __CodeName=alpine
             __UbuntuRepo=
+            __AlpineVersion=3.9
+            ;;
+        alpine3.13)
+            __CodeName=alpine
+            __UbuntuRepo=
+            __AlpineVersion=3.13
+            # Alpine 3.13 has all the packages we need in the 3.13 repository
+            __AlpinePackages+=$__AlpinePackagesEdgeCommunity
+            __AlpinePackagesEdgeCommunity=
+            __AlpinePackages+=$__AlpinePackagesEdgeMain
+            __AlpinePackagesEdgeMain=
+            ;;
+        freebsd11)
+            __FreeBSDBase="11.3-RELEASE"
+            ;&
+        freebsd12)
+            __CodeName=freebsd
+            __BuildArch=x64
+            __SkipUnmount=1
+            ;;
+        illumos)
+            __CodeName=illumos
+            __BuildArch=x64
+            __SkipUnmount=1
             ;;
         --skipunmount)
             __SkipUnmount=1
             ;;
+        --rootfsdir|-rootfsdir)
+            shift
+            __RootfsDir=$1
+            ;;
+        --use-mirror)
+            __UseMirror=1
+            ;;
         *)
-            __UnprocessedBuildArgs="$__UnprocessedBuildArgs $i"
+            __UnprocessedBuildArgs="$__UnprocessedBuildArgs $1"
             ;;
     esac
+
+    shift
 done
 
 if [ "$__BuildArch" == "armel" ]; then
@@ -155,53 +242,141 @@ if [ "$__BuildArch" == "armel" ]; then
 fi
 __UbuntuPackages+=" ${__LLDB_Package:-}"
 
-__RootfsDir="$__CrossDir/rootfs/$__BuildArch"
-
-if [[ -n "$ROOTFS_DIR" ]]; then
+if [ -z "$__RootfsDir" ] && [ ! -z "$ROOTFS_DIR" ]; then
     __RootfsDir=$ROOTFS_DIR
 fi
 
+if [ -z "$__RootfsDir" ]; then
+    __RootfsDir="$__CrossDir/../../../.tools/rootfs/$__BuildArch"
+fi
+
 if [ -d "$__RootfsDir" ]; then
     if [ $__SkipUnmount == 0 ]; then
-        umount $__RootfsDir/*
+        umount $__RootfsDir/* || true
     fi
     rm -rf $__RootfsDir
 fi
 
-if [[ "$__LinuxCodeName" == "alpine" ]]; then
+mkdir -p $__RootfsDir
+__RootfsDir="$( cd "$__RootfsDir" && pwd )"
+
+if [[ "$__CodeName" == "alpine" ]]; then
     __ApkToolsVersion=2.9.1
-    __AlpineVersion=3.7
     __ApkToolsDir=$(mktemp -d)
     wget https://github.com/alpinelinux/apk-tools/releases/download/v$__ApkToolsVersion/apk-tools-$__ApkToolsVersion-x86_64-linux.tar.gz -P $__ApkToolsDir
     tar -xf $__ApkToolsDir/apk-tools-$__ApkToolsVersion-x86_64-linux.tar.gz -C $__ApkToolsDir
     mkdir -p $__RootfsDir/usr/bin
     cp -v /usr/bin/qemu-$__QEMUArch-static $__RootfsDir/usr/bin
+
     $__ApkToolsDir/apk-tools-$__ApkToolsVersion/apk \
       -X http://dl-cdn.alpinelinux.org/alpine/v$__AlpineVersion/main \
       -X http://dl-cdn.alpinelinux.org/alpine/v$__AlpineVersion/community \
-      -X http://dl-cdn.alpinelinux.org/alpine/edge/testing \
       -U --allow-untrusted --root $__RootfsDir --arch $__AlpineArch --initdb \
       add $__AlpinePackages
+
+    if [[ -n "$__AlpinePackagesEdgeMain" ]]; then
+      $__ApkToolsDir/apk-tools-$__ApkToolsVersion/apk \
+        -X http://dl-cdn.alpinelinux.org/alpine/edge/main \
+        -U --allow-untrusted --root $__RootfsDir --arch $__AlpineArch --initdb \
+        add $__AlpinePackagesEdgeMain
+    fi
+
+    if [[ -n "$__AlpinePackagesEdgeCommunity" ]]; then
+      $__ApkToolsDir/apk-tools-$__ApkToolsVersion/apk \
+        -X http://dl-cdn.alpinelinux.org/alpine/edge/community \
+        -U --allow-untrusted --root $__RootfsDir --arch $__AlpineArch --initdb \
+        add $__AlpinePackagesEdgeCommunity
+    fi
+
     rm -r $__ApkToolsDir
-elif [[ -n $__LinuxCodeName ]]; then
-    qemu-debootstrap --arch $__UbuntuArch $__LinuxCodeName $__RootfsDir $__UbuntuRepo
-    cp $__CrossDir/$__BuildArch/sources.list.$__LinuxCodeName $__RootfsDir/etc/apt/sources.list
+elif [[ "$__CodeName" == "freebsd" ]]; then
+    mkdir -p $__RootfsDir/usr/local/etc
+    wget -O - https://download.freebsd.org/ftp/releases/amd64/${__FreeBSDBase}/base.txz | tar -C $__RootfsDir -Jxf - ./lib ./usr/lib ./usr/libdata ./usr/include ./usr/share/keys ./etc ./bin/freebsd-version
+    # For now, ask for 11 ABI even on 12. This can be revisited later.
+    echo "ABI = \"FreeBSD:11:amd64\"; FINGERPRINTS = \"${__RootfsDir}/usr/share/keys\"; REPOS_DIR = [\"${__RootfsDir}/etc/pkg\"]; REPO_AUTOUPDATE = NO; RUN_SCRIPTS = NO;" > ${__RootfsDir}/usr/local/etc/pkg.conf
+    echo "FreeBSD: { url: "pkg+http://pkg.FreeBSD.org/\${ABI}/quarterly", mirror_type: \"srv\", signature_type: \"fingerprints\", fingerprints: \"${__RootfsDir}/usr/share/keys/pkg\", enabled: yes }" > ${__RootfsDir}/etc/pkg/FreeBSD.conf
+    mkdir -p $__RootfsDir/tmp
+    # get and build package manager
+    wget -O -  https://github.com/freebsd/pkg/archive/${__FreeBSDPkg}.tar.gz  |  tar -C $__RootfsDir/tmp -zxf -
+    cd $__RootfsDir/tmp/pkg-${__FreeBSDPkg}
+    # needed for install to succeed
+    mkdir -p $__RootfsDir/host/etc
+    ./autogen.sh && ./configure --prefix=$__RootfsDir/host && make && make install
+    rm -rf $__RootfsDir/tmp/pkg-${__FreeBSDPkg}
+    # install packages we need.
+    INSTALL_AS_USER=$(whoami) $__RootfsDir/host/sbin/pkg -r $__RootfsDir -C $__RootfsDir/usr/local/etc/pkg.conf update
+    INSTALL_AS_USER=$(whoami) $__RootfsDir/host/sbin/pkg -r $__RootfsDir -C $__RootfsDir/usr/local/etc/pkg.conf install --yes $__FreeBSDPackages
+elif [[ "$__CodeName" == "illumos" ]]; then
+    mkdir "$__RootfsDir/tmp"
+    pushd "$__RootfsDir/tmp"
+    JOBS="$(getconf _NPROCESSORS_ONLN)"
+    echo "Downloading sysroot."
+    wget -O - https://github.com/illumos/sysroot/releases/download/20181213-de6af22ae73b-v1/illumos-sysroot-i386-20181213-de6af22ae73b-v1.tar.gz | tar -C "$__RootfsDir" -xzf -
+    echo "Building binutils. Please wait.."
+    wget -O - https://ftp.gnu.org/gnu/binutils/binutils-2.33.1.tar.bz2 | tar -xjf -
+    mkdir build-binutils && cd build-binutils
+    ../binutils-2.33.1/configure --prefix="$__RootfsDir" --target="x86_64-sun-solaris2.10" --program-prefix="x86_64-illumos-" --with-sysroot="$__RootfsDir"
+    make -j "$JOBS" && make install && cd ..
+    echo "Building gcc. Please wait.."
+    wget -O - https://ftp.gnu.org/gnu/gcc/gcc-8.4.0/gcc-8.4.0.tar.xz | tar -xJf -
+    CFLAGS="-fPIC"
+    CXXFLAGS="-fPIC"
+    CXXFLAGS_FOR_TARGET="-fPIC"
+    CFLAGS_FOR_TARGET="-fPIC"
+    export CFLAGS CXXFLAGS CXXFLAGS_FOR_TARGET CFLAGS_FOR_TARGET
+    mkdir build-gcc && cd build-gcc
+    ../gcc-8.4.0/configure --prefix="$__RootfsDir" --target="x86_64-sun-solaris2.10" --program-prefix="x86_64-illumos-" --with-sysroot="$__RootfsDir" --with-gnu-as       \
+        --with-gnu-ld --disable-nls --disable-libgomp --disable-libquadmath --disable-libssp --disable-libvtv --disable-libcilkrts --disable-libada --disable-libsanitizer \
+        --disable-libquadmath-support --disable-shared --enable-tls
+    make -j "$JOBS" && make install && cd ..
+    BaseUrl=https://pkgsrc.joyent.com
+    if [[ "$__UseMirror" == 1 ]]; then
+        BaseUrl=http://pkgsrc.smartos.skylime.net
+    fi
+    BaseUrl="$BaseUrl"/packages/SmartOS/2020Q1/x86_64/All
+    echo "Downloading dependencies."
+    read -ra array <<<"$__IllumosPackages"
+    for package in "${array[@]}"; do
+       echo "Installing $package..."
+        wget "$BaseUrl"/"$package".tgz
+        ar -x "$package".tgz
+        tar --skip-old-files -xzf "$package".tmp.tgz -C "$__RootfsDir" 2>/dev/null
+    done
+    echo "Cleaning up temporary files."
+    popd
+    rm -rf "$__RootfsDir"/{tmp,+*}
+    mkdir -p "$__RootfsDir"/usr/include/net
+    mkdir -p "$__RootfsDir"/usr/include/netpacket
+    wget -P "$__RootfsDir"/usr/include/net https://raw.githubusercontent.com/illumos/illumos-gate/master/usr/src/uts/common/io/bpf/net/bpf.h
+    wget -P "$__RootfsDir"/usr/include/net https://raw.githubusercontent.com/illumos/illumos-gate/master/usr/src/uts/common/io/bpf/net/dlt.h
+    wget -P "$__RootfsDir"/usr/include/netpacket https://raw.githubusercontent.com/illumos/illumos-gate/master/usr/src/uts/common/inet/sockmods/netpacket/packet.h
+    wget -P "$__RootfsDir"/usr/include/sys https://raw.githubusercontent.com/illumos/illumos-gate/master/usr/src/uts/common/sys/sdt.h
+elif [[ -n $__CodeName ]]; then
+    qemu-debootstrap --arch $__UbuntuArch $__CodeName $__RootfsDir $__UbuntuRepo
+    cp $__CrossDir/$__BuildArch/sources.list.$__CodeName $__RootfsDir/etc/apt/sources.list
     chroot $__RootfsDir apt-get update
     chroot $__RootfsDir apt-get -f -y install
     chroot $__RootfsDir apt-get -y install $__UbuntuPackages
     chroot $__RootfsDir symlinks -cr /usr
+    chroot $__RootfsDir apt-get clean
 
     if [ $__SkipUnmount == 0 ]; then
-        umount $__RootfsDir/*
+        umount $__RootfsDir/* || true
     fi
 
-    if [[ "$__BuildArch" == "arm" && "$__LinuxCodeName" == "trusty" ]]; then
+    if [[ "$__BuildArch" == "arm" && "$__CodeName" == "trusty" ]]; then
         pushd $__RootfsDir
         patch -p1 < $__CrossDir/$__BuildArch/trusty.patch
         patch -p1 < $__CrossDir/$__BuildArch/trusty-lttng-2.4.patch
         popd
     fi
-elif [ "$__Tizen" == "tizen" ]; then
+
+    if [[ "$__BuildArch" == "armel" && "$__CodeName" == "jessie" ]]; then
+        pushd $__RootfsDir
+        patch -p1 < $__CrossDir/$__BuildArch/armel.jessie.patch
+        popd
+    fi
+elif [[ "$__Tizen" == "tizen" ]]; then
     ROOTFS_DIR=$__RootfsDir $__CrossDir/$__BuildArch/tizen-build-rootfs.sh
 else
     echo "Unsupported target platform."

eng/common/cross/s390x/sources.list.bionic

@@ -0,0 +1,11 @@
+deb http://ports.ubuntu.com/ubuntu-ports/ bionic main restricted universe
+deb-src http://ports.ubuntu.com/ubuntu-ports/ bionic main restricted universe
+
+deb http://ports.ubuntu.com/ubuntu-ports/ bionic-updates main restricted universe
+deb-src http://ports.ubuntu.com/ubuntu-ports/ bionic-updates main restricted universe
+
+deb http://ports.ubuntu.com/ubuntu-ports/ bionic-backports main restricted
+deb-src http://ports.ubuntu.com/ubuntu-ports/ bionic-backports main restricted
+
+deb http://ports.ubuntu.com/ubuntu-ports/ bionic-security main restricted universe multiverse
+deb-src http://ports.ubuntu.com/ubuntu-ports/ bionic-security main restricted universe multiverse

eng/common/cross/toolchain.cmake

@@ -1,18 +1,27 @@
 set(CROSS_ROOTFS $ENV{ROOTFS_DIR})
 
 set(TARGET_ARCH_NAME $ENV{TARGET_BUILD_ARCH})
-set(CMAKE_SYSTEM_NAME Linux)
+if(EXISTS ${CROSS_ROOTFS}/bin/freebsd-version)
+  set(CMAKE_SYSTEM_NAME FreeBSD)
+elseif(EXISTS ${CROSS_ROOTFS}/usr/platform/i86pc)
+  set(CMAKE_SYSTEM_NAME SunOS)
+  set(ILLUMOS 1)
+else()
+  set(CMAKE_SYSTEM_NAME Linux)
+endif()
 set(CMAKE_SYSTEM_VERSION 1)
 
 if(TARGET_ARCH_NAME STREQUAL "armel")
   set(CMAKE_SYSTEM_PROCESSOR armv7l)
   set(TOOLCHAIN "arm-linux-gnueabi")
   if("$ENV{__DistroRid}" MATCHES "tizen.*")
-    set(TIZEN_TOOLCHAIN "armv7l-tizen-linux-gnueabi/6.2.1")
+    set(TIZEN_TOOLCHAIN "armv7l-tizen-linux-gnueabi/9.2.0")
   endif()
 elseif(TARGET_ARCH_NAME STREQUAL "arm")
   set(CMAKE_SYSTEM_PROCESSOR armv7l)
-  if(EXISTS ${CROSS_ROOTFS}/usr/lib/gcc/armv6-alpine-linux-musleabihf)
+  if(EXISTS ${CROSS_ROOTFS}/usr/lib/gcc/armv7-alpine-linux-musleabihf)
+    set(TOOLCHAIN "armv7-alpine-linux-musleabihf")
+  elseif(EXISTS ${CROSS_ROOTFS}/usr/lib/gcc/armv6-alpine-linux-musleabihf)
     set(TOOLCHAIN "armv6-alpine-linux-musleabihf")
   else()
     set(TOOLCHAIN "arm-linux-gnueabihf")
@@ -24,65 +33,148 @@ elseif(TARGET_ARCH_NAME STREQUAL "arm64")
   else()
     set(TOOLCHAIN "aarch64-linux-gnu")
   endif()
+  if("$ENV{__DistroRid}" MATCHES "tizen.*")
+    set(TIZEN_TOOLCHAIN "aarch64-tizen-linux-gnu/9.2.0")
+  endif()
+elseif(TARGET_ARCH_NAME STREQUAL "s390x")
+  set(CMAKE_SYSTEM_PROCESSOR s390x)
+  set(TOOLCHAIN "s390x-linux-gnu")
 elseif(TARGET_ARCH_NAME STREQUAL "x86")
   set(CMAKE_SYSTEM_PROCESSOR i686)
   set(TOOLCHAIN "i686-linux-gnu")
+elseif (CMAKE_SYSTEM_NAME STREQUAL "FreeBSD")
+  set(CMAKE_SYSTEM_PROCESSOR "x86_64")
+  set(triple "x86_64-unknown-freebsd11")
+elseif (ILLUMOS)
+  set(CMAKE_SYSTEM_PROCESSOR "x86_64")
+  set(TOOLCHAIN "x86_64-illumos")
 else()
-  message(FATAL_ERROR "Arch is ${TARGET_ARCH_NAME}. Only armel, arm, arm64 and x86 are supported!")
+  message(FATAL_ERROR "Arch is ${TARGET_ARCH_NAME}. Only armel, arm, arm64, s390x and x86 are supported!")
+endif()
+
+if(DEFINED ENV{TOOLCHAIN})
+  set(TOOLCHAIN $ENV{TOOLCHAIN})
 endif()
 
 # Specify include paths
-if(TARGET_ARCH_NAME STREQUAL "armel")
-  if(DEFINED TIZEN_TOOLCHAIN)
+if(DEFINED TIZEN_TOOLCHAIN)
+  if(TARGET_ARCH_NAME STREQUAL "armel")
     include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}/include/c++/)
     include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}/include/c++/armv7l-tizen-linux-gnueabi)
   endif()
+  if(TARGET_ARCH_NAME STREQUAL "arm64")
+    include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}/include/c++/)
+    include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}/include/c++/aarch64-tizen-linux-gnu)
+  endif()
 endif()
 
-# add_compile_param - adds only new options without duplicates.
-# arg0 - list with result options, arg1 - list with new options.
-# arg2 - optional argument, quick summary string for optional using CACHE FORCE mode.
-macro(add_compile_param)
-  if(NOT ${ARGC} MATCHES "^(2|3)$")
-    message(FATAL_ERROR "Wrong using add_compile_param! Two or three parameters must be given! See add_compile_param description.")
-  endif()
-  foreach(OPTION ${ARGV1})
-    if(NOT ${ARGV0} MATCHES "${OPTION}($| )")
-      set(${ARGV0} "${${ARGV0}} ${OPTION}")
-      if(${ARGC} EQUAL "3") # CACHE FORCE mode
-        set(${ARGV0} "${${ARGV0}}" CACHE STRING "${ARGV2}" FORCE)
-      endif()
+if("$ENV{__DistroRid}" MATCHES "android.*")
+    if(TARGET_ARCH_NAME STREQUAL "arm")
+        set(ANDROID_ABI armeabi-v7a)
+    elseif(TARGET_ARCH_NAME STREQUAL "arm64")
+        set(ANDROID_ABI arm64-v8a)
     endif()
-  endforeach()
-endmacro()
+
+    # extract platform number required by the NDK's toolchain
+    string(REGEX REPLACE ".*\\.([0-9]+)-.*" "\\1" ANDROID_PLATFORM "$ENV{__DistroRid}")
+
+    set(ANDROID_TOOLCHAIN clang)
+    set(FEATURE_EVENT_TRACE 0) # disable event trace as there is no lttng-ust package in termux repository
+    set(CMAKE_SYSTEM_LIBRARY_PATH "${CROSS_ROOTFS}/usr/lib")
+    set(CMAKE_SYSTEM_INCLUDE_PATH "${CROSS_ROOTFS}/usr/include")
+
+    # include official NDK toolchain script
+    include(${CROSS_ROOTFS}/../build/cmake/android.toolchain.cmake)
+elseif(CMAKE_SYSTEM_NAME STREQUAL "FreeBSD")
+    # we cross-compile by instructing clang
+    set(CMAKE_C_COMPILER_TARGET ${triple})
+    set(CMAKE_CXX_COMPILER_TARGET ${triple})
+    set(CMAKE_ASM_COMPILER_TARGET ${triple})
+    set(CMAKE_SYSROOT "${CROSS_ROOTFS}")
+elseif(ILLUMOS)
+    set(CMAKE_SYSROOT "${CROSS_ROOTFS}")
+
+    include_directories(SYSTEM ${CROSS_ROOTFS}/include)
+
+    set(TOOLSET_PREFIX ${TOOLCHAIN}-)
+    function(locate_toolchain_exec exec var)
+        string(TOUPPER ${exec} EXEC_UPPERCASE)
+        if(NOT "$ENV{CLR_${EXEC_UPPERCASE}}" STREQUAL "")
+            set(${var} "$ENV{CLR_${EXEC_UPPERCASE}}" PARENT_SCOPE)
+            return()
+        endif()
+
+        find_program(EXEC_LOCATION_${exec}
+            NAMES
+            "${TOOLSET_PREFIX}${exec}${CLR_CMAKE_COMPILER_FILE_NAME_VERSION}"
+            "${TOOLSET_PREFIX}${exec}")
+
+        if (EXEC_LOCATION_${exec} STREQUAL "EXEC_LOCATION_${exec}-NOTFOUND")
+            message(FATAL_ERROR "Unable to find toolchain executable. Name: ${exec}, Prefix: ${TOOLSET_PREFIX}.")
+        endif()
+        set(${var} ${EXEC_LOCATION_${exec}} PARENT_SCOPE)
+    endfunction()
+
+    set(CMAKE_SYSTEM_PREFIX_PATH "${CROSS_ROOTFS}")
+
+    locate_toolchain_exec(gcc CMAKE_C_COMPILER)
+    locate_toolchain_exec(g++ CMAKE_CXX_COMPILER)
+
+    set(CMAKE_C_STANDARD_LIBRARIES "${CMAKE_C_STANDARD_LIBRARIES} -lssp")
+    set(CMAKE_CXX_STANDARD_LIBRARIES "${CMAKE_CXX_STANDARD_LIBRARIES} -lssp")
+else()
+    set(CMAKE_SYSROOT "${CROSS_ROOTFS}")
+
+    set(CMAKE_C_COMPILER_EXTERNAL_TOOLCHAIN "${CROSS_ROOTFS}/usr")
+    set(CMAKE_CXX_COMPILER_EXTERNAL_TOOLCHAIN "${CROSS_ROOTFS}/usr")
+    set(CMAKE_ASM_COMPILER_EXTERNAL_TOOLCHAIN "${CROSS_ROOTFS}/usr")
+endif()
 
 # Specify link flags
-add_compile_param(CROSS_LINK_FLAGS "--sysroot=${CROSS_ROOTFS}")
-add_compile_param(CROSS_LINK_FLAGS "--gcc-toolchain=${CROSS_ROOTFS}/usr")
-add_compile_param(CROSS_LINK_FLAGS "--target=${TOOLCHAIN}")
-add_compile_param(CROSS_LINK_FLAGS "-fuse-ld=gold")
+
+function(add_toolchain_linker_flag Flag)
+  set(Config "${ARGV1}")
+  set(CONFIG_SUFFIX "")
+  if (NOT Config STREQUAL "")
+    set(CONFIG_SUFFIX "_${Config}")
+  endif()
+  set("CMAKE_EXE_LINKER_FLAGS${CONFIG_SUFFIX}" "${CMAKE_EXE_LINKER_FLAGS${CONFIG_SUFFIX}} ${Flag}" PARENT_SCOPE)
+  set("CMAKE_SHARED_LINKER_FLAGS${CONFIG_SUFFIX}" "${CMAKE_SHARED_LINKER_FLAGS${CONFIG_SUFFIX}} ${Flag}" PARENT_SCOPE)
+endfunction()
+
+if(CMAKE_SYSTEM_NAME STREQUAL "Linux")
+  add_toolchain_linker_flag("-Wl,--rpath-link=${CROSS_ROOTFS}/lib/${TOOLCHAIN}")
+  add_toolchain_linker_flag("-Wl,--rpath-link=${CROSS_ROOTFS}/usr/lib/${TOOLCHAIN}")
+endif()
 
 if(TARGET_ARCH_NAME STREQUAL "armel")
   if(DEFINED TIZEN_TOOLCHAIN) # For Tizen only
-    add_compile_param(CROSS_LINK_FLAGS "-B${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}")
-    add_compile_param(CROSS_LINK_FLAGS "-L${CROSS_ROOTFS}/lib")
-    add_compile_param(CROSS_LINK_FLAGS "-L${CROSS_ROOTFS}/usr/lib")
-    add_compile_param(CROSS_LINK_FLAGS "-L${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}")
+    add_toolchain_linker_flag("-B${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}")
+    add_toolchain_linker_flag("-L${CROSS_ROOTFS}/lib")
+    add_toolchain_linker_flag("-L${CROSS_ROOTFS}/usr/lib")
+    add_toolchain_linker_flag("-L${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}")
+  endif()
+elseif(TARGET_ARCH_NAME STREQUAL "arm64")
+  if(DEFINED TIZEN_TOOLCHAIN) # For Tizen only
+    add_toolchain_linker_flag("-B${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}")
+    add_toolchain_linker_flag("-L${CROSS_ROOTFS}/lib64")
+    add_toolchain_linker_flag("-L${CROSS_ROOTFS}/usr/lib64")
+    add_toolchain_linker_flag("-L${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}")
+
+    add_toolchain_linker_flag("-Wl,--rpath-link=${CROSS_ROOTFS}/lib64")
+    add_toolchain_linker_flag("-Wl,--rpath-link=${CROSS_ROOTFS}/usr/lib64")
+    add_toolchain_linker_flag("-Wl,--rpath-link=${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}")
   endif()
 elseif(TARGET_ARCH_NAME STREQUAL "x86")
-  add_compile_param(CROSS_LINK_FLAGS "-m32")
+  add_toolchain_linker_flag(-m32)
+elseif(ILLUMOS)
+  add_toolchain_linker_flag("-L${CROSS_ROOTFS}/lib/amd64")
+  add_toolchain_linker_flag("-L${CROSS_ROOTFS}/usr/amd64/lib")
 endif()
 
-add_compile_param(CMAKE_EXE_LINKER_FLAGS "${CROSS_LINK_FLAGS}" "TOOLCHAIN_EXE_LINKER_FLAGS")
-add_compile_param(CMAKE_SHARED_LINKER_FLAGS "${CROSS_LINK_FLAGS}" "TOOLCHAIN_EXE_LINKER_FLAGS")
-add_compile_param(CMAKE_MODULE_LINKER_FLAGS "${CROSS_LINK_FLAGS}" "TOOLCHAIN_EXE_LINKER_FLAGS")
-
 # Specify compile options
-add_compile_options("--sysroot=${CROSS_ROOTFS}")
-add_compile_options("--target=${TOOLCHAIN}")
-add_compile_options("--gcc-toolchain=${CROSS_ROOTFS}/usr")
 
-if(TARGET_ARCH_NAME MATCHES "^(arm|armel|arm64)$")
+if((TARGET_ARCH_NAME MATCHES "^(arm|armel|arm64|s390x)$" AND NOT "$ENV{__DistroRid}" MATCHES "android.*") OR ILLUMOS)
   set(CMAKE_C_COMPILER_TARGET ${TOOLCHAIN})
   set(CMAKE_CXX_COMPILER_TARGET ${TOOLCHAIN})
   set(CMAKE_ASM_COMPILER_TARGET ${TOOLCHAIN})
@@ -90,20 +182,33 @@ endif()
 
 if(TARGET_ARCH_NAME MATCHES "^(arm|armel)$")
   add_compile_options(-mthumb)
-  add_compile_options(-mfpu=vfpv3)
+  if (NOT DEFINED CLR_ARM_FPU_TYPE)
+    set (CLR_ARM_FPU_TYPE vfpv3)
+  endif (NOT DEFINED CLR_ARM_FPU_TYPE)
+
+  add_compile_options (-mfpu=${CLR_ARM_FPU_TYPE})
+  if (NOT DEFINED CLR_ARM_FPU_CAPABILITY)
+    set (CLR_ARM_FPU_CAPABILITY 0x7)
+  endif (NOT DEFINED CLR_ARM_FPU_CAPABILITY)
+
+  add_definitions (-DCLR_ARM_FPU_CAPABILITY=${CLR_ARM_FPU_CAPABILITY})
+
   if(TARGET_ARCH_NAME STREQUAL "armel")
     add_compile_options(-mfloat-abi=softfp)
-    if(DEFINED TIZEN_TOOLCHAIN)
-      add_compile_options(-Wno-deprecated-declarations) # compile-time option
-      add_compile_options(-D__extern_always_inline=inline) # compile-time option
-    endif()
   endif()
 elseif(TARGET_ARCH_NAME STREQUAL "x86")
   add_compile_options(-m32)
   add_compile_options(-Wno-error=unused-command-line-argument)
 endif()
 
-# Set LLDB include and library paths
+if(DEFINED TIZEN_TOOLCHAIN)
+  if(TARGET_ARCH_NAME MATCHES "^(armel|arm64)$")
+    add_compile_options(-Wno-deprecated-declarations) # compile-time option
+    add_compile_options(-D__extern_always_inline=inline) # compile-time option
+  endif()
+endif()
+
+# Set LLDB include and library paths for builds that need lldb.
 if(TARGET_ARCH_NAME MATCHES "^(arm|armel|x86)$")
   if(TARGET_ARCH_NAME STREQUAL "x86")
     set(LLVM_CROSS_DIR "$ENV{LLVM_CROSS_HOME}")
@@ -131,7 +236,6 @@ if(TARGET_ARCH_NAME MATCHES "^(arm|armel|x86)$")
   endif()
 endif()
 
-set(CMAKE_FIND_ROOT_PATH "${CROSS_ROOTFS}")
 set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER)
 set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY)
 set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY)

eng/common/cross/x86/sources.list.vivid

@@ -1,11 +0,0 @@
-deb http://archive.ubuntu.com/ubuntu/ vivid main restricted universe
-deb-src http://archive.ubuntu.com/ubuntu/ vivid main restricted universe
-
-deb http://archive.ubuntu.com/ubuntu/ vivid-updates main restricted universe
-deb-src http://archive.ubuntu.com/ubuntu/ vivid-updates main restricted universe
-
-deb http://archive.ubuntu.com/ubuntu/ vivid-backports main restricted
-deb-src http://archive.ubuntu.com/ubuntu/ vivid-backports main restricted
-
-deb http://archive.ubuntu.com/ubuntu/ vivid-security main restricted universe multiverse
-deb-src http://archive.ubuntu.com/ubuntu/ vivid-security main restricted universe multiverse

eng/common/cross/x86/sources.list.wily

@@ -1,11 +0,0 @@
-deb http://archive.ubuntu.com/ubuntu/ wily main restricted universe
-deb-src http://archive.ubuntu.com/ubuntu/ wily main restricted universe
-
-deb http://archive.ubuntu.com/ubuntu/ wily-updates main restricted universe
-deb-src http://archive.ubuntu.com/ubuntu/ wily-updates main restricted universe
-
-deb http://archive.ubuntu.com/ubuntu/ wily-backports main restricted
-deb-src http://archive.ubuntu.com/ubuntu/ wily-backports main restricted
-
-deb http://archive.ubuntu.com/ubuntu/ wily-security main restricted universe multiverse
-deb-src http://archive.ubuntu.com/ubuntu/ wily-security main restricted universe multiverse

eng/common/darc-init.ps1

@@ -1,32 +1,47 @@
 param (
-    $darcVersion = $null
+    $darcVersion = $null,
+    $versionEndpoint = 'https://maestro-prod.westus2.cloudapp.azure.com/api/assets/darc-version?api-version=2019-01-16',
+    $verbosity = 'minimal',
+    $toolpath = $null
 )
 
-$verbosity = "m"
 . $PSScriptRoot\tools.ps1
 
-function InstallDarcCli ($darcVersion) {
-  $darcCliPackageName = "microsoft.dotnet.darc"
+function InstallDarcCli ($darcVersion, $toolpath) {
+  $darcCliPackageName = 'microsoft.dotnet.darc'
 
   $dotnetRoot = InitializeDotNetCli -install:$true
   $dotnet = "$dotnetRoot\dotnet.exe"
-  $toolList = Invoke-Expression "& `"$dotnet`" tool list -g"
+  $toolList = & "$dotnet" tool list -g
 
   if ($toolList -like "*$darcCliPackageName*") {
-    Invoke-Expression "& `"$dotnet`" tool uninstall $darcCliPackageName -g"
+    & "$dotnet" tool uninstall $darcCliPackageName -g
   }
 
-  # Until we can anonymously query the BAR API for the latest arcade-services
-  # build applied to the PROD channel, this is hardcoded.
+  # If the user didn't explicitly specify the darc version,
+  # query the Maestro API for the correct version of darc to install.
   if (-not $darcVersion) {
-    $darcVersion = '1.1.0-beta.19205.4'
+    $darcVersion = $(Invoke-WebRequest -Uri $versionEndpoint -UseBasicParsing).Content
   }
-  
-  $arcadeServicesSource = 'https://dotnetfeed.blob.core.windows.net/dotnet-arcade/index.json'
+
+  $arcadeServicesSource = 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json'
 
   Write-Host "Installing Darc CLI version $darcVersion..."
-  Write-Host "You may need to restart your command window if this is the first dotnet tool you have installed."
-  Invoke-Expression "& `"$dotnet`" tool install $darcCliPackageName --version $darcVersion --add-source '$arcadeServicesSource' -v $verbosity -g"
+  Write-Host 'You may need to restart your command window if this is the first dotnet tool you have installed.'
+  if (-not $toolpath) {
+    Write-Host "'$dotnet' tool install $darcCliPackageName --version $darcVersion --add-source '$arcadeServicesSource' -v $verbosity -g"
+    & "$dotnet" tool install $darcCliPackageName --version $darcVersion --add-source "$arcadeServicesSource" -v $verbosity -g
+  }else {
+    Write-Host "'$dotnet' tool install $darcCliPackageName --version $darcVersion --add-source '$arcadeServicesSource' -v $verbosity --tool-path '$toolpath'"
+    & "$dotnet" tool install $darcCliPackageName --version $darcVersion --add-source "$arcadeServicesSource" -v $verbosity --tool-path "$toolpath"
+  }
 }
 
-InstallDarcCli $darcVersion
+try {
+  InstallDarcCli $darcVersion $toolpath
+}
+catch {
+  Write-Host $_.ScriptStackTrace
+  Write-PipelineTelemetryError -Category 'Darc' -Message $_
+  ExitWithExitCode 1
+}

eng/common/darc-init.sh

@@ -1,15 +1,29 @@
 #!/usr/bin/env bash
 
 source="${BASH_SOURCE[0]}"
-darcVersion="1.1.0-beta.19205.4"
+darcVersion=''
+versionEndpoint='https://maestro-prod.westus2.cloudapp.azure.com/api/assets/darc-version?api-version=2019-01-16'
+verbosity='minimal'
 
 while [[ $# > 0 ]]; do
-  opt="$(echo "$1" | awk '{print tolower($0)}')"
+  opt="$(echo "$1" | tr "[:upper:]" "[:lower:]")"
   case "$opt" in
     --darcversion)
       darcVersion=$2
       shift
       ;;
+    --versionendpoint)
+      versionEndpoint=$2
+      shift
+      ;;
+    --verbosity)
+      verbosity=$2
+      shift
+      ;;
+    --toolpath)
+      toolpath=$2
+      shift
+      ;;
     *)
       echo "Invalid argument: $1"
       usage
@@ -29,27 +43,40 @@ while [[ -h "$source" ]]; do
   [[ $source != /* ]] && source="$scriptroot/$source"
 done
 scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
-verbosity=m
 
 . "$scriptroot/tools.sh"
 
+if [ -z "$darcVersion" ]; then
+  darcVersion=$(curl -X GET "$versionEndpoint" -H "accept: text/plain")
+fi
+
 function InstallDarcCli {
   local darc_cli_package_name="microsoft.dotnet.darc"
 
   InitializeDotNetCli
   local dotnet_root=$_InitializeDotNetCli
 
-  local uninstall_command=`$dotnet_root/dotnet tool uninstall $darc_cli_package_name -g`
-  local tool_list=$($dotnet_root/dotnet tool list -g)
-  if [[ $tool_list = *$darc_cli_package_name* ]]; then
-    echo $($dotnet_root/dotnet tool uninstall $darc_cli_package_name -g)
+  if [ -z "$toolpath" ]; then
+    local tool_list=$($dotnet_root/dotnet tool list -g)
+    if [[ $tool_list = *$darc_cli_package_name* ]]; then
+      echo $($dotnet_root/dotnet tool uninstall $darc_cli_package_name -g)
+    fi
+  else
+    local tool_list=$($dotnet_root/dotnet tool list --tool-path "$toolpath")
+    if [[ $tool_list = *$darc_cli_package_name* ]]; then
+      echo $($dotnet_root/dotnet tool uninstall $darc_cli_package_name --tool-path "$toolpath")
+    fi
   fi
 
-  local arcadeServicesSource="https://dotnetfeed.blob.core.windows.net/dotnet-arcade/index.json"
+  local arcadeServicesSource="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json"
 
-  echo "Installing Darc CLI version $toolset_version..."
+  echo "Installing Darc CLI version $darcVersion..."
   echo "You may need to restart your command shell if this is the first dotnet tool you have installed."
-  echo $($dotnet_root/dotnet tool install $darc_cli_package_name --version $darcVersion --add-source "$arcadeServicesSource" -v $verbosity -g)
+  if [ -z "$toolpath" ]; then
+    echo $($dotnet_root/dotnet tool install $darc_cli_package_name --version $darcVersion --add-source "$arcadeServicesSource" -v $verbosity -g)
+  else
+    echo $($dotnet_root/dotnet tool install $darc_cli_package_name --version $darcVersion --add-source "$arcadeServicesSource" -v $verbosity --tool-path "$toolpath")
+  fi
 }
 
 InstallDarcCli

eng/common/dotnet-install.cmd

@@ -0,0 +1,2 @@
+@echo off
+powershell -ExecutionPolicy ByPass -NoProfile -command "& """%~dp0dotnet-install.ps1""" %*"

eng/common/dotnet-install.ps1

@@ -0,0 +1,28 @@
+[CmdletBinding(PositionalBinding=$false)]
+Param(
+  [string] $verbosity = 'minimal',
+  [string] $architecture = '',
+  [string] $version = 'Latest',
+  [string] $runtime = 'dotnet',
+  [string] $RuntimeSourceFeed = '',
+  [string] $RuntimeSourceFeedKey = ''
+)
+
+. $PSScriptRoot\tools.ps1
+
+$dotnetRoot = Join-Path $RepoRoot '.dotnet'
+
+$installdir = $dotnetRoot
+try {
+    if ($architecture -and $architecture.Trim() -eq 'x86') {
+        $installdir = Join-Path $installdir 'x86'
+    }
+    InstallDotNet $installdir $version $architecture $runtime $true -RuntimeSourceFeed $RuntimeSourceFeed -RuntimeSourceFeedKey $RuntimeSourceFeedKey
+}
+catch {
+  Write-Host $_.ScriptStackTrace
+  Write-PipelineTelemetryError -Category 'InitializeToolset' -Message $_
+  ExitWithExitCode 1
+}
+
+ExitWithExitCode 0

eng/common/dotnet-install.sh

@@ -0,0 +1,84 @@
+#!/usr/bin/env bash
+
+source="${BASH_SOURCE[0]}"
+# resolve $source until the file is no longer a symlink
+while [[ -h "$source" ]]; do
+  scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
+  source="$(readlink "$source")"
+  # if $source was a relative symlink, we need to resolve it relative to the path where the
+  # symlink file was located
+  [[ $source != /* ]] && source="$scriptroot/$source"
+done
+scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
+
+. "$scriptroot/tools.sh"
+
+version='Latest'
+architecture=''
+runtime='dotnet'
+runtimeSourceFeed=''
+runtimeSourceFeedKey=''
+while [[ $# > 0 ]]; do
+  opt="$(echo "$1" | tr "[:upper:]" "[:lower:]")"
+  case "$opt" in
+    -version|-v)
+      shift
+      version="$1"
+      ;;
+    -architecture|-a)
+      shift
+      architecture="$1"
+      ;;
+    -runtime|-r)
+      shift
+      runtime="$1"
+      ;;
+    -runtimesourcefeed)
+      shift
+      runtimeSourceFeed="$1"
+      ;;
+    -runtimesourcefeedkey)
+      shift
+      runtimeSourceFeedKey="$1"
+      ;;
+    *)
+      Write-PipelineTelemetryError -Category 'Build' -Message "Invalid argument: $1"
+      exit 1
+      ;;
+  esac
+  shift
+done
+
+# Use uname to determine what the CPU is, see https://en.wikipedia.org/wiki/Uname#Examples
+cpuname=$(uname -m)
+case $cpuname in
+  aarch64)
+    buildarch=arm64
+    ;;
+  amd64|x86_64)
+    buildarch=x64
+    ;;
+  armv*l)
+    buildarch=arm
+    ;;
+  i686)
+    buildarch=x86
+    ;;
+  *)
+    echo "Unknown CPU $cpuname detected, treating it as x64"
+    buildarch=x64
+    ;;
+esac
+
+dotnetRoot="$repo_root/.dotnet"
+if [[ $architecture != "" ]] && [[ $architecture != $buildarch ]]; then
+  dotnetRoot="$dotnetRoot/$architecture"
+fi
+
+InstallDotNet $dotnetRoot $version "$architecture" $runtime true $runtimeSourceFeed $runtimeSourceFeedKey || {
+  local exit_code=$?
+  Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "dotnet-install.sh failed (exit code '$exit_code')." >&2
+  ExitWithExitCode $exit_code
+}
+
+ExitWithExitCode 0

eng/common/enable-cross-org-publishing.ps1

@@ -0,0 +1,13 @@
+param(
+  [string] $token
+)
+
+
+. $PSScriptRoot\pipeline-logging-functions.ps1
+
+# Write-PipelineSetVariable will no-op if a variable named $ci is not defined
+# Since this script is only ever called in AzDO builds, just universally set it
+$ci = $true
+
+Write-PipelineSetVariable -Name 'VSS_NUGET_ACCESSTOKEN' -Value $token -IsMultiJobVariable $false
+Write-PipelineSetVariable -Name 'VSS_NUGET_URI_PREFIXES' -Value 'https://dnceng.pkgs.visualstudio.com/;https://pkgs.dev.azure.com/dnceng/;https://devdiv.pkgs.visualstudio.com/;https://pkgs.dev.azure.com/devdiv/' -IsMultiJobVariable $false

eng/common/generate-graph-files.ps1

@@ -3,46 +3,46 @@ Param(
   [Parameter(Mandatory=$true)][string] $gitHubPat,      # GitHub personal access token from https://github.com/settings/tokens (no auth scopes needed)
   [Parameter(Mandatory=$true)][string] $azdoPat,        # Azure Dev Ops tokens from https://dev.azure.com/dnceng/_details/security/tokens (code read scope needed)
   [Parameter(Mandatory=$true)][string] $outputFolder,   # Where the graphviz.txt file will be created
-  [string] $darcVersion = '1.1.0-beta.19175.6',         # darc's version
+  [string] $darcVersion,                                # darc's version
   [string] $graphvizVersion = '2.38',                   # GraphViz version
   [switch] $includeToolset                              # Whether the graph should include toolset dependencies or not. i.e. arcade, optimization. For more about
                                                         # toolset dependencies see https://github.com/dotnet/arcade/blob/master/Documentation/Darc.md#toolset-vs-product-dependencies
 )
 
-$ErrorActionPreference = "Stop"
-. $PSScriptRoot\tools.ps1
-
-Import-Module -Name (Join-Path $PSScriptRoot "native\CommonLibrary.psm1")
-
 function CheckExitCode ([string]$stage)
 {
   $exitCode = $LASTEXITCODE
   if ($exitCode  -ne 0) {
-    Write-Host "Something failed in stage: '$stage'. Check for errors above. Exiting now..."
+    Write-PipelineTelemetryError -Category 'Arcade' -Message "Something failed in stage: '$stage'. Check for errors above. Exiting now..."
     ExitWithExitCode $exitCode
   }
 }
 
 try {
+  $ErrorActionPreference = 'Stop'
+  . $PSScriptRoot\tools.ps1
+  
+  Import-Module -Name (Join-Path $PSScriptRoot 'native\CommonLibrary.psm1')
+
   Push-Location $PSScriptRoot
-    
-  Write-Host "Installing darc..."
+
+  Write-Host 'Installing darc...'
   . .\darc-init.ps1 -darcVersion $darcVersion
-  CheckExitCode "Running darc-init"
+  CheckExitCode 'Running darc-init'
 
-  $engCommonBaseDir = Join-Path $PSScriptRoot "native\"
+  $engCommonBaseDir = Join-Path $PSScriptRoot 'native\'
   $graphvizInstallDir = CommonLibrary\Get-NativeInstallDirectory
-  $nativeToolBaseUri = "https://netcorenativeassets.blob.core.windows.net/resource-packages/external"
-  $installBin = Join-Path $graphvizInstallDir "bin"
+  $nativeToolBaseUri = 'https://netcorenativeassets.blob.core.windows.net/resource-packages/external'
+  $installBin = Join-Path $graphvizInstallDir 'bin'
 
-  Write-Host "Installing dot..."
+  Write-Host 'Installing dot...'
   .\native\install-tool.ps1 -ToolName graphviz -InstallPath $installBin -BaseUri $nativeToolBaseUri -CommonLibraryDirectory $engCommonBaseDir -Version $graphvizVersion -Verbose
 
   $darcExe = "$env:USERPROFILE\.dotnet\tools"
   $darcExe = Resolve-Path "$darcExe\darc.exe"
-  
+
   Create-Directory $outputFolder
-  
+
   # Generate 3 graph descriptions:
   # 1. Flat with coherency information
   # 2. Graphviz (dot) file
@@ -51,37 +51,36 @@ try {
   $graphVizImageFilePath = "$outputFolder\graph.png"
   $normalGraphFilePath = "$outputFolder\graph-full.txt"
   $flatGraphFilePath = "$outputFolder\graph-flat.txt"
-  $baseOptions = "get-dependency-graph --github-pat $gitHubPat --azdev-pat $azdoPat --password $barToken"
-  
+  $baseOptions = @( '--github-pat', "$gitHubPat", '--azdev-pat', "$azdoPat", '--password', "$barToken" )
+
   if ($includeToolset) {
-    Write-Host "Toolsets will be included in the graph..."
-    $baseOptions += " --include-toolset"
+    Write-Host 'Toolsets will be included in the graph...'
+    $baseOptions += @( '--include-toolset' )
   }
 
-  Write-Host "Generating standard dependency graph..."
-  Invoke-Expression "& `"$darcExe`" $baseOptions --output-file $normalGraphFilePath"
-  CheckExitCode "Generating normal dependency graph"
+  Write-Host 'Generating standard dependency graph...'
+  & "$darcExe" get-dependency-graph @baseOptions --output-file $normalGraphFilePath
+  CheckExitCode 'Generating normal dependency graph'
 
-  Write-Host "Generating flat dependency graph and graphviz file..."
-  Invoke-Expression "& `"$darcExe`" $baseOptions --flat --coherency --graphviz $graphVizFilePath --output-file $flatGraphFilePath"
-  CheckExitCode "Generating flat and graphviz dependency graph"
+  Write-Host 'Generating flat dependency graph and graphviz file...'
+  & "$darcExe" get-dependency-graph @baseOptions --flat --coherency --graphviz $graphVizFilePath --output-file $flatGraphFilePath
+  CheckExitCode 'Generating flat and graphviz dependency graph'
 
   Write-Host "Generating graph image $graphVizFilePath"
   $dotFilePath = Join-Path $installBin "graphviz\$graphvizVersion\release\bin\dot.exe"
-  Invoke-Expression "& `"$dotFilePath`" -Tpng -o'$graphVizImageFilePath' `"$graphVizFilePath`""
-  CheckExitCode "Generating graphviz image"
-  
+  & "$dotFilePath" -Tpng -o"$graphVizImageFilePath" "$graphVizFilePath"
+  CheckExitCode 'Generating graphviz image'
+
   Write-Host "'$graphVizFilePath', '$flatGraphFilePath', '$normalGraphFilePath' and '$graphVizImageFilePath' created!"
 }
 catch {
   if (!$includeToolset) {
-    Write-Host "This might be a toolset repo which includes only toolset dependencies. " -NoNewline -ForegroundColor Yellow
-    Write-Host "Since -includeToolset is not set there is no graph to create. Include -includeToolset and try again..." -ForegroundColor Yellow
+    Write-Host 'This might be a toolset repo which includes only toolset dependencies. ' -NoNewline -ForegroundColor Yellow
+    Write-Host 'Since -includeToolset is not set there is no graph to create. Include -includeToolset and try again...' -ForegroundColor Yellow
   }
-  Write-Host $_
-  Write-Host $_.Exception
   Write-Host $_.ScriptStackTrace
+  Write-PipelineTelemetryError -Category 'Arcade' -Message $_
   ExitWithExitCode 1
 } finally {
-    Pop-Location
+  Pop-Location
 }

eng/common/generate-locproject.ps1

@@ -0,0 +1,110 @@
+Param(
+    [Parameter(Mandatory=$true)][string] $SourcesDirectory,     # Directory where source files live; if using a Localize directory it should live in here
+    [string] $LanguageSet = 'VS_Main_Languages',                # Language set to be used in the LocProject.json
+    [switch] $UseCheckedInLocProjectJson,                       # When set, generates a LocProject.json and compares it to one that already exists in the repo; otherwise just generates one
+    [switch] $CreateNeutralXlfs                                 # Creates neutral xlf files. Only set to false when running locally
+)
+
+# Generates LocProject.json files for the OneLocBuild task. OneLocBuildTask is described here:
+# https://ceapex.visualstudio.com/CEINTL/_wiki/wikis/CEINTL.wiki/107/Localization-with-OneLocBuild-Task
+
+Set-StrictMode -Version 2.0
+$ErrorActionPreference = "Stop"
+. $PSScriptRoot\tools.ps1
+
+Import-Module -Name (Join-Path $PSScriptRoot 'native\CommonLibrary.psm1')
+
+$exclusionsFilePath = "$SourcesDirectory\Localize\LocExclusions.json"
+$exclusions = @{ Exclusions = @() }
+if (Test-Path -Path $exclusionsFilePath)
+{
+    $exclusions = Get-Content "$exclusionsFilePath" | ConvertFrom-Json
+}
+
+Push-Location "$SourcesDirectory" # push location for Resolve-Path -Relative to work
+
+# Template files
+$jsonFiles = @()
+$jsonFiles += Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "\.template\.config\\localize\\en\..+\.json" } # .NET templating pattern
+$jsonFiles += Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "en\\strings\.json" } # current winforms pattern
+
+$xlfFiles = @()
+
+$allXlfFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory\*\*.xlf"
+$langXlfFiles = @()
+if ($allXlfFiles) {
+    $null = $allXlfFiles[0].FullName -Match "\.([\w-]+)\.xlf" # matches '[langcode].xlf'
+    $firstLangCode = $Matches.1
+    $langXlfFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory\*\*.$firstLangCode.xlf"
+}
+$langXlfFiles | ForEach-Object {
+    $null = $_.Name -Match "(.+)\.[\w-]+\.xlf" # matches '[filename].[langcode].xlf
+    
+    $destinationFile = "$($_.Directory.FullName)\$($Matches.1).xlf"
+    $xlfFiles += Copy-Item "$($_.FullName)" -Destination $destinationFile -PassThru
+}
+
+$locFiles = $jsonFiles + $xlfFiles
+
+$locJson = @{
+    Projects = @(
+        @{
+            LanguageSet = $LanguageSet
+            LocItems = @(
+                $locFiles | ForEach-Object {
+                    $outputPath = "$(($_.DirectoryName | Resolve-Path -Relative) + "\")" 
+                    $continue = $true
+                    foreach ($exclusion in $exclusions.Exclusions) {
+                        if ($outputPath.Contains($exclusion))
+                        {
+                            $continue = $false
+                        }
+                    }
+                    $sourceFile = ($_.FullName | Resolve-Path -Relative)
+                    if (!$CreateNeutralXlfs -and $_.Extension -eq '.xlf') {
+                        Remove-Item -Path $sourceFile
+                    }
+                    if ($continue)
+                    {
+                        if ($_.Directory.Name -eq 'en' -and $_.Extension -eq '.json') {
+                            return @{
+                                SourceFile = $sourceFile
+                                CopyOption = "LangIDOnPath"
+                                OutputPath = "$($_.Directory.Parent.FullName | Resolve-Path -Relative)\"
+                            }
+                        }
+                        else {
+                            return @{
+                                SourceFile = $sourceFile
+                                CopyOption = "LangIDOnName"
+                                OutputPath = $outputPath
+                            }
+                        }
+                    }
+                }
+            )
+        }
+    )
+}
+
+$json = ConvertTo-Json $locJson -Depth 5
+Write-Host "LocProject.json generated:`n`n$json`n`n"
+Pop-Location
+
+if (!$UseCheckedInLocProjectJson) {
+    New-Item "$SourcesDirectory\Localize\LocProject.json" -Force # Need this to make sure the Localize directory is created
+    Set-Content "$SourcesDirectory\Localize\LocProject.json" $json
+}
+else {
+    New-Item "$SourcesDirectory\Localize\LocProject-generated.json" -Force # Need this to make sure the Localize directory is created
+    Set-Content "$SourcesDirectory\Localize\LocProject-generated.json" $json
+
+    if ((Get-FileHash "$SourcesDirectory\Localize\LocProject-generated.json").Hash -ne (Get-FileHash "$SourcesDirectory\Localize\LocProject.json").Hash) {
+        Write-PipelineTelemetryError -Category "OneLocBuild" -Message "Existing LocProject.json differs from generated LocProject.json. Download LocProject-generated.json and compare them."
+        
+        exit 1
+    }
+    else {
+        Write-Host "Generated LocProject.json and current LocProject.json are identical."
+    }
+}

eng/common/init-tools-native.ps1

@@ -35,32 +35,37 @@ File path to global.json file
 #>
 [CmdletBinding(PositionalBinding=$false)]
 Param (
-  [string] $BaseUri = "https://netcorenativeassets.blob.core.windows.net/resource-packages/external",
+  [string] $BaseUri = 'https://netcorenativeassets.blob.core.windows.net/resource-packages/external',
   [string] $InstallDirectory,
   [switch] $Clean = $False,
   [switch] $Force = $False,
   [int] $DownloadRetries = 5,
   [int] $RetryWaitTimeInSeconds = 30,
-  [string] $GlobalJsonFile = "$PSScriptRoot\..\..\global.json"
+  [string] $GlobalJsonFile
 )
 
-Set-StrictMode -version 2.0
-$ErrorActionPreference="Stop"
+if (!$GlobalJsonFile) {
+  $GlobalJsonFile = Join-Path (Get-Item $PSScriptRoot).Parent.Parent.FullName 'global.json'
+}
 
-Import-Module -Name (Join-Path $PSScriptRoot "native\CommonLibrary.psm1")
+Set-StrictMode -version 2.0
+$ErrorActionPreference='Stop'
+
+. $PSScriptRoot\pipeline-logging-functions.ps1
+Import-Module -Name (Join-Path $PSScriptRoot 'native\CommonLibrary.psm1')
 
 try {
   # Define verbose switch if undefined
-  $Verbose = $VerbosePreference -Eq "Continue"
+  $Verbose = $VerbosePreference -Eq 'Continue'
 
-  $EngCommonBaseDir = Join-Path $PSScriptRoot "native\"
+  $EngCommonBaseDir = Join-Path $PSScriptRoot 'native\'
   $NativeBaseDir = $InstallDirectory
   if (!$NativeBaseDir) {
     $NativeBaseDir = CommonLibrary\Get-NativeInstallDirectory
   }
   $Env:CommonLibrary_NativeInstallDir = $NativeBaseDir
-  $InstallBin = Join-Path $NativeBaseDir "bin"
-  $InstallerPath = Join-Path $EngCommonBaseDir "install-tool.ps1"
+  $InstallBin = Join-Path $NativeBaseDir 'bin'
+  $InstallerPath = Join-Path $EngCommonBaseDir 'install-tool.ps1'
 
   # Process tools list
   Write-Host "Processing $GlobalJsonFile"
@@ -70,51 +75,59 @@ try {
   }
   $NativeTools = Get-Content($GlobalJsonFile) -Raw |
                     ConvertFrom-Json |
-                    Select-Object -Expand "native-tools" -ErrorAction SilentlyContinue
+                    Select-Object -Expand 'native-tools' -ErrorAction SilentlyContinue
   if ($NativeTools) {
     $NativeTools.PSObject.Properties | ForEach-Object {
       $ToolName = $_.Name
       $ToolVersion = $_.Value
-      $LocalInstallerCommand = $InstallerPath
-      $LocalInstallerCommand += " -ToolName $ToolName"
-      $LocalInstallerCommand += " -InstallPath $InstallBin"
-      $LocalInstallerCommand += " -BaseUri $BaseUri"
-      $LocalInstallerCommand += " -CommonLibraryDirectory $EngCommonBaseDir"
-      $LocalInstallerCommand += " -Version $ToolVersion"
+      $LocalInstallerArguments =  @{ ToolName = "$ToolName" }
+      $LocalInstallerArguments += @{ InstallPath = "$InstallBin" }
+      $LocalInstallerArguments += @{ BaseUri = "$BaseUri" }
+      $LocalInstallerArguments += @{ CommonLibraryDirectory = "$EngCommonBaseDir" }
+      $LocalInstallerArguments += @{ Version = "$ToolVersion" }
 
       if ($Verbose) {
-        $LocalInstallerCommand += " -Verbose"
+        $LocalInstallerArguments += @{ Verbose = $True }
       }
       if (Get-Variable 'Force' -ErrorAction 'SilentlyContinue') {
         if($Force) {
-          $LocalInstallerCommand += " -Force"
+          $LocalInstallerArguments += @{ Force = $True }
         }
       }
       if ($Clean) {
-        $LocalInstallerCommand += " -Clean"
+        $LocalInstallerArguments += @{ Clean = $True }
       }
 
       Write-Verbose "Installing $ToolName version $ToolVersion"
-      Write-Verbose "Executing '$LocalInstallerCommand'"
-      Invoke-Expression "$LocalInstallerCommand"
+      Write-Verbose "Executing '$InstallerPath $($LocalInstallerArguments.Keys.ForEach({"-$_ '$($LocalInstallerArguments.$_)'"}) -join ' ')'"
+      & $InstallerPath @LocalInstallerArguments
       if ($LASTEXITCODE -Ne "0") {
         $errMsg = "$ToolName installation failed"
         if ((Get-Variable 'DoNotAbortNativeToolsInstallationOnFailure' -ErrorAction 'SilentlyContinue') -and $DoNotAbortNativeToolsInstallationOnFailure) {
-            Write-Warning $errMsg
+            $showNativeToolsWarning = $true
+            if ((Get-Variable 'DoNotDisplayNativeToolsInstallationWarnings' -ErrorAction 'SilentlyContinue') -and $DoNotDisplayNativeToolsInstallationWarnings) {
+                $showNativeToolsWarning = $false
+            }
+            if ($showNativeToolsWarning) {
+                Write-Warning $errMsg
+            }
             $toolInstallationFailure = $true
         } else {
-            Write-Error $errMsg
+            # We cannot change this to Write-PipelineTelemetryError because of https://github.com/dotnet/arcade/issues/4482
+            Write-Host $errMsg
             exit 1
         }
       }
     }
 
     if ((Get-Variable 'toolInstallationFailure' -ErrorAction 'SilentlyContinue') -and $toolInstallationFailure) {
+        # We cannot change this to Write-PipelineTelemetryError because of https://github.com/dotnet/arcade/issues/4482
+        Write-Host 'Native tools bootstrap failed'
         exit 1
     }
   }
   else {
-    Write-Host "No native tools defined in global.json"
+    Write-Host 'No native tools defined in global.json'
     exit 0
   }
 
@@ -122,17 +135,18 @@ try {
     exit 0
   }
   if (Test-Path $InstallBin) {
-    Write-Host "Native tools are available from" (Convert-Path -Path $InstallBin)
+    Write-Host 'Native tools are available from ' (Convert-Path -Path $InstallBin)
     Write-Host "##vso[task.prependpath]$(Convert-Path -Path $InstallBin)"
+    return $InstallBin
   }
   else {
-    Write-Error "Native tools install directory does not exist, installation failed"
+    Write-PipelineTelemetryError -Category 'NativeToolsBootstrap' -Message 'Native tools install directory does not exist, installation failed'
     exit 1
   }
   exit 0
 }
 catch {
-  Write-Host $_
-  Write-Host $_.Exception
-  exit 1
+  Write-Host $_.ScriptStackTrace
+  Write-PipelineTelemetryError -Category 'NativeToolsBootstrap' -Message $_
+  ExitWithExitCode 1
 }

eng/common/init-tools-native.sh

@@ -9,13 +9,14 @@ clean=false
 force=false
 download_retries=5
 retry_wait_time_seconds=30
-global_json_file="${scriptroot}/../../global.json"
+global_json_file="$(dirname "$(dirname "${scriptroot}")")/global.json"
 declare -A native_assets
 
+. $scriptroot/pipeline-logging-functions.sh
 . $scriptroot/native/common-library.sh
 
 while (($# > 0)); do
-  lowerI="$(echo $1 | awk '{print tolower($0)}')"
+  lowerI="$(echo $1 | tr "[:upper:]" "[:lower:]")"
   case $lowerI in
     --baseuri)
       base_uri=$2
@@ -33,6 +34,14 @@ while (($# > 0)); do
       force=true
       shift 1
       ;;
+    --donotabortonfailure)
+      donotabortonfailure=true
+      shift 1
+      ;;
+    --donotdisplaywarnings)
+      donotdisplaywarnings=true
+      shift 1
+      ;;
     --downloadretries)
       download_retries=$2
       shift 2
@@ -51,6 +60,8 @@ while (($# > 0)); do
       echo "                                          - (default) %USERPROFILE%/.netcoreeng/native"
       echo ""
       echo "  --clean                             Switch specifying not to install anything, but cleanup native asset folders"
+      echo "  --donotabortonfailure               Switch specifiying whether to abort native tools installation on failure"
+      echo "  --donotdisplaywarnings              Switch specifiying whether to display warnings during native tools installation on failure"
       echo "  --force                             Clean and then install tools"
       echo "  --help                              Print help and exit"
       echo ""
@@ -65,24 +76,89 @@ while (($# > 0)); do
 done
 
 function ReadGlobalJsonNativeTools {
-  # Get the native-tools section from the global.json.
-  local native_tools_section=$(cat $global_json_file | awk '/"native-tools"/,/}/')
-  # Only extract the contents of the object.
-  local native_tools_list=$(echo $native_tools_section | awk -F"[{}]" '{print $2}')
-  native_tools_list=${native_tools_list//[\" ]/}
-  native_tools_list=${native_tools_list//,/$'\n'}
+  # happy path: we have a proper JSON parsing tool `jq(1)` in PATH!
+  if command -v jq &> /dev/null; then
 
-  local old_IFS=$IFS
-  while read -r line; do
-    # Lines are of the form: 'tool:version'
-    IFS=:
-    while read -r key value; do
-     native_assets[$key]=$value
-    done <<< "$line"
-  done <<< "$native_tools_list"
-  IFS=$old_IFS
+    # jq: read each key/value pair under "native-tools" entry and emit:
+    #   KEY="<entry-key>" VALUE="<entry-value>"
+    # followed by a null byte.
+    #
+    # bash: read line with null byte delimeter and push to array (for later `eval`uation).
 
-  return 0;
+    while IFS= read -rd '' line; do
+      native_assets+=("$line")
+    done < <(jq -r '. |
+        select(has("native-tools")) |
+        ."native-tools" |
+        keys[] as $k |
+        @sh "KEY=\($k) VALUE=\(.[$k])\u0000"' "$global_json_file")
+
+    return
+  fi
+
+  # Warning: falling back to manually parsing JSON, which is not recommended.
+
+  # Following routine matches the output and escaping logic of jq(1)'s @sh formatter used above.
+  # It has been tested with several weird strings with escaped characters in entries (key and value)
+  # and results were compared with the output of jq(1) in binary representation using xxd(1);
+  # just before the assignment to 'native_assets' array (above and below).
+
+  # try to capture the section under "native-tools".
+  if [[ ! "$(cat "$global_json_file")" =~ \"native-tools\"[[:space:]\:\{]*([^\}]+) ]]; then
+    return
+  fi
+
+  section="${BASH_REMATCH[1]}"
+
+  parseStarted=0
+  possibleEnd=0
+  escaping=0
+  escaped=0
+  isKey=1
+
+  for (( i=0; i<${#section}; i++ )); do
+    char="${section:$i:1}"
+    if ! ((parseStarted)) && [[ "$char" =~ [[:space:],:] ]]; then continue; fi
+
+    if ! ((escaping)) && [[ "$char" == "\\" ]]; then
+      escaping=1
+    elif ((escaping)) && ! ((escaped)); then
+      escaped=1
+    fi
+
+    if ! ((parseStarted)) && [[ "$char" == "\"" ]]; then
+      parseStarted=1
+      possibleEnd=0
+    elif [[ "$char" == "'" ]]; then
+      token="$token'\\\''"
+      possibleEnd=0
+    elif ((escaping)) || [[ "$char" != "\"" ]]; then
+      token="$token$char"
+      possibleEnd=1
+    fi
+
+    if ((possibleEnd)) && ! ((escaping)) && [[ "$char" == "\"" ]]; then
+      # Use printf to unescape token to match jq(1)'s @sh formatting rules.
+      # do not use 'token="$(printf "$token")"' syntax, as $() eats the trailing linefeed.
+      printf -v token "'$token'"
+
+      if ((isKey)); then
+        KEY="$token"
+        isKey=0
+      else
+        line="KEY=$KEY VALUE=$token"
+        native_assets+=("$line")
+        isKey=1
+      fi
+
+      # reset for next token
+      parseStarted=0
+      token=
+    elif ((escaping)) && ((escaped)); then
+      escaping=0
+      escaped=0
+    fi
+  done
 }
 
 native_base_dir=$install_directory
@@ -91,6 +167,7 @@ if [[ -z $install_directory ]]; then
 fi
 
 install_bin="${native_base_dir}/bin"
+installed_any=false
 
 ReadGlobalJsonNativeTools
 
@@ -99,14 +176,15 @@ if [[ ${#native_assets[@]} -eq 0 ]]; then
   exit 0;
 else
   native_installer_dir="$scriptroot/native"
-  for tool in "${!native_assets[@]}"
-  do
-    tool_version=${native_assets[$tool]}
-    installer_name="install-$tool.sh"
-    installer_command="$native_installer_dir/$installer_name"
+  for index in "${!native_assets[@]}"; do
+    eval "${native_assets["$index"]}"
+
+    installer_path="$native_installer_dir/install-$KEY.sh"
+    installer_command="$installer_path"
     installer_command+=" --baseuri $base_uri"
     installer_command+=" --installpath $install_bin"
-    installer_command+=" --version $tool_version"
+    installer_command+=" --version $VALUE"
+    echo $installer_command
 
     if [[ $force = true ]]; then
       installer_command+=" --force"
@@ -116,30 +194,45 @@ else
       installer_command+=" --clean"
     fi
 
-    echo "Installing $tool version $tool_version"
-    echo "Executing '$installer_command'"
-    $installer_command
-
-    if [[ $? != 0 ]]; then
-      echo "Execution Failed" >&2
-      exit 1
+    if [[ -a $installer_path ]]; then
+      $installer_command
+      if [[ $? != 0 ]]; then
+        if [[ $donotabortonfailure = true ]]; then
+          if [[ $donotdisplaywarnings != true ]]; then
+            Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Execution Failed"
+          fi
+        else
+          Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Execution Failed"
+          exit 1
+        fi
+      else
+        $installed_any = true
+      fi
+    else
+      if [[ $donotabortonfailure == true ]]; then
+        if [[ $donotdisplaywarnings != true ]]; then
+          Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Execution Failed: no install script"
+        fi
+      else
+        Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Execution Failed: no install script"
+        exit 1
+      fi
     fi
   done
 fi
 
-if [[ ! -z $clean ]]; then
+if [[ $clean = true ]]; then
   exit 0
 fi
 
 if [[ -d $install_bin ]]; then
   echo "Native tools are available from $install_bin"
-  if [[ !-z BUILD_BUILDNUMBER ]]; then
-    echo "##vso[task.prependpath]$install_bin"
-  fi
+  echo "##vso[task.prependpath]$install_bin"
 else
-  echo "Native tools install directory does not exist, installation failed" >&2
-  exit 1
+  if [[ $installed_any = true ]]; then
+    Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Native tools install directory does not exist, installation failed"
+    exit 1
+  fi
 fi
 
 exit 0
-

eng/common/internal-feed-operations.ps1

@@ -0,0 +1,132 @@
+param(
+  [Parameter(Mandatory=$true)][string] $Operation,
+  [string] $AuthToken,
+  [string] $CommitSha,
+  [string] $RepoName,
+  [switch] $IsFeedPrivate
+)
+
+$ErrorActionPreference = 'Stop'
+Set-StrictMode -Version 2.0
+. $PSScriptRoot\tools.ps1
+
+# Sets VSS_NUGET_EXTERNAL_FEED_ENDPOINTS based on the "darc-int-*" feeds defined in NuGet.config. This is needed
+# in build agents by CredProvider to authenticate the restore requests to internal feeds as specified in
+# https://github.com/microsoft/artifacts-credprovider/blob/0f53327cd12fd893d8627d7b08a2171bf5852a41/README.md#environment-variables. This should ONLY be called from identified
+# internal builds
+function SetupCredProvider {
+  param(
+    [string] $AuthToken
+  )    
+
+  # Install the Cred Provider NuGet plugin
+  Write-Host 'Setting up Cred Provider NuGet plugin in the agent...'
+  Write-Host "Getting 'installcredprovider.ps1' from 'https://github.com/microsoft/artifacts-credprovider'..."
+
+  $url = 'https://raw.githubusercontent.com/microsoft/artifacts-credprovider/master/helpers/installcredprovider.ps1'
+  
+  Write-Host "Writing the contents of 'installcredprovider.ps1' locally..."
+  Invoke-WebRequest $url -OutFile installcredprovider.ps1
+  
+  Write-Host 'Installing plugin...'
+  .\installcredprovider.ps1 -Force
+  
+  Write-Host "Deleting local copy of 'installcredprovider.ps1'..."
+  Remove-Item .\installcredprovider.ps1
+
+  if (-Not("$env:USERPROFILE\.nuget\plugins\netcore")) {
+    Write-PipelineTelemetryError -Category 'Arcade' -Message 'CredProvider plugin was not installed correctly!'
+    ExitWithExitCode 1  
+  } 
+  else {
+    Write-Host 'CredProvider plugin was installed correctly!'
+  }
+
+  # Then, we set the 'VSS_NUGET_EXTERNAL_FEED_ENDPOINTS' environment variable to restore from the stable 
+  # feeds successfully
+
+  $nugetConfigPath = "$RepoRoot\NuGet.config"
+
+  if (-Not (Test-Path -Path $nugetConfigPath)) {
+    Write-PipelineTelemetryError -Category 'Build' -Message 'NuGet.config file not found in repo root!'
+    ExitWithExitCode 1  
+  }
+  
+  $endpoints = New-Object System.Collections.ArrayList
+  $nugetConfigPackageSources = Select-Xml -Path $nugetConfigPath -XPath "//packageSources/add[contains(@key, 'darc-int-')]/@value" | foreach{$_.Node.Value}
+  
+  if (($nugetConfigPackageSources | Measure-Object).Count -gt 0 ) {
+    foreach ($stableRestoreResource in $nugetConfigPackageSources) {
+      $trimmedResource = ([string]$stableRestoreResource).Trim()
+      [void]$endpoints.Add(@{endpoint="$trimmedResource"; password="$AuthToken"}) 
+    }
+  }
+
+  if (($endpoints | Measure-Object).Count -gt 0) {
+      $endpointCredentials = @{endpointCredentials=$endpoints} | ConvertTo-Json -Compress
+
+     # Create the environment variables the AzDo way
+      Write-LoggingCommand -Area 'task' -Event 'setvariable' -Data $endpointCredentials -Properties @{
+        'variable' = 'VSS_NUGET_EXTERNAL_FEED_ENDPOINTS'
+        'issecret' = 'false'
+      } 
+
+      # We don't want sessions cached since we will be updating the endpoints quite frequently
+      Write-LoggingCommand -Area 'task' -Event 'setvariable' -Data 'False' -Properties @{
+        'variable' = 'NUGET_CREDENTIALPROVIDER_SESSIONTOKENCACHE_ENABLED'
+        'issecret' = 'false'
+      } 
+  }
+  else
+  {
+    Write-Host 'No internal endpoints found in NuGet.config'
+  }
+}
+
+#Workaround for https://github.com/microsoft/msbuild/issues/4430
+function InstallDotNetSdkAndRestoreArcade {
+  $dotnetTempDir = "$RepoRoot\dotnet"
+  $dotnetSdkVersion="2.1.507" # After experimentation we know this version works when restoring the SDK (compared to 3.0.*)
+  $dotnet = "$dotnetTempDir\dotnet.exe"
+  $restoreProjPath = "$PSScriptRoot\restore.proj"
+  
+  Write-Host "Installing dotnet SDK version $dotnetSdkVersion to restore Arcade SDK..."
+  InstallDotNetSdk "$dotnetTempDir" "$dotnetSdkVersion"
+  
+  '<Project Sdk="Microsoft.DotNet.Arcade.Sdk"/>' | Out-File "$restoreProjPath"
+
+  & $dotnet restore $restoreProjPath
+
+  Write-Host 'Arcade SDK restored!'
+
+  if (Test-Path -Path $restoreProjPath) {
+    Remove-Item $restoreProjPath
+  }
+
+  if (Test-Path -Path $dotnetTempDir) {
+    Remove-Item $dotnetTempDir -Recurse
+  }
+}
+
+try {
+  Push-Location $PSScriptRoot
+
+  if ($Operation -like 'setup') {
+    SetupCredProvider $AuthToken
+  } 
+  elseif ($Operation -like 'install-restore') {
+    InstallDotNetSdkAndRestoreArcade
+  }
+  else {
+    Write-PipelineTelemetryError -Category 'Arcade' -Message "Unknown operation '$Operation'!"
+    ExitWithExitCode 1  
+  }
+} 
+catch {
+  Write-Host $_.ScriptStackTrace
+  Write-PipelineTelemetryError -Category 'Arcade' -Message $_
+  ExitWithExitCode 1
+} 
+finally {
+  Pop-Location
+}

eng/common/internal-feed-operations.sh

@@ -0,0 +1,141 @@
+#!/usr/bin/env bash
+
+set -e
+
+# Sets VSS_NUGET_EXTERNAL_FEED_ENDPOINTS based on the "darc-int-*" feeds defined in NuGet.config. This is needed
+# in build agents by CredProvider to authenticate the restore requests to internal feeds as specified in
+# https://github.com/microsoft/artifacts-credprovider/blob/0f53327cd12fd893d8627d7b08a2171bf5852a41/README.md#environment-variables. 
+# This should ONLY be called from identified internal builds
+function SetupCredProvider {
+  local authToken=$1
+  
+  # Install the Cred Provider NuGet plugin
+  echo "Setting up Cred Provider NuGet plugin in the agent..."...
+  echo "Getting 'installcredprovider.ps1' from 'https://github.com/microsoft/artifacts-credprovider'..."
+
+  local url="https://raw.githubusercontent.com/microsoft/artifacts-credprovider/master/helpers/installcredprovider.sh"  
+  
+  echo "Writing the contents of 'installcredprovider.ps1' locally..."
+  local installcredproviderPath="installcredprovider.sh"
+  if command -v curl > /dev/null; then
+    curl $url > "$installcredproviderPath"
+  else   
+    wget -q -O "$installcredproviderPath" "$url"
+  fi
+  
+  echo "Installing plugin..."
+  . "$installcredproviderPath"
+  
+  echo "Deleting local copy of 'installcredprovider.sh'..."
+  rm installcredprovider.sh
+
+  if [ ! -d "$HOME/.nuget/plugins" ]; then
+    Write-PipelineTelemetryError -category 'Build' 'CredProvider plugin was not installed correctly!'
+    ExitWithExitCode 1  
+  else 
+    echo "CredProvider plugin was installed correctly!"
+  fi
+
+  # Then, we set the 'VSS_NUGET_EXTERNAL_FEED_ENDPOINTS' environment variable to restore from the stable 
+  # feeds successfully
+
+  local nugetConfigPath="$repo_root/NuGet.config"
+
+  if [ ! "$nugetConfigPath" ]; then
+    Write-PipelineTelemetryError -category 'Build' "NuGet.config file not found in repo's root!"
+    ExitWithExitCode 1  
+  fi
+  
+  local endpoints='['
+  local nugetConfigPackageValues=`cat "$nugetConfigPath" | grep "key=\"darc-int-"`
+  local pattern="value=\"(.*)\""
+
+  for value in $nugetConfigPackageValues 
+  do
+    if [[ $value =~ $pattern ]]; then
+      local endpoint="${BASH_REMATCH[1]}"  
+      endpoints+="{\"endpoint\": \"$endpoint\", \"password\": \"$authToken\"},"
+    fi
+  done
+  
+  endpoints=${endpoints%?}
+  endpoints+=']'
+
+  if [ ${#endpoints} -gt 2 ]; then 
+      local endpointCredentials="{\"endpointCredentials\": "$endpoints"}"
+
+      echo "##vso[task.setvariable variable=VSS_NUGET_EXTERNAL_FEED_ENDPOINTS]$endpointCredentials"
+      echo "##vso[task.setvariable variable=NUGET_CREDENTIALPROVIDER_SESSIONTOKENCACHE_ENABLED]False"
+  else
+    echo "No internal endpoints found in NuGet.config"
+  fi
+} 
+
+# Workaround for https://github.com/microsoft/msbuild/issues/4430
+function InstallDotNetSdkAndRestoreArcade {
+  local dotnetTempDir="$repo_root/dotnet"
+  local dotnetSdkVersion="2.1.507" # After experimentation we know this version works when restoring the SDK (compared to 3.0.*)
+  local restoreProjPath="$repo_root/eng/common/restore.proj"
+  
+  echo "Installing dotnet SDK version $dotnetSdkVersion to restore Arcade SDK..."
+  echo "<Project Sdk=\"Microsoft.DotNet.Arcade.Sdk\"/>" > "$restoreProjPath"
+  
+  InstallDotNetSdk "$dotnetTempDir" "$dotnetSdkVersion"
+
+  local res=`$dotnetTempDir/dotnet restore $restoreProjPath`
+  echo "Arcade SDK restored!"
+
+  # Cleanup
+  if [ "$restoreProjPath" ]; then
+    rm "$restoreProjPath"
+  fi
+
+  if [ "$dotnetTempDir" ]; then
+    rm -r $dotnetTempDir
+  fi
+}
+
+source="${BASH_SOURCE[0]}"
+operation=''
+authToken=''
+repoName=''
+
+while [[ $# > 0 ]]; do
+  opt="$(echo "$1" | tr "[:upper:]" "[:lower:]")"
+  case "$opt" in
+    --operation)
+      operation=$2
+      shift
+      ;;
+    --authtoken)
+      authToken=$2
+      shift
+      ;;
+    *)
+      echo "Invalid argument: $1"
+      usage
+      exit 1
+      ;;
+  esac
+
+  shift
+done
+
+while [[ -h "$source" ]]; do
+  scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
+  source="$(readlink "$source")"
+  # if $source was a relative symlink, we need to resolve it relative to the path where the
+  # symlink file was located
+  [[ $source != /* ]] && source="$scriptroot/$source"
+done
+scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
+
+. "$scriptroot/tools.sh"
+
+if [ "$operation" = "setup" ]; then
+  SetupCredProvider $authToken
+elif [ "$operation" = "install-restore" ]; then
+  InstallDotNetSdkAndRestoreArcade
+else
+  echo "Unknown operation '$operation'!"
+fi

eng/common/internal/Directory.Build.props

@@ -1,4 +1,4 @@
-<!-- Licensed to the .NET Foundation under one or more agreements. The .NET Foundation licenses this file to you under the MIT license. See the LICENSE file in the project root for more information. -->
+<!-- Licensed to the .NET Foundation under one or more agreements. The .NET Foundation licenses this file to you under the MIT license. -->
 <Project>
   <Import Project="Sdk.props" Sdk="Microsoft.DotNet.Arcade.Sdk" />
 </Project>

eng/common/internal/Tools.csproj

@@ -4,6 +4,7 @@
   <PropertyGroup>
     <TargetFramework>net472</TargetFramework>
     <ImportDirectoryBuildTargets>false</ImportDirectoryBuildTargets>
+    <AutomaticallyUseReferenceAssemblyPackages>false</AutomaticallyUseReferenceAssemblyPackages>
   </PropertyGroup>
   <ItemGroup>
     <!-- Clear references, the SDK may add some depending on UsuingToolXxx settings, but we only want to restore the following -->
@@ -12,8 +13,12 @@
     <PackageReference Include="Drop.App" Version="$(DropAppVersion)" ExcludeAssets="all" Condition="'$(UsingToolVisualStudioIbcTraining)' == 'true'"/>
   </ItemGroup>
   <PropertyGroup>
-    <RestoreSources>
-      https://devdiv.pkgs.visualstudio.com/_packaging/8f470c7e-ac49-4afe-a6ee-cf784e438b93/nuget/v3/index.json;
+    <RestoreSources></RestoreSources>
+    <RestoreSources Condition="'$(UsingToolIbcOptimization)' == 'true'">
+      https://devdiv.pkgs.visualstudio.com/_packaging/dotnet-core-internal-tooling/nuget/v3/index.json;
+    </RestoreSources>
+    <RestoreSources Condition="'$(UsingToolVisualStudioIbcTraining)' == 'true'">
+      $(RestoreSources);
       https://devdiv.pkgs.visualstudio.com/_packaging/VS/nuget/v3/index.json;
     </RestoreSources>
   </PropertyGroup>

eng/common/msbuild.ps1

@@ -1,6 +1,6 @@
 [CmdletBinding(PositionalBinding=$false)]
 Param(
-  [string] $verbosity = "minimal",
+  [string] $verbosity = 'minimal',
   [bool] $warnAsError = $true,
   [bool] $nodeReuse = $true,
   [switch] $ci,
@@ -18,9 +18,8 @@ try {
   MSBuild @extraArgs
 } 
 catch {
-  Write-Host $_
-  Write-Host $_.Exception
   Write-Host $_.ScriptStackTrace
+  Write-PipelineTelemetryError -Category 'Build' -Message $_
   ExitWithExitCode 1
 }
 

eng/common/msbuild.sh

@@ -19,7 +19,7 @@ prepare_machine=false
 extra_args=''
 
 while (($# > 0)); do
-  lowerI="$(echo $1 | awk '{print tolower($0)}')"
+  lowerI="$(echo $1 | tr "[:upper:]" "[:lower:]")"
   case $lowerI in
     --verbosity)
       verbosity=$2

eng/common/native/CommonLibrary.psm1

@@ -48,7 +48,7 @@ function DownloadAndExtract {
                                            -Verbose:$Verbose
 
   if ($DownloadStatus -Eq $False) {
-    Write-Error "Download failed"
+    Write-Error "Download failed from $Uri"
     return $False
   }
 
@@ -59,9 +59,38 @@ function DownloadAndExtract {
                                           -Verbose:$Verbose
 
   if ($UnzipStatus -Eq $False) {
-    Write-Error "Unzip failed"
-    return $False
+    # Retry Download one more time with Force=true
+    $DownloadRetryStatus = CommonLibrary\Get-File -Uri $Uri `
+                                             -Path $TempToolPath `
+                                             -DownloadRetries 1 `
+                                             -RetryWaitTimeInSeconds $RetryWaitTimeInSeconds `
+                                             -Force:$True `
+                                             -Verbose:$Verbose
+
+    if ($DownloadRetryStatus -Eq $False) {
+      Write-Error "Last attempt of download failed as well"
+      return $False
+    }
+
+    # Retry unzip again one more time with Force=true
+    $UnzipRetryStatus = CommonLibrary\Expand-Zip -ZipPath $TempToolPath `
+                                            -OutputDirectory $InstallDirectory `
+                                            -Force:$True `
+                                            -Verbose:$Verbose
+    if ($UnzipRetryStatus -Eq $False)
+    {
+      Write-Error "Last attempt of unzip failed as well"
+      # Clean up partial zips and extracts
+      if (Test-Path $TempToolPath) {
+        Remove-Item $TempToolPath -Force
+      }
+      if (Test-Path $InstallDirectory) {
+        Remove-Item $InstallDirectory -Force -Recurse
+      }
+      return $False
+    }
   }
+
   return $True
 }
 
@@ -116,18 +145,25 @@ function Get-File {
     New-Item -path $DownloadDirectory -force -itemType "Directory" | Out-Null
   }
 
+  $TempPath = "$Path.tmp"
   if (Test-Path -IsValid -Path $Uri) {
-    Write-Verbose "'$Uri' is a file path, copying file to '$Path'"
-    Copy-Item -Path $Uri -Destination $Path
+    Write-Verbose "'$Uri' is a file path, copying temporarily to '$TempPath'"
+    Copy-Item -Path $Uri -Destination $TempPath
+    Write-Verbose "Moving temporary file to '$Path'"
+    Move-Item -Path $TempPath -Destination $Path
     return $?
   }
   else {
     Write-Verbose "Downloading $Uri"
+    # Don't display the console progress UI - it's a huge perf hit
+    $ProgressPreference = 'SilentlyContinue'   
     while($Attempt -Lt $DownloadRetries)
     {
       try {
-        Invoke-WebRequest -UseBasicParsing -Uri $Uri -OutFile $Path
-        Write-Verbose "Downloaded to '$Path'"
+        Invoke-WebRequest -UseBasicParsing -Uri $Uri -OutFile $TempPath
+        Write-Verbose "Downloaded to temporary location '$TempPath'"
+        Move-Item -Path $TempPath -Destination $Path
+        Write-Verbose "Moved temporary file to '$Path'"
         return $True
       }
       catch {
@@ -209,7 +245,7 @@ function New-ScriptShim {
       Remove-Item (Join-Path $ShimDirectory "$ShimName.exe")
     }
 
-    Invoke-Expression "$ShimDirectory\WinShimmer\winshimmer.exe $ShimName $ToolFilePath $ShimDirectory"
+    & "$ShimDirectory\WinShimmer\winshimmer.exe" $ShimName $ToolFilePath $ShimDirectory
     return $True
   }
   catch {
@@ -328,16 +364,21 @@ function Expand-Zip {
         return $False
       }
     }
-    if (-Not (Test-Path $OutputDirectory)) {
-      New-Item -path $OutputDirectory -Force -itemType "Directory" | Out-Null
+
+    $TempOutputDirectory = Join-Path "$(Split-Path -Parent $OutputDirectory)" "$(Split-Path -Leaf $OutputDirectory).tmp"
+    if (Test-Path $TempOutputDirectory) {
+      Remove-Item $TempOutputDirectory -Force -Recurse
     }
+    New-Item -Path $TempOutputDirectory -Force -ItemType "Directory" | Out-Null
 
     Add-Type -assembly "system.io.compression.filesystem"
-    [io.compression.zipfile]::ExtractToDirectory("$ZipPath", "$OutputDirectory")
+    [io.compression.zipfile]::ExtractToDirectory("$ZipPath", "$TempOutputDirectory")
     if ($? -Eq $False) {
       Write-Error "Unable to extract '$ZipPath'"
       return $False
     }
+
+    Move-Item -Path $TempOutputDirectory -Destination $OutputDirectory
   }
   catch {
     Write-Host $_

eng/common/native/common-library.sh

@@ -34,7 +34,7 @@ function ExpandZip {
     echo "'Force flag enabled, but '$output_directory' exists. Removing directory"
     rm -rf $output_directory
     if [[ $? != 0 ]]; then
-      echo Unable to remove '$output_directory'>&2
+      Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Unable to remove '$output_directory'"
       return 1
     fi
   fi
@@ -45,7 +45,7 @@ function ExpandZip {
   echo "Extracting archive"
   tar -xf $zip_path -C $output_directory
   if [[ $? != 0 ]]; then
-    echo "Unable to extract '$zip_path'" >&2
+    Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Unable to extract '$zip_path'"
     return 1
   fi
 
@@ -117,7 +117,7 @@ function DownloadAndExtract {
   # Download file
   GetFile "$uri" "$temp_tool_path" $force $download_retries $retry_wait_time_seconds
   if [[ $? != 0 ]]; then
-    echo "Failed to download '$uri' to '$temp_tool_path'." >&2
+    Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Failed to download '$uri' to '$temp_tool_path'."
     return 1
   fi
 
@@ -125,7 +125,7 @@ function DownloadAndExtract {
   echo "extracting from  $temp_tool_path to $installDir"
   ExpandZip "$temp_tool_path" "$installDir" $force $download_retries $retry_wait_time_seconds
   if [[ $? != 0 ]]; then
-    echo "Failed to extract '$temp_tool_path' to '$installDir'." >&2
+    Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Failed to extract '$temp_tool_path' to '$installDir'."
     return 1
   fi
 
@@ -148,7 +148,7 @@ function NewScriptShim {
   fi
   
   if [[ ! -f $tool_file_path ]]; then
-    echo "Specified tool file path:'$tool_file_path' does not exist" >&2
+    Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Specified tool file path:'$tool_file_path' does not exist"
     return 1
   fi
 

eng/common/native/find-native-compiler.sh

@@ -0,0 +1,121 @@
+#!/usr/bin/env bash
+#
+# This file locates the native compiler with the given name and version and sets the environment variables to locate it.
+#
+
+source="${BASH_SOURCE[0]}"
+
+# resolve $SOURCE until the file is no longer a symlink
+while [[ -h $source ]]; do
+  scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
+  source="$(readlink "$source")"
+
+  # if $source was a relative symlink, we need to resolve it relative to the path where the
+  # symlink file was located
+  [[ $source != /* ]] && source="$scriptroot/$source"
+done
+scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
+
+if [ $# -lt 0 ]
+then
+  echo "Usage..."
+  echo "find-native-compiler.sh <compiler> <compiler major version> <compiler minor version>"
+  echo "Specify the name of compiler (clang or gcc)."
+  echo "Specify the major version of compiler."
+  echo "Specify the minor version of compiler."
+  exit 1
+fi
+
+. $scriptroot/../pipeline-logging-functions.sh
+
+compiler="$1"
+cxxCompiler="$compiler++"
+majorVersion="$2"
+minorVersion="$3"
+
+if [ "$compiler" = "gcc" ]; then cxxCompiler="g++"; fi
+
+check_version_exists() {
+    desired_version=-1
+
+    # Set up the environment to be used for building with the desired compiler.
+    if command -v "$compiler-$1.$2" > /dev/null; then
+        desired_version="-$1.$2"
+    elif command -v "$compiler$1$2" > /dev/null; then
+        desired_version="$1$2"
+    elif command -v "$compiler-$1$2" > /dev/null; then
+        desired_version="-$1$2"
+    fi
+
+    echo "$desired_version"
+}
+
+if [ -z "$CLR_CC" ]; then
+
+    # Set default versions
+    if [ -z "$majorVersion" ]; then
+        # note: gcc (all versions) and clang versions higher than 6 do not have minor version in file name, if it is zero.
+        if [ "$compiler" = "clang" ]; then versions=( 9 8 7 6.0 5.0 4.0 3.9 3.8 3.7 3.6 3.5 )
+        elif [ "$compiler" = "gcc" ]; then versions=( 9 8 7 6 5 4.9 ); fi
+
+        for version in "${versions[@]}"; do
+            parts=(${version//./ })
+            desired_version="$(check_version_exists "${parts[0]}" "${parts[1]}")"
+            if [ "$desired_version" != "-1" ]; then majorVersion="${parts[0]}"; break; fi
+        done
+
+        if [ -z "$majorVersion" ]; then
+            if command -v "$compiler" > /dev/null; then
+                if [ "$(uname)" != "Darwin" ]; then
+                    Write-PipelineTelemetryError -category "Build" -type "warning" "Specific version of $compiler not found, falling back to use the one in PATH."
+                fi
+                export CC="$(command -v "$compiler")"
+                export CXX="$(command -v "$cxxCompiler")"
+            else
+                Write-PipelineTelemetryError -category "Build" "No usable version of $compiler found."
+                exit 1
+            fi
+        else
+            if [ "$compiler" = "clang" ] && [ "$majorVersion" -lt 5 ]; then
+                if [ "$build_arch" = "arm" ] || [ "$build_arch" = "armel" ]; then
+                    if command -v "$compiler" > /dev/null; then
+                        Write-PipelineTelemetryError -category "Build" -type "warning" "Found clang version $majorVersion which is not supported on arm/armel architectures, falling back to use clang from PATH."
+                        export CC="$(command -v "$compiler")"
+                        export CXX="$(command -v "$cxxCompiler")"
+                    else
+                        Write-PipelineTelemetryError -category "Build" "Found clang version $majorVersion which is not supported on arm/armel architectures, and there is no clang in PATH."
+                        exit 1
+                    fi
+                fi
+            fi
+        fi
+    else
+        desired_version="$(check_version_exists "$majorVersion" "$minorVersion")"
+        if [ "$desired_version" = "-1" ]; then
+            Write-PipelineTelemetryError -category "Build" "Could not find specific version of $compiler: $majorVersion $minorVersion."
+            exit 1
+        fi
+    fi
+
+    if [ -z "$CC" ]; then
+        export CC="$(command -v "$compiler$desired_version")"
+        export CXX="$(command -v "$cxxCompiler$desired_version")"
+        if [ -z "$CXX" ]; then export CXX="$(command -v "$cxxCompiler")"; fi
+    fi
+else
+    if [ ! -f "$CLR_CC" ]; then
+        Write-PipelineTelemetryError -category "Build" "CLR_CC is set but path '$CLR_CC' does not exist"
+        exit 1
+    fi
+    export CC="$CLR_CC"
+    export CXX="$CLR_CXX"
+fi
+
+if [ -z "$CC" ]; then
+   Write-PipelineTelemetryError -category "Build" "Unable to find $compiler."
+    exit 1
+fi
+
+export CCC_CC="$CC"
+export CCC_CXX="$CXX"
+export SCAN_BUILD_COMMAND="$(command -v "scan-build$desired_version")"

eng/common/native/install-cmake-test.sh

@@ -0,0 +1,117 @@
+#!/usr/bin/env bash
+
+source="${BASH_SOURCE[0]}"
+scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
+
+. $scriptroot/common-library.sh
+
+base_uri=
+install_path=
+version=
+clean=false
+force=false
+download_retries=5
+retry_wait_time_seconds=30
+
+while (($# > 0)); do
+  lowerI="$(echo $1 | tr "[:upper:]" "[:lower:]")"
+  case $lowerI in
+    --baseuri)
+      base_uri=$2
+      shift 2
+      ;;
+    --installpath)
+      install_path=$2
+      shift 2
+      ;;
+    --version)
+      version=$2
+      shift 2
+      ;;
+    --clean)
+      clean=true
+      shift 1
+      ;;
+    --force)
+      force=true
+      shift 1
+      ;;
+    --downloadretries)
+      download_retries=$2
+      shift 2
+      ;;
+    --retrywaittimeseconds)
+      retry_wait_time_seconds=$2
+      shift 2
+      ;;
+    --help)
+      echo "Common settings:"
+      echo "  --baseuri <value>        Base file directory or Url wrom which to acquire tool archives"
+      echo "  --installpath <value>    Base directory to install native tool to"
+      echo "  --clean                  Don't install the tool, just clean up the current install of the tool"
+      echo "  --force                  Force install of tools even if they previously exist"
+      echo "  --help                   Print help and exit"
+      echo ""
+      echo "Advanced settings:"
+      echo "  --downloadretries        Total number of retry attempts"
+      echo "  --retrywaittimeseconds   Wait time between retry attempts in seconds"
+      echo ""
+      exit 0
+      ;;
+  esac
+done
+
+tool_name="cmake-test"
+tool_os=$(GetCurrentOS)
+tool_folder="$(echo $tool_os | tr "[:upper:]" "[:lower:]")"
+tool_arch="x86_64"
+tool_name_moniker="$tool_name-$version-$tool_os-$tool_arch"
+tool_install_directory="$install_path/$tool_name/$version"
+tool_file_path="$tool_install_directory/$tool_name_moniker/bin/$tool_name"
+shim_path="$install_path/$tool_name.sh"
+uri="${base_uri}/$tool_folder/$tool_name/$tool_name_moniker.tar.gz"
+
+# Clean up tool and installers
+if [[ $clean = true ]]; then
+  echo "Cleaning $tool_install_directory"
+  if [[ -d $tool_install_directory ]]; then
+    rm -rf $tool_install_directory
+  fi
+
+  echo "Cleaning $shim_path"
+  if [[ -f $shim_path ]]; then
+    rm -rf $shim_path
+  fi
+
+  tool_temp_path=$(GetTempPathFileName $uri)
+  echo "Cleaning $tool_temp_path"
+  if [[ -f $tool_temp_path ]]; then
+    rm -rf $tool_temp_path
+  fi
+
+  exit 0
+fi
+
+# Install tool
+if [[ -f $tool_file_path ]] && [[ $force = false ]]; then
+  echo "$tool_name ($version) already exists, skipping install"
+  exit 0
+fi
+
+DownloadAndExtract $uri $tool_install_directory $force $download_retries $retry_wait_time_seconds
+
+if [[ $? != 0 ]]; then
+  Write-PipelineTelemetryError -category 'NativeToolsBootstrap' 'Installation failed'
+  exit 1
+fi
+
+# Generate Shim
+# Always rewrite shims so that we are referencing the expected version
+NewScriptShim $shim_path $tool_file_path true
+
+if [[ $? != 0 ]]; then
+  Write-PipelineTelemetryError -category 'NativeToolsBootstrap' 'Shim generation failed'
+  exit 1
+fi
+
+exit 0

eng/common/native/install-cmake.sh

@@ -14,7 +14,7 @@ download_retries=5
 retry_wait_time_seconds=30
 
 while (($# > 0)); do
-  lowerI="$(echo $1 | awk '{print tolower($0)}')"
+  lowerI="$(echo $1 | tr "[:upper:]" "[:lower:]")"
   case $lowerI in
     --baseuri)
       base_uri=$2
@@ -63,13 +63,13 @@ done
 
 tool_name="cmake"
 tool_os=$(GetCurrentOS)
-tool_folder=$(echo $tool_os | awk '{print tolower($0)}')
+tool_folder="$(echo $tool_os | tr "[:upper:]" "[:lower:]")"
 tool_arch="x86_64"
 tool_name_moniker="$tool_name-$version-$tool_os-$tool_arch"
 tool_install_directory="$install_path/$tool_name/$version"
 tool_file_path="$tool_install_directory/$tool_name_moniker/bin/$tool_name"
 shim_path="$install_path/$tool_name.sh"
-uri="${base_uri}/$tool_folder/cmake/$tool_name_moniker.tar.gz"
+uri="${base_uri}/$tool_folder/$tool_name/$tool_name_moniker.tar.gz"
 
 # Clean up tool and installers
 if [[ $clean = true ]]; then
@@ -101,7 +101,7 @@ fi
 DownloadAndExtract $uri $tool_install_directory $force $download_retries $retry_wait_time_seconds
 
 if [[ $? != 0 ]]; then
-  echo "Installation failed" >&2
+  Write-PipelineTelemetryError -category 'NativeToolsBootstrap' 'Installation failed'
   exit 1
 fi
 
@@ -110,8 +110,8 @@ fi
 NewScriptShim $shim_path $tool_file_path true
 
 if [[ $? != 0 ]]; then
-  echo "Shim generation failed" >&2
+  Write-PipelineTelemetryError -category 'NativeToolsBootstrap' 'Shim generation failed'
   exit 1
 fi
 
-exit 0
+exit 0

eng/common/native/install-tool.ps1

@@ -46,6 +46,8 @@ Param (
   [int] $RetryWaitTimeInSeconds = 30
 )
 
+. $PSScriptRoot\..\pipeline-logging-functions.ps1
+
 # Import common library modules
 Import-Module -Name (Join-Path $CommonLibraryDirectory "CommonLibrary.psm1")
 
@@ -93,7 +95,7 @@ try {
                                                       -Verbose:$Verbose
 
     if ($InstallStatus -Eq $False) {
-      Write-Error "Installation failed"
+      Write-PipelineTelemetryError "Installation failed" -Category "NativeToolsetBootstrapping"
       exit 1
     }
   }
@@ -103,7 +105,7 @@ try {
     Write-Error "There are multiple copies of $ToolName in $($ToolInstallDirectory): `n$(@($ToolFilePath | out-string))"
     exit 1
   } elseif (@($ToolFilePath).Length -Lt 1) {
-    Write-Error "$ToolName was not found in $ToolFilePath."
+    Write-Host "$ToolName was not found in $ToolInstallDirectory."
     exit 1
   }
 
@@ -117,14 +119,14 @@ try {
                                                      -Verbose:$Verbose
 
   if ($GenerateShimStatus -Eq $False) {
-    Write-Error "Generate shim failed"
+    Write-PipelineTelemetryError "Generate shim failed" -Category "NativeToolsetBootstrapping"
     return 1
   }
 
   exit 0
 }
 catch {
-  Write-Host $_
-  Write-Host $_.Exception
+  Write-Host $_.ScriptStackTrace
+  Write-PipelineTelemetryError -Category "NativeToolsetBootstrapping" -Message $_
   exit 1
 }

eng/common/pipeline-logging-functions.ps1

@@ -0,0 +1,260 @@
+# Source for this file was taken from https://github.com/microsoft/azure-pipelines-task-lib/blob/11c9439d4af17e6475d9fe058e6b2e03914d17e6/powershell/VstsTaskSdk/LoggingCommandFunctions.ps1 and modified.
+
+# NOTE: You should not be calling these method directly as they are likely to change.  Instead you should be calling the Write-Pipeline* functions defined in tools.ps1
+
+$script:loggingCommandPrefix = '##vso['
+$script:loggingCommandEscapeMappings = @( # TODO: WHAT ABOUT "="? WHAT ABOUT "%"?
+    New-Object psobject -Property @{ Token = ';' ; Replacement = '%3B' }
+    New-Object psobject -Property @{ Token = "`r" ; Replacement = '%0D' }
+    New-Object psobject -Property @{ Token = "`n" ; Replacement = '%0A' }
+    New-Object psobject -Property @{ Token = "]" ; Replacement = '%5D' }
+)
+# TODO: BUG: Escape % ???
+# TODO: Add test to verify don't need to escape "=".
+
+# Specify "-Force" to force pipeline formatted output even if "$ci" is false or not set
+function Write-PipelineTelemetryError {
+    [CmdletBinding()]
+    param(
+        [Parameter(Mandatory = $true)]
+        [string]$Category,
+        [Parameter(Mandatory = $true)]
+        [string]$Message,
+        [Parameter(Mandatory = $false)]
+        [string]$Type = 'error',
+        [string]$ErrCode,
+        [string]$SourcePath,
+        [string]$LineNumber,
+        [string]$ColumnNumber,
+        [switch]$AsOutput,
+        [switch]$Force)
+
+    $PSBoundParameters.Remove('Category') | Out-Null
+
+    if ($Force -Or ((Test-Path variable:ci) -And $ci)) {
+        $Message = "(NETCORE_ENGINEERING_TELEMETRY=$Category) $Message"
+    }
+    $PSBoundParameters.Remove('Message') | Out-Null
+    $PSBoundParameters.Add('Message', $Message)
+    Write-PipelineTaskError @PSBoundParameters
+}
+
+# Specify "-Force" to force pipeline formatted output even if "$ci" is false or not set
+function Write-PipelineTaskError {
+    [CmdletBinding()]
+    param(
+        [Parameter(Mandatory = $true)]
+        [string]$Message,
+        [Parameter(Mandatory = $false)]
+        [string]$Type = 'error',
+        [string]$ErrCode,
+        [string]$SourcePath,
+        [string]$LineNumber,
+        [string]$ColumnNumber,
+        [switch]$AsOutput,
+        [switch]$Force
+    )
+
+    if (!$Force -And (-Not (Test-Path variable:ci) -Or !$ci)) {
+        if ($Type -eq 'error') {
+            Write-Host $Message -ForegroundColor Red
+            return
+        }
+        elseif ($Type -eq 'warning') {
+            Write-Host $Message -ForegroundColor Yellow
+            return
+        }
+    }
+
+    if (($Type -ne 'error') -and ($Type -ne 'warning')) {
+        Write-Host $Message
+        return
+    }
+    $PSBoundParameters.Remove('Force') | Out-Null      
+    if (-not $PSBoundParameters.ContainsKey('Type')) {
+        $PSBoundParameters.Add('Type', 'error')
+    }
+    Write-LogIssue @PSBoundParameters
+}
+  
+function Write-PipelineSetVariable {
+    [CmdletBinding()]
+    param(
+        [Parameter(Mandatory = $true)]
+        [string]$Name,
+        [string]$Value,
+        [switch]$Secret,
+        [switch]$AsOutput,
+        [bool]$IsMultiJobVariable = $true)
+
+    if ((Test-Path variable:ci) -And $ci) {
+        Write-LoggingCommand -Area 'task' -Event 'setvariable' -Data $Value -Properties @{
+            'variable' = $Name
+            'isSecret' = $Secret
+            'isOutput' = $IsMultiJobVariable
+        } -AsOutput:$AsOutput
+    }
+}
+  
+function Write-PipelinePrependPath {
+    [CmdletBinding()]
+    param(
+        [Parameter(Mandatory = $true)]
+        [string]$Path,
+        [switch]$AsOutput)
+
+    if ((Test-Path variable:ci) -And $ci) {
+        Write-LoggingCommand -Area 'task' -Event 'prependpath' -Data $Path -AsOutput:$AsOutput
+    }
+}
+
+function Write-PipelineSetResult {
+    [CmdletBinding()]
+    param(
+        [ValidateSet("Succeeded", "SucceededWithIssues", "Failed", "Cancelled", "Skipped")]
+        [Parameter(Mandatory = $true)]
+        [string]$Result,
+        [string]$Message)
+    if ((Test-Path variable:ci) -And $ci) {
+        Write-LoggingCommand -Area 'task' -Event 'complete' -Data $Message -Properties @{
+            'result' = $Result
+        }
+    }
+}
+
+<########################################
+# Private functions.
+########################################>
+function Format-LoggingCommandData {
+    [CmdletBinding()]
+    param([string]$Value, [switch]$Reverse)
+
+    if (!$Value) {
+        return ''
+    }
+
+    if (!$Reverse) {
+        foreach ($mapping in $script:loggingCommandEscapeMappings) {
+            $Value = $Value.Replace($mapping.Token, $mapping.Replacement)
+        }
+    }
+    else {
+        for ($i = $script:loggingCommandEscapeMappings.Length - 1 ; $i -ge 0 ; $i--) {
+            $mapping = $script:loggingCommandEscapeMappings[$i]
+            $Value = $Value.Replace($mapping.Replacement, $mapping.Token)
+        }
+    }
+
+    return $Value
+}
+
+function Format-LoggingCommand {
+    [CmdletBinding()]
+    param(
+        [Parameter(Mandatory = $true)]
+        [string]$Area,
+        [Parameter(Mandatory = $true)]
+        [string]$Event,
+        [string]$Data,
+        [hashtable]$Properties)
+
+    # Append the preamble.
+    [System.Text.StringBuilder]$sb = New-Object -TypeName System.Text.StringBuilder
+    $null = $sb.Append($script:loggingCommandPrefix).Append($Area).Append('.').Append($Event)
+
+    # Append the properties.
+    if ($Properties) {
+        $first = $true
+        foreach ($key in $Properties.Keys) {
+            [string]$value = Format-LoggingCommandData $Properties[$key]
+            if ($value) {
+                if ($first) {
+                    $null = $sb.Append(' ')
+                    $first = $false
+                }
+                else {
+                    $null = $sb.Append(';')
+                }
+
+                $null = $sb.Append("$key=$value")
+            }
+        }
+    }
+
+    # Append the tail and output the value.
+    $Data = Format-LoggingCommandData $Data
+    $sb.Append(']').Append($Data).ToString()
+}
+
+function Write-LoggingCommand {
+    [CmdletBinding(DefaultParameterSetName = 'Parameters')]
+    param(
+        [Parameter(Mandatory = $true, ParameterSetName = 'Parameters')]
+        [string]$Area,
+        [Parameter(Mandatory = $true, ParameterSetName = 'Parameters')]
+        [string]$Event,
+        [Parameter(ParameterSetName = 'Parameters')]
+        [string]$Data,
+        [Parameter(ParameterSetName = 'Parameters')]
+        [hashtable]$Properties,
+        [Parameter(Mandatory = $true, ParameterSetName = 'Object')]
+        $Command,
+        [switch]$AsOutput)
+
+    if ($PSCmdlet.ParameterSetName -eq 'Object') {
+        Write-LoggingCommand -Area $Command.Area -Event $Command.Event -Data $Command.Data -Properties $Command.Properties -AsOutput:$AsOutput
+        return
+    }
+
+    $command = Format-LoggingCommand -Area $Area -Event $Event -Data $Data -Properties $Properties
+    if ($AsOutput) {
+        $command
+    }
+    else {
+        Write-Host $command
+    }
+}
+
+function Write-LogIssue {
+    [CmdletBinding()]
+    param(
+        [ValidateSet('warning', 'error')]
+        [Parameter(Mandatory = $true)]
+        [string]$Type,
+        [string]$Message,
+        [string]$ErrCode,
+        [string]$SourcePath,
+        [string]$LineNumber,
+        [string]$ColumnNumber,
+        [switch]$AsOutput)
+
+    $command = Format-LoggingCommand -Area 'task' -Event 'logissue' -Data $Message -Properties @{
+        'type'         = $Type
+        'code'         = $ErrCode
+        'sourcepath'   = $SourcePath
+        'linenumber'   = $LineNumber
+        'columnnumber' = $ColumnNumber
+    }
+    if ($AsOutput) {
+        return $command
+    }
+
+    if ($Type -eq 'error') {
+        $foregroundColor = $host.PrivateData.ErrorForegroundColor
+        $backgroundColor = $host.PrivateData.ErrorBackgroundColor
+        if ($foregroundColor -isnot [System.ConsoleColor] -or $backgroundColor -isnot [System.ConsoleColor]) {
+            $foregroundColor = [System.ConsoleColor]::Red
+            $backgroundColor = [System.ConsoleColor]::Black
+        }
+    }
+    else {
+        $foregroundColor = $host.PrivateData.WarningForegroundColor
+        $backgroundColor = $host.PrivateData.WarningBackgroundColor
+        if ($foregroundColor -isnot [System.ConsoleColor] -or $backgroundColor -isnot [System.ConsoleColor]) {
+            $foregroundColor = [System.ConsoleColor]::Yellow
+            $backgroundColor = [System.ConsoleColor]::Black
+        }
+    }
+
+    Write-Host $command -ForegroundColor $foregroundColor -BackgroundColor $backgroundColor
+}

eng/common/pipeline-logging-functions.sh

@@ -0,0 +1,206 @@
+#!/usr/bin/env bash
+
+function Write-PipelineTelemetryError {
+  local telemetry_category=''
+  local force=false
+  local function_args=()
+  local message=''
+  while [[ $# -gt 0 ]]; do
+    opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")"
+    case "$opt" in
+      -category|-c)
+        telemetry_category=$2
+        shift
+        ;;
+      -force|-f)
+        force=true
+        ;;
+      -*)
+        function_args+=("$1 $2")
+        shift
+        ;;
+      *)
+        message=$*
+        ;;
+    esac
+    shift
+  done
+
+  if [[ $force != true ]] && [[ "$ci" != true ]]; then
+    echo "$message" >&2
+    return
+  fi
+
+  if [[ $force == true ]]; then
+    function_args+=("-force")
+  fi
+  message="(NETCORE_ENGINEERING_TELEMETRY=$telemetry_category) $message"
+  function_args+=("$message")
+  Write-PipelineTaskError ${function_args[@]}
+}
+
+function Write-PipelineTaskError {
+  local message_type="error"
+  local sourcepath=''
+  local linenumber=''
+  local columnnumber=''
+  local error_code=''
+  local force=false
+
+  while [[ $# -gt 0 ]]; do
+    opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")"
+    case "$opt" in
+      -type|-t)
+        message_type=$2
+        shift
+        ;;
+      -sourcepath|-s)
+        sourcepath=$2
+        shift
+        ;;
+      -linenumber|-ln)
+        linenumber=$2
+        shift
+        ;;
+      -columnnumber|-cn)
+        columnnumber=$2
+        shift
+        ;;
+      -errcode|-e)
+        error_code=$2
+        shift
+        ;;
+      -force|-f)
+        force=true
+        ;;
+      *)
+        break
+        ;;
+    esac
+
+    shift
+  done
+
+  if [[ $force != true ]] && [[ "$ci" != true ]]; then
+    echo "$@" >&2
+    return
+  fi
+
+  local message="##vso[task.logissue"
+
+  message="$message type=$message_type"
+
+  if [ -n "$sourcepath" ]; then
+    message="$message;sourcepath=$sourcepath"
+  fi
+
+  if [ -n "$linenumber" ]; then
+    message="$message;linenumber=$linenumber"
+  fi
+
+  if [ -n "$columnnumber" ]; then
+    message="$message;columnnumber=$columnnumber"
+  fi
+
+  if [ -n "$error_code" ]; then
+    message="$message;code=$error_code"
+  fi
+
+  message="$message]$*"
+  echo "$message"
+}
+
+function Write-PipelineSetVariable {
+  if [[ "$ci" != true ]]; then
+    return
+  fi
+
+  local name=''
+  local value=''
+  local secret=false
+  local as_output=false
+  local is_multi_job_variable=true
+
+  while [[ $# -gt 0 ]]; do
+    opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")"
+    case "$opt" in
+      -name|-n)
+        name=$2
+        shift
+        ;;
+      -value|-v)
+        value=$2
+        shift
+        ;;
+      -secret|-s)
+        secret=true
+        ;;
+      -as_output|-a)
+        as_output=true
+        ;;
+      -is_multi_job_variable|-i)
+        is_multi_job_variable=$2
+        shift
+        ;;
+    esac
+    shift
+  done
+
+  value=${value/;/%3B}
+  value=${value/\\r/%0D}
+  value=${value/\\n/%0A}
+  value=${value/]/%5D}
+
+  local message="##vso[task.setvariable variable=$name;isSecret=$secret;isOutput=$is_multi_job_variable]$value"
+
+  if [[ "$as_output" == true ]]; then
+    $message
+  else
+    echo "$message"
+  fi
+}
+
+function Write-PipelinePrependPath {
+  local prepend_path=''
+
+  while [[ $# -gt 0 ]]; do
+    opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")"
+    case "$opt" in
+      -path|-p)
+        prepend_path=$2
+        shift
+        ;;
+    esac
+    shift
+  done
+
+  export PATH="$prepend_path:$PATH"
+
+  if [[ "$ci" == true ]]; then
+    echo "##vso[task.prependpath]$prepend_path"
+  fi
+}
+
+function Write-PipelineSetResult {
+  local result=''
+  local message=''
+
+  while [[ $# -gt 0 ]]; do
+    opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")"
+    case "$opt" in
+      -result|-r)
+        result=$2
+        shift
+        ;;
+      -message|-m)
+        message=$2
+        shift
+        ;;
+    esac
+    shift
+  done
+
+  if [[ "$ci" == true ]]; then
+    echo "##vso[task.complete result=$result;]$message"
+  fi
+}

eng/common/post-build/add-build-to-channel.ps1

@@ -0,0 +1,48 @@
+param(
+  [Parameter(Mandatory=$true)][int] $BuildId,
+  [Parameter(Mandatory=$true)][int] $ChannelId,
+  [Parameter(Mandatory=$true)][string] $MaestroApiAccessToken,
+  [Parameter(Mandatory=$false)][string] $MaestroApiEndPoint = 'https://maestro-prod.westus2.cloudapp.azure.com',
+  [Parameter(Mandatory=$false)][string] $MaestroApiVersion = '2019-01-16'
+)
+
+try {
+  . $PSScriptRoot\post-build-utils.ps1
+
+  # Check that the channel we are going to promote the build to exist
+  $channelInfo = Get-MaestroChannel -ChannelId $ChannelId
+
+  if (!$channelInfo) {
+    Write-PipelineTelemetryCategory -Category 'PromoteBuild' -Message "Channel with BAR ID $ChannelId was not found in BAR!"
+    ExitWithExitCode 1
+  }
+
+  # Get info about which channel(s) the build has already been promoted to
+  $buildInfo = Get-MaestroBuild -BuildId $BuildId
+  
+  if (!$buildInfo) {
+    Write-PipelineTelemetryError -Category 'PromoteBuild' -Message "Build with BAR ID $BuildId was not found in BAR!"
+    ExitWithExitCode 1
+  }
+
+  # Find whether the build is already assigned to the channel or not
+  if ($buildInfo.channels) {
+    foreach ($channel in $buildInfo.channels) {
+      if ($channel.Id -eq $ChannelId) {
+        Write-Host "The build with BAR ID $BuildId is already on channel $ChannelId!"
+        ExitWithExitCode 0
+      }
+    }
+  }
+
+  Write-Host "Promoting build '$BuildId' to channel '$ChannelId'."
+
+  Assign-BuildToChannel -BuildId $BuildId -ChannelId $ChannelId
+
+  Write-Host 'done.'
+} 
+catch {
+  Write-Host $_
+  Write-PipelineTelemetryError -Category 'PromoteBuild' -Message "There was an error while trying to promote build '$BuildId' to channel '$ChannelId'"
+  ExitWithExitCode 1
+}

eng/common/post-build/check-channel-consistency.ps1

@@ -0,0 +1,40 @@
+param(
+  [Parameter(Mandatory=$true)][string] $PromoteToChannels,            # List of channels that the build should be promoted to
+  [Parameter(Mandatory=$true)][array] $AvailableChannelIds            # List of channel IDs available in the YAML implementation
+)
+
+try {
+  . $PSScriptRoot\post-build-utils.ps1
+
+  if ($PromoteToChannels -eq "") {
+    Write-PipelineTaskError -Type 'warning' -Message "This build won't publish assets as it's not configured to any Maestro channel. If that wasn't intended use Darc to configure a default channel using add-default-channel for this branch or to promote it to a channel using add-build-to-channel. See https://github.com/dotnet/arcade/blob/master/Documentation/Darc.md#assigning-an-individual-build-to-a-channel for more info."
+    ExitWithExitCode 0
+  }
+
+  # Check that every channel that Maestro told to promote the build to 
+  # is available in YAML
+  $PromoteToChannelsIds = $PromoteToChannels -split "\D" | Where-Object { $_ }
+
+  $hasErrors = $false
+
+  foreach ($id in $PromoteToChannelsIds) {
+    if (($id -ne 0) -and ($id -notin $AvailableChannelIds)) {
+      Write-PipelineTaskError -Message "Channel $id is not present in the post-build YAML configuration! This is an error scenario. Please contact @dnceng."
+      $hasErrors = $true
+    }
+  }
+
+  # The `Write-PipelineTaskError` doesn't error the script and we might report several errors
+  # in the previous lines. The check below makes sure that we return an error state from the
+  # script if we reported any validation error
+  if ($hasErrors) {
+    ExitWithExitCode 1 
+  }
+
+  Write-Host 'done.'
+} 
+catch {
+  Write-Host $_
+  Write-PipelineTelemetryError -Category 'CheckChannelConsistency' -Message "There was an error while trying to check consistency of Maestro default channels for the build and post-build YAML configuration."
+  ExitWithExitCode 1
+}

eng/common/post-build/nuget-validation.ps1

@@ -0,0 +1,24 @@
+# This script validates NuGet package metadata information using this 
+# tool: https://github.com/NuGet/NuGetGallery/tree/jver-verify/src/VerifyMicrosoftPackage
+
+param(
+  [Parameter(Mandatory=$true)][string] $PackagesPath,           # Path to where the packages to be validated are
+  [Parameter(Mandatory=$true)][string] $ToolDestinationPath     # Where the validation tool should be downloaded to
+)
+
+try {
+  . $PSScriptRoot\post-build-utils.ps1
+
+  $url = 'https://raw.githubusercontent.com/NuGet/NuGetGallery/3e25ad135146676bcab0050a516939d9958bfa5d/src/VerifyMicrosoftPackage/verify.ps1'
+
+  New-Item -ItemType 'directory' -Path ${ToolDestinationPath} -Force
+
+  Invoke-WebRequest $url -OutFile ${ToolDestinationPath}\verify.ps1 
+
+  & ${ToolDestinationPath}\verify.ps1 ${PackagesPath}\*.nupkg
+} 
+catch {
+  Write-Host $_.ScriptStackTrace
+  Write-PipelineTelemetryError -Category 'NuGetValidation' -Message $_
+  ExitWithExitCode 1
+}

eng/common/post-build/post-build-utils.ps1

@@ -0,0 +1,91 @@
+# Most of the functions in this file require the variables `MaestroApiEndPoint`, 
+# `MaestroApiVersion` and `MaestroApiAccessToken` to be globally available.
+
+$ErrorActionPreference = 'Stop'
+Set-StrictMode -Version 2.0
+
+# `tools.ps1` checks $ci to perform some actions. Since the post-build
+# scripts don't necessarily execute in the same agent that run the
+# build.ps1/sh script this variable isn't automatically set.
+$ci = $true
+$disableConfigureToolsetImport = $true
+. $PSScriptRoot\..\tools.ps1
+
+function Create-MaestroApiRequestHeaders([string]$ContentType = 'application/json') {
+  Validate-MaestroVars
+
+  $headers = New-Object 'System.Collections.Generic.Dictionary[[String],[String]]'
+  $headers.Add('Accept', $ContentType)
+  $headers.Add('Authorization',"Bearer $MaestroApiAccessToken")
+  return $headers
+}
+
+function Get-MaestroChannel([int]$ChannelId) {
+  Validate-MaestroVars
+
+  $apiHeaders = Create-MaestroApiRequestHeaders
+  $apiEndpoint = "$MaestroApiEndPoint/api/channels/${ChannelId}?api-version=$MaestroApiVersion"
+  
+  $result = try { Invoke-WebRequest -Method Get -Uri $apiEndpoint -Headers $apiHeaders | ConvertFrom-Json } catch { Write-Host "Error: $_" }
+  return $result
+}
+
+function Get-MaestroBuild([int]$BuildId) {
+  Validate-MaestroVars
+
+  $apiHeaders = Create-MaestroApiRequestHeaders -AuthToken $MaestroApiAccessToken
+  $apiEndpoint = "$MaestroApiEndPoint/api/builds/${BuildId}?api-version=$MaestroApiVersion"
+
+  $result = try { return Invoke-WebRequest -Method Get -Uri $apiEndpoint -Headers $apiHeaders | ConvertFrom-Json } catch { Write-Host "Error: $_" }
+  return $result
+}
+
+function Get-MaestroSubscriptions([string]$SourceRepository, [int]$ChannelId) {
+  Validate-MaestroVars
+
+  $SourceRepository = [System.Web.HttpUtility]::UrlEncode($SourceRepository) 
+  $apiHeaders = Create-MaestroApiRequestHeaders -AuthToken $MaestroApiAccessToken
+  $apiEndpoint = "$MaestroApiEndPoint/api/subscriptions?sourceRepository=$SourceRepository&channelId=$ChannelId&api-version=$MaestroApiVersion"
+
+  $result = try { Invoke-WebRequest -Method Get -Uri $apiEndpoint -Headers $apiHeaders | ConvertFrom-Json } catch { Write-Host "Error: $_" }
+  return $result
+}
+
+function Assign-BuildToChannel([int]$BuildId, [int]$ChannelId) {
+  Validate-MaestroVars
+
+  $apiHeaders = Create-MaestroApiRequestHeaders -AuthToken $MaestroApiAccessToken
+  $apiEndpoint = "$MaestroApiEndPoint/api/channels/${ChannelId}/builds/${BuildId}?api-version=$MaestroApiVersion"
+  Invoke-WebRequest -Method Post -Uri $apiEndpoint -Headers $apiHeaders | Out-Null
+}
+
+function Trigger-Subscription([string]$SubscriptionId) {
+  Validate-MaestroVars
+
+  $apiHeaders = Create-MaestroApiRequestHeaders -AuthToken $MaestroApiAccessToken
+  $apiEndpoint = "$MaestroApiEndPoint/api/subscriptions/$SubscriptionId/trigger?api-version=$MaestroApiVersion"
+  Invoke-WebRequest -Uri $apiEndpoint -Headers $apiHeaders -Method Post | Out-Null
+}
+
+function Validate-MaestroVars {
+  try {
+    Get-Variable MaestroApiEndPoint | Out-Null
+    Get-Variable MaestroApiVersion | Out-Null
+    Get-Variable MaestroApiAccessToken | Out-Null
+
+    if (!($MaestroApiEndPoint -Match '^http[s]?://maestro-(int|prod).westus2.cloudapp.azure.com$')) {
+      Write-PipelineTelemetryError -Category 'MaestroVars' -Message "MaestroApiEndPoint is not a valid Maestro URL. '$MaestroApiEndPoint'"
+      ExitWithExitCode 1  
+    }
+
+    if (!($MaestroApiVersion -Match '^[0-9]{4}-[0-9]{2}-[0-9]{2}$')) {
+      Write-PipelineTelemetryError -Category 'MaestroVars' -Message "MaestroApiVersion does not match a version string in the format yyyy-MM-DD. '$MaestroApiVersion'"
+      ExitWithExitCode 1
+    }
+  }
+  catch {
+    Write-PipelineTelemetryError -Category 'MaestroVars' -Message 'Error: Variables `MaestroApiEndPoint`, `MaestroApiVersion` and `MaestroApiAccessToken` are required while using this script.'
+    Write-Host $_
+    ExitWithExitCode 1
+  }
+}

eng/common/post-build/publish-using-darc.ps1

@@ -0,0 +1,80 @@
+param(
+  [Parameter(Mandatory=$true)][int] $BuildId,
+  [Parameter(Mandatory=$true)][int] $PublishingInfraVersion,
+  [Parameter(Mandatory=$true)][string] $AzdoToken,
+  [Parameter(Mandatory=$true)][string] $MaestroToken,
+  [Parameter(Mandatory=$false)][string] $MaestroApiEndPoint = 'https://maestro-prod.westus2.cloudapp.azure.com',
+  [Parameter(Mandatory=$true)][string] $WaitPublishingFinish,
+  [Parameter(Mandatory=$false)][string] $EnableSourceLinkValidation,
+  [Parameter(Mandatory=$false)][string] $EnableSigningValidation,
+  [Parameter(Mandatory=$false)][string] $EnableNugetValidation,
+  [Parameter(Mandatory=$false)][string] $PublishInstallersAndChecksums,
+  [Parameter(Mandatory=$false)][string] $ArtifactsPublishingAdditionalParameters,
+  [Parameter(Mandatory=$false)][string] $SymbolPublishingAdditionalParameters,
+  [Parameter(Mandatory=$false)][string] $SigningValidationAdditionalParameters
+)
+
+try {
+  . $PSScriptRoot\post-build-utils.ps1
+
+  $darc = Get-Darc 
+
+  $optionalParams = [System.Collections.ArrayList]::new()
+
+  if ("" -ne $ArtifactsPublishingAdditionalParameters) {
+    $optionalParams.Add("--artifact-publishing-parameters") | Out-Null
+    $optionalParams.Add($ArtifactsPublishingAdditionalParameters) | Out-Null
+  }
+
+  if ("" -ne $SymbolPublishingAdditionalParameters) {
+    $optionalParams.Add("--symbol-publishing-parameters") | Out-Null
+    $optionalParams.Add($SymbolPublishingAdditionalParameters) | Out-Null
+  }
+
+  if ("false" -eq $WaitPublishingFinish) {
+    $optionalParams.Add("--no-wait") | Out-Null
+  }
+
+  if ("false" -ne $PublishInstallersAndChecksums) {
+    $optionalParams.Add("--publish-installers-and-checksums") | Out-Null
+  }
+
+  if ("true" -eq $EnableNugetValidation) {
+    $optionalParams.Add("--validate-nuget") | Out-Null
+  }
+
+  if ("true" -eq $EnableSourceLinkValidation) {
+    $optionalParams.Add("--validate-sourcelinkchecksums") | Out-Null
+  }
+
+  if ("true" -eq $EnableSigningValidation) {
+    $optionalParams.Add("--validate-signingchecksums") | Out-Null
+
+    if ("" -ne $SigningValidationAdditionalParameters) {
+      $optionalParams.Add("--signing-validation-parameters") | Out-Null
+      $optionalParams.Add($SigningValidationAdditionalParameters) | Out-Null
+    }
+  }
+
+  & $darc add-build-to-channel `
+  --id $buildId `
+  --publishing-infra-version $PublishingInfraVersion `
+  --default-channels `
+  --source-branch main `
+  --azdev-pat $AzdoToken `
+  --bar-uri $MaestroApiEndPoint `
+  --password $MaestroToken `
+	@optionalParams
+
+  if ($LastExitCode -ne 0) {
+    Write-Host "Problems using Darc to promote build ${buildId} to default channels. Stopping execution..."
+    exit 1
+  }
+
+  Write-Host 'done.'
+} 
+catch {
+  Write-Host $_
+  Write-PipelineTelemetryError -Category 'PromoteBuild' -Message "There was an error while trying to publish build '$BuildId' to default channels."
+  ExitWithExitCode 1
+}

eng/common/post-build/sourcelink-validation.ps1

@@ -0,0 +1,277 @@
+param(
+  [Parameter(Mandatory=$true)][string] $InputPath,              # Full path to directory where Symbols.NuGet packages to be checked are stored
+  [Parameter(Mandatory=$true)][string] $ExtractPath,            # Full path to directory where the packages will be extracted during validation
+  [Parameter(Mandatory=$false)][string] $GHRepoName,            # GitHub name of the repo including the Org. E.g., dotnet/arcade
+  [Parameter(Mandatory=$false)][string] $GHCommit,              # GitHub commit SHA used to build the packages
+  [Parameter(Mandatory=$true)][string] $SourcelinkCliVersion    # Version of SourceLink CLI to use
+)
+
+. $PSScriptRoot\post-build-utils.ps1
+
+# Cache/HashMap (File -> Exist flag) used to consult whether a file exist 
+# in the repository at a specific commit point. This is populated by inserting
+# all files present in the repo at a specific commit point.
+$global:RepoFiles = @{}
+
+# Maximum number of jobs to run in parallel
+$MaxParallelJobs = 16
+
+# Wait time between check for system load
+$SecondsBetweenLoadChecks = 10
+
+$ValidatePackage = {
+  param( 
+    [string] $PackagePath                                 # Full path to a Symbols.NuGet package
+  )
+
+  . $using:PSScriptRoot\..\tools.ps1
+
+  # Ensure input file exist
+  if (!(Test-Path $PackagePath)) {
+    Write-Host "Input file does not exist: $PackagePath"
+    return 1
+  }
+
+  # Extensions for which we'll look for SourceLink information
+  # For now we'll only care about Portable & Embedded PDBs
+  $RelevantExtensions = @('.dll', '.exe', '.pdb')
+ 
+  Write-Host -NoNewLine 'Validating ' ([System.IO.Path]::GetFileName($PackagePath)) '...'
+
+  $PackageId = [System.IO.Path]::GetFileNameWithoutExtension($PackagePath)
+  $ExtractPath = Join-Path -Path $using:ExtractPath -ChildPath $PackageId
+  $FailedFiles = 0
+
+  Add-Type -AssemblyName System.IO.Compression.FileSystem
+
+  [System.IO.Directory]::CreateDirectory($ExtractPath)  | Out-Null
+
+  try {
+    $zip = [System.IO.Compression.ZipFile]::OpenRead($PackagePath)
+
+    $zip.Entries | 
+      Where-Object {$RelevantExtensions -contains [System.IO.Path]::GetExtension($_.Name)} |
+        ForEach-Object {
+          $FileName = $_.FullName
+          $Extension = [System.IO.Path]::GetExtension($_.Name)
+          $FakeName = -Join((New-Guid), $Extension)
+          $TargetFile = Join-Path -Path $ExtractPath -ChildPath $FakeName 
+
+          # We ignore resource DLLs
+          if ($FileName.EndsWith('.resources.dll')) {
+            return
+          }
+
+          [System.IO.Compression.ZipFileExtensions]::ExtractToFile($_, $TargetFile, $true)
+
+          $ValidateFile = {
+            param( 
+              [string] $FullPath,                                # Full path to the module that has to be checked
+              [string] $RealPath,
+              [ref] $FailedFiles
+            )
+
+            $sourcelinkExe = "$env:USERPROFILE\.dotnet\tools"
+            $sourcelinkExe = Resolve-Path "$sourcelinkExe\sourcelink.exe"
+            $SourceLinkInfos = & $sourcelinkExe print-urls $FullPath | Out-String
+
+            if ($LASTEXITCODE -eq 0 -and -not ([string]::IsNullOrEmpty($SourceLinkInfos))) {
+              $NumFailedLinks = 0
+
+              # We only care about Http addresses
+              $Matches = (Select-String '(http[s]?)(:\/\/)([^\s,]+)' -Input $SourceLinkInfos -AllMatches).Matches
+
+              if ($Matches.Count -ne 0) {
+                $Matches.Value |
+                  ForEach-Object {
+                    $Link = $_
+                    $CommitUrl = "https://raw.githubusercontent.com/${using:GHRepoName}/${using:GHCommit}/"
+                    
+                    $FilePath = $Link.Replace($CommitUrl, "")
+                    $Status = 200
+                    $Cache = $using:RepoFiles
+
+                    if ( !($Cache.ContainsKey($FilePath)) ) {
+                      try {
+                        $Uri = $Link -as [System.URI]
+                      
+                        # Only GitHub links are valid
+                        if ($Uri.AbsoluteURI -ne $null -and ($Uri.Host -match 'github' -or $Uri.Host -match 'githubusercontent')) {
+                          $Status = (Invoke-WebRequest -Uri $Link -UseBasicParsing -Method HEAD -TimeoutSec 5).StatusCode
+                        }
+                        else {
+                          $Status = 0
+                        }
+                      }
+                      catch {
+                        write-host $_
+                        $Status = 0
+                      }
+                    }
+
+                    if ($Status -ne 200) {
+                      if ($NumFailedLinks -eq 0) {
+                        if ($FailedFiles.Value -eq 0) {
+                          Write-Host
+                        }
+
+                        Write-Host "`tFile $RealPath has broken links:"
+                      }
+
+                      Write-Host "`t`tFailed to retrieve $Link"
+
+                      $NumFailedLinks++
+                    }
+                  }
+              }
+
+              if ($NumFailedLinks -ne 0) {
+                $FailedFiles.value++
+                $global:LASTEXITCODE = 1
+              }
+            }
+          }
+        
+          &$ValidateFile $TargetFile $FileName ([ref]$FailedFiles)
+        }
+  }
+  catch {
+  
+  }
+  finally {
+    $zip.Dispose() 
+  }
+
+  if ($FailedFiles -eq 0) {
+    Write-Host 'Passed.'
+    return [pscustomobject]@{
+      result = 0
+      packagePath = $PackagePath
+    }
+  }
+  else {
+    Write-PipelineTelemetryError -Category 'SourceLink' -Message "$PackagePath has broken SourceLink links."
+    return [pscustomobject]@{
+      result = 1
+      packagePath = $PackagePath
+    }
+  }
+}
+
+function CheckJobResult(
+    $result, 
+    $packagePath,
+    [ref]$ValidationFailures,
+    [switch]$logErrors) {
+  if ($result -ne '0') {
+    if ($logErrors) {
+      Write-PipelineTelemetryError -Category 'SourceLink' -Message "$packagePath has broken SourceLink links."
+    }
+    $ValidationFailures.Value++
+  }
+}
+
+function ValidateSourceLinkLinks {
+  if ($GHRepoName -ne '' -and !($GHRepoName -Match '^[^\s\/]+/[^\s\/]+$')) {
+    if (!($GHRepoName -Match '^[^\s-]+-[^\s]+$')) {
+      Write-PipelineTelemetryError -Category 'SourceLink' -Message "GHRepoName should be in the format <org>/<repo> or <org>-<repo>. '$GHRepoName'"
+      ExitWithExitCode 1
+    }
+    else {
+      $GHRepoName = $GHRepoName -replace '^([^\s-]+)-([^\s]+)$', '$1/$2';
+    }
+  }
+
+  if ($GHCommit -ne '' -and !($GHCommit -Match '^[0-9a-fA-F]{40}$')) {
+    Write-PipelineTelemetryError -Category 'SourceLink' -Message "GHCommit should be a 40 chars hexadecimal string. '$GHCommit'"
+    ExitWithExitCode 1
+  }
+
+  if ($GHRepoName -ne '' -and $GHCommit -ne '') {
+    $RepoTreeURL = -Join('http://api.github.com/repos/', $GHRepoName, '/git/trees/', $GHCommit, '?recursive=1')
+    $CodeExtensions = @('.cs', '.vb', '.fs', '.fsi', '.fsx', '.fsscript')
+
+    try {
+      # Retrieve the list of files in the repo at that particular commit point and store them in the RepoFiles hash
+      $Data = Invoke-WebRequest $RepoTreeURL -UseBasicParsing | ConvertFrom-Json | Select-Object -ExpandProperty tree
+  
+      foreach ($file in $Data) {
+        $Extension = [System.IO.Path]::GetExtension($file.path)
+
+        if ($CodeExtensions.Contains($Extension)) {
+          $RepoFiles[$file.path] = 1
+        }
+      }
+    }
+    catch {
+      Write-Host "Problems downloading the list of files from the repo. Url used: $RepoTreeURL . Execution will proceed without caching."
+    }
+  }
+  elseif ($GHRepoName -ne '' -or $GHCommit -ne '') {
+    Write-Host 'For using the http caching mechanism both GHRepoName and GHCommit should be informed.'
+  }
+  
+  if (Test-Path $ExtractPath) {
+    Remove-Item $ExtractPath -Force -Recurse -ErrorAction SilentlyContinue
+  }
+
+  $ValidationFailures = 0
+
+  # Process each NuGet package in parallel
+  Get-ChildItem "$InputPath\*.symbols.nupkg" |
+    ForEach-Object {
+      Start-Job -ScriptBlock $ValidatePackage -ArgumentList $_.FullName | Out-Null
+      $NumJobs = @(Get-Job -State 'Running').Count
+      
+      while ($NumJobs -ge $MaxParallelJobs) {
+        Write-Host "There are $NumJobs validation jobs running right now. Waiting $SecondsBetweenLoadChecks seconds to check again."
+        sleep $SecondsBetweenLoadChecks
+        $NumJobs = @(Get-Job -State 'Running').Count
+      }
+
+      foreach ($Job in @(Get-Job -State 'Completed')) {
+        $jobResult = Wait-Job -Id $Job.Id | Receive-Job
+        CheckJobResult $jobResult.result $jobResult.packagePath ([ref]$ValidationFailures) -LogErrors
+        Remove-Job -Id $Job.Id
+      }
+    }
+
+  foreach ($Job in @(Get-Job)) {
+    $jobResult = Wait-Job -Id $Job.Id | Receive-Job
+    CheckJobResult $jobResult.result $jobResult.packagePath ([ref]$ValidationFailures)
+    Remove-Job -Id $Job.Id
+  }
+  if ($ValidationFailures -gt 0) {
+    Write-PipelineTelemetryError -Category 'SourceLink' -Message "$ValidationFailures package(s) failed validation."
+    ExitWithExitCode 1
+  }
+}
+
+function InstallSourcelinkCli {
+  $sourcelinkCliPackageName = 'sourcelink'
+
+  $dotnetRoot = InitializeDotNetCli -install:$true
+  $dotnet = "$dotnetRoot\dotnet.exe"
+  $toolList = & "$dotnet" tool list --global
+
+  if (($toolList -like "*$sourcelinkCliPackageName*") -and ($toolList -like "*$sourcelinkCliVersion*")) {
+    Write-Host "SourceLink CLI version $sourcelinkCliVersion is already installed."
+  }
+  else {
+    Write-Host "Installing SourceLink CLI version $sourcelinkCliVersion..."
+    Write-Host 'You may need to restart your command window if this is the first dotnet tool you have installed.'
+    & "$dotnet" tool install $sourcelinkCliPackageName --version $sourcelinkCliVersion --verbosity "minimal" --global 
+  }
+}
+
+try {
+  InstallSourcelinkCli
+
+  ValidateSourceLinkLinks 
+}
+catch {
+  Write-Host $_.Exception
+  Write-Host $_.ScriptStackTrace
+  Write-PipelineTelemetryError -Category 'SourceLink' -Message $_
+  ExitWithExitCode 1
+}

eng/common/post-build/symbols-validation.ps1

@@ -0,0 +1,304 @@
+param(
+  [Parameter(Mandatory = $true)][string] $InputPath, # Full path to directory where NuGet packages to be checked are stored
+  [Parameter(Mandatory = $true)][string] $ExtractPath, # Full path to directory where the packages will be extracted during validation
+  [Parameter(Mandatory = $true)][string] $DotnetSymbolVersion, # Version of dotnet symbol to use
+  [Parameter(Mandatory = $false)][switch] $CheckForWindowsPdbs, # If we should check for the existence of windows pdbs in addition to portable PDBs
+  [Parameter(Mandatory = $false)][switch] $ContinueOnError, # If we should keep checking symbols after an error
+  [Parameter(Mandatory = $false)][switch] $Clean                  # Clean extracted symbols directory after checking symbols
+)
+
+# Maximum number of jobs to run in parallel
+$MaxParallelJobs = 16
+
+# Max number of retries
+$MaxRetry = 5
+
+# Wait time between check for system load
+$SecondsBetweenLoadChecks = 10
+
+# Set error codes
+Set-Variable -Name "ERROR_BADEXTRACT" -Option Constant -Value -1
+Set-Variable -Name "ERROR_FILEDOESNOTEXIST" -Option Constant -Value -2
+
+$WindowsPdbVerificationParam = ""
+if ($CheckForWindowsPdbs) {
+  $WindowsPdbVerificationParam = "--windows-pdbs"
+}
+
+$CountMissingSymbols = {
+  param( 
+    [string] $PackagePath, # Path to a NuGet package
+    [string] $WindowsPdbVerificationParam # If we should check for the existence of windows pdbs in addition to portable PDBs
+  )
+
+  . $using:PSScriptRoot\..\tools.ps1
+
+  Add-Type -AssemblyName System.IO.Compression.FileSystem
+
+  Write-Host "Validating $PackagePath "
+
+  # Ensure input file exist
+  if (!(Test-Path $PackagePath)) {
+    Write-PipelineTaskError "Input file does not exist: $PackagePath"
+    return [pscustomobject]@{
+      result      = $using:ERROR_FILEDOESNOTEXIST
+      packagePath = $PackagePath
+    }
+  }
+  
+  # Extensions for which we'll look for symbols
+  $RelevantExtensions = @('.dll', '.exe', '.so', '.dylib')
+
+  # How many files are missing symbol information
+  $MissingSymbols = 0
+
+  $PackageId = [System.IO.Path]::GetFileNameWithoutExtension($PackagePath)
+  $PackageGuid = New-Guid
+  $ExtractPath = Join-Path -Path $using:ExtractPath -ChildPath $PackageGuid
+  $SymbolsPath = Join-Path -Path $ExtractPath -ChildPath 'Symbols'
+  
+  try {
+    [System.IO.Compression.ZipFile]::ExtractToDirectory($PackagePath, $ExtractPath)
+  }
+  catch {
+    Write-Host "Something went wrong extracting $PackagePath"
+    Write-Host $_
+    return [pscustomobject]@{
+      result      = $using:ERROR_BADEXTRACT
+      packagePath = $PackagePath
+    }
+  }
+
+  Get-ChildItem -Recurse $ExtractPath |
+  Where-Object { $RelevantExtensions -contains $_.Extension } |
+  ForEach-Object {
+    $FileName = $_.FullName
+    if ($FileName -Match '\\ref\\') {
+      Write-Host "`t Ignoring reference assembly file " $FileName
+      return
+    }
+
+    $FirstMatchingSymbolDescriptionOrDefault = {
+      param( 
+        [string] $FullPath, # Full path to the module that has to be checked
+        [string] $TargetServerParam, # Parameter to pass to `Symbol Tool` indicating the server to lookup for symbols
+        [string] $WindowsPdbVerificationParam, # Parameter to pass to potential check for windows-pdbs.
+        [string] $SymbolsPath
+      )
+
+      $FileName = [System.IO.Path]::GetFileName($FullPath)
+      $Extension = [System.IO.Path]::GetExtension($FullPath)
+
+      # Those below are potential symbol files that the `dotnet symbol` might
+      # return. Which one will be returned depend on the type of file we are
+      # checking and which type of file was uploaded.
+
+      # The file itself is returned
+      $SymbolPath = $SymbolsPath + '\' + $FileName
+
+      # PDB file for the module
+      $PdbPath = $SymbolPath.Replace($Extension, '.pdb')
+
+      # PDB file for R2R module (created by crossgen)
+      $NGenPdb = $SymbolPath.Replace($Extension, '.ni.pdb')
+
+      # DBG file for a .so library
+      $SODbg = $SymbolPath.Replace($Extension, '.so.dbg')
+
+      # DWARF file for a .dylib
+      $DylibDwarf = $SymbolPath.Replace($Extension, '.dylib.dwarf')
+
+      $dotnetSymbolExe = "$env:USERPROFILE\.dotnet\tools"
+      $dotnetSymbolExe = Resolve-Path "$dotnetSymbolExe\dotnet-symbol.exe"
+
+      $totalRetries = 0
+
+      while ($totalRetries -lt $using:MaxRetry) {
+        # Save the output and get diagnostic output
+        $output = & $dotnetSymbolExe --symbols --modules $WindowsPdbVerificationParam $TargetServerParam $FullPath -o $SymbolsPath --diagnostics | Out-String
+
+        if (Test-Path $PdbPath) {
+          return 'PDB'
+        }
+        elseif (Test-Path $NGenPdb) {
+          return 'NGen PDB'
+        }
+        elseif (Test-Path $SODbg) {
+          return 'DBG for SO'
+        }  
+        elseif (Test-Path $DylibDwarf) {
+          return 'Dwarf for Dylib'
+        }  
+        elseif (Test-Path $SymbolPath) {
+          return 'Module'
+        }
+        elseif ($output.Contains("503 Service Unavailable")) {
+          # If we got a 503 error, we should retry.
+          $totalRetries++
+        }
+        else {
+          return $null
+        }
+      }
+      
+      return $null
+    }
+
+    $SymbolsOnMSDL = & $FirstMatchingSymbolDescriptionOrDefault $FileName '--microsoft-symbol-server' $SymbolsPath $WindowsPdbVerificationParam
+    $SymbolsOnSymWeb = & $FirstMatchingSymbolDescriptionOrDefault $FileName '--internal-server' $SymbolsPath $WindowsPdbVerificationParam
+
+    Write-Host -NoNewLine "`t Checking file " $FileName "... "
+  
+    if ($SymbolsOnMSDL -ne $null -and $SymbolsOnSymWeb -ne $null) {
+      Write-Host "Symbols found on MSDL ($SymbolsOnMSDL) and SymWeb ($SymbolsOnSymWeb)"
+    }
+    else {
+      $MissingSymbols++
+
+      if ($SymbolsOnMSDL -eq $null -and $SymbolsOnSymWeb -eq $null) {
+        Write-Host 'No symbols found on MSDL or SymWeb!'
+      }
+      else {
+        if ($SymbolsOnMSDL -eq $null) {
+          Write-Host 'No symbols found on MSDL!'
+        }
+        else {
+          Write-Host 'No symbols found on SymWeb!'
+        }
+      }
+    }
+  }
+  
+  if ($using:Clean) {
+    Remove-Item $ExtractPath -Recurse -Force
+  }
+  
+  Pop-Location
+
+  return [pscustomobject]@{
+    result      = $MissingSymbols
+    packagePath = $PackagePath
+  }
+}
+
+function CheckJobResult(
+  $result, 
+  $packagePath,
+  [ref]$DupedSymbols,
+  [ref]$TotalFailures) {
+  if ($result -eq $ERROR_BADEXTRACT) {
+    Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "$packagePath has duplicated symbol files"
+    $DupedSymbols.Value++
+  } 
+  elseif ($result -eq $ERROR_FILEDOESNOTEXIST) {
+    Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "$packagePath does not exist"
+    $TotalFailures.Value++
+  }
+  elseif ($result -gt '0') {
+    Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "Missing symbols for $result modules in the package $packagePath"
+    $TotalFailures.Value++
+  }
+  else {
+    Write-Host "All symbols verified for package $packagePath"
+  }
+}
+
+function CheckSymbolsAvailable {
+  if (Test-Path $ExtractPath) {
+    Remove-Item $ExtractPath -Force  -Recurse -ErrorAction SilentlyContinue
+  }
+
+  $TotalFailures = 0
+  $DupedSymbols = 0
+
+  Get-ChildItem "$InputPath\*.nupkg" |
+    ForEach-Object {
+      $FileName = $_.Name
+      $FullName = $_.FullName
+
+      # These packages from Arcade-Services include some native libraries that
+      # our current symbol uploader can't handle. Below is a workaround until
+      # we get issue: https://github.com/dotnet/arcade/issues/2457 sorted.
+      if ($FileName -Match 'Microsoft\.DotNet\.Darc\.') {
+        Write-Host "Ignoring Arcade-services file: $FileName"
+        Write-Host
+        return
+      }
+      elseif ($FileName -Match 'Microsoft\.DotNet\.Maestro\.Tasks\.') {
+        Write-Host "Ignoring Arcade-services file: $FileName"
+        Write-Host
+        return
+      }
+
+      Start-Job -ScriptBlock $CountMissingSymbols -ArgumentList @($FullName,$WindowsPdbVerificationParam) | Out-Null
+
+      $NumJobs = @(Get-Job -State 'Running').Count
+
+      while ($NumJobs -ge $MaxParallelJobs) {
+        Write-Host "There are $NumJobs validation jobs running right now. Waiting $SecondsBetweenLoadChecks seconds to check again."
+        sleep $SecondsBetweenLoadChecks
+        $NumJobs = @(Get-Job -State 'Running').Count
+      }
+
+      foreach ($Job in @(Get-Job -State 'Completed')) {
+        $jobResult = Wait-Job -Id $Job.Id | Receive-Job
+        CheckJobResult $jobResult.result $jobResult.packagePath ([ref]$DupedSymbols) ([ref]$TotalFailures)
+        Remove-Job -Id $Job.Id
+      }
+      Write-Host
+    }
+
+  foreach ($Job in @(Get-Job)) {
+    $jobResult = Wait-Job -Id $Job.Id | Receive-Job
+    CheckJobResult $jobResult.result $jobResult.packagePath ([ref]$DupedSymbols) ([ref]$TotalFailures)
+  }
+
+  if ($TotalFailures -gt 0 -or $DupedSymbols -gt 0) {
+    if ($TotalFailures -gt 0) {
+      Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "Symbols missing for $TotalFailures packages"
+    }
+
+    if ($DupedSymbols -gt 0) {
+      Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "$DupedSymbols packages had duplicated symbol files"
+    }
+    
+    ExitWithExitCode 1
+  }
+  else {
+    Write-Host "All symbols validated!"
+  }
+}
+
+function InstallDotnetSymbol {
+  $dotnetSymbolPackageName = 'dotnet-symbol'
+
+  $dotnetRoot = InitializeDotNetCli -install:$true
+  $dotnet = "$dotnetRoot\dotnet.exe"
+  $toolList = & "$dotnet" tool list --global
+
+  if (($toolList -like "*$dotnetSymbolPackageName*") -and ($toolList -like "*$dotnetSymbolVersion*")) {
+    Write-Host "dotnet-symbol version $dotnetSymbolVersion is already installed."
+  }
+  else {
+    Write-Host "Installing dotnet-symbol version $dotnetSymbolVersion..."
+    Write-Host 'You may need to restart your command window if this is the first dotnet tool you have installed.'
+    & "$dotnet" tool install $dotnetSymbolPackageName --version $dotnetSymbolVersion --verbosity "minimal" --global
+  }
+}
+
+try {
+  . $PSScriptRoot\post-build-utils.ps1
+  
+  InstallDotnetSymbol
+
+  foreach ($Job in @(Get-Job)) {
+    Remove-Job -Id $Job.Id
+  }
+
+  CheckSymbolsAvailable
+}
+catch {
+  Write-Host $_.ScriptStackTrace
+  Write-PipelineTelemetryError -Category 'CheckSymbols' -Message $_
+  ExitWithExitCode 1
+}

eng/common/post-build/trigger-subscriptions.ps1

@@ -0,0 +1,64 @@
+param(
+  [Parameter(Mandatory=$true)][string] $SourceRepo,
+  [Parameter(Mandatory=$true)][int] $ChannelId,
+  [Parameter(Mandatory=$true)][string] $MaestroApiAccessToken,
+  [Parameter(Mandatory=$false)][string] $MaestroApiEndPoint = 'https://maestro-prod.westus2.cloudapp.azure.com',
+  [Parameter(Mandatory=$false)][string] $MaestroApiVersion = '2019-01-16'
+)
+
+try {
+  . $PSScriptRoot\post-build-utils.ps1
+
+  # Get all the $SourceRepo subscriptions
+  $normalizedSourceRepo = $SourceRepo.Replace('dnceng@', '')
+  $subscriptions = Get-MaestroSubscriptions -SourceRepository $normalizedSourceRepo -ChannelId $ChannelId
+
+  if (!$subscriptions) {
+    Write-PipelineTelemetryError -Category 'TriggerSubscriptions' -Message "No subscriptions found for source repo '$normalizedSourceRepo' in channel '$ChannelId'"
+    ExitWithExitCode 0
+  }
+
+  $subscriptionsToTrigger = New-Object System.Collections.Generic.List[string]
+  $failedTriggeredSubscription = $false
+
+  # Get all enabled subscriptions that need dependency flow on 'everyBuild'
+  foreach ($subscription in $subscriptions) {
+    if ($subscription.enabled -and $subscription.policy.updateFrequency -like 'everyBuild' -and $subscription.channel.id -eq $ChannelId) {
+      Write-Host "Should trigger this subscription: ${$subscription.id}"
+      [void]$subscriptionsToTrigger.Add($subscription.id)
+    }
+  }
+
+  foreach ($subscriptionToTrigger in $subscriptionsToTrigger) {
+    try {
+      Write-Host "Triggering subscription '$subscriptionToTrigger'."
+
+      Trigger-Subscription -SubscriptionId $subscriptionToTrigger
+    
+      Write-Host 'done.'
+    } 
+    catch
+    {
+      Write-Host "There was an error while triggering subscription '$subscriptionToTrigger'"
+      Write-Host $_
+      Write-Host $_.ScriptStackTrace
+      $failedTriggeredSubscription = $true
+    }
+  }
+
+  if ($subscriptionsToTrigger.Count -eq 0) {
+    Write-Host "No subscription matched source repo '$normalizedSourceRepo' and channel ID '$ChannelId'."
+  }
+  elseif ($failedTriggeredSubscription) {
+    Write-PipelineTelemetryError -Category 'TriggerSubscriptions' -Message 'At least one subscription failed to be triggered...'
+    ExitWithExitCode 1
+  }
+  else {
+    Write-Host 'All subscriptions were triggered successfully!'
+  }
+}
+catch {
+  Write-Host $_.ScriptStackTrace
+  Write-PipelineTelemetryError -Category 'TriggerSubscriptions' -Message $_
+  ExitWithExitCode 1
+}

eng/common/sdk-task.ps1

@@ -1,8 +1,8 @@
 [CmdletBinding(PositionalBinding=$false)]
 Param(
-  [string] $configuration = "Debug",
+  [string] $configuration = 'Debug',
   [string] $task,
-  [string] $verbosity = "minimal",
+  [string] $verbosity = 'minimal',
   [string] $msbuildEngine = $null,
   [switch] $restore,
   [switch] $prepareMachine,
@@ -32,7 +32,7 @@ function Print-Usage() {
 }
 
 function Build([string]$target) {
-  $logSuffix = if ($target -eq "Execute") { "" } else { ".$target" }
+  $logSuffix = if ($target -eq 'Execute') { '' } else { ".$target" }
   $log = Join-Path $LogDir "$task$logSuffix.binlog"
   $outputPath = Join-Path $ToolsetDir "$task\\"
 
@@ -42,37 +42,55 @@ function Build([string]$target) {
     /p:Configuration=$configuration `
     /p:RepoRoot=$RepoRoot `
     /p:BaseIntermediateOutputPath=$outputPath `
+    /v:$verbosity `
     @properties
 }
 
 try {
-  if ($help -or (($null -ne $properties) -and ($properties.Contains("/help") -or $properties.Contains("/?")))) {
+  if ($help -or (($null -ne $properties) -and ($properties.Contains('/help') -or $properties.Contains('/?')))) {
     Print-Usage
     exit 0
   }
 
   if ($task -eq "") {
-    Write-Host "Missing required parameter '-task <value>'" -ForegroundColor Red
+    Write-PipelineTelemetryError -Category 'Build' -Message "Missing required parameter '-task <value>'" -ForegroundColor Red
     Print-Usage
     ExitWithExitCode 1
   }
 
+  if( $msbuildEngine -eq "vs") {
+    # Ensure desktop MSBuild is available for sdk tasks.
+    if( -not ($GlobalJson.tools.PSObject.Properties.Name -contains "vs" )) {
+      $GlobalJson.tools | Add-Member -Name "vs" -Value (ConvertFrom-Json "{ `"version`": `"16.5`" }") -MemberType NoteProperty
+    }
+    if( -not ($GlobalJson.tools.PSObject.Properties.Name -match "xcopy-msbuild" )) {
+      $GlobalJson.tools | Add-Member -Name "xcopy-msbuild" -Value "16.8.0-preview3" -MemberType NoteProperty
+    }
+    if ($GlobalJson.tools."xcopy-msbuild".Trim() -ine "none") {
+        $xcopyMSBuildToolsFolder = InitializeXCopyMSBuild $GlobalJson.tools."xcopy-msbuild" -install $true
+    }
+    if ($xcopyMSBuildToolsFolder -eq $null) {
+      throw 'Unable to get xcopy downloadable version of msbuild'
+    }
+
+    $global:_MSBuildExe = "$($xcopyMSBuildToolsFolder)\MSBuild\Current\Bin\MSBuild.exe"
+  }
+
   $taskProject = GetSdkTaskProject $task
   if (!(Test-Path $taskProject)) {
-    Write-Host "Unknown task: $task" -ForegroundColor Red
+    Write-PipelineTelemetryError -Category 'Build' -Message "Unknown task: $task" -ForegroundColor Red
     ExitWithExitCode 1
   }
 
   if ($restore) {
-    Build "Restore"
+    Build 'Restore'
   }
 
-  Build "Execute"
+  Build 'Execute'
 }
 catch {
-  Write-Host $_
-  Write-Host $_.Exception
   Write-Host $_.ScriptStackTrace
+  Write-PipelineTelemetryError -Category 'Build' -Message $_
   ExitWithExitCode 1
 }
 

eng/common/sdl/NuGet.config

@@ -0,0 +1,13 @@
+<?xml version="1.0" encoding="utf-8"?>
+<configuration>
+  <solution>
+    <add key="disableSourceControlIntegration" value="true" />
+  </solution>
+  <packageSources>
+    <clear />
+    <add key="guardian" value="https://securitytools.pkgs.visualstudio.com/_packaging/Guardian/nuget/v3/index.json" />
+  </packageSources>
+  <disabledPackageSources>
+    <clear />
+  </disabledPackageSources>
+</configuration>

eng/common/sdl/execute-all-sdl-tools.ps1

@@ -0,0 +1,116 @@
+Param(
+  [string] $GuardianPackageName,                                                                 # Required: the name of guardian CLI package (not needed if GuardianCliLocation is specified)
+  [string] $NugetPackageDirectory,                                                               # Required: directory where NuGet packages are installed (not needed if GuardianCliLocation is specified)
+  [string] $GuardianCliLocation,                                                                 # Optional: Direct location of Guardian CLI executable if GuardianPackageName & NugetPackageDirectory are not specified
+  [string] $Repository=$env:BUILD_REPOSITORY_NAME,                                               # Required: the name of the repository (e.g. dotnet/arcade)
+  [string] $BranchName=$env:BUILD_SOURCEBRANCH,                                                  # Optional: name of branch or version of gdn settings; defaults to master
+  [string] $SourceDirectory=$env:BUILD_SOURCESDIRECTORY,                                         # Required: the directory where source files are located
+  [string] $ArtifactsDirectory = (Join-Path $env:BUILD_ARTIFACTSTAGINGDIRECTORY ('artifacts')),  # Required: the directory where build artifacts are located
+  [string] $AzureDevOpsAccessToken,                                                              # Required: access token for dnceng; should be provided via KeyVault
+  [string[]] $SourceToolsList,                                                                   # Optional: list of SDL tools to run on source code
+  [string[]] $ArtifactToolsList,                                                                 # Optional: list of SDL tools to run on built artifacts
+  [bool] $TsaPublish=$False,                                                                     # Optional: true will publish results to TSA; only set to true after onboarding to TSA; TSA is the automated framework used to upload test results as bugs.
+  [string] $TsaBranchName=$env:BUILD_SOURCEBRANCH,                                               # Optional: required for TSA publish; defaults to $(Build.SourceBranchName); TSA is the automated framework used to upload test results as bugs.
+  [string] $TsaRepositoryName=$env:BUILD_REPOSITORY_NAME,                                        # Optional: TSA repository name; will be generated automatically if not submitted; TSA is the automated framework used to upload test results as bugs.
+  [string] $BuildNumber=$env:BUILD_BUILDNUMBER,                                                  # Optional: required for TSA publish; defaults to $(Build.BuildNumber)
+  [bool] $UpdateBaseline=$False,                                                                 # Optional: if true, will update the baseline in the repository; should only be run after fixing any issues which need to be fixed
+  [bool] $TsaOnboard=$False,                                                                     # Optional: if true, will onboard the repository to TSA; should only be run once; TSA is the automated framework used to upload test results as bugs.
+  [string] $TsaInstanceUrl,                                                                      # Optional: only needed if TsaOnboard or TsaPublish is true; the instance-url registered with TSA; TSA is the automated framework used to upload test results as bugs.
+  [string] $TsaCodebaseName,                                                                     # Optional: only needed if TsaOnboard or TsaPublish is true; the name of the codebase registered with TSA; TSA is the automated framework used to upload test results as bugs.
+  [string] $TsaProjectName,                                                                      # Optional: only needed if TsaOnboard or TsaPublish is true; the name of the project registered with TSA; TSA is the automated framework used to upload test results as bugs.
+  [string] $TsaNotificationEmail,                                                                # Optional: only needed if TsaOnboard is true; the email(s) which will receive notifications of TSA bug filings (e.g. alias@microsoft.com); TSA is the automated framework used to upload test results as bugs.
+  [string] $TsaCodebaseAdmin,                                                                    # Optional: only needed if TsaOnboard is true; the aliases which are admins of the TSA codebase (e.g. DOMAIN\alias); TSA is the automated framework used to upload test results as bugs.
+  [string] $TsaBugAreaPath,                                                                      # Optional: only needed if TsaOnboard is true; the area path where TSA will file bugs in AzDO; TSA is the automated framework used to upload test results as bugs.
+  [string] $TsaIterationPath,                                                                    # Optional: only needed if TsaOnboard is true; the iteration path where TSA will file bugs in AzDO; TSA is the automated framework used to upload test results as bugs.
+  [string] $GuardianLoggerLevel='Standard',                                                      # Optional: the logger level for the Guardian CLI; options are Trace, Verbose, Standard, Warning, and Error
+  [string[]] $CrScanAdditionalRunConfigParams,                                                   # Optional: Additional Params to custom build a CredScan run config in the format @("xyz:abc","sdf:1")
+  [string[]] $PoliCheckAdditionalRunConfigParams,                                                # Optional: Additional Params to custom build a Policheck run config in the format @("xyz:abc","sdf:1")
+  [bool] $BreakOnFailure=$False                                                                  # Optional: Fail the build if there were errors during the run
+)
+
+try {
+  $ErrorActionPreference = 'Stop'
+  Set-StrictMode -Version 2.0
+  $disableConfigureToolsetImport = $true
+  $LASTEXITCODE = 0
+
+  # `tools.ps1` checks $ci to perform some actions. Since the SDL
+  # scripts don't necessarily execute in the same agent that run the
+  # build.ps1/sh script this variable isn't automatically set.
+  $ci = $true
+  . $PSScriptRoot\..\tools.ps1
+
+  #Replace repo names to the format of org/repo
+  if (!($Repository.contains('/'))) {
+    $RepoName = $Repository -replace '(.*?)-(.*)', '$1/$2';
+  }
+  else{
+    $RepoName = $Repository;
+  }
+
+  if ($GuardianPackageName) {
+    $guardianCliLocation = Join-Path $NugetPackageDirectory (Join-Path $GuardianPackageName (Join-Path 'tools' 'guardian.cmd'))
+  } else {
+    $guardianCliLocation = $GuardianCliLocation
+  }
+
+  $workingDirectory = (Split-Path $SourceDirectory -Parent)
+  $ValidPath = Test-Path $guardianCliLocation
+
+  if ($ValidPath -eq $False)
+  {
+    Write-PipelineTelemetryError -Force -Category 'Sdl' -Message 'Invalid Guardian CLI Location.'
+    ExitWithExitCode 1
+  }
+
+  & $(Join-Path $PSScriptRoot 'init-sdl.ps1') -GuardianCliLocation $guardianCliLocation -Repository $RepoName -BranchName $BranchName -WorkingDirectory $workingDirectory -AzureDevOpsAccessToken $AzureDevOpsAccessToken -GuardianLoggerLevel $GuardianLoggerLevel
+  $gdnFolder = Join-Path $workingDirectory '.gdn'
+
+  if ($TsaOnboard) {
+    if ($TsaCodebaseName -and $TsaNotificationEmail -and $TsaCodebaseAdmin -and $TsaBugAreaPath) {
+      Write-Host "$guardianCliLocation tsa-onboard --codebase-name `"$TsaCodebaseName`" --notification-alias `"$TsaNotificationEmail`" --codebase-admin `"$TsaCodebaseAdmin`" --instance-url `"$TsaInstanceUrl`" --project-name `"$TsaProjectName`" --area-path `"$TsaBugAreaPath`" --iteration-path `"$TsaIterationPath`" --working-directory $workingDirectory --logger-level $GuardianLoggerLevel"
+      & $guardianCliLocation tsa-onboard --codebase-name "$TsaCodebaseName" --notification-alias "$TsaNotificationEmail" --codebase-admin "$TsaCodebaseAdmin" --instance-url "$TsaInstanceUrl" --project-name "$TsaProjectName" --area-path "$TsaBugAreaPath" --iteration-path "$TsaIterationPath" --working-directory $workingDirectory --logger-level $GuardianLoggerLevel
+      if ($LASTEXITCODE -ne 0) {
+        Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Guardian tsa-onboard failed with exit code $LASTEXITCODE."
+        ExitWithExitCode $LASTEXITCODE
+      }
+    } else {
+      Write-PipelineTelemetryError -Force -Category 'Sdl' -Message 'Could not onboard to TSA -- not all required values ($TsaCodebaseName, $TsaNotificationEmail, $TsaCodebaseAdmin, $TsaBugAreaPath) were specified.'
+      ExitWithExitCode 1
+    }
+  }
+
+  if ($ArtifactToolsList -and $ArtifactToolsList.Count -gt 0) {
+    & $(Join-Path $PSScriptRoot 'run-sdl.ps1') -GuardianCliLocation $guardianCliLocation -WorkingDirectory $workingDirectory -TargetDirectory $ArtifactsDirectory -GdnFolder $gdnFolder -ToolsList $ArtifactToolsList -AzureDevOpsAccessToken $AzureDevOpsAccessToken -UpdateBaseline $UpdateBaseline -GuardianLoggerLevel $GuardianLoggerLevel -CrScanAdditionalRunConfigParams $CrScanAdditionalRunConfigParams -PoliCheckAdditionalRunConfigParams $PoliCheckAdditionalRunConfigParams
+  }
+  if ($SourceToolsList -and $SourceToolsList.Count -gt 0) {
+    & $(Join-Path $PSScriptRoot 'run-sdl.ps1') -GuardianCliLocation $guardianCliLocation -WorkingDirectory $workingDirectory -TargetDirectory $SourceDirectory -GdnFolder $gdnFolder -ToolsList $SourceToolsList -AzureDevOpsAccessToken $AzureDevOpsAccessToken -UpdateBaseline $UpdateBaseline -GuardianLoggerLevel $GuardianLoggerLevel -CrScanAdditionalRunConfigParams $CrScanAdditionalRunConfigParams -PoliCheckAdditionalRunConfigParams $PoliCheckAdditionalRunConfigParams
+  }
+
+  if ($TsaPublish) {
+    if ($TsaBranchName -and $BuildNumber) {
+      if (-not $TsaRepositoryName) {
+        $TsaRepositoryName = "$($Repository)-$($BranchName)"
+      }
+      Write-Host "$guardianCliLocation tsa-publish --all-tools --repository-name `"$TsaRepositoryName`" --branch-name `"$TsaBranchName`" --build-number `"$BuildNumber`" --codebase-name `"$TsaCodebaseName`" --notification-alias `"$TsaNotificationEmail`" --codebase-admin `"$TsaCodebaseAdmin`" --instance-url `"$TsaInstanceUrl`" --project-name `"$TsaProjectName`" --area-path `"$TsaBugAreaPath`" --iteration-path `"$TsaIterationPath`" --working-directory $workingDirectory --logger-level $GuardianLoggerLevel"
+      & $guardianCliLocation tsa-publish --all-tools --repository-name "$TsaRepositoryName" --branch-name "$TsaBranchName" --build-number "$BuildNumber" --onboard $True --codebase-name "$TsaCodebaseName" --notification-alias "$TsaNotificationEmail" --codebase-admin "$TsaCodebaseAdmin" --instance-url "$TsaInstanceUrl" --project-name "$TsaProjectName" --area-path "$TsaBugAreaPath" --iteration-path "$TsaIterationPath" --working-directory $workingDirectory  --logger-level $GuardianLoggerLevel
+      if ($LASTEXITCODE -ne 0) {
+        Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Guardian tsa-publish failed with exit code $LASTEXITCODE."
+        ExitWithExitCode $LASTEXITCODE
+      }
+    } else {
+      Write-PipelineTelemetryError -Force -Category 'Sdl' -Message 'Could not publish to TSA -- not all required values ($TsaBranchName, $BuildNumber) were specified.'
+      ExitWithExitCode 1
+    }
+  }
+
+  if ($BreakOnFailure) {
+    Write-Host "Failing the build in case of breaking results..."
+    & $guardianCliLocation break
+  }
+}
+catch {
+  Write-Host $_.ScriptStackTrace
+  Write-PipelineTelemetryError -Force -Category 'Sdl' -Message $_
+  exit 1
+}

eng/common/sdl/extract-artifact-packages.ps1

@@ -0,0 +1,80 @@
+param(
+  [Parameter(Mandatory=$true)][string] $InputPath,              # Full path to directory where artifact packages are stored
+  [Parameter(Mandatory=$true)][string] $ExtractPath            # Full path to directory where the packages will be extracted
+)
+
+$ErrorActionPreference = 'Stop'
+Set-StrictMode -Version 2.0
+
+$disableConfigureToolsetImport = $true
+
+function ExtractArtifacts {
+  if (!(Test-Path $InputPath)) {
+    Write-Host "Input Path does not exist: $InputPath"
+    ExitWithExitCode 0
+  }
+  $Jobs = @()
+  Get-ChildItem "$InputPath\*.nupkg" |
+    ForEach-Object {
+      $Jobs += Start-Job -ScriptBlock $ExtractPackage -ArgumentList $_.FullName
+    }
+
+  foreach ($Job in $Jobs) {
+    Wait-Job -Id $Job.Id | Receive-Job
+  }
+}
+
+try {
+  # `tools.ps1` checks $ci to perform some actions. Since the SDL
+  # scripts don't necessarily execute in the same agent that run the
+  # build.ps1/sh script this variable isn't automatically set.
+  $ci = $true
+  . $PSScriptRoot\..\tools.ps1
+
+  $ExtractPackage = {
+    param( 
+      [string] $PackagePath                                 # Full path to a NuGet package
+    )
+    
+    if (!(Test-Path $PackagePath)) {
+      Write-PipelineTelemetryError -Category 'Build' -Message "Input file does not exist: $PackagePath"
+      ExitWithExitCode 1
+    }
+    
+    $RelevantExtensions = @('.dll', '.exe', '.pdb')
+    Write-Host -NoNewLine 'Extracting ' ([System.IO.Path]::GetFileName($PackagePath)) '...'
+  
+    $PackageId = [System.IO.Path]::GetFileNameWithoutExtension($PackagePath)
+    $ExtractPath = Join-Path -Path $using:ExtractPath -ChildPath $PackageId
+  
+    Add-Type -AssemblyName System.IO.Compression.FileSystem
+  
+    [System.IO.Directory]::CreateDirectory($ExtractPath);
+  
+    try {
+      $zip = [System.IO.Compression.ZipFile]::OpenRead($PackagePath)
+  
+      $zip.Entries | 
+      Where-Object {$RelevantExtensions -contains [System.IO.Path]::GetExtension($_.Name)} |
+        ForEach-Object {
+            $TargetFile = Join-Path -Path $ExtractPath -ChildPath $_.Name
+  
+            [System.IO.Compression.ZipFileExtensions]::ExtractToFile($_, $TargetFile, $true)
+          }
+    }
+    catch {
+      Write-Host $_
+      Write-PipelineTelemetryError -Force -Category 'Sdl' -Message $_
+      ExitWithExitCode 1
+    }
+    finally {
+      $zip.Dispose() 
+    }
+  }
+  Measure-Command { ExtractArtifacts }
+}
+catch {
+  Write-Host $_
+  Write-PipelineTelemetryError -Force -Category 'Sdl' -Message $_
+  ExitWithExitCode 1
+}

eng/common/sdl/init-sdl.ps1

@@ -0,0 +1,55 @@
+Param(
+  [string] $GuardianCliLocation,
+  [string] $Repository,
+  [string] $BranchName='master',
+  [string] $WorkingDirectory,
+  [string] $AzureDevOpsAccessToken,
+  [string] $GuardianLoggerLevel='Standard'
+)
+
+$ErrorActionPreference = 'Stop'
+Set-StrictMode -Version 2.0
+$disableConfigureToolsetImport = $true
+$LASTEXITCODE = 0
+
+# `tools.ps1` checks $ci to perform some actions. Since the SDL
+# scripts don't necessarily execute in the same agent that run the
+# build.ps1/sh script this variable isn't automatically set.
+$ci = $true
+. $PSScriptRoot\..\tools.ps1
+
+# Don't display the console progress UI - it's a huge perf hit
+$ProgressPreference = 'SilentlyContinue'
+
+# Construct basic auth from AzDO access token; construct URI to the repository's gdn folder stored in that repository; construct location of zip file
+$encodedPat = [Convert]::ToBase64String([System.Text.Encoding]::ASCII.GetBytes(":$AzureDevOpsAccessToken"))
+$escapedRepository = [Uri]::EscapeDataString("/$Repository/$BranchName/.gdn")
+$uri = "https://dev.azure.com/dnceng/internal/_apis/git/repositories/sdl-tool-cfg/Items?path=$escapedRepository&versionDescriptor[versionOptions]=0&`$format=zip&api-version=5.0"
+$zipFile = "$WorkingDirectory/gdn.zip"
+
+Add-Type -AssemblyName System.IO.Compression.FileSystem
+$gdnFolder = (Join-Path $WorkingDirectory '.gdn')
+
+try {
+  # if the folder does not exist, we'll do a guardian init and push it to the remote repository
+  Write-Host 'Initializing Guardian...'
+  Write-Host "$GuardianCliLocation init --working-directory $WorkingDirectory --logger-level $GuardianLoggerLevel"
+  & $GuardianCliLocation init --working-directory $WorkingDirectory --logger-level $GuardianLoggerLevel
+  if ($LASTEXITCODE -ne 0) {
+    Write-PipelineTelemetryError -Force -Category 'Build' -Message "Guardian init failed with exit code $LASTEXITCODE."
+    ExitWithExitCode $LASTEXITCODE
+  }
+  # We create the mainbaseline so it can be edited later
+  Write-Host "$GuardianCliLocation baseline --working-directory $WorkingDirectory --name mainbaseline"
+  & $GuardianCliLocation baseline --working-directory $WorkingDirectory --name mainbaseline
+  if ($LASTEXITCODE -ne 0) {
+    Write-PipelineTelemetryError -Force -Category 'Build' -Message "Guardian baseline failed with exit code $LASTEXITCODE."
+    ExitWithExitCode $LASTEXITCODE
+  }
+  ExitWithExitCode 0
+}
+catch {
+  Write-Host $_.ScriptStackTrace
+  Write-PipelineTelemetryError -Force -Category 'Sdl' -Message $_
+  ExitWithExitCode 1
+}

eng/common/sdl/packages.config

@@ -0,0 +1,4 @@
+<?xml version="1.0" encoding="utf-8"?>
+<packages>
+  <package id="Microsoft.Guardian.Cli" version="0.53.3"/>
+</packages>

eng/common/sdl/run-sdl.ps1

@@ -0,0 +1,73 @@
+Param(
+  [string] $GuardianCliLocation,
+  [string] $WorkingDirectory,
+  [string] $TargetDirectory,
+  [string] $GdnFolder,
+  [string[]] $ToolsList,
+  [string] $UpdateBaseline,
+  [string] $GuardianLoggerLevel='Standard',
+  [string[]] $CrScanAdditionalRunConfigParams,
+  [string[]] $PoliCheckAdditionalRunConfigParams
+)
+
+$ErrorActionPreference = 'Stop'
+Set-StrictMode -Version 2.0
+$disableConfigureToolsetImport = $true
+$LASTEXITCODE = 0
+
+try {
+  # `tools.ps1` checks $ci to perform some actions. Since the SDL
+  # scripts don't necessarily execute in the same agent that run the
+  # build.ps1/sh script this variable isn't automatically set.
+  $ci = $true
+  . $PSScriptRoot\..\tools.ps1
+
+  # We store config files in the r directory of .gdn
+  Write-Host $ToolsList
+  $gdnConfigPath = Join-Path $GdnFolder 'r'
+  $ValidPath = Test-Path $GuardianCliLocation
+
+  if ($ValidPath -eq $False)
+  {
+    Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Invalid Guardian CLI Location."
+    ExitWithExitCode 1
+  }
+
+  $configParam = @('--config')
+
+  foreach ($tool in $ToolsList) {
+    $gdnConfigFile = Join-Path $gdnConfigPath "$tool-configure.gdnconfig"
+    Write-Host $tool
+    # We have to manually configure tools that run on source to look at the source directory only
+    if ($tool -eq 'credscan') {
+      Write-Host "$GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args `" TargetDirectory < $TargetDirectory `" `" OutputType < pre `" $(If ($CrScanAdditionalRunConfigParams) {$CrScanAdditionalRunConfigParams})"
+      & $GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args " TargetDirectory < $TargetDirectory " "OutputType < pre" $(If ($CrScanAdditionalRunConfigParams) {$CrScanAdditionalRunConfigParams})
+      if ($LASTEXITCODE -ne 0) {
+        Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Guardian configure for $tool failed with exit code $LASTEXITCODE."
+        ExitWithExitCode $LASTEXITCODE
+      }
+    }
+    if ($tool -eq 'policheck') {
+      Write-Host "$GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args `" Target < $TargetDirectory `" $(If ($PoliCheckAdditionalRunConfigParams) {$PoliCheckAdditionalRunConfigParams})"
+      & $GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args " Target < $TargetDirectory " $(If ($PoliCheckAdditionalRunConfigParams) {$PoliCheckAdditionalRunConfigParams})
+      if ($LASTEXITCODE -ne 0) {
+        Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Guardian configure for $tool failed with exit code $LASTEXITCODE."
+        ExitWithExitCode $LASTEXITCODE
+      }
+    }
+
+    $configParam+=$gdnConfigFile
+  }
+
+  Write-Host "$GuardianCliLocation run --working-directory $WorkingDirectory --baseline mainbaseline --update-baseline $UpdateBaseline --logger-level $GuardianLoggerLevel $configParam"
+  & $GuardianCliLocation run --working-directory $WorkingDirectory --tool $tool --baseline mainbaseline --update-baseline $UpdateBaseline --logger-level $GuardianLoggerLevel $configParam
+  if ($LASTEXITCODE -ne 0) {
+    Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Guardian run for $ToolsList using $configParam failed with exit code $LASTEXITCODE."
+    ExitWithExitCode $LASTEXITCODE
+  }
+}
+catch {
+  Write-Host $_.ScriptStackTrace
+  Write-PipelineTelemetryError -Force -Category 'Sdl' -Message $_
+  ExitWithExitCode 1
+}

eng/common/templates/job/execute-sdl.yml

@@ -0,0 +1,93 @@
+parameters:
+  enable: 'false'                                             # Whether the SDL validation job should execute or not
+  overrideParameters: ''                                       # Optional: to override values for parameters.
+  additionalParameters: ''                                     # Optional: parameters that need user specific values eg: '-SourceToolsList @("abc","def") -ArtifactToolsList @("ghi","jkl")'
+  # There is some sort of bug (has been reported) in Azure DevOps where if this parameter is named
+  # 'continueOnError', the parameter value is not correctly picked up.
+  # This can also be remedied by the caller (post-build.yml) if it does not use a nested parameter
+  sdlContinueOnError: false                                    # optional: determines whether to continue the build if the step errors;
+  downloadArtifacts: true                                      # optional: determines if the artifacts should be dowloaded
+  dependsOn: ''                                                # Optional: dependencies of the job
+  artifactNames: ''                                            # Optional: patterns supplied to DownloadBuildArtifacts
+                                                               # Usage:
+                                                               #  artifactNames:
+                                                               #    - 'BlobArtifacts'
+                                                               #    - 'Artifacts_Windows_NT_Release'
+
+jobs:
+- job: Run_SDL
+  dependsOn: ${{ parameters.dependsOn }}
+  displayName: Run SDL tool
+  condition: eq( ${{ parameters.enable }}, 'true')
+  variables:
+    - group: DotNet-VSTS-Bot
+    - name: AzDOProjectName
+      value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ]
+    - name: AzDOPipelineId
+      value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ]
+    - name: AzDOBuildId
+      value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ]
+  pool:
+    name: Hosted VS2017
+  steps:
+  - checkout: self
+    clean: true
+  - ${{ if ne(parameters.downloadArtifacts, 'false')}}:
+    - ${{ if ne(parameters.artifactNames, '') }}:
+      - ${{ each artifactName in parameters.artifactNames }}:
+        - task: DownloadBuildArtifacts@0
+          displayName: Download Build Artifacts
+          inputs:
+            buildType: specific
+            buildVersionToDownload: specific
+            project: $(AzDOProjectName)
+            pipeline: $(AzDOPipelineId)
+            buildId: $(AzDOBuildId)
+            artifactName: ${{ artifactName }}
+            downloadPath: $(Build.ArtifactStagingDirectory)\artifacts
+            checkDownloadedFiles: true
+    - ${{ if eq(parameters.artifactNames, '') }}:
+      - task: DownloadBuildArtifacts@0
+        displayName: Download Build Artifacts
+        inputs:
+          buildType: specific
+          buildVersionToDownload: specific
+          project: $(AzDOProjectName)
+          pipeline: $(AzDOPipelineId)
+          buildId: $(AzDOBuildId)
+          downloadType: specific files
+          itemPattern: "**"
+          downloadPath: $(Build.ArtifactStagingDirectory)\artifacts
+          checkDownloadedFiles: true
+  - powershell: eng/common/sdl/extract-artifact-packages.ps1
+      -InputPath $(Build.ArtifactStagingDirectory)\artifacts\BlobArtifacts
+      -ExtractPath $(Build.ArtifactStagingDirectory)\artifacts\BlobArtifacts
+    displayName: Extract Blob Artifacts
+    continueOnError: ${{ parameters.sdlContinueOnError }}
+  - powershell: eng/common/sdl/extract-artifact-packages.ps1
+      -InputPath $(Build.ArtifactStagingDirectory)\artifacts\PackageArtifacts
+      -ExtractPath $(Build.ArtifactStagingDirectory)\artifacts\PackageArtifacts
+    displayName: Extract Package Artifacts
+    continueOnError: ${{ parameters.sdlContinueOnError }}
+  - task: NuGetToolInstaller@1
+    displayName: 'Install NuGet.exe'
+  - task: NuGetCommand@2
+    displayName: 'Install Guardian'
+    inputs:
+      restoreSolution: $(Build.SourcesDirectory)\eng\common\sdl\packages.config
+      feedsToUse: config
+      nugetConfigPath: $(Build.SourcesDirectory)\eng\common\sdl\NuGet.config
+      externalFeedCredentials: GuardianConnect
+      restoreDirectory: $(Build.SourcesDirectory)\.packages
+  - ${{ if ne(parameters.overrideParameters, '') }}:
+    - powershell: eng/common/sdl/execute-all-sdl-tools.ps1 ${{ parameters.overrideParameters }}
+      displayName: Execute SDL
+      continueOnError: ${{ parameters.sdlContinueOnError }}
+  - ${{ if eq(parameters.overrideParameters, '') }}:
+    - powershell: eng/common/sdl/execute-all-sdl-tools.ps1
+        -GuardianPackageName Microsoft.Guardian.Cli.0.53.3
+        -NugetPackageDirectory $(Build.SourcesDirectory)\.packages
+        -AzureDevOpsAccessToken $(dn-bot-dotnet-build-rw-code-rw)
+        ${{ parameters.additionalParameters }}
+      displayName: Execute SDL
+      continueOnError: ${{ parameters.sdlContinueOnError }}

eng/common/templates/job/job.yml

@@ -1,61 +1,36 @@
+# Internal resources (telemetry, microbuild) can only be accessed from non-public projects,
+# and some (Microbuild) should only be applied to non-PR cases for internal builds.
+
 parameters:
 # Job schema parameters - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#job
   cancelTimeoutInMinutes: ''
-
   condition: ''
-
-  continueOnError: false
-
   container: ''
-
+  continueOnError: false
   dependsOn: ''
-
   displayName: ''
-
-  steps: []
-
   pool: ''
-
+  steps: []
   strategy: ''
-
   timeoutInMinutes: ''
-
   variables: []
-
   workspace: ''
 
 # Job base template specific parameters
-  # Optional: Enable installing Microbuild plugin
-  #           if 'true', these "variables" must be specified in the variables object or as part of the queue matrix
-  #             _TeamName - the name of your team
-  #             _SignType - 'test' or 'real'
+  # See schema documentation - https://github.com/dotnet/arcade/blob/master/Documentation/AzureDevOps/TemplateSchema.md
+  artifacts: ''
   enableMicrobuild: false
-
-  # Optional: Include PublishBuildArtifacts task
   enablePublishBuildArtifacts: false
-
-  # Optional: Enable publishing to the build asset registry
   enablePublishBuildAssets: false
-
-  # Optional: Include PublishTestResults task
   enablePublishTestResults: false
-
-  # Optional: enable sending telemetry
-  enableTelemetry: false
-
-  # Optional: define the helix repo for telemeetry (example: 'dotnet/arcade')
-  helixRepo: ''
-
-  # Required: name of the job
+  enablePublishUsingPipelines: false
+  mergeTestResults: false
+  testRunTitle: ''
+  testResultsFormat: ''
   name: ''
-
-  # Optional: should run as a public build even in the internal project
-  #           if 'true', the build won't run any of the internal only steps, even if it is running in non-public projects.
+  preSteps: []
   runAsPublic: false
 
-# Internal resources (telemetry, microbuild) can only be accessed from non-public projects,
-# and some (Microbuild) should only be applied to non-PR cases for internal builds.
-
 jobs:
 - job: ${{ parameters.name }}
 
@@ -87,6 +62,12 @@ jobs:
     timeoutInMinutes: ${{ parameters.timeoutInMinutes }}
 
   variables:
+  - ${{ if ne(parameters.enableTelemetry, 'false') }}:
+    - name: DOTNET_CLI_TELEMETRY_PROFILE
+      value: '$(Build.Repository.Uri)'
+  - ${{ if eq(parameters.enableRichCodeNavigation, 'true') }}:
+    - name: EnableRichCodeNavigation
+      value: 'true'
   - ${{ each variable in parameters.variables }}:
     # handle name-value variable syntax
     # example:
@@ -116,19 +97,12 @@ jobs:
     workspace: ${{ parameters.workspace }}
 
   steps:
-  - ${{ if eq(parameters.enableTelemetry, 'true') }}:
-    # Telemetry tasks are built from https://github.com/dotnet/arcade-extensions
-    - task: sendStartTelemetry@0
-      displayName: 'Send Helix Start Telemetry'
-      inputs:
-        helixRepo: ${{ parameters.helixRepo }}
-        buildConfig: $(_BuildConfig)
-        runAsPublic: ${{ parameters.runAsPublic }}
-      continueOnError: ${{ parameters.continueOnError }}
-      condition: always()
+  - ${{ if ne(parameters.preSteps, '') }}:
+    - ${{ each preStep in parameters.preSteps }}:
+      - ${{ preStep }}
 
-  - ${{ if eq(parameters.enableMicrobuild, 'true') }}:
-    - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
+  - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
+    - ${{ if eq(parameters.enableMicrobuild, 'true') }}:
       - task: MicroBuildSigningPlugin@2
         displayName: Install MicroBuild plugin
         inputs:
@@ -140,9 +114,28 @@ jobs:
         continueOnError: ${{ parameters.continueOnError }}
         condition: and(succeeded(), in(variables['_SignType'], 'real', 'test'), eq(variables['Agent.Os'], 'Windows_NT'))
 
+    - task: NuGetAuthenticate@0
+
+  - ${{ if or(eq(parameters.artifacts.download, 'true'), ne(parameters.artifacts.download, '')) }}:
+    - task: DownloadPipelineArtifact@2
+      inputs:
+        buildType: current
+        artifactName: ${{ coalesce(parameters.artifacts.download.name, 'Artifacts_$(Agent.OS)_$(_BuildConfig)') }}
+        targetPath: ${{ coalesce(parameters.artifacts.download.path, 'artifacts') }}
+        itemPattern: ${{ coalesce(parameters.artifacts.download.pattern, '**') }}
+
   - ${{ each step in parameters.steps }}:
     - ${{ step }}
 
+  - ${{ if eq(parameters.enableRichCodeNavigation, true) }}:
+    - task: RichCodeNavIndexer@0
+      displayName: RichCodeNav Upload
+      inputs:
+        languages: ${{ coalesce(parameters.richCodeNavigationLanguage, 'csharp') }}
+        environment: ${{ coalesce(parameters.richCodeNavigationEnvironment, 'production') }}
+        richNavLogOutputDirectory: $(Build.SourcesDirectory)/artifacts/bin
+      continueOnError: true
+
   - ${{ if eq(parameters.enableMicrobuild, 'true') }}:
     - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
       - task: MicroBuildCleanup@1
@@ -152,34 +145,87 @@ jobs:
         env:
           TeamName: $(_TeamName)
 
-  - ${{ if eq(parameters.enableTelemetry, 'true') }}:
-    # Telemetry tasks are built from https://github.com/dotnet/arcade-extensions
-    - task: sendEndTelemetry@0
-      displayName: 'Send Helix End Telemetry'
-      continueOnError: ${{ parameters.continueOnError }}
-      condition: always()
+  - ${{ if ne(parameters.artifacts.publish, '') }}:
+    - ${{ if or(eq(parameters.artifacts.publish.artifacts, 'true'), ne(parameters.artifacts.publish.artifacts, '')) }}:
+      - task: CopyFiles@2
+        displayName: Gather binaries for publish to artifacts
+        inputs:
+          SourceFolder: 'artifacts/bin'
+          Contents: '**'
+          TargetFolder: '$(Build.ArtifactStagingDirectory)/artifacts/bin'
+      - task: CopyFiles@2
+        displayName: Gather packages for publish to artifacts
+        inputs:
+          SourceFolder: 'artifacts/packages'
+          Contents: '**'
+          TargetFolder: '$(Build.ArtifactStagingDirectory)/artifacts/packages'
+      - task: PublishBuildArtifacts@1
+        displayName: Publish pipeline artifacts
+        inputs:
+          PathtoPublish: '$(Build.ArtifactStagingDirectory)/artifacts'
+          PublishLocation: Container
+          ArtifactName: ${{ coalesce(parameters.artifacts.publish.artifacts.name , 'Artifacts_$(Agent.Os)_$(_BuildConfig)') }}
+        continueOnError: true
+        condition: always()
+    - ${{ if or(eq(parameters.artifacts.publish.logs, 'true'), ne(parameters.artifacts.publish.logs, '')) }}:
+      - publish: artifacts/log
+        artifact: ${{ coalesce(parameters.artifacts.publish.logs.name, 'Logs_Build_$(Agent.Os)_$(_BuildConfig)') }}
+        displayName: Publish logs
+        continueOnError: true
+        condition: always()
+    - ${{ if or(eq(parameters.artifacts.publish.manifests, 'true'), ne(parameters.artifacts.publish.manifests, '')) }}:
+      - ${{ if and(ne(parameters.enablePublishUsingPipelines, 'true'), eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:    
+        - task: CopyFiles@2
+          displayName: Gather Asset Manifests
+          inputs:
+            SourceFolder: '$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)/AssetManifest'
+            TargetFolder: '$(Build.ArtifactStagingDirectory)/AssetManifests'
+          continueOnError: ${{ parameters.continueOnError }}
+          condition: and(succeeded(), eq(variables['_DotNetPublishToBlobFeed'], 'true'))
 
-  - ${{ if eq(parameters.enablePublishBuildArtifacts, 'true') }}:
+        - task: PublishBuildArtifacts@1
+          displayName: Push Asset Manifests
+          inputs:
+            PathtoPublish: '$(Build.ArtifactStagingDirectory)/AssetManifests'
+            PublishLocation: Container
+            ArtifactName: AssetManifests
+          continueOnError: ${{ parameters.continueOnError }}
+          condition: and(succeeded(), eq(variables['_DotNetPublishToBlobFeed'], 'true'))
+
+  - ${{ if ne(parameters.enablePublishBuildArtifacts, 'false') }}:
     - task: PublishBuildArtifacts@1
       displayName: Publish Logs
       inputs:
         PathtoPublish: '$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)'
         PublishLocation: Container
-        ArtifactName: $(Agent.Os)_$(Agent.JobName)
+        ArtifactName: ${{ coalesce(parameters.enablePublishBuildArtifacts.artifactName, '$(Agent.Os)_$(Agent.JobName)' ) }}
       continueOnError: true
       condition: always()
 
-  - ${{ if eq(parameters.enablePublishTestResults, 'true') }}:
+  - ${{ if or(and(eq(parameters.enablePublishTestResults, 'true'), eq(parameters.testResultsFormat, '')), eq(parameters.testResultsFormat, 'xunit')) }}:
     - task: PublishTestResults@2
-      displayName: Publish Test Results
+      displayName: Publish XUnit Test Results
       inputs:
         testResultsFormat: 'xUnit'
         testResultsFiles: '*.xml' 
         searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)'
+        testRunTitle: ${{ coalesce(parameters.testRunTitle, parameters.name, '$(System.JobName)') }}-xunit
+        mergeTestResults: ${{ parameters.mergeTestResults }}
+      continueOnError: true
+      condition: always()
+  - ${{ if or(and(eq(parameters.enablePublishTestResults, 'true'), eq(parameters.testResultsFormat, '')), eq(parameters.testResultsFormat, 'vstest')) }}:
+    - task: PublishTestResults@2
+      displayName: Publish TRX Test Results
+      inputs:
+        testResultsFormat: 'VSTest'
+        testResultsFiles: '*.trx' 
+        searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)'
+        testRunTitle: ${{ coalesce(parameters.testRunTitle, parameters.name, '$(System.JobName)') }}-trx
+        mergeTestResults: ${{ parameters.mergeTestResults }}
       continueOnError: true
       condition: always()
     
-  - ${{ if and(eq(parameters.enablePublishBuildAssets, true), ne(variables['_PublishUsingPipelines'], 'true'), eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
+  - ${{ if and(eq(parameters.enablePublishBuildAssets, true), ne(parameters.enablePublishUsingPipelines, 'true'), eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
     - task: CopyFiles@2
       displayName: Gather Asset Manifests
       inputs:
@@ -187,6 +233,7 @@ jobs:
         TargetFolder: '$(Build.StagingDirectory)/AssetManifests'
       continueOnError: ${{ parameters.continueOnError }}
       condition: and(succeeded(), eq(variables['_DotNetPublishToBlobFeed'], 'true'))
+
     - task: PublishBuildArtifacts@1
       displayName: Push Asset Manifests
       inputs:

eng/common/templates/job/onelocbuild.yml

@@ -0,0 +1,85 @@
+parameters:
+  # Optional: dependencies of the job
+  dependsOn: ''
+
+  # Optional: A defined YAML pool - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#pool
+  pool:
+    vmImage: vs2017-win2016
+
+  CeapexPat: $(dn-bot-ceapex-package-r) # PAT for the loc AzDO instance https://dev.azure.com/ceapex
+  GithubPat: $(BotAccount-dotnet-bot-repo-PAT)
+
+  SourcesDirectory: $(Build.SourcesDirectory)
+  CreatePr: true
+  AutoCompletePr: false
+  UseLfLineEndings: true
+  UseCheckedInLocProjectJson: false
+  LanguageSet: VS_Main_Languages
+  LclSource: lclFilesInRepo
+  LclPackageId: ''
+  RepoType: gitHub
+  condition: ''
+
+jobs:
+- job: OneLocBuild
+  
+  dependsOn: ${{ parameters.dependsOn }}
+
+  displayName: OneLocBuild
+
+  pool: ${{ parameters.pool }}
+
+  variables:
+    - group: OneLocBuildVariables # Contains the CeapexPat and GithubPat
+    - name: _GenerateLocProjectArguments
+      value: -SourcesDirectory ${{ parameters.SourcesDirectory }}
+        -LanguageSet "${{ parameters.LanguageSet }}"
+        -CreateNeutralXlfs
+    - ${{ if eq(parameters.UseCheckedInLocProjectJson, 'true') }}:
+      - name: _GenerateLocProjectArguments
+        value: ${{ variables._GenerateLocProjectArguments }} -UseCheckedInLocProjectJson
+      
+
+  steps:
+    - task: Powershell@2
+      inputs:
+        filePath: $(Build.SourcesDirectory)/eng/common/generate-locproject.ps1
+        arguments: $(_GenerateLocProjectArguments)
+      displayName: Generate LocProject.json
+      condition: ${{ parameters.condition }}
+
+    - task: OneLocBuild@2
+      displayName: OneLocBuild
+      env:
+        SYSTEM_ACCESSTOKEN: $(System.AccessToken)
+      inputs:
+        locProj: Localize/LocProject.json
+        outDir: $(Build.ArtifactStagingDirectory)
+        lclSource: ${{ parameters.LclSource }}
+        lclPackageId: ${{ parameters.LclPackageId }}
+        isCreatePrSelected: ${{ parameters.CreatePr }}
+        ${{ if eq(parameters.CreatePr, true) }}:
+          isAutoCompletePrSelected: ${{ parameters.AutoCompletePr }}
+          isUseLfLineEndingsSelected: ${{ parameters.UseLfLineEndings }}
+        packageSourceAuth: patAuth
+        patVariable: ${{ parameters.CeapexPat }}
+        ${{ if eq(parameters.RepoType, 'gitHub') }}:
+          repoType: ${{ parameters.RepoType }}
+          gitHubPatVariable: "${{ parameters.GithubPat }}"
+      condition: ${{ parameters.condition }}
+
+    - task: PublishBuildArtifacts@1
+      displayName: Publish Localization Files
+      inputs:
+        PathtoPublish: '$(Build.ArtifactStagingDirectory)/loc'
+        PublishLocation: Container
+        ArtifactName: Loc
+      condition: ${{ parameters.condition }}
+
+    - task: PublishBuildArtifacts@1
+      displayName: Publish LocProject.json
+      inputs:
+        PathtoPublish: '$(Build.SourcesDirectory)/Localize/'
+        PublishLocation: Container
+        ArtifactName: Loc
+      condition: ${{ parameters.condition }}

eng/common/templates/job/publish-build-assets.yml

@@ -37,6 +37,13 @@ jobs:
     - name: _BuildConfig
       value: ${{ parameters.configuration }}
     - group: Publish-Build-Assets
+    - group: AzureDevOps-Artifact-Feeds-Pats
+    # Skip component governance and codesign validation for SDL. These jobs
+    # create no content.
+    - name: skipComponentGovernanceDetection
+      value: true
+    - name: runCodesignValidationInjection
+      value: false
 
   steps:
   - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
@@ -45,8 +52,19 @@ jobs:
       inputs:
         artifactName: AssetManifests
         downloadPath: '$(Build.StagingDirectory)/Download'
+        checkDownloadedFiles: true
       condition: ${{ parameters.condition }}
       continueOnError: ${{ parameters.continueOnError }}
+    
+    - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
+      - task: NuGetAuthenticate@0
+
+      - task: PowerShell@2 
+        displayName: Enable cross-org NuGet feed authentication 
+        inputs: 
+          filePath: $(Build.SourcesDirectory)/eng/common/enable-cross-org-publishing.ps1 
+          arguments: -token $(dn-bot-all-orgs-artifact-feeds-rw) 
+
     - task: PowerShell@2
       displayName: Publish Build Assets
       inputs:
@@ -57,14 +75,27 @@ jobs:
           /p:MaestroApiEndpoint=https://maestro-prod.westus2.cloudapp.azure.com
           /p:PublishUsingPipelines=${{ parameters.publishUsingPipelines }}
           /p:Configuration=$(_BuildConfig)
+          /p:OfficialBuildId=$(Build.BuildNumber)
       condition: ${{ parameters.condition }}
       continueOnError: ${{ parameters.continueOnError }}
+    
+    - task: powershell@2
+      displayName: Create ReleaseConfigs Artifact
+      inputs:
+        targetType: inline
+        script: |
+          Add-Content -Path "$(Build.StagingDirectory)/ReleaseConfigs.txt" -Value $(BARBuildId)
+          Add-Content -Path "$(Build.StagingDirectory)/ReleaseConfigs.txt" -Value "$(DefaultChannels)"
+          Add-Content -Path "$(Build.StagingDirectory)/ReleaseConfigs.txt" -Value $(IsStableBuild)
+    
+    - task: PublishBuildArtifacts@1
+      displayName: Publish ReleaseConfigs Artifact
+      inputs:
+        PathtoPublish: '$(Build.StagingDirectory)/ReleaseConfigs.txt'
+        PublishLocation: Container
+        ArtifactName: ReleaseConfigs
+    
     - ${{ if eq(parameters.enablePublishBuildArtifacts, 'true') }}:
-      - task: PublishBuildArtifacts@1
-        displayName: Publish Logs to VSTS
-        inputs:
-          PathtoPublish: '$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)'
-          PublishLocation: Container
-          ArtifactName: $(Agent.Os)_PublishBuildAssets
-        continueOnError: true
-        condition: always()      
+      - template: /eng/common/templates/steps/publish-logs.yml
+        parameters:
+          JobLabel: 'Publish_Artifacts_Logs'     

eng/common/templates/job/source-build.yml

@@ -0,0 +1,57 @@
+parameters:
+  # This template adds arcade-powered source-build to CI. The template produces a server job with a
+  # default ID 'Source_Build_Complete' to put in a dependency list if necessary.
+
+  # Specifies the prefix for source-build jobs added to pipeline. Use this if disambiguation needed.
+  jobNamePrefix: 'Source_Build'
+
+  # Defines the platform on which to run the job. By default, a linux-x64 machine, suitable for
+  # managed-only repositories. This is an object with these properties:
+  #
+  # name: ''
+  #   The name of the job. This is included in the job ID.
+  # targetRID: ''
+  #   The name of the target RID to use, instead of the one auto-detected by Arcade.
+  # nonPortable: false
+  #   Enables non-portable mode. This means a more specific RID (e.g. fedora.32-x64 rather than
+  #   linux-x64), and compiling against distro-provided packages rather than portable ones.
+  # container: ''
+  #   A container to use. Runs in docker.
+  # pool: {}
+  #   A pool to use. Runs directly on an agent.
+  # buildScript: ''
+  #   Specifies the build script to invoke to perform the build in the repo. The default
+  #   './build.sh' should work for typical Arcade repositories, but this is customizable for
+  #   difficult situations.
+  # jobProperties: {}
+  #   A list of job properties to inject at the top level, for potential extensibility beyond
+  #   container and pool.
+  platform: {}
+
+  # The default VM host AzDO pool. This should be capable of running Docker containers: almost all
+  # source-build builds run in Docker, including the default managed platform.
+  defaultContainerHostPool:
+    vmImage: ubuntu-20.04
+
+jobs:
+- job: ${{ parameters.jobNamePrefix }}_${{ parameters.platform.name }}
+  displayName: Source-Build (${{ parameters.platform.name }})
+
+  ${{ each property in parameters.platform.jobProperties }}:
+    ${{ property.key }}: ${{ property.value }}
+
+  ${{ if ne(parameters.platform.container, '') }}:
+    container: ${{ parameters.platform.container }}
+
+  ${{ if eq(parameters.platform.pool, '') }}:
+    pool: ${{ parameters.defaultContainerHostPool }}
+  ${{ if ne(parameters.platform.pool, '') }}:
+    pool: ${{ parameters.platform.pool }}
+
+  workspace:
+    clean: all
+
+  steps:
+  - template: /eng/common/templates/steps/source-build.yml
+    parameters:
+      platform: ${{ parameters.platform }}

eng/common/templates/job/source-index-stage1.yml

@@ -0,0 +1,58 @@
+parameters:
+  runAsPublic: false
+  sourceIndexPackageVersion: 1.0.1-20210421.1
+  sourceIndexPackageSource: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json
+  sourceIndexBuildCommand: powershell -NoLogo -NoProfile -ExecutionPolicy Bypass -Command "eng/common/build.ps1 -restore -build -binarylog -ci"
+  preSteps: []
+  binlogPath: artifacts/log/Debug/Build.binlog
+  pool:
+    vmImage: vs2017-win2016
+
+jobs:
+- job: SourceIndexStage1
+  variables:
+  - name: SourceIndexPackageVersion
+    value: ${{ parameters.sourceIndexPackageVersion }}
+  - name: SourceIndexPackageSource
+    value: ${{ parameters.sourceIndexPackageSource }}
+  - name: BinlogPath
+    value: ${{ parameters.binlogPath }}
+  - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
+    - group: source-dot-net stage1 variables
+
+  pool: ${{ parameters.pool }}
+  steps:
+  - ${{ each preStep in parameters.preSteps }}:
+    - ${{ preStep }}
+
+  - task: UseDotNet@2
+    displayName: Use .NET Core sdk 3.1
+    inputs:
+      packageType: sdk
+      version: 3.1.x
+
+  - task: UseDotNet@2
+    displayName: Use .NET Core sdk
+    inputs:
+      useGlobalJson: true
+
+  - script: |
+      dotnet tool install BinLogToSln --version $(SourceIndexPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path .source-index/tools
+      dotnet tool install UploadIndexStage1 --version $(SourceIndexPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path .source-index/tools
+      echo ##vso[task.prependpath]$(Build.SourcesDirectory)/.source-index/tools
+    displayName: Download Tools
+
+  - script: ${{ parameters.sourceIndexBuildCommand }}
+    displayName: Build Repository
+
+  - script: BinLogToSln -i $(BinlogPath) -r $(Build.SourcesDirectory) -n $(Build.Repository.Name) -o .source-index/stage1output
+    displayName: Process Binlog into indexable sln
+    env:
+      DOTNET_ROLL_FORWARD_ON_NO_CANDIDATE_FX: 2
+
+  - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
+    - script: UploadIndexStage1 -i .source-index/stage1output -n $(Build.Repository.Name)
+      displayName: Upload stage1 artifacts to source index
+      env:
+        BLOB_CONTAINER_URL: $(source-dot-net-stage1-blob-container-url)
+        DOTNET_ROLL_FORWARD_ON_NO_CANDIDATE_FX: 2

eng/common/templates/jobs/jobs.yml

@@ -1,41 +1,29 @@
 parameters:
-  # Optional: 'true' if failures in job.yml job should not fail the job
+  # See schema documentation in /Documentation/AzureDevOps/TemplateSchema.md
   continueOnError: false
 
-  # Optional: Enable installing Microbuild plugin
-  #           if 'true', these "variables" must be specified in the variables object or as part of the queue matrix
-  #             _TeamName - the name of your team
-  #             _SignType - 'test' or 'real'
-  enableMicrobuild: false
-
   # Optional: Include PublishBuildArtifacts task
   enablePublishBuildArtifacts: false
 
-  # Optional: Enable publishing to the build asset registry
-  enablePublishBuildAssets: false
-
   # Optional: Enable publishing using release pipelines
   enablePublishUsingPipelines: false
-  
+
+  # Optional: Enable running the source-build jobs to build repo from source
+  enableSourceBuild: false
+
+  # Optional: Parameters for source-build template.
+  #           See /eng/common/templates/jobs/source-build.yml for options
+  sourceBuildParameters: []
+
   graphFileGeneration:
     # Optional: Enable generating the graph files at the end of the build
     enabled: false
     # Optional: Include toolset dependencies in the generated graph files
     includeToolset: false
     
-  # Optional: Include PublishTestResults task
-  enablePublishTestResults: false
-
-  # Optional: enable sending telemetry
-  # if enabled then the 'helixRepo' parameter should also be specified
-  enableTelemetry: false
-
   # Required: A collection of jobs to run - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#job
   jobs: []
 
-  # Optional: define the helix repo for telemetry (example: 'dotnet/arcade')
-  helixRepo: ''
-
   # Optional: Override automatically derived dependsOn value for "publish build assets" job
   publishBuildAssetsDependsOn: ''
 
@@ -43,6 +31,9 @@ parameters:
   #           if 'true', the build won't run any of the internal only steps, even if it is running in non-public projects.
   runAsPublic: false
 
+  enableSourceIndex: false
+  sourceIndexParams: {}
+
 # Internal resources (telemetry, microbuild) can only be accessed from non-public projects,
 # and some (Microbuild) should only be applied to non-PR cases for internal builds.
 
@@ -62,29 +53,47 @@ jobs:
 
       name: ${{ job.job }}
 
-- ${{ if and(eq(parameters.enablePublishBuildAssets, true), eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
-  - template: ../job/publish-build-assets.yml
+- ${{ if eq(parameters.enableSourceBuild, true) }}:
+  - template: /eng/common/templates/jobs/source-build.yml
+    parameters:
+      allCompletedJobId: Source_Build_Complete
+      ${{ each parameter in parameters.sourceBuildParameters }}:
+        ${{ parameter.key }}: ${{ parameter.value }}
+
+- ${{ if eq(parameters.enableSourceIndex, 'true') }}:
+  - template: ../job/source-index-stage1.yml
     parameters:
-      continueOnError: ${{ parameters.continueOnError }}
-      dependsOn:
-      - ${{ if ne(parameters.publishBuildAssetsDependsOn, '') }}:
-        - ${{ each job in parameters.publishBuildAssetsDependsOn }}:
-          - ${{ job.job }}
-      - ${{ if eq(parameters.publishBuildAssetsDependsOn, '') }}:
-        - ${{ each job in parameters.jobs }}:
-          - ${{ job.job }}
-      pool:
-        vmImage: vs2017-win2016
       runAsPublic: ${{ parameters.runAsPublic }}
-      publishUsingPipelines: ${{ parameters.enablePublishUsingPipelines }}
-      enablePublishBuildArtifacts: ${{ parameters.enablePublishBuildArtifacts }}
-      
-- ${{ if and(eq(parameters.graphFileGeneration.enabled, true), eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
-  - template: ../job/generate-graph-files.yml
-    parameters:
-      continueOnError: ${{ parameters.continueOnError }}
-      includeToolset: ${{ parameters.graphFileGeneration.includeToolset }}
-      dependsOn:
-        - Asset_Registry_Publish
-      pool:
-        vmImage: vs2017-win2016
+      ${{ each parameter in parameters.sourceIndexParams }}:
+        ${{ parameter.key }}: ${{ parameter.value }}
+
+- ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
+  
+  - ${{ if or(eq(parameters.enablePublishBuildAssets, true), eq(parameters.artifacts.publish.manifests, 'true'), ne(parameters.artifacts.publish.manifests, '')) }}:
+    - template: ../job/publish-build-assets.yml
+      parameters:
+        continueOnError: ${{ parameters.continueOnError }}
+        dependsOn:
+        - ${{ if ne(parameters.publishBuildAssetsDependsOn, '') }}:
+          - ${{ each job in parameters.publishBuildAssetsDependsOn }}:
+            - ${{ job.job }}
+        - ${{ if eq(parameters.publishBuildAssetsDependsOn, '') }}:
+          - ${{ each job in parameters.jobs }}:
+            - ${{ job.job }}
+        - ${{ if eq(parameters.enableSourceBuild, true) }}:
+          - Source_Build_Complete
+        pool:
+          vmImage: vs2017-win2016
+        runAsPublic: ${{ parameters.runAsPublic }}
+        publishUsingPipelines: ${{ parameters.enablePublishUsingPipelines }}
+        enablePublishBuildArtifacts: ${{ parameters.enablePublishBuildArtifacts }}
+
+  - ${{ if eq(parameters.graphFileGeneration.enabled, true) }}:
+    - template: ../job/generate-graph-files.yml
+      parameters:
+        continueOnError: ${{ parameters.continueOnError }}
+        includeToolset: ${{ parameters.graphFileGeneration.includeToolset }}
+        dependsOn:
+          - Asset_Registry_Publish
+        pool:
+          vmImage: vs2017-win2016

eng/common/templates/jobs/source-build.yml

@@ -0,0 +1,46 @@
+parameters:
+  # This template adds arcade-powered source-build to CI. A job is created for each platform, as
+  # well as an optional server job that completes when all platform jobs complete.
+
+  # The name of the "join" job for all source-build platforms. If set to empty string, the job is
+  # not included. Existing repo pipelines can use this job depend on all source-build jobs
+  # completing without maintaining a separate list of every single job ID: just depend on this one
+  # server job. By default, not included. Recommended name if used: 'Source_Build_Complete'.
+  allCompletedJobId: ''
+
+  # See /eng/common/templates/job/source-build.yml
+  jobNamePrefix: 'Source_Build'
+
+  # This is the default platform provided by Arcade, intended for use by a managed-only repo.
+  defaultManagedPlatform:
+    name: 'Managed'
+    container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:centos-7-3e800f1-20190501005343'
+
+  # Defines the platforms on which to run build jobs. One job is created for each platform, and the
+  # object in this array is sent to the job template as 'platform'. If no platforms are specified,
+  # one job runs on 'defaultManagedPlatform'.
+  platforms: []
+
+jobs:
+
+- ${{ if ne(parameters.allCompletedJobId, '') }}:
+  - job: ${{ parameters.allCompletedJobId }}
+    displayName: Source-Build Complete
+    pool: server
+    dependsOn:
+    - ${{ each platform in parameters.platforms }}:
+      - ${{ parameters.jobNamePrefix }}_${{ platform.name }}
+    - ${{ if eq(length(parameters.platforms), 0) }}:
+      - ${{ parameters.jobNamePrefix }}_${{ parameters.defaultManagedPlatform.name }}
+
+- ${{ each platform in parameters.platforms }}:
+  - template: /eng/common/templates/job/source-build.yml
+    parameters:
+      jobNamePrefix: ${{ parameters.jobNamePrefix }}
+      platform: ${{ platform }}
+
+- ${{ if eq(length(parameters.platforms), 0) }}:
+  - template: /eng/common/templates/job/source-build.yml
+    parameters:
+      jobNamePrefix: ${{ parameters.jobNamePrefix }}
+      platform: ${{ parameters.defaultManagedPlatform }}

eng/common/templates/phases/publish-build-assets.yml

@@ -5,6 +5,7 @@ parameters:
   condition: succeeded()
   continueOnError: false
   runAsPublic: false
+  publishUsingPipelines: false
 phases:
   - phase: Asset_Registry_Publish
     displayName: Publish to Build Asset Registry
@@ -19,6 +20,7 @@ phases:
           inputs:
             artifactName: AssetManifests
             downloadPath: '$(Build.StagingDirectory)/Download'
+            checkDownloadedFiles: true
           condition: ${{ parameters.condition }}
           continueOnError: ${{ parameters.continueOnError }}
         - task: AzureKeyVault@1
@@ -36,6 +38,7 @@ phases:
               /p:ManifestsPath='$(Build.StagingDirectory)/Download/AssetManifests'
               /p:BuildAssetRegistryToken=$(MaestroAccessToken)
               /p:MaestroApiEndpoint=https://maestro-prod.westus2.cloudapp.azure.com
+              /p:PublishUsingPipelines=${{ parameters.publishUsingPipelines }}
               /p:Configuration=$(_BuildConfig)
           condition: ${{ parameters.condition }}
           continueOnError: ${{ parameters.continueOnError }}

eng/common/templates/post-build/channels/generic-internal-channel.yml

@@ -0,0 +1,184 @@
+parameters:
+  BARBuildId: ''
+  PromoteToChannelIds: ''
+  artifactsPublishingAdditionalParameters: ''
+  dependsOn:
+  - Validate
+  publishInstallersAndChecksums: true
+  symbolPublishingAdditionalParameters: ''
+  stageName: ''
+  channelName: ''
+  channelId: ''
+  transportFeed: ''
+  shippingFeed: ''
+  symbolsFeed: ''
+
+stages:
+- stage: ${{ parameters.stageName }}
+  dependsOn: ${{ parameters.dependsOn }}
+  variables:
+    - template: ../common-variables.yml
+  displayName: ${{ parameters.channelName }} Publishing
+  jobs:
+  - template: ../setup-maestro-vars.yml
+    parameters:
+      BARBuildId: ${{ parameters.BARBuildId }}
+      PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
+
+  - job: publish_symbols
+    displayName: Symbol Publishing
+    dependsOn: setupMaestroVars
+    condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.TargetChannels'], format('[{0}]', ${{ parameters.channelId }} ))
+    variables:
+      - group: DotNet-Symbol-Server-Pats
+      - name: AzDOProjectName
+        value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ]
+      - name: AzDOPipelineId
+        value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ]
+      - name: AzDOBuildId
+        value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ]
+    pool:
+      vmImage: 'windows-2019'
+    steps:
+      # This is necessary whenever we want to publish/restore to an AzDO private feed
+      - task: NuGetAuthenticate@0
+        displayName: 'Authenticate to AzDO Feeds'
+
+      - task: DownloadBuildArtifacts@0
+        displayName: Download Build Assets
+        continueOnError: true
+        inputs:
+          buildType: specific
+          buildVersionToDownload: specific
+          project: $(AzDOProjectName)
+          pipeline: $(AzDOPipelineId)
+          buildId: $(AzDOBuildId)
+          downloadType: 'specific'
+          itemPattern: |
+            PdbArtifacts/**
+            BlobArtifacts/**
+          downloadPath: '$(Build.ArtifactStagingDirectory)'
+          checkDownloadedFiles: true
+
+      # This is necessary whenever we want to publish/restore to an AzDO private feed
+      # Since sdk-task.ps1 tries to restore packages we need to do this authentication here
+      # otherwise it'll complain about accessing a private feed.
+      - task: NuGetAuthenticate@0
+        displayName: 'Authenticate to AzDO Feeds'
+
+      - task: PowerShell@2
+        displayName: Enable cross-org publishing
+        inputs:
+          filePath: eng\common\enable-cross-org-publishing.ps1
+          arguments: -token $(dn-bot-dnceng-artifact-feeds-rw)
+
+      - task: PowerShell@2
+        displayName: Publish
+        inputs:
+          filePath: eng\common\sdk-task.ps1
+          arguments: -task PublishToSymbolServers -restore -msbuildEngine dotnet
+            /p:DotNetSymbolServerTokenMsdl=$(microsoft-symbol-server-pat)
+            /p:DotNetSymbolServerTokenSymWeb=$(symweb-symbol-server-pat)
+            /p:PDBArtifactsDirectory='$(Build.ArtifactStagingDirectory)/PDBArtifacts/'
+            /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/'
+            /p:SymbolPublishingExclusionsFile='$(Build.SourcesDirectory)/eng/SymbolPublishingExclusionsFile.txt'
+            /p:Configuration=Release
+            /p:PublishToMSDL=false
+            ${{ parameters.symbolPublishingAdditionalParameters }}
+
+      - template: ../../steps/publish-logs.yml
+        parameters:
+          StageLabel: '${{ parameters.stageName }}'
+          JobLabel: 'SymbolPublishing'
+
+  - job: publish_assets
+    displayName: Publish Assets
+    dependsOn: setupMaestroVars
+    timeoutInMinutes: 120
+    variables:
+      - name: BARBuildId
+        value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ]
+      - name: IsStableBuild
+        value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.IsStableBuild'] ]
+      - name: AzDOProjectName
+        value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ]
+      - name: AzDOPipelineId
+        value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ]
+      - name: AzDOBuildId
+        value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ]
+    condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.TargetChannels'], format('[{0}]', ${{ parameters.channelId }} ))
+    pool:
+      vmImage: 'windows-2019'
+    steps:
+      - task: DownloadBuildArtifacts@0
+        displayName: Download Build Assets
+        continueOnError: true
+        inputs:
+          buildType: specific
+          buildVersionToDownload: specific
+          project: $(AzDOProjectName)
+          pipeline: $(AzDOPipelineId)
+          buildId: $(AzDOBuildId)
+          downloadType: 'specific'
+          itemPattern: |
+            PackageArtifacts/**
+            BlobArtifacts/**
+            AssetManifests/**
+          downloadPath: '$(Build.ArtifactStagingDirectory)'
+          checkDownloadedFiles: true
+
+      - task: NuGetToolInstaller@1
+        displayName: 'Install NuGet.exe'
+
+      # This is necessary whenever we want to publish/restore to an AzDO private feed
+      - task: NuGetAuthenticate@0
+        displayName: 'Authenticate to AzDO Feeds'
+
+      - task: PowerShell@2
+        displayName: Enable cross-org publishing
+        inputs:
+          filePath: eng\common\enable-cross-org-publishing.ps1
+          arguments: -token $(dn-bot-dnceng-artifact-feeds-rw)
+
+      - task: PowerShell@2
+        displayName: Publish Assets
+        inputs:
+          filePath: eng\common\sdk-task.ps1
+          arguments: -task PublishArtifactsInManifest -restore -msbuildEngine dotnet
+            /p:PublishingInfraVersion=2
+            /p:IsStableBuild=$(IsStableBuild)
+            /p:IsInternalBuild=$(IsInternalBuild)
+            /p:RepositoryName=$(Build.Repository.Name)
+            /p:CommitSha=$(Build.SourceVersion)
+            /p:NugetPath=$(NuGetExeToolPath)
+            /p:AzdoTargetFeedPAT='$(dn-bot-dnceng-universal-packages-rw)'
+            /p:AzureStorageTargetFeedPAT='$(dotnetfeed-storage-access-key-1)'
+            /p:BARBuildId=$(BARBuildId)
+            /p:MaestroApiEndpoint='$(MaestroApiEndPoint)'
+            /p:BuildAssetRegistryToken='$(MaestroApiAccessToken)'
+            /p:ManifestsBasePath='$(Build.ArtifactStagingDirectory)/AssetManifests/'
+            /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/'
+            /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts/'
+            /p:Configuration=Release
+            /p:PublishInstallersAndChecksums=${{ parameters.publishInstallersAndChecksums }}
+            /p:ChecksumsTargetStaticFeed=$(InternalChecksumsBlobFeedUrl)
+            /p:ChecksumsAzureAccountKey=$(InternalChecksumsBlobFeedKey)
+            /p:InstallersTargetStaticFeed=$(InternalInstallersBlobFeedUrl)
+            /p:InstallersAzureAccountKey=$(InternalInstallersBlobFeedKey)
+            /p:AzureDevOpsStaticShippingFeed='${{ parameters.shippingFeed }}'
+            /p:AzureDevOpsStaticShippingFeedKey='$(dn-bot-dnceng-artifact-feeds-rw)'
+            /p:AzureDevOpsStaticTransportFeed='${{ parameters.transportFeed }}'
+            /p:AzureDevOpsStaticTransportFeedKey='$(dn-bot-dnceng-artifact-feeds-rw)'
+            /p:AzureDevOpsStaticSymbolsFeed='${{ parameters.symbolsFeed }}'
+            /p:AzureDevOpsStaticSymbolsFeedKey='$(dn-bot-dnceng-artifact-feeds-rw)'
+            /p:PublishToMSDL=false
+            ${{ parameters.artifactsPublishingAdditionalParameters }}
+
+      - template: ../../steps/publish-logs.yml
+        parameters:
+          StageLabel: '${{ parameters.stageName }}'
+          JobLabel: 'AssetsPublishing'
+
+      - template: ../../steps/add-build-to-channel.yml
+        parameters:
+          ChannelId: ${{ parameters.channelId }}

eng/common/templates/post-build/channels/generic-public-channel.yml

@@ -0,0 +1,186 @@
+parameters:
+  BARBuildId: ''
+  PromoteToChannelIds: ''
+  artifactsPublishingAdditionalParameters: ''
+  dependsOn:
+  - Validate
+  publishInstallersAndChecksums: true
+  symbolPublishingAdditionalParameters: ''
+  stageName: ''
+  channelName: ''
+  channelId: ''
+  transportFeed: ''
+  shippingFeed: ''
+  symbolsFeed: ''
+  # If the channel name is empty, no links will be generated
+  akaMSChannelName: ''
+
+stages:
+- stage: ${{ parameters.stageName }}
+  dependsOn: ${{ parameters.dependsOn }}
+  variables:
+    - template: ../common-variables.yml
+  displayName: ${{ parameters.channelName }} Publishing
+  jobs:
+  - template: ../setup-maestro-vars.yml
+    parameters:
+      BARBuildId: ${{ parameters.BARBuildId }}
+      PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
+
+  - job: publish_symbols
+    displayName: Symbol Publishing
+    dependsOn: setupMaestroVars
+    condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.TargetChannels'], format('[{0}]', ${{ parameters.channelId }} ))
+    variables:
+      - group: DotNet-Symbol-Server-Pats
+      - name: AzDOProjectName
+        value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ]
+      - name: AzDOPipelineId
+        value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ]
+      - name: AzDOBuildId
+        value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ]
+    pool:
+      vmImage: 'windows-2019'
+    steps:
+      - task: DownloadBuildArtifacts@0
+        displayName: Download Build Assets
+        continueOnError: true
+        inputs:
+          buildType: specific
+          buildVersionToDownload: specific
+          project: $(AzDOProjectName)
+          pipeline: $(AzDOPipelineId)
+          buildId: $(AzDOBuildId)
+          downloadType: 'specific'
+          itemPattern: |
+            PdbArtifacts/**
+            BlobArtifacts/**
+          downloadPath: '$(Build.ArtifactStagingDirectory)'
+          checkDownloadedFiles: true
+
+      # This is necessary whenever we want to publish/restore to an AzDO private feed
+      # Since sdk-task.ps1 tries to restore packages we need to do this authentication here
+      # otherwise it'll complain about accessing a private feed.
+      - task: NuGetAuthenticate@0
+        displayName: 'Authenticate to AzDO Feeds'
+
+      - task: PowerShell@2
+        displayName: Enable cross-org publishing
+        inputs:
+          filePath: eng\common\enable-cross-org-publishing.ps1
+          arguments: -token $(dn-bot-dnceng-artifact-feeds-rw)
+
+      - task: PowerShell@2
+        displayName: Publish
+        inputs:
+          filePath: eng\common\sdk-task.ps1
+          arguments: -task PublishToSymbolServers -restore -msbuildEngine dotnet
+            /p:DotNetSymbolServerTokenMsdl=$(microsoft-symbol-server-pat)
+            /p:DotNetSymbolServerTokenSymWeb=$(symweb-symbol-server-pat)
+            /p:PDBArtifactsDirectory='$(Build.ArtifactStagingDirectory)/PDBArtifacts/'
+            /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/'
+            /p:SymbolPublishingExclusionsFile='$(Build.SourcesDirectory)/eng/SymbolPublishingExclusionsFile.txt'
+            /p:Configuration=Release
+            ${{ parameters.symbolPublishingAdditionalParameters }}
+
+      - template: ../../steps/publish-logs.yml
+        parameters:
+          StageLabel: '${{ parameters.stageName }}'
+          JobLabel: 'SymbolPublishing'
+
+  - job: publish_assets
+    displayName: Publish Assets
+    dependsOn: setupMaestroVars
+    timeoutInMinutes: 120
+    variables:
+      - name: BARBuildId
+        value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ]
+      - name: IsStableBuild
+        value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.IsStableBuild'] ]
+      - name: AzDOProjectName
+        value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ]
+      - name: AzDOPipelineId
+        value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ]
+      - name: AzDOBuildId
+        value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ]
+      - name: ArtifactsCategory
+        value: ${{ coalesce(variables._DotNetArtifactsCategory, '.NETCore') }}
+    condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.TargetChannels'], format('[{0}]', ${{ parameters.channelId }} ))
+    pool:
+      vmImage: 'windows-2019'
+    steps:
+      - task: DownloadBuildArtifacts@0
+        displayName: Download Build Assets
+        continueOnError: true
+        inputs:
+          buildType: specific
+          buildVersionToDownload: specific
+          project: $(AzDOProjectName)
+          pipeline: $(AzDOPipelineId)
+          buildId: $(AzDOBuildId)
+          downloadType: 'specific'
+          itemPattern: |
+            PackageArtifacts/**
+            BlobArtifacts/**
+            AssetManifests/**
+          downloadPath: '$(Build.ArtifactStagingDirectory)'
+          checkDownloadedFiles: true
+
+      - task: NuGetToolInstaller@1
+        displayName: 'Install NuGet.exe'
+
+      # This is necessary whenever we want to publish/restore to an AzDO private feed
+      - task: NuGetAuthenticate@0
+        displayName: 'Authenticate to AzDO Feeds'
+
+      - task: PowerShell@2
+        displayName: Enable cross-org publishing
+        inputs:
+          filePath: eng\common\enable-cross-org-publishing.ps1
+          arguments: -token $(dn-bot-dnceng-artifact-feeds-rw)
+
+      - task: PowerShell@2
+        displayName: Publish Assets
+        inputs:
+          filePath: eng\common\sdk-task.ps1
+          arguments: -task PublishArtifactsInManifest -restore -msbuildEngine dotnet
+            /p:PublishingInfraVersion=2
+            /p:ArtifactsCategory=$(ArtifactsCategory)
+            /p:IsStableBuild=$(IsStableBuild)
+            /p:IsInternalBuild=$(IsInternalBuild)
+            /p:RepositoryName=$(Build.Repository.Name)
+            /p:CommitSha=$(Build.SourceVersion)
+            /p:NugetPath=$(NuGetExeToolPath)
+            /p:AzdoTargetFeedPAT='$(dn-bot-dnceng-universal-packages-rw)'
+            /p:AzureStorageTargetFeedPAT='$(dotnetfeed-storage-access-key-1)'
+            /p:BARBuildId=$(BARBuildId)
+            /p:MaestroApiEndpoint='$(MaestroApiEndPoint)'
+            /p:BuildAssetRegistryToken='$(MaestroApiAccessToken)'
+            /p:ManifestsBasePath='$(Build.ArtifactStagingDirectory)/AssetManifests/'
+            /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/'
+            /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts/'
+            /p:Configuration=Release
+            /p:PublishInstallersAndChecksums=${{ parameters.publishInstallersAndChecksums }}
+            /p:InstallersTargetStaticFeed=$(InstallersBlobFeedUrl)
+            /p:InstallersAzureAccountKey=$(dotnetcli-storage-key)
+            /p:ChecksumsTargetStaticFeed=$(ChecksumsBlobFeedUrl)
+            /p:ChecksumsAzureAccountKey=$(dotnetclichecksums-storage-key)
+            /p:AzureDevOpsStaticShippingFeed='${{ parameters.shippingFeed }}'
+            /p:AzureDevOpsStaticShippingFeedKey='$(dn-bot-dnceng-artifact-feeds-rw)'
+            /p:AzureDevOpsStaticTransportFeed='${{ parameters.transportFeed }}'
+            /p:AzureDevOpsStaticTransportFeedKey='$(dn-bot-dnceng-artifact-feeds-rw)'
+            /p:AzureDevOpsStaticSymbolsFeed='${{ parameters.symbolsFeed }}'
+            /p:AzureDevOpsStaticSymbolsFeedKey='$(dn-bot-dnceng-artifact-feeds-rw)'
+            /p:LatestLinkShortUrlPrefix=dotnet/'${{ parameters.akaMSChannelName }}'
+            /p:AkaMSClientId=$(akams-client-id)
+            /p:AkaMSClientSecret=$(akams-client-secret)
+            ${{ parameters.artifactsPublishingAdditionalParameters }}
+
+      - template: ../../steps/publish-logs.yml
+        parameters:
+          StageLabel: '${{ parameters.stageName }}'
+          JobLabel: 'AssetsPublishing'
+
+      - template: ../../steps/add-build-to-channel.yml
+        parameters:
+          ChannelId: ${{ parameters.channelId }}

eng/common/templates/post-build/common-variables.yml

@@ -0,0 +1,99 @@
+variables:
+  - group: AzureDevOps-Artifact-Feeds-Pats
+  - group: DotNet-Blob-Feed
+  - group: DotNet-DotNetCli-Storage
+  - group: DotNet-MSRC-Storage
+  - group: Publish-Build-Assets
+    
+  # .NET Core 3.1 Dev
+  - name: PublicDevRelease_31_Channel_Id
+    value: 128
+
+  # .NET 5 Dev
+  - name: Net_5_Dev_Channel_Id
+    value: 131
+
+  # .NET Eng - Validation
+  - name: Net_Eng_Validation_Channel_Id
+    value: 9
+
+  # .NET Eng - Latest
+  - name: Net_Eng_Latest_Channel_Id
+    value: 2
+
+  # .NET 3 Eng - Validation
+  - name: NET_3_Eng_Validation_Channel_Id
+    value: 390
+
+  # .NET 3 Eng
+  - name: NetCore_3_Tools_Channel_Id
+    value: 344
+
+  # .NET Core 3.0 Internal Servicing
+  - name: InternalServicing_30_Channel_Id
+    value: 184
+
+  # .NET Core 3.0 Release
+  - name: PublicRelease_30_Channel_Id
+    value: 19
+
+  # .NET Core 3.1 Release
+  - name: PublicRelease_31_Channel_Id
+    value: 129
+
+  # General Testing
+  - name: GeneralTesting_Channel_Id
+    value: 529
+
+  # .NET Core 3.1 Blazor Features
+  - name: NetCore_31_Blazor_Features_Channel_Id
+    value: 531
+
+  # .NET Core Experimental
+  - name: NetCore_Experimental_Channel_Id
+    value: 562
+
+  # Whether the build is internal or not
+  - name: IsInternalBuild
+    value: ${{ and(ne(variables['System.TeamProject'], 'public'), contains(variables['Build.SourceBranch'], 'internal')) }}
+
+  # Default Maestro++ API Endpoint and API Version
+  - name: MaestroApiEndPoint
+    value: "https://maestro-prod.westus2.cloudapp.azure.com"
+  - name: MaestroApiAccessToken
+    value: $(MaestroAccessToken)
+  - name: MaestroApiVersion
+    value: "2020-02-20"
+
+  - name: SourceLinkCLIVersion
+    value: 3.0.0
+  - name: SymbolToolVersion
+    value: 1.0.1
+
+  # Feed Configurations
+  # These should include the suffix "/index.json"
+
+  # Default locations for Installers and checksums
+  # Public Locations
+  - name: ChecksumsBlobFeedUrl
+    value: https://dotnetclichecksums.blob.core.windows.net/dotnet/index.json
+  - name: InstallersBlobFeedUrl
+    value: https://dotnetcli.blob.core.windows.net/dotnet/index.json
+
+  # Private Locations
+  - name: InternalChecksumsBlobFeedUrl
+    value: https://dotnetclichecksumsmsrc.blob.core.windows.net/dotnet/index.json
+  - name: InternalChecksumsBlobFeedKey
+    value: $(dotnetclichecksumsmsrc-storage-key)
+
+  - name: InternalInstallersBlobFeedUrl
+    value: https://dotnetclimsrc.blob.core.windows.net/dotnet/index.json
+  - name: InternalInstallersBlobFeedKey
+    value: $(dotnetclimsrc-access-key)
+
+  # Skip component governance and codesign validation for SDL. These jobs
+  # create no content.
+  - name: skipComponentGovernanceDetection
+    value: true
+  - name: runCodesignValidationInjection
+    value: false

eng/common/templates/post-build/post-build.yml

@@ -0,0 +1,589 @@
+parameters:
+ # Which publishing infra should be used. THIS SHOULD MATCH THE VERSION ON THE BUILD MANIFEST.
+  # Publishing V2 accepts optionally outlining the publishing stages - default is inline.
+  # Publishing V3 DOES NOT accept inlining the publishing stages.
+  publishingInfraVersion: 2
+  # When set to true the publishing templates from the repo will be used
+  # otherwise Darc add-build-to-channel will be used to trigger the promotion pipeline
+  inline: true
+
+  # Only used if inline==false. When set to true will stall the current build until
+  # the Promotion Pipeline build finishes. Otherwise, the current build will continue 
+  # execution concurrently with the promotion build.
+  waitPublishingFinish: true
+
+  BARBuildId: ''
+  PromoteToChannelIds: ''
+
+  enableSourceLinkValidation: false
+  enableSigningValidation: true
+  enableSymbolValidation: false
+  enableNugetValidation: true
+  publishInstallersAndChecksums: true
+  SDLValidationParameters:
+    enable: false
+    continueOnError: false
+    params: ''
+    artifactNames: ''
+    downloadArtifacts: true
+
+  # These parameters let the user customize the call to sdk-task.ps1 for publishing
+  # symbols & general artifacts as well as for signing validation
+  symbolPublishingAdditionalParameters: ''
+  artifactsPublishingAdditionalParameters: ''
+  signingValidationAdditionalParameters: ''
+
+  # Which stages should finish execution before post-build stages start
+  validateDependsOn:
+  - build
+  publishDependsOn: 
+  - Validate
+
+  # Channel ID's instantiated in this file.
+  # When adding a new channel implementation the call to `check-channel-consistency.ps1` 
+  # needs to be updated with the new channel ID
+  NetEngLatestChannelId: 2
+  NetEngValidationChannelId: 9
+  NetDev5ChannelId: 131
+  NetDev6ChannelId: 1296
+  GeneralTestingChannelId: 529
+  NETCoreToolingDevChannelId: 548
+  NETCoreToolingReleaseChannelId: 549
+  NETInternalToolingChannelId: 551
+  NETCoreExperimentalChannelId: 562
+  NetEngServicesIntChannelId: 678
+  NetEngServicesProdChannelId: 679
+  NetCoreSDK313xxChannelId: 759
+  NetCoreSDK313xxInternalChannelId: 760
+  NetCoreSDK314xxChannelId: 921
+  NetCoreSDK314xxInternalChannelId: 922
+  VS166ChannelId: 1010
+  VS167ChannelId: 1011
+  VS168ChannelId: 1154
+  VSMasterChannelId: 1012
+  VS169ChannelId: 1473
+  VS1610ChannelId: 1692
+
+stages:
+- ${{ if or(and(le(parameters.publishingInfraVersion, 2), eq(parameters.inline, 'true')), eq( parameters.enableNugetValidation, 'true'), eq(parameters.enableSigningValidation, 'true'), eq(parameters.enableSourceLinkValidation, 'true'), eq(parameters.SDLValidationParameters.enable, 'true')) }}:
+  - stage: Validate
+    dependsOn: ${{ parameters.validateDependsOn }}
+    displayName: Validate Build Assets
+    variables:
+      - template: common-variables.yml
+    jobs:
+    - template: setup-maestro-vars.yml
+      parameters:
+        BARBuildId: ${{ parameters.BARBuildId }}
+        PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
+
+    - ${{ if and(le(parameters.publishingInfraVersion, 2), eq(parameters.inline, 'true')) }}:
+      - job:
+        displayName: Post-build Checks
+        dependsOn: setupMaestroVars
+        variables:
+          - name: TargetChannels
+            value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.TargetChannels'] ]
+        pool:
+          vmImage: 'windows-2019'
+        steps:
+          - task: PowerShell@2
+            displayName: Maestro Channels Consistency
+            inputs:
+              filePath: $(Build.SourcesDirectory)/eng/common/post-build/check-channel-consistency.ps1
+              arguments: -PromoteToChannels "$(TargetChannels)"
+                -AvailableChannelIds ${{parameters.NetEngLatestChannelId}},${{parameters.NetEngValidationChannelId}},${{parameters.NetDev5ChannelId}},${{parameters.NetDev6ChannelId}},${{parameters.GeneralTestingChannelId}},${{parameters.NETCoreToolingDevChannelId}},${{parameters.NETCoreToolingReleaseChannelId}},${{parameters.NETInternalToolingChannelId}},${{parameters.NETCoreExperimentalChannelId}},${{parameters.NetEngServicesIntChannelId}},${{parameters.NetEngServicesProdChannelId}},${{parameters.NetCoreSDK313xxChannelId}},${{parameters.NetCoreSDK313xxInternalChannelId}},${{parameters.NetCoreSDK314xxChannelId}},${{parameters.NetCoreSDK314xxInternalChannelId}},${{parameters.VS166ChannelId}},${{parameters.VS167ChannelId}},${{parameters.VS168ChannelId}},${{parameters.VSMasterChannelId}},${{parameters.VS169ChannelId}},${{parameters.VS1610ChannelId}}
+
+    - job:
+      displayName: NuGet Validation
+      dependsOn: setupMaestroVars
+      condition: eq( ${{ parameters.enableNugetValidation }}, 'true')
+      pool:
+        vmImage: 'windows-2019'
+      variables:
+        - name: AzDOProjectName
+          value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ]
+        - name: AzDOPipelineId
+          value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ]
+        - name: AzDOBuildId
+          value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ]
+      steps:
+        - task: DownloadBuildArtifacts@0
+          displayName: Download Package Artifacts
+          inputs:
+            buildType: specific
+            buildVersionToDownload: specific
+            project: $(AzDOProjectName)
+            pipeline: $(AzDOPipelineId)
+            buildId: $(AzDOBuildId)
+            artifactName: PackageArtifacts
+            checkDownloadedFiles: true
+
+        - task: PowerShell@2
+          displayName: Validate
+          inputs:
+            filePath: $(Build.SourcesDirectory)/eng/common/post-build/nuget-validation.ps1
+            arguments: -PackagesPath $(Build.ArtifactStagingDirectory)/PackageArtifacts/ 
+              -ToolDestinationPath $(Agent.BuildDirectory)/Extract/ 
+
+    - job:
+      displayName: Signing Validation
+      dependsOn: setupMaestroVars
+      condition: and( eq( ${{ parameters.enableSigningValidation }}, 'true'), ne( variables['PostBuildSign'], 'true'))
+      variables:
+        - template: common-variables.yml
+        - name: AzDOProjectName
+          value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ]
+        - name: AzDOPipelineId
+          value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ]
+        - name: AzDOBuildId
+          value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ]
+      pool:
+        vmImage: 'windows-2019'
+      steps:
+        - task: DownloadBuildArtifacts@0
+          displayName: Download Package Artifacts
+          inputs:
+            buildType: specific
+            buildVersionToDownload: specific
+            project: $(AzDOProjectName)
+            pipeline: $(AzDOPipelineId)
+            buildId: $(AzDOBuildId)
+            artifactName: PackageArtifacts
+            checkDownloadedFiles: true
+            itemPattern: |
+              **
+              !**/Microsoft.SourceBuild.Intermediate.*.nupkg
+
+        # This is necessary whenever we want to publish/restore to an AzDO private feed
+        # Since sdk-task.ps1 tries to restore packages we need to do this authentication here
+        # otherwise it'll complain about accessing a private feed.
+        - task: NuGetAuthenticate@0
+          displayName: 'Authenticate to AzDO Feeds'
+
+        - task: PowerShell@2
+          displayName: Enable cross-org publishing
+          inputs:
+            filePath: eng\common\enable-cross-org-publishing.ps1
+            arguments: -token $(dn-bot-dnceng-artifact-feeds-rw)
+
+        # Signing validation will optionally work with the buildmanifest file which is downloaded from
+        # Azure DevOps above.
+        - task: PowerShell@2
+          displayName: Validate
+          inputs:
+            filePath: eng\common\sdk-task.ps1
+            arguments: -task SigningValidation -restore -msbuildEngine vs
+              /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts'
+              /p:SignCheckExclusionsFile='$(Build.SourcesDirectory)/eng/SignCheckExclusionsFile.txt'
+              ${{ parameters.signingValidationAdditionalParameters }}
+
+        - template: ../steps/publish-logs.yml
+          parameters:
+            StageLabel: 'Validation'
+            JobLabel: 'Signing'
+
+    - job:
+      displayName: SourceLink Validation
+      dependsOn: setupMaestroVars
+      condition: eq( ${{ parameters.enableSourceLinkValidation }}, 'true')
+      variables:
+        - template: common-variables.yml
+        - name: AzDOProjectName
+          value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ]
+        - name: AzDOPipelineId
+          value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ]
+        - name: AzDOBuildId
+          value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ]
+      pool:
+        vmImage: 'windows-2019'
+      steps:
+        - task: DownloadBuildArtifacts@0
+          displayName: Download Blob Artifacts
+          inputs:
+            buildType: specific
+            buildVersionToDownload: specific
+            project: $(AzDOProjectName)
+            pipeline: $(AzDOPipelineId)
+            buildId: $(AzDOBuildId)
+            artifactName: BlobArtifacts
+            checkDownloadedFiles: true
+
+        - task: PowerShell@2
+          displayName: Validate
+          inputs:
+            filePath: $(Build.SourcesDirectory)/eng/common/post-build/sourcelink-validation.ps1
+            arguments: -InputPath $(Build.ArtifactStagingDirectory)/BlobArtifacts/ 
+              -ExtractPath $(Agent.BuildDirectory)/Extract/ 
+              -GHRepoName $(Build.Repository.Name) 
+              -GHCommit $(Build.SourceVersion)
+              -SourcelinkCliVersion $(SourceLinkCLIVersion)
+          continueOnError: true
+
+    - template: /eng/common/templates/job/execute-sdl.yml
+      parameters:
+        enable: ${{ parameters.SDLValidationParameters.enable }}
+        dependsOn: setupMaestroVars
+        additionalParameters: ${{ parameters.SDLValidationParameters.params }}
+        continueOnError: ${{ parameters.SDLValidationParameters.continueOnError }}
+        artifactNames: ${{ parameters.SDLValidationParameters.artifactNames }}
+        downloadArtifacts: ${{ parameters.SDLValidationParameters.downloadArtifacts }}
+
+- ${{ if or(ge(parameters.publishingInfraVersion, 3), eq(parameters.inline, 'false')) }}:
+  - stage: publish_using_darc
+    ${{ if or(eq(parameters.enableNugetValidation, 'true'), eq(parameters.enableSigningValidation, 'true'), eq(parameters.enableSourceLinkValidation, 'true'), eq(parameters.SDLValidationParameters.enable, 'true')) }}:
+      dependsOn: ${{ parameters.publishDependsOn }}
+    ${{ if and(ne(parameters.enableNugetValidation, 'true'), ne(parameters.enableSigningValidation, 'true'), ne(parameters.enableSourceLinkValidation, 'true'), ne(parameters.SDLValidationParameters.enable, 'true')) }}:
+      dependsOn: ${{ parameters.validateDependsOn }}
+    displayName: Publish using Darc
+    variables:
+      - template: common-variables.yml
+    jobs:
+    - template: setup-maestro-vars.yml
+      parameters:
+        BARBuildId: ${{ parameters.BARBuildId }}
+        PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
+
+    - job:
+      displayName: Publish Using Darc
+      dependsOn: setupMaestroVars
+      timeoutInMinutes: 120
+      variables:
+        - name: BARBuildId
+          value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ]
+      pool:
+        vmImage: 'windows-2019'
+      steps:
+        - task: PowerShell@2
+          displayName: Publish Using Darc
+          inputs:
+            filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1
+            arguments: -BuildId $(BARBuildId) 
+              -PublishingInfraVersion ${{ parameters.PublishingInfraVersion }}
+              -AzdoToken '$(publishing-dnceng-devdiv-code-r-build-re)'
+              -MaestroToken '$(MaestroApiAccessToken)'
+              -WaitPublishingFinish ${{ parameters.waitPublishingFinish }}
+              -PublishInstallersAndChecksums ${{ parameters.publishInstallersAndChecksums }}
+              -ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}'
+              -SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}'
+
+- ${{ if and(le(parameters.publishingInfraVersion, 2), eq(parameters.inline, 'true')) }}:
+  - template: \eng\common\templates\post-build\channels\generic-public-channel.yml
+    parameters:
+      BARBuildId: ${{ parameters.BARBuildId }}
+      PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}    
+      artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
+      dependsOn: ${{ parameters.publishDependsOn }}
+      publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
+      symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
+      stageName: 'NetCore_Dev5_Publish'
+      channelName: '.NET 5 Dev'
+      akaMSChannelName: 'net5/dev'
+      channelId: ${{ parameters.NetDev5ChannelId }}
+      transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-transport/nuget/v3/index.json'
+      shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5/nuget/v3/index.json'
+      symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-symbols/nuget/v3/index.json'
+
+  - template: \eng\common\templates\post-build\channels\generic-public-channel.yml
+    parameters:
+      BARBuildId: ${{ parameters.BARBuildId }}
+      PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
+      artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
+      dependsOn: ${{ parameters.publishDependsOn }}
+      publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
+      symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
+      stageName: 'NetCore_Dev6_Publish'
+      channelName: '.NET 6 Dev'
+      akaMSChannelName: 'net6/dev'
+      channelId: ${{ parameters.NetDev6ChannelId }}
+      transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet6-transport/nuget/v3/index.json'
+      shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet6/nuget/v3/index.json'
+      symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet6-symbols/nuget/v3/index.json'
+
+  - template: \eng\common\templates\post-build\channels\generic-public-channel.yml
+    parameters:
+      BARBuildId: ${{ parameters.BARBuildId }}
+      PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
+      artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
+      dependsOn: ${{ parameters.publishDependsOn }}
+      publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
+      symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
+      stageName: 'Net_Eng_Latest_Publish'
+      channelName: '.NET Eng - Latest'
+      akaMSChannelName: 'eng/daily'
+      channelId: ${{ parameters.NetEngLatestChannelId }}
+      transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json'
+      shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json'
+      symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng-symbols/nuget/v3/index.json'
+
+  - template: \eng\common\templates\post-build\channels\generic-public-channel.yml
+    parameters:
+      BARBuildId: ${{ parameters.BARBuildId }}
+      PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}    
+      artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
+      dependsOn: ${{ parameters.publishDependsOn }}
+      publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
+      symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
+      stageName: 'Net_Eng_Validation_Publish'
+      channelName: '.NET Eng - Validation'
+      akaMSChannelName: 'eng/validation'
+      channelId: ${{ parameters.NetEngValidationChannelId }}
+      transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json'
+      shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json'
+      symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng-symbols/nuget/v3/index.json'
+
+  - template: \eng\common\templates\post-build\channels\generic-public-channel.yml
+    parameters:
+      BARBuildId: ${{ parameters.BARBuildId }}
+      PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}    
+      artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
+      dependsOn: ${{ parameters.publishDependsOn }}
+      publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
+      symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
+      stageName: 'General_Testing_Publish'
+      channelName: 'General Testing'
+      akaMSChannelName: 'generaltesting'
+      channelId: ${{ parameters.GeneralTestingChannelId }}
+      transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/general-testing/nuget/v3/index.json'
+      shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/general-testing/nuget/v3/index.json'
+      symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/general-testing-symbols/nuget/v3/index.json'
+
+  - template: \eng\common\templates\post-build\channels\generic-public-channel.yml
+    parameters:
+      BARBuildId: ${{ parameters.BARBuildId }}
+      PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}    
+      artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
+      dependsOn: ${{ parameters.publishDependsOn }}
+      publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
+      symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
+      stageName: 'NETCore_Tooling_Dev_Publishing'
+      channelName: '.NET Core Tooling Dev'
+      channelId: ${{ parameters.NETCoreToolingDevChannelId }}
+      transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json'
+      shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json'
+      symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-symbols/nuget/v3/index.json'
+
+  - template: \eng\common\templates\post-build\channels\generic-public-channel.yml
+    parameters:
+      BARBuildId: ${{ parameters.BARBuildId }}
+      PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}    
+      artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
+      dependsOn: ${{ parameters.publishDependsOn }}
+      publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
+      symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
+      stageName: 'NETCore_Tooling_Release_Publishing'
+      channelName: '.NET Core Tooling Release'
+      channelId: ${{ parameters.NETCoreToolingReleaseChannelId }}
+      transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json'
+      shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json'
+      symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-symbols/nuget/v3/index.json'
+
+  - template: \eng\common\templates\post-build\channels\generic-internal-channel.yml
+    parameters:
+      BARBuildId: ${{ parameters.BARBuildId }}
+      PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}    
+      artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
+      dependsOn: ${{ parameters.publishDependsOn }}
+      publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
+      symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
+      stageName: 'NET_Internal_Tooling_Publishing'
+      channelName: '.NET Internal Tooling'
+      channelId: ${{ parameters.NETInternalToolingChannelId }}
+      transportFeed: 'https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet-tools-internal/nuget/v3/index.json'
+      shippingFeed: 'https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet-tools-internal/nuget/v3/index.json'
+      symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet-tools-internal-symbols/nuget/v3/index.json'
+
+  - template: \eng\common\templates\post-build\channels\generic-public-channel.yml
+    parameters:
+      BARBuildId: ${{ parameters.BARBuildId }}
+      PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}    
+      artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
+      dependsOn: ${{ parameters.publishDependsOn }}
+      publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
+      symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
+      stageName: 'NETCore_Experimental_Publishing'
+      channelName: '.NET Core Experimental'
+      channelId: ${{ parameters.NETCoreExperimentalChannelId }}
+      transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-experimental/nuget/v3/index.json'
+      shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-experimental/nuget/v3/index.json'
+      symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-experimental-symbols/nuget/v3/index.json'
+
+  - template: \eng\common\templates\post-build\channels\generic-public-channel.yml
+    parameters:
+      BARBuildId: ${{ parameters.BARBuildId }}
+      PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}    
+      artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
+      dependsOn: ${{ parameters.publishDependsOn }}
+      publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
+      symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
+      stageName: 'Net_Eng_Services_Int_Publish'
+      channelName: '.NET Eng Services - Int'
+      channelId: ${{ parameters.NetEngServicesIntChannelId }}
+      transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json'
+      shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json'
+      symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng-symbols/nuget/v3/index.json'
+
+  - template: \eng\common\templates\post-build\channels\generic-public-channel.yml
+    parameters:
+      BARBuildId: ${{ parameters.BARBuildId }}
+      PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}    
+      artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
+      dependsOn: ${{ parameters.publishDependsOn }}
+      publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
+      symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
+      stageName: 'Net_Eng_Services_Prod_Publish'
+      channelName: '.NET Eng Services - Prod'
+      channelId: ${{ parameters.NetEngServicesProdChannelId }}
+      transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json'
+      shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json'
+      symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng-symbols/nuget/v3/index.json'
+
+  - template: \eng\common\templates\post-build\channels\generic-public-channel.yml
+    parameters:
+      BARBuildId: ${{ parameters.BARBuildId }}
+      PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}    
+      artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
+      dependsOn: ${{ parameters.publishDependsOn }}
+      publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
+      symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
+      stageName: 'NETCore_SDK_314xx_Publishing'
+      channelName: '.NET Core SDK 3.1.4xx'
+      channelId: ${{ parameters.NetCoreSDK314xxChannelId }}
+      transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet3.1-transport/nuget/v3/index.json'
+      shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet3.1/nuget/v3/index.json'
+      symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet3.1-symbols/nuget/v3/index.json'
+
+  - template: \eng\common\templates\post-build\channels\generic-internal-channel.yml
+    parameters:
+      BARBuildId: ${{ parameters.BARBuildId }}
+      PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}    
+      artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
+      dependsOn: ${{ parameters.publishDependsOn }}
+      publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
+      symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
+      stageName: 'NETCore_SDK_314xx_Internal_Publishing'
+      channelName: '.NET Core SDK 3.1.4xx Internal'
+      channelId: ${{ parameters.NetCoreSDK314xxInternalChannelId }}
+      transportFeed: 'https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-transport/nuget/v3/index.json'
+      shippingFeed: 'https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal/nuget/v3/index.json'
+      symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-symbols/nuget/v3/index.json' 
+
+  - template: \eng\common\templates\post-build\channels\generic-public-channel.yml
+    parameters:
+      BARBuildId: ${{ parameters.BARBuildId }}
+      PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}    
+      artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
+      dependsOn: ${{ parameters.publishDependsOn }}
+      publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
+      symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
+      stageName: 'NETCore_SDK_313xx_Publishing'
+      channelName: '.NET Core SDK 3.1.3xx'
+      channelId: ${{ parameters.NetCoreSDK313xxChannelId }}
+      transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet3.1-transport/nuget/v3/index.json'
+      shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet3.1/nuget/v3/index.json'
+      symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet3.1-symbols/nuget/v3/index.json'
+
+  - template: \eng\common\templates\post-build\channels\generic-internal-channel.yml
+    parameters:
+      BARBuildId: ${{ parameters.BARBuildId }}
+      PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}    
+      artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
+      dependsOn: ${{ parameters.publishDependsOn }}
+      publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
+      symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
+      stageName: 'NETCore_SDK_313xx_Internal_Publishing'
+      channelName: '.NET Core SDK 3.1.3xx Internal'
+      channelId: ${{ parameters.NetCoreSDK313xxInternalChannelId }}
+      transportFeed: 'https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-transport/nuget/v3/index.json'
+      shippingFeed: 'https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal/nuget/v3/index.json'
+      symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-symbols/nuget/v3/index.json' 
+
+  - template: \eng\common\templates\post-build\channels\generic-public-channel.yml
+    parameters:
+      BARBuildId: ${{ parameters.BARBuildId }}
+      PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}    
+      artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
+      dependsOn: ${{ parameters.publishDependsOn }}
+      publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
+      symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
+      stageName: 'VS16_6_Publishing'
+      channelName: 'VS 16.6'
+      channelId: ${{ parameters.VS166ChannelId }}
+      transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-transport/nuget/v3/index.json'
+      shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json'
+      symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-symbols/nuget/v3/index.json'
+
+  - template: \eng\common\templates\post-build\channels\generic-public-channel.yml
+    parameters:
+      BARBuildId: ${{ parameters.BARBuildId }}
+      PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}    
+      artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
+      dependsOn: ${{ parameters.publishDependsOn }}
+      publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
+      symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
+      stageName: 'VS16_7_Publishing'
+      channelName: 'VS 16.7'
+      channelId: ${{ parameters.VS167ChannelId }}
+      transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-transport/nuget/v3/index.json'
+      shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json'
+      symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-symbols/nuget/v3/index.json'
+      
+  - template: \eng\common\templates\post-build\channels\generic-public-channel.yml
+    parameters:
+      BARBuildId: ${{ parameters.BARBuildId }}
+      PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}    
+      artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
+      dependsOn: ${{ parameters.publishDependsOn }}
+      publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
+      symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
+      stageName: 'VS16_8_Publishing'
+      channelName: 'VS 16.8'
+      channelId: ${{ parameters.VS168ChannelId }}
+      transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-transport/nuget/v3/index.json'
+      shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json'
+      symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-symbols/nuget/v3/index.json'
+
+  - template: \eng\common\templates\post-build\channels\generic-public-channel.yml
+    parameters:
+      BARBuildId: ${{ parameters.BARBuildId }}
+      PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}    
+      artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
+      dependsOn: ${{ parameters.publishDependsOn }}
+      publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
+      symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
+      stageName: 'VS_Master_Publishing'
+      channelName: 'VS Master'
+      channelId: ${{ parameters.VSMasterChannelId }}
+      transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-transport/nuget/v3/index.json'
+      shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json'
+      symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-symbols/nuget/v3/index.json'
+
+  - template: \eng\common\templates\post-build\channels\generic-public-channel.yml
+    parameters:
+      BARBuildId: ${{ parameters.BARBuildId }}
+      PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}    
+      artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
+      dependsOn: ${{ parameters.publishDependsOn }}
+      publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
+      symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
+      stageName: 'VS_16_9_Publishing'
+      channelName: 'VS 16.9'
+      channelId: ${{ parameters.VS169ChannelId }}
+      transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-transport/nuget/v3/index.json'
+      shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json'
+      symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-symbols/nuget/v3/index.json'
+
+  - template: \eng\common\templates\post-build\channels\generic-public-channel.yml
+    parameters:
+      BARBuildId: ${{ parameters.BARBuildId }}
+      PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}    
+      artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
+      dependsOn: ${{ parameters.publishDependsOn }}
+      publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
+      symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
+      stageName: 'VS_16_10_Publishing'
+      channelName: 'VS 16.10'
+      channelId: ${{ parameters.VS1610ChannelId }}
+      transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-transport/nuget/v3/index.json'
+      shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json'
+      symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-symbols/nuget/v3/index.json'

eng/common/templates/post-build/setup-maestro-vars.yml

@@ -0,0 +1,78 @@
+parameters:
+  BARBuildId: ''
+  PromoteToChannelIds: ''
+
+jobs:
+- job: setupMaestroVars
+  displayName: Setup Maestro Vars
+  variables:
+    - template: common-variables.yml
+  pool:
+    vmImage: 'windows-2019'
+  steps:
+    - checkout: none
+
+    - ${{ if eq(coalesce(parameters.PromoteToChannelIds, 0), 0) }}:
+      - task: DownloadBuildArtifacts@0
+        displayName: Download Release Configs
+        inputs:
+          buildType: current
+          artifactName: ReleaseConfigs
+          checkDownloadedFiles: true
+
+    - task: PowerShell@2
+      name: setReleaseVars
+      displayName: Set Release Configs Vars
+      inputs:
+        targetType: inline
+        script: |
+          try {
+            if (!$Env:PromoteToMaestroChannels -or $Env:PromoteToMaestroChannels.Trim() -eq '') {
+              $Content = Get-Content $(Build.StagingDirectory)/ReleaseConfigs/ReleaseConfigs.txt
+
+              $BarId = $Content | Select -Index 0
+              $Channels = $Content | Select -Index 1             
+              $IsStableBuild = $Content | Select -Index 2
+
+              $AzureDevOpsProject = $Env:System_TeamProject
+              $AzureDevOpsBuildDefinitionId = $Env:System_DefinitionId
+              $AzureDevOpsBuildId = $Env:Build_BuildId
+            }
+            else {
+              $buildApiEndpoint = "${Env:MaestroApiEndPoint}/api/builds/${Env:BARBuildId}?api-version=${Env:MaestroApiVersion}"
+
+              $apiHeaders = New-Object 'System.Collections.Generic.Dictionary[[String],[String]]'
+              $apiHeaders.Add('Accept', 'application/json')
+              $apiHeaders.Add('Authorization',"Bearer ${Env:MAESTRO_API_TOKEN}")
+
+              $buildInfo = try { Invoke-WebRequest -Method Get -Uri $buildApiEndpoint -Headers $apiHeaders | ConvertFrom-Json } catch { Write-Host "Error: $_" }
+             
+              $BarId = $Env:BARBuildId
+              $Channels = $Env:PromoteToMaestroChannels -split ","
+              $Channels = $Channels -join "]["
+              $Channels = "[$Channels]"
+
+              $IsStableBuild = $buildInfo.stable
+              $AzureDevOpsProject = $buildInfo.azureDevOpsProject
+              $AzureDevOpsBuildDefinitionId = $buildInfo.azureDevOpsBuildDefinitionId
+              $AzureDevOpsBuildId = $buildInfo.azureDevOpsBuildId
+            }
+
+            Write-Host "##vso[task.setvariable variable=BARBuildId;isOutput=true]$BarId"
+            Write-Host "##vso[task.setvariable variable=TargetChannels;isOutput=true]$Channels"
+            Write-Host "##vso[task.setvariable variable=IsStableBuild;isOutput=true]$IsStableBuild"
+
+            Write-Host "##vso[task.setvariable variable=AzDOProjectName;isOutput=true]$AzureDevOpsProject"
+            Write-Host "##vso[task.setvariable variable=AzDOPipelineId;isOutput=true]$AzureDevOpsBuildDefinitionId"
+            Write-Host "##vso[task.setvariable variable=AzDOBuildId;isOutput=true]$AzureDevOpsBuildId"
+          }
+          catch {
+            Write-Host $_
+            Write-Host $_.Exception
+            Write-Host $_.ScriptStackTrace
+            exit 1
+          }
+      env:
+        MAESTRO_API_TOKEN: $(MaestroApiAccessToken)
+        BARBuildId: ${{ parameters.BARBuildId }}
+        PromoteToMaestroChannels: ${{ parameters.PromoteToChannelIds }}

eng/common/templates/post-build/trigger-subscription.yml

@@ -0,0 +1,13 @@
+parameters:
+  ChannelId: 0
+
+steps:
+- task: PowerShell@2
+  displayName: Triggering subscriptions
+  inputs:
+    filePath: $(Build.SourcesDirectory)/eng/common/post-build/trigger-subscriptions.ps1
+    arguments: -SourceRepo $(Build.Repository.Uri)
+      -ChannelId ${{ parameters.ChannelId }}
+      -MaestroApiAccessToken $(MaestroAccessToken)
+      -MaestroApiEndPoint $(MaestroApiEndPoint)
+      -MaestroApiVersion $(MaestroApiVersion)

eng/common/templates/steps/add-build-to-channel.yml

@@ -0,0 +1,13 @@
+parameters:
+  ChannelId: 0
+
+steps:
+- task: PowerShell@2
+  displayName: Add Build to Channel
+  inputs:
+    filePath: $(Build.SourcesDirectory)/eng/common/post-build/add-build-to-channel.ps1
+    arguments: -BuildId $(BARBuildId) 
+      -ChannelId ${{ parameters.ChannelId }}
+      -MaestroApiAccessToken $(MaestroApiAccessToken)
+      -MaestroApiEndPoint $(MaestroApiEndPoint)
+      -MaestroApiVersion $(MaestroApiVersion) 

eng/common/templates/steps/helix-publish.yml

@@ -1,51 +0,0 @@
-parameters:
-  HelixSource: 'pr/dotnet-github-anon-kaonashi-bot'
-  HelixType: ̓'tests/default'
-  HelixBuild: $(Build.BuildNumber)
-  HelixTargetQueues: ''
-  HelixAccessToken: ''
-  HelixPreCommands: ''
-  HelixPostCommands: ''
-  WorkItemDirectory: ''
-  WorkItemCommand: ''
-  CorrelationPayloadDirectory: ''
-  XUnitProjects: ''
-  XUnitTargetFramework: ''
-  XUnitRunnerVersion: ''
-  IncludeDotNetCli: false
-  DotNetCliPackageType: ''
-  DotNetCliVersion: ''
-  EnableXUnitReporter: false
-  WaitForWorkItemCompletion: true
-  condition: succeeded()
-  continueOnError: false
-
-steps:
-  - task: DotNetCoreCLI@2
-    inputs:
-      command: custom
-      projects: eng/common/helixpublish.proj
-      custom: msbuild
-      arguments: '/bl:$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)/SendToHelix.binlog'
-    displayName: Send job to Helix
-    env:
-      HelixSource: ${{ parameters.HelixSource }}
-      HelixType: ${{ parameters.HelixType }}
-      HelixBuild: ${{ parameters.HelixBuild }}
-      HelixTargetQueues: ${{ parameters.HelixTargetQueues }}
-      HelixAccessToken: ${{ parameters.HelixAccessToken }}
-      HelixPreCommands: ${{ parameters.HelixPreCommands }}
-      HelixPostCommands: ${{ parameters.HelixPostCommands }}
-      WorkItemDirectory: ${{ parameters.WorkItemDirectory }}
-      WorkItemCommand: ${{ parameters.WorkItemCommand }}
-      CorrelationPayloadDirectory: ${{ parameters.CorrelationPayloadDirectory }}
-      XUnitProjects: ${{ parameters.XUnitProjects }}
-      XUnitRuntimeTargetFramework: ${{ parameters.XUnitTargetFramework }}
-      XUnitRunnerVersion: ${{ parameters.XUnitRunnerVersion }}
-      IncludeDotNetCli: ${{ parameters.IncludeDotNetCli }}
-      DotNetCliPackageType: ${{ parameters.DotNetCliPackageType }}
-      DotNetCliVersion: ${{ parameters.DotNetCliVersion }}
-      EnableXUnitReporter: ${{ parameters.EnableXUnitReporter }}
-      WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }}
-    condition: ${{ parameters.condition }}
-    continueOnError: ${{ parameters.continueOnError }}

eng/common/templates/steps/publish-logs.yml

@@ -0,0 +1,23 @@
+parameters:
+  StageLabel: ''
+  JobLabel: ''
+
+steps:
+- task: Powershell@2
+  displayName: Prepare Binlogs to Upload
+  inputs:
+    targetType: inline
+    script: |
+      New-Item -ItemType Directory $(Build.SourcesDirectory)/PostBuildLogs/${{parameters.StageLabel}}/${{parameters.JobLabel}}/
+      Move-Item -Path $(Build.SourcesDirectory)/artifacts/log/Debug/* $(Build.SourcesDirectory)/PostBuildLogs/${{parameters.StageLabel}}/${{parameters.JobLabel}}/
+  continueOnError: true
+  condition: always()
+
+- task: PublishBuildArtifacts@1
+  displayName: Publish Logs
+  inputs:
+    PathtoPublish: '$(Build.SourcesDirectory)/PostBuildLogs'
+    PublishLocation: Container
+    ArtifactName: PostBuildLogs
+  continueOnError: true
+  condition: always()

eng/common/templates/steps/send-to-helix.yml

@@ -5,11 +5,12 @@ parameters:
   HelixBuild: $(Build.BuildNumber)       # required -- the build number Helix will use to identify this -- automatically set to the AzDO build number
   HelixTargetQueues: ''                  # required -- semicolon delimited list of Helix queues to test on; see https://helix.dot.net/ for a list of queues
   HelixAccessToken: ''                   # required -- access token to make Helix API requests; should be provided by the appropriate variable group
+  HelixConfiguration: ''                 # optional -- additional property attached to a job
   HelixPreCommands: ''                   # optional -- commands to run before Helix work item execution
   HelixPostCommands: ''                  # optional -- commands to run after Helix work item execution
   WorkItemDirectory: ''                  # optional -- a payload directory to zip up and send to Helix; requires WorkItemCommand; incompatible with XUnitProjects
   WorkItemCommand: ''                    # optional -- a command to execute on the payload; requires WorkItemDirectory; incompatible with XUnitProjects
-  WorkItemTimeout: ''                    # optional -- a timeout in seconds for the work item command; requires WorkItemDirectory; incompatible with XUnitProjects
+  WorkItemTimeout: ''                    # optional -- a timeout in TimeSpan.Parse-ready value (e.g. 00:02:00) for the work item command; requires WorkItemDirectory; incompatible with XUnitProjects
   CorrelationPayloadDirectory: ''        # optional -- a directory to zip up and send to Helix as a correlation payload
   XUnitProjects: ''                      # optional -- semicolon delimited list of XUnitProjects to parse and send to Helix; requires XUnitRuntimeTargetFramework, XUnitPublishTargetFramework, XUnitRunnerVersion, and IncludeDotNetCli=true
   XUnitWorkItemTimeout: ''               # optional -- the workitem timeout in seconds for all workitems created from the xUnit projects specified by XUnitProjects
@@ -17,13 +18,14 @@ parameters:
   XUnitRuntimeTargetFramework: ''        # optional -- framework to use for the xUnit console runner
   XUnitRunnerVersion: ''                 # optional -- version of the xUnit nuget package you wish to use on Helix; required for XUnitProjects
   IncludeDotNetCli: false                # optional -- true will download a version of the .NET CLI onto the Helix machine as a correlation payload; requires DotNetCliPackageType and DotNetCliVersion
-  DotNetCliPackageType: ''               # optional -- either 'sdk' or 'runtime'; determines whether the sdk or runtime will be sent to Helix; see https://raw.githubusercontent.com/dotnet/core/master/release-notes/releases.json
-  DotNetCliVersion: ''                   # optional -- version of the CLI to send to Helix; based on this: https://raw.githubusercontent.com/dotnet/core/master/release-notes/releases.json
+  DotNetCliPackageType: ''               # optional -- either 'sdk', 'runtime' or 'aspnetcore-runtime'; determines whether the sdk or runtime will be sent to Helix; see https://raw.githubusercontent.com/dotnet/core/main/release-notes/releases-index.json
+  DotNetCliVersion: ''                   # optional -- version of the CLI to send to Helix; based on this: https://raw.githubusercontent.com/dotnet/core/main/release-notes/releases-index.json
   EnableXUnitReporter: false             # optional -- true enables XUnit result reporting to Mission Control
   WaitForWorkItemCompletion: true        # optional -- true will make the task wait until work items have been completed and fail the build if work items fail. False is "fire and forget."
   IsExternal: false                      # [DEPRECATED] -- doesn't do anything, jobs are external if HelixAccessToken is empty and Creator is set
+  HelixBaseUri: 'https://helix.dot.net/' # optional -- sets the Helix API base URI (allows targeting int)
   Creator: ''                            # optional -- if the build is external, use this to specify who is sending the job
-  DisplayNamePrefix: 'Send job to Helix' # optional -- rename the beginning of the displayName of the steps in AzDO 
+  DisplayNamePrefix: 'Run Tests'         # optional -- rename the beginning of the displayName of the steps in AzDO 
   condition: succeeded()                 # optional -- condition for step to execute; defaults to succeeded()
   continueOnError: false                 # optional -- determines whether to continue the build if the step errors; defaults to false
 
@@ -35,6 +37,7 @@ steps:
       HelixSource: ${{ parameters.HelixSource }}
       HelixType: ${{ parameters.HelixType }}
       HelixBuild: ${{ parameters.HelixBuild }}
+      HelixConfiguration:  ${{ parameters.HelixConfiguration }}
       HelixTargetQueues: ${{ parameters.HelixTargetQueues }}
       HelixAccessToken: ${{ parameters.HelixAccessToken }}
       HelixPreCommands: ${{ parameters.HelixPreCommands }}
@@ -53,6 +56,7 @@ steps:
       DotNetCliVersion: ${{ parameters.DotNetCliVersion }}
       EnableXUnitReporter: ${{ parameters.EnableXUnitReporter }}
       WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }}
+      HelixBaseUri: ${{ parameters.HelixBaseUri }}
       Creator: ${{ parameters.Creator }}
       SYSTEM_ACCESSTOKEN: $(System.AccessToken)
     condition: and(${{ parameters.condition }}, eq(variables['Agent.Os'], 'Windows_NT'))
@@ -64,6 +68,7 @@ steps:
       HelixSource: ${{ parameters.HelixSource }}
       HelixType: ${{ parameters.HelixType }}
       HelixBuild: ${{ parameters.HelixBuild }}
+      HelixConfiguration:  ${{ parameters.HelixConfiguration }}
       HelixTargetQueues: ${{ parameters.HelixTargetQueues }}
       HelixAccessToken: ${{ parameters.HelixAccessToken }}
       HelixPreCommands: ${{ parameters.HelixPreCommands }}
@@ -82,6 +87,7 @@ steps:
       DotNetCliVersion: ${{ parameters.DotNetCliVersion }}
       EnableXUnitReporter: ${{ parameters.EnableXUnitReporter }}
       WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }}
+      HelixBaseUri: ${{ parameters.HelixBaseUri }}
       Creator: ${{ parameters.Creator }}
       SYSTEM_ACCESSTOKEN: $(System.AccessToken)
     condition: and(${{ parameters.condition }}, ne(variables['Agent.Os'], 'Windows_NT'))

eng/common/templates/steps/source-build.yml

@@ -0,0 +1,66 @@
+parameters:
+  # This template adds arcade-powered source-build to CI.
+
+  # This is a 'steps' template, and is intended for advanced scenarios where the existing build
+  # infra has a careful build methodology that must be followed. For example, a repo
+  # (dotnet/runtime) might choose to clone the GitHub repo only once and store it as a pipeline
+  # artifact for all subsequent jobs to use, to reduce dependence on a strong network connection to
+  # GitHub. Using this steps template leaves room for that infra to be included.
+
+  # Defines the platform on which to run the steps. See 'eng/common/templates/job/source-build.yml'
+  # for details. The entire object is described in the 'job' template for simplicity, even though
+  # the usage of the properties on this object is split between the 'job' and 'steps' templates.
+  platform: {}
+
+steps:
+# Build. Keep it self-contained for simple reusability. (No source-build-specific job variables.)
+- script: |
+    set -x
+    df -h
+
+    buildConfig=Release
+    # Check if AzDO substitutes in a build config from a variable, and use it if so.
+    if [ '$(_BuildConfig)' != '$''(_BuildConfig)' ]; then
+      buildConfig='$(_BuildConfig)'
+    fi
+
+    officialBuildArgs=
+    if [ '${{ and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}' = 'True' ]; then
+      officialBuildArgs='/p:DotNetPublishUsingPipelines=true /p:OfficialBuildId=$(BUILD.BUILDNUMBER)'
+    fi
+
+    targetRidArgs=
+    if [ '${{ parameters.platform.targetRID }}' != '' ]; then
+      targetRidArgs='/p:TargetRid=${{ parameters.platform.targetRID }}'
+    fi
+
+    ${{ coalesce(parameters.platform.buildScript, './build.sh') }} --ci \
+      --configuration $buildConfig \
+      --restore --build --pack --publish -bl \
+      $officialBuildArgs \
+      $targetRidArgs \
+      /p:SourceBuildNonPortable=${{ parameters.platform.nonPortable }} \
+      /p:ArcadeBuildFromSource=true
+  displayName: Build
+
+# Upload build logs for diagnosis.
+- task: CopyFiles@2
+  displayName: Prepare BuildLogs staging directory
+  inputs:
+    SourceFolder: '$(Build.SourcesDirectory)'
+    Contents: |
+      **/*.log
+      **/*.binlog
+      artifacts/source-build/self/prebuilt-report/**
+    TargetFolder: '$(Build.StagingDirectory)/BuildLogs'
+    CleanTargetFolder: true
+  continueOnError: true
+  condition: succeededOrFailed()
+
+- task: PublishPipelineArtifact@1
+  displayName: Publish BuildLogs
+  inputs:
+    targetPath: '$(Build.StagingDirectory)/BuildLogs'
+    artifactName: BuildLogs_SourceBuild_${{ parameters.platform.name }}_Attempt$(System.JobAttempt)
+  continueOnError: true
+  condition: succeededOrFailed()

eng/common/tools.ps1

@@ -5,11 +5,19 @@
 [bool]$ci = if (Test-Path variable:ci) { $ci } else { $false }
 
 # Build configuration. Common values include 'Debug' and 'Release', but the repository may use other names.
-[string]$configuration = if (Test-Path variable:configuration) { $configuration } else { "Debug" }
+[string]$configuration = if (Test-Path variable:configuration) { $configuration } else { 'Debug' }
+
+# Set to true to opt out of outputting binary log while running in CI
+[bool]$excludeCIBinarylog = if (Test-Path variable:excludeCIBinarylog) { $excludeCIBinarylog } else { $false }
 
 # Set to true to output binary log from msbuild. Note that emitting binary log slows down the build.
-# Binary log must be enabled on CI.
-[bool]$binaryLog = if (Test-Path variable:binaryLog) { $binaryLog } else { $ci }
+[bool]$binaryLog = if (Test-Path variable:binaryLog) { $binaryLog } else { $ci -and !$excludeCIBinarylog }
+
+# Set to true to use the pipelines logger which will enable Azure logging output.
+# https://github.com/Microsoft/azure-pipelines-tasks/blob/master/docs/authoring/commands.md
+# This flag is meant as a temporary opt-opt for the feature while validate it across
+# our consumers. It will be deleted in the future.
+[bool]$pipelinesLog = if (Test-Path variable:pipelinesLog) { $pipelinesLog } else { $ci }
 
 # Turns on machine preparation/clean up code that changes the machine state (e.g. kills build processes).
 [bool]$prepareMachine = if (Test-Path variable:prepareMachine) { $prepareMachine } else { $false }
@@ -18,7 +26,7 @@
 [bool]$restore = if (Test-Path variable:restore) { $restore } else { $true }
 
 # Adjusts msbuild verbosity level.
-[string]$verbosity = if (Test-Path variable:verbosity) { $verbosity } else { "minimal" }
+[string]$verbosity = if (Test-Path variable:verbosity) { $verbosity } else { 'minimal' }
 
 # Set to true to reuse msbuild nodes. Recommended to not reuse on CI.
 [bool]$nodeReuse = if (Test-Path variable:nodeReuse) { $nodeReuse } else { !$ci }
@@ -29,24 +37,33 @@
 # Specifies which msbuild engine to use for build: 'vs', 'dotnet' or unspecified (determined based on presence of tools.vs in global.json).
 [string]$msbuildEngine = if (Test-Path variable:msbuildEngine) { $msbuildEngine } else { $null }
 
-# True to attempt using .NET Core already that meets requirements specified in global.json 
+# True to attempt using .NET Core already that meets requirements specified in global.json
 # installed on the machine instead of downloading one.
 [bool]$useInstalledDotNetCli = if (Test-Path variable:useInstalledDotNetCli) { $useInstalledDotNetCli } else { $true }
 
+# Enable repos to use a particular version of the on-line dotnet-install scripts.
+#    default URL: https://dot.net/v1/dotnet-install.ps1
+[string]$dotnetInstallScriptVersion = if (Test-Path variable:dotnetInstallScriptVersion) { $dotnetInstallScriptVersion } else { 'v1' }
+
 # True to use global NuGet cache instead of restoring packages to repository-local directory.
 [bool]$useGlobalNuGetCache = if (Test-Path variable:useGlobalNuGetCache) { $useGlobalNuGetCache } else { !$ci }
 
 # An array of names of processes to stop on script exit if prepareMachine is true.
-$processesToStopOnExit = if (Test-Path variable:processesToStopOnExit) { $processesToStopOnExit } else { @("msbuild", "dotnet", "vbcscompiler") }
+$processesToStopOnExit = if (Test-Path variable:processesToStopOnExit) { $processesToStopOnExit } else { @('msbuild', 'dotnet', 'vbcscompiler') }
+
+$disableConfigureToolsetImport = if (Test-Path variable:disableConfigureToolsetImport) { $disableConfigureToolsetImport } else { $null }
 
 set-strictmode -version 2.0
-$ErrorActionPreference = "Stop"
+$ErrorActionPreference = 'Stop'
 [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
 
-function Create-Directory([string[]] $path) {
-  if (!(Test-Path $path)) {
-    New-Item -path $path -force -itemType "Directory" | Out-Null
-  }
+# If specifies, provides an alternate path for getting .NET Core SDKs and Runtimes. This script will still try public sources first.
+[string]$runtimeSourceFeed = if (Test-Path variable:runtimeSourceFeed) { $runtimeSourceFeed } else { $null }
+# Base-64 encoded SAS token that has permission to storage container described by $runtimeSourceFeed
+[string]$runtimeSourceFeedKey = if (Test-Path variable:runtimeSourceFeedKey) { $runtimeSourceFeedKey } else { $null }
+
+function Create-Directory ([string[]] $path) {
+    New-Item -Path $path -Force -ItemType 'Directory' | Out-Null
 }
 
 function Unzip([string]$zipfile, [string]$outpath) {
@@ -70,7 +87,7 @@ function Exec-Process([string]$command, [string]$commandArgs) {
 
   $finished = $false
   try {
-    while (-not $process.WaitForExit(100)) { 
+    while (-not $process.WaitForExit(100)) {
       # Non-blocking loop done to allow ctr-c interrupts
     }
 
@@ -78,7 +95,7 @@ function Exec-Process([string]$command, [string]$commandArgs) {
     return $global:LASTEXITCODE = $process.ExitCode
   }
   finally {
-    # If we didn't finish then an error occured or the user hit ctrl-c.  Either
+    # If we didn't finish then an error occurred or the user hit ctrl-c.  Either
     # way kill the process
     if (-not $finished) {
       $process.Kill()
@@ -86,7 +103,10 @@ function Exec-Process([string]$command, [string]$commandArgs) {
   }
 }
 
-function InitializeDotNetCli([bool]$install) {
+# createSdkLocationFile parameter enables a file being generated under the toolset directory
+# which writes the sdk's location into. This is only necessary for cmd --> powershell invocations
+# as dot sourcing isn't possible.
+function InitializeDotNetCli([bool]$install, [bool]$createSdkLocationFile) {
   if (Test-Path variable:global:_DotNetInstallDir) {
     return $global:_DotNetInstallDir
   }
@@ -108,8 +128,10 @@ function InitializeDotNetCli([bool]$install) {
   }
 
   # Find the first path on %PATH% that contains the dotnet.exe
-  if ($useInstalledDotNetCli -and ($env:DOTNET_INSTALL_DIR -eq $null)) {
-    $dotnetCmd = Get-Command "dotnet.exe" -ErrorAction SilentlyContinue
+  if ($useInstalledDotNetCli -and (-not $globalJsonHasRuntimes) -and ($env:DOTNET_INSTALL_DIR -eq $null)) {
+    $dotnetExecutable = GetExecutableFileName 'dotnet'
+    $dotnetCmd = Get-Command $dotnetExecutable -ErrorAction SilentlyContinue
+
     if ($dotnetCmd -ne $null) {
       $env:DOTNET_INSTALL_DIR = Split-Path $dotnetCmd.Path -Parent
     }
@@ -119,16 +141,16 @@ function InitializeDotNetCli([bool]$install) {
 
   # Use dotnet installation specified in DOTNET_INSTALL_DIR if it contains the required SDK version,
   # otherwise install the dotnet CLI and SDK to repo local .dotnet directory to avoid potential permission issues.
-  if (($env:DOTNET_INSTALL_DIR -ne $null) -and (Test-Path(Join-Path $env:DOTNET_INSTALL_DIR "sdk\$dotnetSdkVersion"))) {
+  if ((-not $globalJsonHasRuntimes) -and (-not [string]::IsNullOrEmpty($env:DOTNET_INSTALL_DIR)) -and (Test-Path(Join-Path $env:DOTNET_INSTALL_DIR "sdk\$dotnetSdkVersion"))) {
     $dotnetRoot = $env:DOTNET_INSTALL_DIR
   } else {
-    $dotnetRoot = Join-Path $RepoRoot ".dotnet"
+    $dotnetRoot = Join-Path $RepoRoot '.dotnet'
 
     if (-not (Test-Path(Join-Path $dotnetRoot "sdk\$dotnetSdkVersion"))) {
       if ($install) {
         InstallDotNetSdk $dotnetRoot $dotnetSdkVersion
       } else {
-        Write-Host "Unable to find dotnet with SDK version '$dotnetSdkVersion'" -ForegroundColor Red
+        Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "Unable to find dotnet with SDK version '$dotnetSdkVersion'"
         ExitWithExitCode 1
       }
     }
@@ -136,43 +158,129 @@ function InitializeDotNetCli([bool]$install) {
     $env:DOTNET_INSTALL_DIR = $dotnetRoot
   }
 
+  # Creates a temporary file under the toolset dir.
+  # The following code block is protecting against concurrent access so that this function can
+  # be called in parallel.
+  if ($createSdkLocationFile) {
+    do {
+      $sdkCacheFileTemp = Join-Path $ToolsetDir $([System.IO.Path]::GetRandomFileName())
+    }
+    until (!(Test-Path $sdkCacheFileTemp))
+    Set-Content -Path $sdkCacheFileTemp -Value $dotnetRoot
+
+    try {
+      Move-Item -Force $sdkCacheFileTemp (Join-Path $ToolsetDir 'sdk.txt')
+    } catch {
+      # Somebody beat us
+      Remove-Item -Path $sdkCacheFileTemp
+    }
+  }
+
   # Add dotnet to PATH. This prevents any bare invocation of dotnet in custom
   # build steps from using anything other than what we've downloaded.
   # It also ensures that VS msbuild will use the downloaded sdk targets.
   $env:PATH = "$dotnetRoot;$env:PATH"
 
-  if ($ci) {
-    # Make Sure that our bootstrapped dotnet cli is avaliable in future steps of the Azure Pipelines build
-    Write-Host "##vso[task.prependpath]$dotnetRoot"
-    Write-Host "##vso[task.setvariable variable=DOTNET_MULTILEVEL_LOOKUP]0"
-    Write-Host "##vso[task.setvariable variable=DOTNET_SKIP_FIRST_TIME_EXPERIENCE]1"
-  }
+  # Make Sure that our bootstrapped dotnet cli is available in future steps of the Azure Pipelines build
+  Write-PipelinePrependPath -Path $dotnetRoot
+
+  Write-PipelineSetVariable -Name 'DOTNET_MULTILEVEL_LOOKUP' -Value '0'
+  Write-PipelineSetVariable -Name 'DOTNET_SKIP_FIRST_TIME_EXPERIENCE' -Value '1'
 
   return $global:_DotNetInstallDir = $dotnetRoot
 }
 
 function GetDotNetInstallScript([string] $dotnetRoot) {
-  $installScript = "$dotnetRoot\dotnet-install.ps1"
+  $installScript = Join-Path $dotnetRoot 'dotnet-install.ps1'
   if (!(Test-Path $installScript)) {
     Create-Directory $dotnetRoot
-    Invoke-WebRequest "https://dot.net/v1/dotnet-install.ps1" -OutFile $installScript
+    $ProgressPreference = 'SilentlyContinue' # Don't display the console progress UI - it's a huge perf hit
+
+    $maxRetries = 5
+    $retries = 1
+
+    $uri = "https://dot.net/$dotnetInstallScriptVersion/dotnet-install.ps1"
+
+    while($true) {
+      try {
+        Write-Host "GET $uri"
+        Invoke-WebRequest $uri -OutFile $installScript
+        break
+      }
+      catch {
+        Write-Host "Failed to download '$uri'"
+        Write-Error $_.Exception.Message -ErrorAction Continue
+      }
+
+      if (++$retries -le $maxRetries) {
+        $delayInSeconds = [math]::Pow(2, $retries) - 1 # Exponential backoff
+        Write-Host "Retrying. Waiting for $delayInSeconds seconds before next attempt ($retries of $maxRetries)."
+        Start-Sleep -Seconds $delayInSeconds
+      }
+      else {
+        throw "Unable to download file in $maxRetries attempts."
+      }
+
+    }
   }
 
   return $installScript
 }
 
-function InstallDotNetSdk([string] $dotnetRoot, [string] $version, [string] $architecture = "") {
+function InstallDotNetSdk([string] $dotnetRoot, [string] $version, [string] $architecture = '', [switch] $noPath) {
+  InstallDotNet $dotnetRoot $version $architecture '' $false $runtimeSourceFeed $runtimeSourceFeedKey -noPath:$noPath
+}
+
+function InstallDotNet([string] $dotnetRoot,
+  [string] $version,
+  [string] $architecture = '',
+  [string] $runtime = '',
+  [bool] $skipNonVersionedFiles = $false,
+  [string] $runtimeSourceFeed = '',
+  [string] $runtimeSourceFeedKey = '',
+  [switch] $noPath) {
+
   $installScript = GetDotNetInstallScript $dotnetRoot
-  $archArg = if ($architecture) { $architecture } else { "<auto>" }
-  & $installScript -Version $version -InstallDir $dotnetRoot -Architecture $archArg
-  if ($lastExitCode -ne 0) {
-    Write-Host "Failed to install dotnet cli (exit code '$lastExitCode')." -ForegroundColor Red
-    ExitWithExitCode $lastExitCode
+  $installParameters = @{
+    Version = $version
+    InstallDir = $dotnetRoot
+  }
+
+  if ($architecture) { $installParameters.Architecture = $architecture }
+  if ($runtime) { $installParameters.Runtime = $runtime }
+  if ($skipNonVersionedFiles) { $installParameters.SkipNonVersionedFiles = $skipNonVersionedFiles }
+  if ($noPath) { $installParameters.NoPath = $True }
+
+  try {
+    & $installScript @installParameters
+  }
+  catch {
+    if ($runtimeSourceFeed -or $runtimeSourceFeedKey) {
+      Write-Host "Failed to install dotnet from public location. Trying from '$runtimeSourceFeed'"
+      if ($runtimeSourceFeed) { $installParameters.AzureFeed = $runtimeSourceFeed }
+
+      if ($runtimeSourceFeedKey) {
+        $decodedBytes = [System.Convert]::FromBase64String($runtimeSourceFeedKey)
+        $decodedString = [System.Text.Encoding]::UTF8.GetString($decodedBytes)
+        $installParameters.FeedCredential = $decodedString
+      }
+
+      try {
+        & $installScript @installParameters
+      }
+      catch {
+        Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "Failed to install dotnet from custom location '$runtimeSourceFeed'."
+        ExitWithExitCode 1
+      }
+    } else {
+      Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "Failed to install dotnet from public location."
+      ExitWithExitCode 1
+    }
   }
 }
 
 #
-# Locates Visual Studio MSBuild installation. 
+# Locates Visual Studio MSBuild installation.
 # The preference order for MSBuild to use is as follows:
 #
 #   1. MSBuild from an active VS command prompt
@@ -183,19 +291,36 @@ function InstallDotNetSdk([string] $dotnetRoot, [string] $version, [string] $arc
 # Throws on failure.
 #
 function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements = $null) {
+  if (-not (IsWindowsPlatform)) {
+    throw "Cannot initialize Visual Studio on non-Windows"
+  }
+
   if (Test-Path variable:global:_MSBuildExe) {
     return $global:_MSBuildExe
   }
 
+  # Minimum VS version to require.
+  $vsMinVersionReqdStr = '16.8'
+  $vsMinVersionReqd = [Version]::new($vsMinVersionReqdStr)
+
+  # If the version of msbuild is going to be xcopied,
+  # use this version. Version matches a package here:
+  # https://dev.azure.com/dnceng/public/_packaging?_a=package&feed=dotnet-eng&package=RoslynTools.MSBuild&protocolType=NuGet&version=16.8.0-preview3&view=overview
+  $defaultXCopyMSBuildVersion = '16.8.0-preview3'
+
   if (!$vsRequirements) { $vsRequirements = $GlobalJson.tools.vs }
-  $vsMinVersionStr = if ($vsRequirements.version) { $vsRequirements.version } else { "15.9" }
-  $vsMinVersion = [Version]::new($vsMinVersionStr) 
+  $vsMinVersionStr = if ($vsRequirements.version) { $vsRequirements.version } else { $vsMinVersionReqdStr }
+  $vsMinVersion = [Version]::new($vsMinVersionStr)
 
   # Try msbuild command available in the environment.
   if ($env:VSINSTALLDIR -ne $null) {
-    $msbuildCmd = Get-Command "msbuild.exe" -ErrorAction SilentlyContinue
+    $msbuildCmd = Get-Command 'msbuild.exe' -ErrorAction SilentlyContinue
     if ($msbuildCmd -ne $null) {
-      if ($msbuildCmd.Version -ge $vsMinVersion) {
+      # Workaround for https://github.com/dotnet/roslyn/issues/35793
+      # Due to this issue $msbuildCmd.Version returns 0.0.0.0 for msbuild.exe 16.2+
+      $msbuildVersion = [Version]::new((Get-Item $msbuildCmd.Path).VersionInfo.ProductVersion.Split([char[]]@('-', '+'))[0])
+
+      if ($msbuildVersion -ge $vsMinVersion) {
         return $global:_MSBuildExe = $msbuildCmd.Path
       }
 
@@ -213,17 +338,35 @@ function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements =
     InitializeVisualStudioEnvironmentVariables $vsInstallDir $vsMajorVersion
   } else {
 
-    if (Get-Member -InputObject $GlobalJson.tools -Name "xcopy-msbuild") {
+    if (Get-Member -InputObject $GlobalJson.tools -Name 'xcopy-msbuild') {
       $xcopyMSBuildVersion = $GlobalJson.tools.'xcopy-msbuild'
       $vsMajorVersion = $xcopyMSBuildVersion.Split('.')[0]
     } else {
-      $vsMajorVersion = $vsMinVersion.Major
-      $xcopyMSBuildVersion = "$vsMajorVersion.$($vsMinVersion.Minor).0-alpha"
+      #if vs version provided in global.json is incompatible (too low) then use the default version for xcopy msbuild download
+      if($vsMinVersion -lt $vsMinVersionReqd){
+        Write-Host "Using xcopy-msbuild version of $defaultXCopyMSBuildVersion since VS version $vsMinVersionStr provided in global.json is not compatible"
+        $xcopyMSBuildVersion = $defaultXCopyMSBuildVersion
+      }
+      else{
+        # If the VS version IS compatible, look for an xcopy msbuild package
+        # with a version matching VS.
+        # Note: If this version does not exist, then an explicit version of xcopy msbuild
+        # can be specified in global.json. This will be required for pre-release versions of msbuild.
+        $vsMajorVersion = $vsMinVersion.Major
+        $vsMinorVersion = $vsMinVersion.Minor
+        $xcopyMSBuildVersion = "$vsMajorVersion.$vsMinorVersion.0"
+      }
     }
 
-    $vsInstallDir = InitializeXCopyMSBuild $xcopyMSBuildVersion $install
+    $vsInstallDir = $null
+    if ($xcopyMSBuildVersion.Trim() -ine "none") {
+        $vsInstallDir = InitializeXCopyMSBuild $xcopyMSBuildVersion $install
+        if ($vsInstallDir -eq $null) {
+            throw "Could not xcopy msbuild. Please check that package 'RoslynTools.MSBuild @ $xcopyMSBuildVersion' exists on feed 'dotnet-eng'."
+        }
+    }
     if ($vsInstallDir -eq $null) {
-      throw "Unable to find Visual Studio that has required version and components installed"
+      throw 'Unable to find Visual Studio that has required version and components installed'
     }
   }
 
@@ -234,7 +377,7 @@ function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements =
 function InitializeVisualStudioEnvironmentVariables([string] $vsInstallDir, [string] $vsMajorVersion) {
   $env:VSINSTALLDIR = $vsInstallDir
   Set-Item "env:VS$($vsMajorVersion)0COMNTOOLS" (Join-Path $vsInstallDir "Common7\Tools\")
-  
+
   $vsSdkInstallDir = Join-Path $vsInstallDir "VSSDK\"
   if (Test-Path $vsSdkInstallDir) {
     Set-Item "env:VSSDK$($vsMajorVersion)0Install" $vsSdkInstallDir
@@ -247,7 +390,7 @@ function InstallXCopyMSBuild([string]$packageVersion) {
 }
 
 function InitializeXCopyMSBuild([string]$packageVersion, [bool]$install) {
-  $packageName = "RoslynTools.MSBuild"
+  $packageName = 'RoslynTools.MSBuild'
   $packageDir = Join-Path $ToolsDir "msbuild\$packageVersion"
   $packagePath = Join-Path $packageDir "$packageName.$packageVersion.nupkg"
 
@@ -258,55 +401,80 @@ function InitializeXCopyMSBuild([string]$packageVersion, [bool]$install) {
 
     Create-Directory $packageDir
     Write-Host "Downloading $packageName $packageVersion"
-    Invoke-WebRequest "https://dotnet.myget.org/F/roslyn-tools/api/v2/package/$packageName/$packageVersion/" -OutFile $packagePath
+    $ProgressPreference = 'SilentlyContinue' # Don't display the console progress UI - it's a huge perf hit
+    Invoke-WebRequest "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/flat2/$packageName/$packageVersion/$packageName.$packageVersion.nupkg" -OutFile $packagePath
     Unzip $packagePath $packageDir
   }
 
-  return Join-Path $packageDir "tools"
+  return Join-Path $packageDir 'tools'
 }
 
 #
 # Locates Visual Studio instance that meets the minimal requirements specified by tools.vs object in global.json.
 #
 # The following properties of tools.vs are recognized:
-#   "version": "{major}.{minor}"    
+#   "version": "{major}.{minor}"
 #       Two part minimal VS version, e.g. "15.9", "16.0", etc.
-#   "components": ["componentId1", "componentId2", ...] 
+#   "components": ["componentId1", "componentId2", ...]
 #       Array of ids of workload components that must be available in the VS instance.
 #       See e.g. https://docs.microsoft.com/en-us/visualstudio/install/workload-component-id-vs-enterprise?view=vs-2017
 #
-# Returns JSON describing the located VS instance (same format as returned by vswhere), 
+# Returns JSON describing the located VS instance (same format as returned by vswhere),
 # or $null if no instance meeting the requirements is found on the machine.
 #
 function LocateVisualStudio([object]$vsRequirements = $null){
-  if (Get-Member -InputObject $GlobalJson.tools -Name "vswhere") {
+  if (-not (IsWindowsPlatform)) {
+    throw "Cannot run vswhere on non-Windows platforms."
+  }
+
+  if (Get-Member -InputObject $GlobalJson.tools -Name 'vswhere') {
     $vswhereVersion = $GlobalJson.tools.vswhere
   } else {
-    $vswhereVersion = "2.5.2"
+    $vswhereVersion = '2.5.2'
   }
 
   $vsWhereDir = Join-Path $ToolsDir "vswhere\$vswhereVersion"
-  $vsWhereExe = Join-Path $vsWhereDir "vswhere.exe"
+  $vsWhereExe = Join-Path $vsWhereDir 'vswhere.exe'
 
   if (!(Test-Path $vsWhereExe)) {
     Create-Directory $vsWhereDir
-    Write-Host "Downloading vswhere"
-    Invoke-WebRequest "https://github.com/Microsoft/vswhere/releases/download/$vswhereVersion/vswhere.exe" -OutFile $vswhereExe
+    Write-Host 'Downloading vswhere'
+    $maxRetries = 5
+    $retries = 1
+
+    while($true) {
+      try {
+        Invoke-WebRequest "https://netcorenativeassets.blob.core.windows.net/resource-packages/external/windows/vswhere/$vswhereVersion/vswhere.exe" -OutFile $vswhereExe
+        break
+      }
+      catch{
+        Write-PipelineTelemetryError -Category 'InitializeToolset' -Message $_
+      }
+
+      if (++$retries -le $maxRetries) {
+        $delayInSeconds = [math]::Pow(2, $retries) - 1 # Exponential backoff
+        Write-Host "Retrying. Waiting for $delayInSeconds seconds before next attempt ($retries of $maxRetries)."
+        Start-Sleep -Seconds $delayInSeconds
+      }
+      else {
+        Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "Unable to download file in $maxRetries attempts."
+      }
+    }
   }
 
   if (!$vsRequirements) { $vsRequirements = $GlobalJson.tools.vs }
-  $args = @("-latest", "-prerelease", "-format", "json", "-requires", "Microsoft.Component.MSBuild")
-  
-  if (Get-Member -InputObject $vsRequirements -Name "version") {
-    $args += "-version"
+  $args = @('-latest', '-prerelease', '-format', 'json', '-requires', 'Microsoft.Component.MSBuild', '-products', '*')
+
+  if (Get-Member -InputObject $vsRequirements -Name 'version') {
+    $args += '-version'
     $args += $vsRequirements.version
   }
 
-  if (Get-Member -InputObject $vsRequirements -Name "components") {
+  if (Get-Member -InputObject $vsRequirements -Name 'components') {
     foreach ($component in $vsRequirements.components) {
-      $args += "-requires"
+      $args += '-requires'
       $args += $component
-    }    
+    }
   }
 
   $vsInfo =& $vsWhereExe $args | ConvertFrom-Json
@@ -330,28 +498,28 @@ function InitializeBuildTool() {
 
   # Initialize dotnet cli if listed in 'tools'
   $dotnetRoot = $null
-  if (Get-Member -InputObject $GlobalJson.tools -Name "dotnet") {
+  if (Get-Member -InputObject $GlobalJson.tools -Name 'dotnet') {
     $dotnetRoot = InitializeDotNetCli -install:$restore
   }
 
-  if ($msbuildEngine -eq "dotnet") {
+  if ($msbuildEngine -eq 'dotnet') {
     if (!$dotnetRoot) {
-      Write-Host "/global.json must specify 'tools.dotnet'." -ForegroundColor Red
+      Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "/global.json must specify 'tools.dotnet'."
       ExitWithExitCode 1
     }
-
-    $buildTool = @{ Path = Join-Path $dotnetRoot "dotnet.exe"; Command = "msbuild" }
+    $dotnetPath = Join-Path $dotnetRoot (GetExecutableFileName 'dotnet')
+    $buildTool = @{ Path = $dotnetPath; Command = 'msbuild'; Tool = 'dotnet'; Framework = 'netcoreapp3.1' }
   } elseif ($msbuildEngine -eq "vs") {
     try {
       $msbuildPath = InitializeVisualStudioMSBuild -install:$restore
     } catch {
-      Write-Host $_ -ForegroundColor Red
+      Write-PipelineTelemetryError -Category 'InitializeToolset' -Message $_
       ExitWithExitCode 1
     }
 
-    $buildTool = @{ Path = $msbuildPath; Command = "" }
+    $buildTool = @{ Path = $msbuildPath; Command = ""; Tool = "vs"; Framework = "net472" }
   } else {
-    Write-Host "Unexpected value of -msbuildEngine: '$msbuildEngine'." -ForegroundColor Red
+    Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "Unexpected value of -msbuildEngine: '$msbuildEngine'."
     ExitWithExitCode 1
   }
 
@@ -360,26 +528,29 @@ function InitializeBuildTool() {
 
 function GetDefaultMSBuildEngine() {
   # Presence of tools.vs indicates the repo needs to build using VS msbuild on Windows.
-  if (Get-Member -InputObject $GlobalJson.tools -Name "vs") {
-    return "vs"
-  }
-  
-  if (Get-Member -InputObject $GlobalJson.tools -Name "dotnet") {
-    return "dotnet"
+  if (Get-Member -InputObject $GlobalJson.tools -Name 'vs') {
+    return 'vs'
   }
 
-  Write-Host "-msbuildEngine must be specified, or /global.json must specify 'tools.dotnet' or 'tools.vs'." -ForegroundColor Red
+  if (Get-Member -InputObject $GlobalJson.tools -Name 'dotnet') {
+    return 'dotnet'
+  }
+
+  Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "-msbuildEngine must be specified, or /global.json must specify 'tools.dotnet' or 'tools.vs'."
   ExitWithExitCode 1
 }
 
 function GetNuGetPackageCachePath() {
   if ($env:NUGET_PACKAGES -eq $null) {
-    # Use local cache on CI to ensure deterministic build,
+    # Use local cache on CI to ensure deterministic build. 
+    # Avoid using the http cache as workaround for https://github.com/NuGet/Home/issues/3116
     # use global cache in dev builds to avoid cost of downloading packages.
+    # For directory normalization, see also: https://github.com/NuGet/Home/issues/7968
     if ($useGlobalNuGetCache) {
-      $env:NUGET_PACKAGES = Join-Path $env:UserProfile ".nuget\packages"
+      $env:NUGET_PACKAGES = Join-Path $env:UserProfile '.nuget\packages\'
     } else {
-      $env:NUGET_PACKAGES = Join-Path $RepoRoot ".packages"
+      $env:NUGET_PACKAGES = Join-Path $RepoRoot '.packages\'
+      $env:RESTORENOCACHE = $true
     }
   }
 
@@ -391,6 +562,18 @@ function GetSdkTaskProject([string]$taskName) {
   return Join-Path (Split-Path (InitializeToolset) -Parent) "SdkTasks\$taskName.proj"
 }
 
+function InitializeNativeTools() {
+  if (-Not (Test-Path variable:DisableNativeToolsetInstalls) -And (Get-Member -InputObject $GlobalJson -Name "native-tools")) {
+    $nativeArgs= @{}
+    if ($ci) {
+      $nativeArgs = @{
+        InstallDirectory = "$ToolsDir"
+      }
+    }
+    & "$PSScriptRoot/init-tools-native.ps1" @nativeArgs
+  }
+}
+
 function InitializeToolset() {
   if (Test-Path variable:global:_ToolsetBuildProj) {
     return $global:_ToolsetBuildProj
@@ -409,19 +592,20 @@ function InitializeToolset() {
   }
 
   if (-not $restore) {
-    Write-Host "Toolset version $toolsetVersion has not been restored." -ForegroundColor Red
+    Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "Toolset version $toolsetVersion has not been restored."
     ExitWithExitCode 1
   }
 
   $buildTool = InitializeBuildTool
 
-  $proj = Join-Path $ToolsetDir "restore.proj"
-  $bl = if ($binaryLog) { "/bl:" + (Join-Path $LogDir "ToolsetRestore.binlog") } else { "" }
+  $proj = Join-Path $ToolsetDir 'restore.proj'
+  $bl = if ($binaryLog) { '/bl:' + (Join-Path $LogDir 'ToolsetRestore.binlog') } else { '' }
 
   '<Project Sdk="Microsoft.DotNet.Arcade.Sdk"/>' | Set-Content $proj
-  MSBuild $proj $bl /t:__WriteToolsetLocation /clp:ErrorsOnly`;NoSummary /p:__ToolsetLocationOutputFile=$toolsetLocationFile
 
-  $path = Get-Content $toolsetLocationFile -TotalCount 1
+  MSBuild-Core $proj $bl /t:__WriteToolsetLocation /clp:ErrorsOnly`;NoSummary /p:__ToolsetLocationOutputFile=$toolsetLocationFile
+
+  $path = Get-Content $toolsetLocationFile -Encoding UTF8 -TotalCount 1
   if (!(Test-Path $path)) {
     throw "Invalid toolset path: $path"
   }
@@ -437,7 +621,7 @@ function ExitWithExitCode([int] $exitCode) {
 }
 
 function Stop-Processes() {
-  Write-Host "Killing running build processes..."
+  Write-Host 'Killing running build processes...'
   foreach ($processName in $processesToStopOnExit) {
     Get-Process -Name $processName -ErrorAction SilentlyContinue | Stop-Process
   }
@@ -449,13 +633,57 @@ function Stop-Processes() {
 # Terminates the script if the build fails.
 #
 function MSBuild() {
+  if ($pipelinesLog) {
+    $buildTool = InitializeBuildTool
+
+    if ($ci -and $buildTool.Tool -eq 'dotnet') {
+      $env:NUGET_PLUGIN_HANDSHAKE_TIMEOUT_IN_SECONDS = 20
+      $env:NUGET_PLUGIN_REQUEST_TIMEOUT_IN_SECONDS = 20
+      Write-PipelineSetVariable -Name 'NUGET_PLUGIN_HANDSHAKE_TIMEOUT_IN_SECONDS' -Value '20'
+      Write-PipelineSetVariable -Name 'NUGET_PLUGIN_REQUEST_TIMEOUT_IN_SECONDS' -Value '20'
+    }
+
+    $toolsetBuildProject = InitializeToolset
+    $basePath = Split-Path -parent $toolsetBuildProject
+    $possiblePaths = @(
+      # new scripts need to work with old packages, so we need to look for the old names/versions
+      (Join-Path $basePath (Join-Path $buildTool.Framework 'Microsoft.DotNet.ArcadeLogging.dll')),
+      (Join-Path $basePath (Join-Path $buildTool.Framework 'Microsoft.DotNet.Arcade.Sdk.dll')),
+      (Join-Path $basePath (Join-Path netcoreapp2.1 'Microsoft.DotNet.ArcadeLogging.dll')),
+      (Join-Path $basePath (Join-Path netcoreapp2.1 'Microsoft.DotNet.Arcade.Sdk.dll'))
+    )
+    $selectedPath = $null
+    foreach ($path in $possiblePaths) {
+      if (Test-Path $path -PathType Leaf) {
+        $selectedPath = $path
+        break
+      }
+    }
+    if (-not $selectedPath) {
+      Write-PipelineTelemetryError -Category 'Build' -Message 'Unable to find arcade sdk logger assembly.'
+      ExitWithExitCode 1
+    }
+    $args += "/logger:$selectedPath"
+  }
+
+  MSBuild-Core @args
+}
+
+#
+# Executes msbuild (or 'dotnet msbuild') with arguments passed to the function.
+# The arguments are automatically quoted.
+# Terminates the script if the build fails.
+#
+function MSBuild-Core() {
   if ($ci) {
-    if (!$binaryLog) {
-      throw "Binary log must be enabled in CI build."
+    if (!$binaryLog -and !$excludeCIBinarylog) {
+      Write-PipelineTelemetryError -Category 'Build' -Message 'Binary log must be enabled in CI build, or explicitly opted-out from with the -excludeCIBinarylog switch.'
+      ExitWithExitCode 1
     }
 
     if ($nodeReuse) {
-      throw "Node reuse must be disabled in CI build."
+      Write-PipelineTelemetryError -Category 'Build' -Message 'Node reuse must be disabled in CI build.'
+      ExitWithExitCode 1
     }
   }
 
@@ -463,8 +691,11 @@ function MSBuild() {
 
   $cmdArgs = "$($buildTool.Command) /m /nologo /clp:Summary /v:$verbosity /nr:$nodeReuse /p:ContinuousIntegrationBuild=$ci"
 
-  if ($warnAsError) { 
-    $cmdArgs += " /warnaserror /p:TreatWarningsAsErrors=true" 
+  if ($warnAsError) {
+    $cmdArgs += ' /warnaserror /p:TreatWarningsAsErrors=true'
+  }
+  else {
+    $cmdArgs += ' /p:TreatWarningsAsErrors=false'
   }
 
   foreach ($arg in $args) {
@@ -472,31 +703,42 @@ function MSBuild() {
       $cmdArgs += " `"$arg`""
     }
   }
-  
+
+  $env:ARCADE_BUILD_TOOL_COMMAND = "$($buildTool.Path) $cmdArgs"
+
   $exitCode = Exec-Process $buildTool.Path $cmdArgs
 
   if ($exitCode -ne 0) {
-    Write-Host "Build failed." -ForegroundColor Red
+    # We should not Write-PipelineTaskError here because that message shows up in the build summary
+    # The build already logged an error, that's the reason it failed. Producing an error here only adds noise.
+    Write-Host "Build failed with exit code $exitCode. Check errors above." -ForegroundColor Red
 
     $buildLog = GetMSBuildBinaryLogCommandLineArgument $args
-    if ($buildLog -ne $null) {      
-      Write-Host "See log: $buildLog" -ForegroundColor DarkGray 
+    if ($null -ne $buildLog) {
+      Write-Host "See log: $buildLog" -ForegroundColor DarkGray
     }
 
-    ExitWithExitCode $exitCode
+    if ($ci) {
+      Write-PipelineSetResult -Result "Failed" -Message "msbuild execution failed."
+      # Exiting with an exit code causes the azure pipelines task to log yet another "noise" error
+      # The above Write-PipelineSetResult will cause the task to be marked as failure without adding yet another error
+      ExitWithExitCode 0
+    } else {
+      ExitWithExitCode $exitCode
+    }
   }
 }
 
-function GetMSBuildBinaryLogCommandLineArgument($arguments) {  
+function GetMSBuildBinaryLogCommandLineArgument($arguments) {
   foreach ($argument in $arguments) {
     if ($argument -ne $null) {
       $arg = $argument.Trim()
-      if ($arg.StartsWith("/bl:", "OrdinalIgnoreCase")) {
-        return $arg.Substring("/bl:".Length)
-      } 
-        
-      if ($arg.StartsWith("/binaryLogger:", "OrdinalIgnoreCase")) {
-        return $arg.Substring("/binaryLogger:".Length)
+      if ($arg.StartsWith('/bl:', "OrdinalIgnoreCase")) {
+        return $arg.Substring('/bl:'.Length)
+      }
+
+      if ($arg.StartsWith('/binaryLogger:', 'OrdinalIgnoreCase')) {
+        return $arg.Substring('/binaryLogger:'.Length)
       }
     }
   }
@@ -504,24 +746,63 @@ function GetMSBuildBinaryLogCommandLineArgument($arguments) {
   return $null
 }
 
-$RepoRoot = Resolve-Path (Join-Path $PSScriptRoot "..\..")
-$EngRoot = Resolve-Path (Join-Path $PSScriptRoot "..")
-$ArtifactsDir = Join-Path $RepoRoot "artifacts"
-$ToolsetDir = Join-Path $ArtifactsDir "toolset"
-$ToolsDir = Join-Path $RepoRoot ".tools"
-$LogDir = Join-Path (Join-Path $ArtifactsDir "log") $configuration
-$TempDir = Join-Path (Join-Path $ArtifactsDir "tmp") $configuration
-$GlobalJson = Get-Content -Raw -Path (Join-Path $RepoRoot "global.json") | ConvertFrom-Json
+function GetExecutableFileName($baseName) {
+  if (IsWindowsPlatform) {
+    return "$baseName.exe"
+  }
+  else {
+    return $baseName
+  }
+}
+
+function IsWindowsPlatform() {
+  return [environment]::OSVersion.Platform -eq [PlatformID]::Win32NT
+}
+
+function Get-Darc($version) {
+  $darcPath  = "$TempDir\darc\$(New-Guid)"
+  if ($version -ne $null) {
+    & $PSScriptRoot\darc-init.ps1 -toolpath $darcPath -darcVersion $version | Out-Host
+  } else {
+    & $PSScriptRoot\darc-init.ps1 -toolpath $darcPath | Out-Host
+  }
+  return "$darcPath\darc.exe"
+}
+
+. $PSScriptRoot\pipeline-logging-functions.ps1
+
+$RepoRoot = Resolve-Path (Join-Path $PSScriptRoot '..\..')
+$EngRoot = Resolve-Path (Join-Path $PSScriptRoot '..')
+$ArtifactsDir = Join-Path $RepoRoot 'artifacts'
+$ToolsetDir = Join-Path $ArtifactsDir 'toolset'
+$ToolsDir = Join-Path $RepoRoot '.tools'
+$LogDir = Join-Path (Join-Path $ArtifactsDir 'log') $configuration
+$TempDir = Join-Path (Join-Path $ArtifactsDir 'tmp') $configuration
+$GlobalJson = Get-Content -Raw -Path (Join-Path $RepoRoot 'global.json') | ConvertFrom-Json
+# true if global.json contains a "runtimes" section
+$globalJsonHasRuntimes = if ($GlobalJson.tools.PSObject.Properties.Name -Match 'runtimes') { $true } else { $false }
 
 Create-Directory $ToolsetDir
 Create-Directory $TempDir
 Create-Directory $LogDir
 
-if ($ci) {
-  Write-Host "##vso[task.setvariable variable=Artifacts]$ArtifactsDir"
-  Write-Host "##vso[task.setvariable variable=Artifacts.Toolset]$ToolsetDir"
-  Write-Host "##vso[task.setvariable variable=Artifacts.Log]$LogDir"
+Write-PipelineSetVariable -Name 'Artifacts' -Value $ArtifactsDir
+Write-PipelineSetVariable -Name 'Artifacts.Toolset' -Value $ToolsetDir
+Write-PipelineSetVariable -Name 'Artifacts.Log' -Value $LogDir
+Write-PipelineSetVariable -Name 'TEMP' -Value $TempDir
+Write-PipelineSetVariable -Name 'TMP' -Value $TempDir
 
-  $env:TEMP = $TempDir
-  $env:TMP = $TempDir
+# Import custom tools configuration, if present in the repo.
+# Note: Import in global scope so that the script set top-level variables without qualification.
+if (!$disableConfigureToolsetImport) {
+  $configureToolsetScript = Join-Path $EngRoot 'configure-toolset.ps1'
+  if (Test-Path $configureToolsetScript) {
+    . $configureToolsetScript
+    if ((Test-Path variable:failOnConfigureToolsetError) -And $failOnConfigureToolsetError) {
+      if ((Test-Path variable:LastExitCode) -And ($LastExitCode -ne 0)) {
+        Write-PipelineTelemetryError -Category 'Build' -Message 'configure-toolset.ps1 returned a non-zero exit code'
+        ExitWithExitCode $LastExitCode
+      }
+    }
+  }
 }

eng/common/tools.sh

@@ -1,14 +1,34 @@
+#!/usr/bin/env bash
+
 # Initialize variables if they aren't already defined.
 
 # CI mode - set to true on CI server for PR validation build or official build.
 ci=${ci:-false}
 
+# Set to true to use the pipelines logger which will enable Azure logging output.
+# https://github.com/Microsoft/azure-pipelines-tasks/blob/master/docs/authoring/commands.md
+# This flag is meant as a temporary opt-opt for the feature while validate it across
+# our consumers. It will be deleted in the future.
+if [[ "$ci" == true ]]; then
+  pipelines_log=${pipelines_log:-true}
+else
+  pipelines_log=${pipelines_log:-false}
+fi
+
 # Build configuration. Common values include 'Debug' and 'Release', but the repository may use other names.
 configuration=${configuration:-'Debug'}
 
+# Set to true to opt out of outputting binary log while running in CI
+exclude_ci_binary_log=${exclude_ci_binary_log:-false}
+
+if [[ "$ci" == true && "$exclude_ci_binary_log" == false ]]; then
+  binary_log_default=true
+else
+  binary_log_default=false
+fi
+
 # Set to true to output binary log from msbuild. Note that emitting binary log slows down the build.
-# Binary log must be enabled on CI.
-binary_log=${binary_log:-$ci}
+binary_log=${binary_log:-$binary_log_default}
 
 # Turns on machine preparation/clean up code that changes the machine state (e.g. kills build processes).
 prepare_machine=${prepare_machine:-false}
@@ -29,10 +49,14 @@ fi
 # Configures warning treatment in msbuild.
 warn_as_error=${warn_as_error:-true}
 
-# True to attempt using .NET Core already that meets requirements specified in global.json 
+# True to attempt using .NET Core already that meets requirements specified in global.json
 # installed on the machine instead of downloading one.
 use_installed_dotnet_cli=${use_installed_dotnet_cli:-true}
 
+# Enable repos to use a particular version of the on-line dotnet-install scripts.
+#    default URL: https://dot.net/v1/dotnet-install.sh
+dotnetInstallScriptVersion=${dotnetInstallScriptVersion:-'v1'}
+
 # True to use global NuGet cache instead of restoring packages to repository-local directory.
 if [[ "$ci" == true ]]; then
   use_global_nuget_cache=${use_global_nuget_cache:-false}
@@ -40,6 +64,10 @@ else
   use_global_nuget_cache=${use_global_nuget_cache:-true}
 fi
 
+# Used when restoring .NET SDK from alternative feeds
+runtime_source_feed=${runtime_source_feed:-''}
+runtime_source_feed_key=${runtime_source_feed_key:-''}
+
 # Resolve any symlinks in the given path.
 function ResolvePath {
   local path=$1
@@ -61,16 +89,16 @@ function ResolvePath {
 function ReadGlobalVersion {
   local key=$1
 
-  local line=`grep -m 1 "$key" "$global_json_file"`
-  local pattern="\"$key\" *: *\"(.*)\""
-
-  if [[ ! $line =~ $pattern ]]; then
-    echo "Error: Cannot find \"$key\" in $global_json_file" >&2
-    ExitWithExitCode 1
+  if command -v jq &> /dev/null; then
+    _ReadGlobalVersion="$(jq -r ".[] | select(has(\"$key\")) | .\"$key\"" "$global_json_file")"
+  elif [[ "$(cat "$global_json_file")" =~ \"$key\"[[:space:]\:]*\"([^\"]+) ]]; then
+    _ReadGlobalVersion=${BASH_REMATCH[1]}
   fi
 
-  # return value
-  _ReadGlobalVersion=${BASH_REMATCH[1]}
+  if [[ -z "$_ReadGlobalVersion" ]]; then
+    Write-PipelineTelemetryError -category 'Build' "Error: Cannot find \"$key\" in $global_json_file"
+    ExitWithExitCode 1
+  fi
 }
 
 function InitializeDotNetCli {
@@ -101,7 +129,7 @@ function InitializeDotNetCli {
   fi
 
   # Find the first path on $PATH that contains the dotnet.exe
-  if [[ "$use_installed_dotnet_cli" == true && -z "${DOTNET_INSTALL_DIR:-}" ]]; then
+  if [[ "$use_installed_dotnet_cli" == true && $global_json_has_runtimes == false && -z "${DOTNET_INSTALL_DIR:-}" ]]; then
     local dotnet_path=`command -v dotnet`
     if [[ -n "$dotnet_path" ]]; then
       ResolvePath "$dotnet_path"
@@ -115,17 +143,18 @@ function InitializeDotNetCli {
 
   # Use dotnet installation specified in DOTNET_INSTALL_DIR if it contains the required SDK version,
   # otherwise install the dotnet CLI and SDK to repo local .dotnet directory to avoid potential permission issues.
-  if [[ -n "${DOTNET_INSTALL_DIR:-}" && -d "$DOTNET_INSTALL_DIR/sdk/$dotnet_sdk_version" ]]; then
+  if [[ $global_json_has_runtimes == false && -n "${DOTNET_INSTALL_DIR:-}" && -d "$DOTNET_INSTALL_DIR/sdk/$dotnet_sdk_version" ]]; then
     dotnet_root="$DOTNET_INSTALL_DIR"
   else
     dotnet_root="$repo_root/.dotnet"
+
     export DOTNET_INSTALL_DIR="$dotnet_root"
 
     if [[ ! -d "$DOTNET_INSTALL_DIR/sdk/$dotnet_sdk_version" ]]; then
       if [[ "$install" == true ]]; then
         InstallDotNetSdk "$dotnet_root" "$dotnet_sdk_version"
       else
-        echo "Unable to find dotnet with SDK version '$dotnet_sdk_version'" >&2
+        Write-PipelineTelemetryError -category 'InitializeToolset' "Unable to find dotnet with SDK version '$dotnet_sdk_version'"
         ExitWithExitCode 1
       fi
     fi
@@ -133,14 +162,10 @@ function InitializeDotNetCli {
 
   # Add dotnet to PATH. This prevents any bare invocation of dotnet in custom
   # build steps from using anything other than what we've downloaded.
-  export PATH="$dotnet_root:$PATH"
+  Write-PipelinePrependPath -path "$dotnet_root"
 
-  if [[ $ci == true ]]; then
-    # Make Sure that our bootstrapped dotnet cli is avaliable in future steps of the Azure Pipelines build
-    echo "##vso[task.prependpath]$dotnet_root"
-    echo "##vso[task.setvariable variable=DOTNET_MULTILEVEL_LOOKUP]0"
-    echo "##vso[task.setvariable variable=DOTNET_SKIP_FIRST_TIME_EXPERIENCE]1"
-  fi
+  Write-PipelineSetVariable -name "DOTNET_MULTILEVEL_LOOKUP" -value "0"
+  Write-PipelineSetVariable -name "DOTNET_SKIP_FIRST_TIME_EXPERIENCE" -value "1"
 
   # return value
   _InitializeDotNetCli="$dotnet_root"
@@ -149,26 +174,95 @@ function InitializeDotNetCli {
 function InstallDotNetSdk {
   local root=$1
   local version=$2
+  local architecture="unset"
+  if [[ $# -ge 3 ]]; then
+    architecture=$3
+  fi
+  InstallDotNet "$root" "$version" $architecture 'sdk' 'false' $runtime_source_feed $runtime_source_feed_key
+}
+
+function InstallDotNet {
+  local root=$1
+  local version=$2
 
   GetDotNetInstallScript "$root"
   local install_script=$_GetDotNetInstallScript
 
-  local arch_arg=""
-  if [[ $# == 3 ]]; then
-    arch_arg="--architecture $3"
+  local archArg=''
+  if [[ -n "${3:-}" ]] && [ "$3" != 'unset' ]; then
+    archArg="--architecture $3"
   fi
-
-  bash "$install_script" --version $version --install-dir "$root" $arch_arg || {
+  local runtimeArg=''
+  if [[ -n "${4:-}" ]] && [ "$4" != 'sdk' ]; then
+    runtimeArg="--runtime $4"
+  fi
+  local skipNonVersionedFilesArg=""
+  if [[ "$#" -ge "5" ]] && [[ "$5" != 'false' ]]; then
+    skipNonVersionedFilesArg="--skip-non-versioned-files"
+  fi
+  bash "$install_script" --version $version --install-dir "$root" $archArg $runtimeArg $skipNonVersionedFilesArg || {
     local exit_code=$?
-    echo "Failed to install dotnet SDK (exit code '$exit_code')." >&2
-    ExitWithExitCode $exit_code
+    echo "Failed to install dotnet SDK from public location (exit code '$exit_code')."
+
+    local runtimeSourceFeed=''
+    if [[ -n "${6:-}" ]]; then
+      runtimeSourceFeed="--azure-feed $6"
+    fi
+
+    local runtimeSourceFeedKey=''
+    if [[ -n "${7:-}" ]]; then
+      # The 'base64' binary on alpine uses '-d' and doesn't support '--decode'
+      # '-d'. To work around this, do a simple detection and switch the parameter
+      # accordingly.
+      decodeArg="--decode"
+      if base64 --help 2>&1 | grep -q "BusyBox"; then
+          decodeArg="-d"
+      fi
+      decodedFeedKey=`echo $7 | base64 $decodeArg`
+      runtimeSourceFeedKey="--feed-credential $decodedFeedKey"
+    fi
+
+    if [[ -n "$runtimeSourceFeed" || -n "$runtimeSourceFeedKey" ]]; then
+      bash "$install_script" --version $version --install-dir "$root" $archArg $runtimeArg $skipNonVersionedFilesArg $runtimeSourceFeed $runtimeSourceFeedKey || {
+        local exit_code=$?
+        Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to install dotnet SDK from custom location '$runtimeSourceFeed' (exit code '$exit_code')."
+        ExitWithExitCode $exit_code
+      }
+    else
+      if [[ $exit_code != 0 ]]; then
+        Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to install dotnet SDK from public location (exit code '$exit_code')."
+      fi
+      ExitWithExitCode $exit_code
+    fi
   }
 }
 
+function with_retries {
+  local maxRetries=5
+  local retries=1
+  echo "Trying to run '$@' for maximum of $maxRetries attempts."
+  while [[ $((retries++)) -le $maxRetries ]]; do
+    "$@"
+
+    if [[ $? == 0 ]]; then
+      echo "Ran '$@' successfully."
+      return 0
+    fi
+
+    timeout=$((3**$retries-1))
+    echo "Failed to execute '$@'. Waiting $timeout seconds before next attempt ($retries out of $maxRetries)." 1>&2
+    sleep $timeout
+  done
+
+  echo "Failed to execute '$@' for $maxRetries times." 1>&2
+
+  return 1
+}
+
 function GetDotNetInstallScript {
   local root=$1
   local install_script="$root/dotnet-install.sh"
-  local install_script_url="https://dot.net/v1/dotnet-install.sh"
+  local install_script_url="https://dot.net/$dotnetInstallScriptVersion/dotnet-install.sh"
 
   if [[ ! -a "$install_script" ]]; then
     mkdir -p "$root"
@@ -177,12 +271,27 @@ function GetDotNetInstallScript {
 
     # Use curl if available, otherwise use wget
     if command -v curl > /dev/null; then
-      curl "$install_script_url" -sSL --retry 10 --create-dirs -o "$install_script"
+      # first, try directly, if this fails we will retry with verbose logging
+      curl "$install_script_url" -sSL --retry 10 --create-dirs -o "$install_script" || {
+        if command -v openssl &> /dev/null; then
+          echo "Curl failed; dumping some information about dotnet.microsoft.com for later investigation"
+          echo | openssl s_client -showcerts -servername dotnet.microsoft.com  -connect dotnet.microsoft.com:443
+        fi
+        echo "Will now retry the same URL with verbose logging."
+        with_retries curl "$install_script_url" -sSL --verbose --retry 10 --create-dirs -o "$install_script" || {
+          local exit_code=$?
+          Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to acquire dotnet install script (exit code '$exit_code')."
+          ExitWithExitCode $exit_code
+        }
+      }
     else
-      wget -q -O "$install_script" "$install_script_url"
+      with_retries wget -v -O "$install_script" "$install_script_url" || {
+        local exit_code=$?
+        Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to acquire dotnet install script (exit code '$exit_code')."
+        ExitWithExitCode $exit_code
+      }
     fi
   fi
-
   # return value
   _GetDotNetInstallScript="$install_script"
 }
@@ -191,20 +300,23 @@ function InitializeBuildTool {
   if [[ -n "${_InitializeBuildTool:-}" ]]; then
     return
   fi
-  
+
   InitializeDotNetCli $restore
 
   # return values
-  _InitializeBuildTool="$_InitializeDotNetCli/dotnet"  
+  _InitializeBuildTool="$_InitializeDotNetCli/dotnet"
   _InitializeBuildToolCommand="msbuild"
+  _InitializeBuildToolFramework="netcoreapp3.1"
 }
 
+# Set RestoreNoCache as a workaround for https://github.com/NuGet/Home/issues/3116
 function GetNuGetPackageCachePath {
   if [[ -z ${NUGET_PACKAGES:-} ]]; then
     if [[ "$use_global_nuget_cache" == true ]]; then
       export NUGET_PACKAGES="$HOME/.nuget/packages"
     else
       export NUGET_PACKAGES="$repo_root/.packages"
+      export RESTORENOCACHE=true
     fi
   fi
 
@@ -212,6 +324,20 @@ function GetNuGetPackageCachePath {
   _GetNuGetPackageCachePath=$NUGET_PACKAGES
 }
 
+function InitializeNativeTools() {
+  if [[ -n "${DisableNativeToolsetInstalls:-}" ]]; then
+    return
+  fi
+  if grep -Fq "native-tools" $global_json_file
+  then
+    local nativeArgs=""
+    if [[ "$ci" == true ]]; then
+      nativeArgs="--installDirectory $tools_dir"
+    fi
+    "$_script_dir/init-tools-native.sh" $nativeArgs
+  fi
+}
+
 function InitializeToolset {
   if [[ -n "${_InitializeToolset:-}" ]]; then
     return
@@ -234,7 +360,7 @@ function InitializeToolset {
   fi
 
   if [[ "$restore" != true ]]; then
-    echo "Toolset version $toolsetVersion has not been restored." >&2
+    Write-PipelineTelemetryError -category 'InitializeToolset' "Toolset version $toolset_version has not been restored."
     ExitWithExitCode 2
   fi
 
@@ -244,14 +370,14 @@ function InitializeToolset {
   if [[ "$binary_log" == true ]]; then
     bl="/bl:$log_dir/ToolsetRestore.binlog"
   fi
-  
+
   echo '<Project Sdk="Microsoft.DotNet.Arcade.Sdk"/>' > "$proj"
-  MSBuild "$proj" $bl /t:__WriteToolsetLocation /clp:ErrorsOnly\;NoSummary /p:__ToolsetLocationOutputFile="$toolset_location_file"
+  MSBuild-Core "$proj" $bl /t:__WriteToolsetLocation /clp:ErrorsOnly\;NoSummary /p:__ToolsetLocationOutputFile="$toolset_location_file"
 
   local toolset_build_proj=`cat "$toolset_location_file"`
 
   if [[ ! -a "$toolset_build_proj" ]]; then
-    echo "Invalid toolset path: $toolset_build_proj" >&2
+    Write-PipelineTelemetryError -category 'Build' "Invalid toolset path: $toolset_build_proj"
     ExitWithExitCode 3
   fi
 
@@ -274,14 +400,51 @@ function StopProcesses {
 }
 
 function MSBuild {
+  local args=$@
+  if [[ "$pipelines_log" == true ]]; then
+    InitializeBuildTool
+    InitializeToolset
+
+    if [[ "$ci" == true ]]; then
+      export NUGET_PLUGIN_HANDSHAKE_TIMEOUT_IN_SECONDS=20
+      export NUGET_PLUGIN_REQUEST_TIMEOUT_IN_SECONDS=20
+      Write-PipelineSetVariable -name "NUGET_PLUGIN_HANDSHAKE_TIMEOUT_IN_SECONDS" -value "20"
+      Write-PipelineSetVariable -name "NUGET_PLUGIN_REQUEST_TIMEOUT_IN_SECONDS" -value "20"
+    fi
+
+    local toolset_dir="${_InitializeToolset%/*}"
+    # new scripts need to work with old packages, so we need to look for the old names/versions
+    local selectedPath=
+    local possiblePaths=()
+    possiblePaths+=( "$toolset_dir/$_InitializeBuildToolFramework/Microsoft.DotNet.ArcadeLogging.dll" )
+    possiblePaths+=( "$toolset_dir/$_InitializeBuildToolFramework/Microsoft.DotNet.Arcade.Sdk.dll" )
+    possiblePaths+=( "$toolset_dir/netcoreapp2.1/Microsoft.DotNet.ArcadeLogging.dll" )
+    possiblePaths+=( "$toolset_dir/netcoreapp2.1/Microsoft.DotNet.Arcade.Sdk.dll" )
+    for path in "${possiblePaths[@]}"; do
+      if [[ -f $path ]]; then
+        selectedPath=$path
+        break
+      fi
+    done
+    if [[ -z "$selectedPath" ]]; then
+      Write-PipelineTelemetryError -category 'Build'  "Unable to find arcade sdk logger assembly."
+      ExitWithExitCode 1
+    fi
+    args+=( "-logger:$selectedPath" )
+  fi
+
+  MSBuild-Core ${args[@]}
+}
+
+function MSBuild-Core {
   if [[ "$ci" == true ]]; then
-    if [[ "$binary_log" != true ]]; then
-      echo "Binary log must be enabled in CI build." >&2
+    if [[ "$binary_log" != true && "$exclude_ci_binary_log" != true ]]; then
+      Write-PipelineTelemetryError -category 'Build'  "Binary log must be enabled in CI build, or explicitly opted-out from with the -noBinaryLog switch."
       ExitWithExitCode 1
     fi
 
     if [[ "$node_reuse" == true ]]; then
-      echo "Node reuse must be disabled in CI build." >&2
+      Write-PipelineTelemetryError -category 'Build'  "Node reuse must be disabled in CI build."
       ExitWithExitCode 1
     fi
   fi
@@ -293,24 +456,51 @@ function MSBuild {
     warnaserror_switch="/warnaserror"
   fi
 
-  "$_InitializeBuildTool" "$_InitializeBuildToolCommand" /m /nologo /clp:Summary /v:$verbosity /nr:$node_reuse $warnaserror_switch /p:TreatWarningsAsErrors=$warn_as_error /p:ContinuousIntegrationBuild=$ci "$@" || {
-    local exit_code=$?
-    echo "Build failed (exit code '$exit_code')." >&2
-    ExitWithExitCode $exit_code
+  function RunBuildTool {
+    export ARCADE_BUILD_TOOL_COMMAND="$_InitializeBuildTool $@"
+
+    "$_InitializeBuildTool" "$@" || {
+      local exit_code=$?
+      # We should not Write-PipelineTaskError here because that message shows up in the build summary
+      # The build already logged an error, that's the reason it failed. Producing an error here only adds noise.
+      echo "Build failed with exit code $exit_code. Check errors above."
+      if [[ "$ci" == "true" ]]; then
+        Write-PipelineSetResult -result "Failed" -message "msbuild execution failed."
+        # Exiting with an exit code causes the azure pipelines task to log yet another "noise" error
+        # The above Write-PipelineSetResult will cause the task to be marked as failure without adding yet another error
+        ExitWithExitCode 0
+      else
+        ExitWithExitCode $exit_code
+      fi
+    }
   }
+
+  RunBuildTool "$_InitializeBuildToolCommand" /m /nologo /clp:Summary /v:$verbosity /nr:$node_reuse $warnaserror_switch /p:TreatWarningsAsErrors=$warn_as_error /p:ContinuousIntegrationBuild=$ci "$@"
 }
 
 ResolvePath "${BASH_SOURCE[0]}"
 _script_dir=`dirname "$_ResolvePath"`
 
+. "$_script_dir/pipeline-logging-functions.sh"
+
 eng_root=`cd -P "$_script_dir/.." && pwd`
 repo_root=`cd -P "$_script_dir/../.." && pwd`
 artifacts_dir="$repo_root/artifacts"
 toolset_dir="$artifacts_dir/toolset"
+tools_dir="$repo_root/.tools"
 log_dir="$artifacts_dir/log/$configuration"
 temp_dir="$artifacts_dir/tmp/$configuration"
 
 global_json_file="$repo_root/global.json"
+# determine if global.json contains a "runtimes" entry
+global_json_has_runtimes=false
+if command -v jq &> /dev/null; then
+  if jq -er '. | select(has("runtimes"))' "$global_json_file" &> /dev/null; then
+    global_json_has_runtimes=true
+  fi
+elif [[ "$(cat "$global_json_file")" =~ \"runtimes\"[[:space:]\:]*\{ ]]; then
+  global_json_has_runtimes=true
+fi
 
 # HOME may not be defined in some scenarios, but it is required by NuGet
 if [[ -z $HOME ]]; then
@@ -322,7 +512,23 @@ mkdir -p "$toolset_dir"
 mkdir -p "$temp_dir"
 mkdir -p "$log_dir"
 
-if [[ $ci == true ]]; then
-  export TEMP="$temp_dir"
-  export TMP="$temp_dir"
+Write-PipelineSetVariable -name "Artifacts" -value "$artifacts_dir"
+Write-PipelineSetVariable -name "Artifacts.Toolset" -value "$toolset_dir"
+Write-PipelineSetVariable -name "Artifacts.Log" -value "$log_dir"
+Write-PipelineSetVariable -name "Temp" -value "$temp_dir"
+Write-PipelineSetVariable -name "TMP" -value "$temp_dir"
+
+# Import custom tools configuration, if present in the repo.
+if [ -z "${disable_configure_toolset_import:-}" ]; then
+  configure_toolset_script="$eng_root/configure-toolset.sh"
+  if [[ -a "$configure_toolset_script" ]]; then
+    . "$configure_toolset_script"
+  fi
+fi
+
+# TODO: https://github.com/dotnet/arcade/issues/1468
+# Temporary workaround to avoid breaking change.
+# Remove once repos are updated.
+if [[ -n "${useInstalledDotNetCli:-}" ]]; then
+  use_installed_dotnet_cli="$useInstalledDotNetCli"
 fi

global.json

@@ -3,6 +3,6 @@
     "dotnet": "5.0.201"
   },
   "msbuild-sdks": {
-    "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19207.1"
+    "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.21256.3"
   }
 }