New build pools for official builds (#6103)

build/vsts-ci.yml

@@ -24,9 +24,8 @@ jobs:
     DOTNET_SKIP_FIRST_TIME_EXPERIENCE: 1
     DOTNET_MULTILEVEL_LOOKUP: 0
   pool:
-    name: DotnetBuild-1ES-Linux
-    demands:
-    - agent.os -equals linux
+    name: NetCore1ESPool-Internal
+    demands: ImageOverride -equals Build.Ubuntu.1804.Amd64
   container: CentosContainer
   steps:
   - script: ./restore.sh
@@ -57,9 +56,8 @@ jobs:
     DOTNET_MULTILEVEL_LOOKUP: 0
     ROOTFS_DIR: '/crossrootfs/arm'
   pool:
-    name: DotnetBuild-1ES-Linux
-    demands:
-    - agent.os -equals linux
+    name: NetCore1ESPool-Internal
+    demands: ImageOverride -equals Build.Ubuntu.1804.Amd64
   container: UbuntuCrossArmContainer
   steps:
   - script: ./restore.sh
@@ -90,9 +88,8 @@ jobs:
     DOTNET_MULTILEVEL_LOOKUP: 0
     ROOTFS_DIR: '/crossrootfs/arm64'
   pool:
-    name: DotnetBuild-1ES-Linux
-    demands:
-    - agent.os -equals linux
+    name: NetCore1ESPool-Internal
+    demands: ImageOverride -equals Build.Ubuntu.1804.Amd64
   container: UbuntuCrossArm64Container
   steps:
   - script: ./restore.sh
@@ -197,17 +194,17 @@ jobs:
     _UseEsrpSigning: true
     _TeamName: DotNetCore
   pool:
-    name: DotNetCore-Build-1ES
-    demands:
-      - agent.os -equals Windows_NT
+    name: NetCore1ESPool-Internal
+    demands: ImageOverride -equals Build.Server.Amd64.VS2019
   steps:
 
-  - task: ms-vseng.MicroBuildTasks.30666190-6959-11e5-9f96-f56098202fef.MicroBuildSigningPlugin@1
+  - task: ms-vseng.MicroBuildTasks.30666190-6959-11e5-9f96-f56098202fef.MicroBuildSigningPlugin@4
     displayName: Install MicroBuild Signing Plugin
     inputs:
       signType: '$(_SignType)'
       zipSources: false
       esrpSigning: '$(_UseEsrpSigning)'
+      feedSource: 'https://dnceng.pkgs.visualstudio.com/DefaultCollection/_packaging/MicroBuildToolset/nuget/v3/index.json'
     env:
       TeamName: $(_TeamName)
     continueOnError: false
@@ -250,17 +247,17 @@ jobs:
     _UseEsrpSigning: true
     _TeamName: DotNetCore
   pool:
-    name: DotNetCore-Build-1ES
-    demands:
-      - agent.os -equals Windows_NT
+    name: NetCore1ESPool-Internal
+    demands: ImageOverride -equals Build.Server.Amd64.VS2019
   steps:
 
-  - task: ms-vseng.MicroBuildTasks.30666190-6959-11e5-9f96-f56098202fef.MicroBuildSigningPlugin@1
+  - task: ms-vseng.MicroBuildTasks.30666190-6959-11e5-9f96-f56098202fef.MicroBuildSigningPlugin@4
     displayName: Install MicroBuild Signing Plugin
     inputs:
       signType: '$(_SignType)'
       zipSources: false
       esrpSigning: '$(_UseEsrpSigning)'
+      feedSource: 'https://dnceng.pkgs.visualstudio.com/DefaultCollection/_packaging/MicroBuildToolset/nuget/v3/index.json'
     env:
       TeamName: $(_TeamName)
     continueOnError: false
@@ -305,32 +302,51 @@ jobs:
   - Windows_x64
   - MacOS_Apple_Silicon
   variables:
-    BuildConfig: Release
-    OfficialBuildId: $(BUILD.BUILDNUMBER)
-    DotnetVersionKind: $[variables.VERSIONKIND] # If no "VERSIONKIND" variable is set when queuing the publishing task, this defaults to empty string.
-    DOTNET_CLI_TELEMETRY_OPTOUT: 1
-    DOTNET_SKIP_FIRST_TIME_EXPERIENCE: 1
-    DOTNET_MULTILEVEL_LOOKUP: 0
-    NUGET_PACKAGES: $(Build.SourcesDirectory)/.packages
-    _SignType: real
-    _UseEsrpSigning: true
-    _TeamName: DotNetCore
-    _AzureDevopsFeedUrl: https://pkgs.dev.azure.com/dnceng/public/_packaging/MachineLearning/nuget/v3/index.json
-    _SymwebSymbolServerPath: https://microsoft.artifacts.visualstudio.com/DefaultCollection
-    _MsdlSymbolServerPath: https://microsoftpublicsymbols.artifacts.visualstudio.com/DefaultCollection
+    - name: BuildConfig
+      value:  Release
+    - name: OfficialBuildId
+      value: $(BUILD.BUILDNUMBER)
+    - name: DotnetVersionKind
+      value: $[variables.VERSIONKIND] # If no "VERSIONKIND" variable is set when queuing the publishing task, this defaults to empty string.
+    - name: DOTNET_CLI_TELEMETRY_OPTOUT
+      value: 1
+    - name: DOTNET_SKIP_FIRST_TIME_EXPERIENCE
+      value: 1
+    - name: DOTNET_MULTILEVEL_LOOKUP
+      value: 0
+    - name: NUGET_PACKAGES
+      value: $(Build.SourcesDirectory)/.packages
+    - name: _SignType
+      value: real
+    - name: _UseEsrpSigning
+      value: true
+    - name: _TeamName
+      value: DotNetCore
+    - name: _AzureDevopsFeedUrl
+      value: https://pkgs.dev.azure.com/dnceng/public/_packaging/MachineLearning/nuget/v3/index.json
+    - name: _SymwebSymbolServerPath
+      value: https://microsoft.artifacts.visualstudio.com/DefaultCollection
+    - name: _MsdlSymbolServerPath
+      value: https://microsoftpublicsymbols.artifacts.visualstudio.com/DefaultCollection
+
+    # Only enable publishing in non-public, non PR scenarios.
+    - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
+      # DotNet-Symbol-Server-Pats provides: microsoft-symbol-server-pat symweb-symbol-server-pat
+      - group: DotNet-Symbol-Server-Pats
+
   pool:
-    name: DotNetCore-Build-1ES
-    demands:
-      - agent.os -equals Windows_NT
+    name: NetCore1ESPool-Internal
+    demands: ImageOverride -equals Build.Server.Amd64.VS2019
   steps:
 
   # Install MicroBuild plugin
-  - task: ms-vseng.MicroBuildTasks.30666190-6959-11e5-9f96-f56098202fef.MicroBuildSigningPlugin@1
+  - task: ms-vseng.MicroBuildTasks.30666190-6959-11e5-9f96-f56098202fef.MicroBuildSigningPlugin@4
     displayName: Install MicroBuild Signing Plugin
     inputs:
       signType: '$(_SignType)'
       zipSources: false
       esrpSigning: '$(_UseEsrpSigning)'
+      feedSource: 'https://dnceng.pkgs.visualstudio.com/DefaultCollection/_packaging/MicroBuildToolset/nuget/v3/index.json'
     env:
       TeamName: $(_TeamName)
     continueOnError: false
@@ -373,23 +389,19 @@ jobs:
       artifactType: container
     continueOnError: true
 
-  - task: NuGetAuthenticate@0
-    inputs:
-      nuGetServiceConnections: machinelearning-dnceng-public-feed # To allow publishing to a feed of another organization
-
   - task: NuGetCommand@2
     displayName: Push packages to AzureDevOps feed
     inputs:
       command: push
+      nuGetFeedType: 'internal'
+      publishVstsFeed: 'public/MachineLearning'
       packagesToPush: $(Build.SourcesDirectory)/artifacts/**/*.nupkg;!$(Build.SourcesDirectory)/artifacts/**/*.snupkg
-      nuGetFeedType: external
-      publishFeedCredentials: machinelearning-dnceng-public-feed
 
   - task: MSBuild@1
     displayName: Publish Symbols to SymWeb Symbol Server
     inputs:
       solution: build/publish.proj
-      msbuildArguments: /t:PublishSymbolPackages /p:SymbolServerPath=$(_SymwebSymbolServerPath) /p:SymbolServerPAT=$(SymwebSymbolServerPAT)
+      msbuildArguments: /t:PublishSymbolPackages /p:SymbolServerPath=$(_SymwebSymbolServerPath) /p:SymbolServerPAT=$(symweb-symbol-server-pat)
       msbuildVersion: 15.0
     continueOnError: true
 
@@ -397,7 +409,7 @@ jobs:
     displayName: Publish Symbols to Msdl Symbol Server
     inputs:
       solution: build/publish.proj
-      msbuildArguments: /t:PublishSymbolPackages /p:SymbolServerPath=$(_MsdlSymbolServerPath) /p:SymbolServerPAT=$(MsdlSymbolServerPAT)
+      msbuildArguments: /t:PublishSymbolPackages /p:SymbolServerPath=$(_MsdlSymbolServerPath) /p:SymbolServerPAT=$(microsoft-symbol-server-pat)
       msbuildVersion: 15.0
     continueOnError: true
 

src/Microsoft.ML.TensorFlow/Microsoft.ML.TensorFlow.csproj

@@ -13,6 +13,7 @@
     <PackageReference Include="System.IO.FileSystem.AccessControl" Version="$(SystemIOFileSystemAccessControl)" />
     <PackageReference Include="System.Security.Principal.Windows" Version="$(SystemSecurityPrincipalWindows)" />
     <PackageReference Include="TensorFlow.NET" Version="$(TensorflowDotNETVersion)" />
+    <PackageReference Include="Google.Protobuf" Version="$(GoogleProtobufPackageVersion)" />
   </ItemGroup>
 
   <ItemGroup>

test/Microsoft.ML.FSharp.Tests/Microsoft.ML.FSharp.Tests.fsproj

@@ -2,8 +2,10 @@
 
   <PropertyGroup>
   <!-- NU1604 inclusive lower bound for dependencies.
-  .NET Framework 4.6.1 includes System.ValueTuple without a version, so we need to ignore the warning here -->
-    <NoWarn>NU1604;2003;$(NoWarn)</NoWarn>
+  .NET Framework 4.6.1 includes System.ValueTuple without a version, so we need to ignore the warning here.
+  Bug in the latest version of VS points to a Version of FSharp.Core that doesn't exist. Temporarily ignoring
+  NU1603 so it will use the latest version found.-->
+    <NoWarn>NU1603;NU1604;2003;$(NoWarn)</NoWarn>
     <PublicSign>false</PublicSign>
     <SourceLink></SourceLink>
     <PlatformTarget>$(TargetArchitecture)</PlatformTarget>
@@ -26,6 +28,7 @@
     <ProjectReference Include="..\..\src\Microsoft.ML.FastTree\Microsoft.ML.FastTree.csproj" />
     <ProjectReference Include="..\..\src\Microsoft.ML.Transforms\Microsoft.ML.Transforms.csproj" />
   </ItemGroup>
+
   <ItemGroup>
     <NativeAssemblyReference Include="FastTreeNative" />
     <NativeAssemblyReference Include="CpuMathNative" />

tools-local/Microsoft.ML.StableApi/Microsoft.ML.StableApi.csproj

@@ -6,18 +6,19 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.Extensions.ML" Version="1.4.0" />
-    <PackageReference Include="Microsoft.ML" Version="1.4.0" />
-    <PackageReference Include="Microsoft.ML.DataView" Version="1.4.0" />
-    <PackageReference Include="Microsoft.ML.CpuMath" Version="1.4.0" />
-    <PackageReference Include="Microsoft.ML.FastTree" Version="1.4.0" />
-    <PackageReference Include="Microsoft.ML.LightGbm" Version="1.4.0" />
-    <PackageReference Include="Microsoft.ML.ImageAnalytics" Version="1.4.0" />
-    <PackageReference Include="Microsoft.ML.Mkl.Components" Version="1.4.0" />
-    <PackageReference Include="Microsoft.ML.TimeSeries" Version="1.4.0" />
-    <PackageReference Include="Microsoft.ML.TensorFlow" Version="1.4.0" />
-    <PackageReference Include="Microsoft.ML.OnnxTransformer" Version="1.4.0" />
-    <PackageReference Include="Microsoft.ML.Vision" Version="1.4.0" />
+    <PackageReference Include="Microsoft.Extensions.ML" Version="1.7.0" />
+    <PackageReference Include="Microsoft.ML" Version="1.7.0" />
+    <PackageReference Include="Microsoft.ML.DataView" Version="1.7.0" />
+    <PackageReference Include="Microsoft.ML.CpuMath" Version="1.7.0" />
+    <PackageReference Include="Microsoft.ML.FastTree" Version="1.7.0" />
+    <PackageReference Include="Microsoft.ML.LightGbm" Version="1.7.0" />
+    <PackageReference Include="Microsoft.ML.ImageAnalytics" Version="1.7.0" />
+    <PackageReference Include="Microsoft.ML.Mkl.Components" Version="1.7.0" />
+    <PackageReference Include="Microsoft.ML.TimeSeries" Version="1.7.0" />
+    <PackageReference Include="Microsoft.ML.TensorFlow" Version="1.7.0" />
+    <PackageReference Include="Microsoft.ML.OnnxTransformer" Version="1.7.0" />
+    <PackageReference Include="Microsoft.ML.Vision" Version="1.7.0" />
+    <PackageReference Include="Google.Protobuf" Version="$(GoogleProtobufPackageVersion)" />
   </ItemGroup>
 
   <!-- The purpose of this target is to return a path from a referenced