[master] Update dependencies from dotnet/arcade (#735)

[master] Update dependencies from dotnet/arcade
2020-10-31 12:38:06 +00:00 · 2020-10-31 12:38:06 +00:00 · b068e39f1a
--- a/.config/dotnet-tools.json
+++ b/.config/dotnet-tools.json
@ -9,4 +9,4 @@
      ]
    }
  }
-}
+}
--- a/eng/Version.Details.xml
+++ b/eng/Version.Details.xml
@ -3,25 +3,25 @@
  <ProductDependencies>
  </ProductDependencies>
  <ToolsetDependencies>
-    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="6.0.0-beta.20526.10">
+    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="6.0.0-beta.20529.1">
      <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>c0f916d7f45834c080bf6cce60cf2bcfa00bc0a7</Sha>
+      <Sha>19e7e769f7ca2ece42221f7ff951e7ec705498ec</Sha>
    </Dependency>
-    <Dependency Name="Microsoft.DotNet.Build.Tasks.Feed" Version="6.0.0-beta.20526.10">
+    <Dependency Name="Microsoft.DotNet.Build.Tasks.Feed" Version="6.0.0-beta.20529.1">
      <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>c0f916d7f45834c080bf6cce60cf2bcfa00bc0a7</Sha>
+      <Sha>19e7e769f7ca2ece42221f7ff951e7ec705498ec</Sha>
    </Dependency>
-    <Dependency Name="Microsoft.DotNet.SignTool" Version="6.0.0-beta.20526.10">
+    <Dependency Name="Microsoft.DotNet.SignTool" Version="6.0.0-beta.20529.1">
      <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>c0f916d7f45834c080bf6cce60cf2bcfa00bc0a7</Sha>
+      <Sha>19e7e769f7ca2ece42221f7ff951e7ec705498ec</Sha>
    </Dependency>
-    <Dependency Name="Microsoft.DotNet.Helix.Sdk" Version="6.0.0-beta.20526.10">
+    <Dependency Name="Microsoft.DotNet.Helix.Sdk" Version="6.0.0-beta.20529.1">
      <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>c0f916d7f45834c080bf6cce60cf2bcfa00bc0a7</Sha>
+      <Sha>19e7e769f7ca2ece42221f7ff951e7ec705498ec</Sha>
    </Dependency>
-    <Dependency Name="Microsoft.DotNet.SwaggerGenerator.MSBuild" Version="6.0.0-beta.20526.10">
+    <Dependency Name="Microsoft.DotNet.SwaggerGenerator.MSBuild" Version="6.0.0-beta.20529.1">
      <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>c0f916d7f45834c080bf6cce60cf2bcfa00bc0a7</Sha>
+      <Sha>19e7e769f7ca2ece42221f7ff951e7ec705498ec</Sha>
    </Dependency>
    <Dependency Name="Microsoft.DotNet.Maestro.Client" Version="1.1.0-beta.19556.4">
      <Uri>https://github.com/dotnet/arcade-services</Uri>
--- a/eng/common/cross/build-rootfs.sh
+++ b/eng/common/cross/build-rootfs.sh
@ -106,6 +106,13 @@ while :; do
            __UbuntuRepo="http://ftp.debian.org/debian/"
            __CodeName=jessie
            ;;
+        s390x)
+            __BuildArch=s390x
+            __UbuntuArch=s390x
+            __UbuntuRepo="http://ports.ubuntu.com/ubuntu-ports/"
+            __UbuntuPackages=$(echo ${__UbuntuPackages} | sed 's/ libunwind8-dev//')
+            unset __LLDB_Package
+            ;;
        x86)
            __BuildArch=x86
            __UbuntuArch=i386
--- a/eng/common/cross/s390x/sources.list.bionic
+++ b/eng/common/cross/s390x/sources.list.bionic
@ -0,0 +1,11 @@
+deb http://ports.ubuntu.com/ubuntu-ports/ bionic main restricted universe
+deb-src http://ports.ubuntu.com/ubuntu-ports/ bionic main restricted universe
+
+deb http://ports.ubuntu.com/ubuntu-ports/ bionic-updates main restricted universe
+deb-src http://ports.ubuntu.com/ubuntu-ports/ bionic-updates main restricted universe
+
+deb http://ports.ubuntu.com/ubuntu-ports/ bionic-backports main restricted
+deb-src http://ports.ubuntu.com/ubuntu-ports/ bionic-backports main restricted
+
+deb http://ports.ubuntu.com/ubuntu-ports/ bionic-security main restricted universe multiverse
+deb-src http://ports.ubuntu.com/ubuntu-ports/ bionic-security main restricted universe multiverse
--- a/eng/common/cross/toolchain.cmake
+++ b/eng/common/cross/toolchain.cmake
@ -36,6 +36,9 @@ elseif(TARGET_ARCH_NAME STREQUAL "arm64")
  if("$ENV{__DistroRid}" MATCHES "tizen.*")
    set(TIZEN_TOOLCHAIN "aarch64-tizen-linux-gnu/9.2.0")
  endif()
+elseif(TARGET_ARCH_NAME STREQUAL "s390x")
+  set(CMAKE_SYSTEM_PROCESSOR s390x)
+  set(TOOLCHAIN "s390x-linux-gnu")
 elseif(TARGET_ARCH_NAME STREQUAL "x86")
  set(CMAKE_SYSTEM_PROCESSOR i686)
  set(TOOLCHAIN "i686-linux-gnu")
@ -46,7 +49,7 @@ elseif (ILLUMOS)
  set(CMAKE_SYSTEM_PROCESSOR "x86_64")
  set(TOOLCHAIN "x86_64-illumos")
 else()
-  message(FATAL_ERROR "Arch is ${TARGET_ARCH_NAME}. Only armel, arm, arm64 and x86 are supported!")
+  message(FATAL_ERROR "Arch is ${TARGET_ARCH_NAME}. Only armel, arm, arm64, s390x and x86 are supported!")
 endif()

 if(DEFINED ENV{TOOLCHAIN})
@ -171,7 +174,7 @@ endif()

 # Specify compile options

-if((TARGET_ARCH_NAME MATCHES "^(arm|armel|arm64)$" AND NOT "$ENV{__DistroRid}" MATCHES "android.*") OR ILLUMOS)
+if((TARGET_ARCH_NAME MATCHES "^(arm|armel|arm64|s390x)$" AND NOT "$ENV{__DistroRid}" MATCHES "android.*") OR ILLUMOS)
  set(CMAKE_C_COMPILER_TARGET ${TOOLCHAIN})
  set(CMAKE_CXX_COMPILER_TARGET ${TOOLCHAIN})
  set(CMAKE_ASM_COMPILER_TARGET ${TOOLCHAIN})
--- a/eng/common/performance/crossgen_perf.proj
+++ b/eng/common/performance/crossgen_perf.proj
@ -65,6 +65,24 @@
    </Crossgen2SingleThreadedWorkItem>
  </ItemGroup>

+  <ItemGroup>
+    <CrossgenSizeOnDiskWorkItem Include="@(SingleAssembly)" Condition="'$(Architecture)' == 'x64'">
+      <PayloadDirectory>$(WorkItemDirectory)</PayloadDirectory>
+      <PreCommands>$(Python) pre.py crossgen --core-root $(CoreRoot) --single %(Identity) </PreCommands>
+      <Command>$(Python) test.py sod --scenario-name &quot;Crossgen %(Identity) Size&quot; --dirs ./crossgen/</Command>
+      <PostCommands>$(Python) post.py</PostCommands>
+    </CrossgenSizeOnDiskWorkItem>
+  </ItemGroup>
+
+  <ItemGroup>
+    <Crossgen2SizeOnDiskWorkItem Include="@(SingleAssembly)" Condition="'$(Architecture)' == 'x64'">
+      <PayloadDirectory>$(WorkItemDirectory)</PayloadDirectory>
+      <PreCommands>$(Python) $(Crossgen2Directory)pre.py crossgen2 --core-root $(CoreRoot) --single %(Identity) </PreCommands>
+      <Command>$(Python) test.py sod --scenario-name &quot;Crossgen2 %(Identity) Size&quot; --dirs ./crossgen/</Command>
+      <PostCommands>$(Python) post.py</PostCommands>
+    </Crossgen2SizeOnDiskWorkItem>
+  </ItemGroup>
+
  <ItemGroup>
    <!-- Enable crossgen tests on Windows x64 and Windows x86 -->
    <HelixWorkItem Include="@(CrossgenWorkItem -> 'Crossgen %(Identity)')" Condition="'$(AGENT_OS)' == 'Windows_NT'">
@ -82,5 +100,11 @@
      <Command>$(Python) $(Crossgen2Directory)test.py crossgen2 --core-root $(CoreRoot) --composite $(Crossgen2Directory)framework-r2r.dll.rsp</Command>
      <Timeout>1:00</Timeout>  
    </HelixWorkItem>
+    <HelixWorkItem Include="@(CrossgenSizeOnDiskWorkItem -> 'Crossgen Size on Disk %(Identity)')" Condition="'$(Architecture)' == 'x64'">
+      <Timeout>4:00</Timeout>  
+    </HelixWorkItem>
+    <HelixWorkItem Include="@(Crossgen2SizeOnDiskWorkItem -> 'Crossgen2 Size on Disk %(Identity)')" Condition="'$(Architecture)' == 'x64'">
+      <Timeout>4:00</Timeout>  
+    </HelixWorkItem>
  </ItemGroup>
 </Project>
--- a/eng/common/post-build/symbols-validation.ps1
+++ b/eng/common/post-build/symbols-validation.ps1
@ -9,9 +9,16 @@ param(
 # Maximum number of jobs to run in parallel
 $MaxParallelJobs = 6

+# Max number of retries
+$MaxRetry = 5
+
 # Wait time between check for system load
 $SecondsBetweenLoadChecks = 10

+# Set error codes
+Set-Variable -Name "ERROR_BADEXTRACT" -Option Constant -Value -1
+Set-Variable -Name "ERROR_FILEDOESNOTEXIST" -Option Constant -Value -2
+
 $CountMissingSymbols = {
  param( 
    [string] $PackagePath          # Path to a NuGet package
@ -21,10 +28,15 @@ $CountMissingSymbols = {

  Add-Type -AssemblyName System.IO.Compression.FileSystem

+  Write-Host "Validating $PackagePath "
+
  # Ensure input file exist
  if (!(Test-Path $PackagePath)) {
    Write-PipelineTaskError "Input file does not exist: $PackagePath"
-    return -2
+    return [pscustomobject]@{
+      result = $using:ERROR_FILEDOESNOTEXIST
+      packagePath = $PackagePath
+    }
  }
  
  # Extensions for which we'll look for symbols
@ -45,7 +57,7 @@ $CountMissingSymbols = {
    Write-Host "Something went wrong extracting $PackagePath"
    Write-Host $_
    return [pscustomobject]@{
-      result = -1
+      result = $using:ERROR_BADEXTRACT
      packagePath = $PackagePath
    }
  }
@ -91,26 +103,37 @@ $CountMissingSymbols = {
      $dotnetSymbolExe = "$env:USERPROFILE\.dotnet\tools"
      $dotnetSymbolExe = Resolve-Path "$dotnetSymbolExe\dotnet-symbol.exe"

-      & $dotnetSymbolExe --symbols --modules --windows-pdbs $TargetServerParam $FullPath -o $SymbolsPath | Out-Null
+      $totalRetries = 0

-      if (Test-Path $PdbPath) {
-        return 'PDB'
-      }
-      elseif (Test-Path $NGenPdb) {
-        return 'NGen PDB'
-      }
-      elseif (Test-Path $SODbg) {
-        return 'DBG for SO'
-      }  
-      elseif (Test-Path $DylibDwarf) {
-        return 'Dwarf for Dylib'
-      }  
-      elseif (Test-Path $SymbolPath) {
-        return 'Module'
-      }
-      else {
-        return $null
+      while ($totalRetries -lt $using:MaxRetry) {
+        # Save the output and get diagnostic output
+        $output = & $dotnetSymbolExe --symbols --modules --windows-pdbs $TargetServerParam $FullPath -o $SymbolsPath --diagnostics | Out-String
+
+        if (Test-Path $PdbPath) {
+          return 'PDB'
+        }
+        elseif (Test-Path $NGenPdb) {
+          return 'NGen PDB'
+        }
+        elseif (Test-Path $SODbg) {
+          return 'DBG for SO'
+        }  
+        elseif (Test-Path $DylibDwarf) {
+          return 'Dwarf for Dylib'
+        }  
+        elseif (Test-Path $SymbolPath) {
+          return 'Module'
+        }
+        elseif ($output.Contains("503 Service Unavailable")) {
+          # If we got a 503 error, we should retry.
+          $totalRetries++
+        }
+        else {
+          return $null
+        }
      }
+      
+      return $null
    }

      $SymbolsOnMSDL = & $FirstMatchingSymbolDescriptionOrDefault $FileName '--microsoft-symbol-server' $SymbolsPath
@ -155,14 +178,21 @@ function CheckJobResult(
    $packagePath,
    [ref]$DupedSymbols,
    [ref]$TotalFailures) {
-  if ($result -eq '-1') {
+  if ($result -eq $ERROR_BADEXTRACT) {
    Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "$packagePath has duplicated symbol files"
    $DupedSymbols.Value++
  } 
-  elseif ($jobResult.result -ne '0') {
+  elseif ($result -eq $ERROR_FILEDOESNOTEXIST) {
+    Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "$packagePath does not exist"
+    $TotalFailures.Value++
+  }
+  elseif ($result -gt '0') {
    Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "Missing symbols for $result modules in the package $packagePath"
    $TotalFailures.Value++
  }
+  else {
+    Write-Host "All symbols verified for package $packagePath"
+  }
 }

 function CheckSymbolsAvailable {
@ -192,8 +222,6 @@ function CheckSymbolsAvailable {
        return
      }

-      Write-Host "Validating $FileName "
-
      Start-Job -ScriptBlock $CountMissingSymbols -ArgumentList $FullName | Out-Null

      $NumJobs = @(Get-Job -State 'Running').Count
--- a/global.json
+++ b/global.json
@ -11,6 +11,6 @@
    }
  },
  "msbuild-sdks": {
-    "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.20526.10"
+    "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.20529.1"
  }
 }