[opentracing] Update VMs 2020-12 (#15151)

* Add meson from https://github.com/microsoft/vcpkg/pull/12860/ * Add autoconf-archive from https://github.com/microsoft/vcpkg/pull/13081/ * Add kf5windowsystem libs from https://github.com/microsoft/vcpkg/pull/13467/ * Open the FTP and SFTP ports from https://github.com/microsoft/vcpkg/pull/14412/ * Add libxcb-util0-dev from https://github.com/microsoft/vcpkg/pull/14678/ * Add libasound2-dev from https://github.com/microsoft/vcpkg/pull/14774 * Remove no longer necessary apt-mark calls. * Update nasm on Linux. * Fix longstanding bug where Storage was publicly accessible and change to generate SAS token rather than File Share * Delete no longer necessary azure storage firewall rules. * Install the newer Windows SDK with the VS installer instead of manually. * Install the VS2015 and VS2017 compilers. * Update Powershell-Core to 7.1.0. * Update source of WDK. * Update pools. * [opentracing] Repair arm64-windows failures caused by mojibake in `expected.hpp` and errors in opentracing-cpp's lint for arm64 where it thinks exceptions are disabled when they are enabled. Fixes: C:\Dev\vcpkg\buildtrees\opentracing\src\b67575dab0-0250653c81.clean\3rd_party\include\opentracing/expected/expected.hpp(1): warning C4828: The file contains a character starting at offset 0x4a77 that is illegal in the current source character set (codepage 65001). Fixes: D:\buildtrees\opentracing\src\b67575dab0-0250653c81.clean\include\opentracing/tracer.h:223:5: error: cannot use 'try' with exceptions disabled [clang-diagnostic-error] try { ^ * [mmloader] Patch out overrides of CMAKE_C_FLAGS and CMAKE_CXX_FLAGS that inserted /WX. Note that this port sets /GS-, possibly because it may be for authoring shellcode. * Bump storage API version to 2020-04-08.
2020-12-22 14:52:58 -08:00 · 2020-12-22 14:52:58 -08:00 · 229f537c94
--- a/ports/mmloader/CONTROL
+++ b/ports/mmloader/CONTROL
@ -1,8 +0,0 @@
-Source: mmloader
-Version: 2020-05-15
-Description: A library for loading dll module bypassing windows PE loader from memory (x86/x64)
-Homepage: http://tishion.github.io/mmLoader/
-Supports: (x86|x64)&windows&static
-
-Feature: shellcode
-Description: Generate mmLoader shell code headers
--- a/ports/mmloader/fix_c_and_cxx_flags.patch
+++ b/ports/mmloader/fix_c_and_cxx_flags.patch
@ -0,0 +1,32 @@
+diff --git a/CMakeLists.txt b/CMakeLists.txt
+index 7a22b56..2ce1bd8 100644
+--- a/CMakeLists.txt
+++ b/CMakeLists.txt
+@@ -47,24 +47,11 @@ set(CMAKE_CXX_STANDARD 11)
+ set(CMAKE_CONFIGURATION_TYPES Debug Release)
+ 
+ # Default compile flags
+-set(CMAKE_CXX_FLAGS 
+-    "/DWIN32 /D_WINDOWS /W3 /WX /GR /GS- /Gy /FAcs /Z7 /DEBUG"
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /GS-"
+     CACHE STRING "default C++ compile flags" FORCE)
+-set(CMAKE_CXX_FLAGS_DEBUG 
+-    "/MTd /Ob0 /Od" 
+-    CACHE STRING "default C++ compile flags for debug build" FORCE)
+-set(CMAKE_CXX_FLAGS_RELEASE 
+-    "/MT /O2 /Ob2 /DNDEBUG" 
+-    CACHE STRING "default C++ compile flags for release build" FORCE)
+-set(CMAKE_C_FLAGS 
+-    "/DWIN32 /D_WINDOWS /W3 /WX /GR /GS- /Gy /FAcs /Z7 /DEBUG"
+set(CMAKE_C_FLAGS
+    "${CMAKE_C_FLAGS} /GS-"
+     CACHE STRING "default C++ compile flags" FORCE)
+-set(CMAKE_C_FLAGS_DEBUG
+-    "/MTd /Ob0 /Od" 
+-    CACHE STRING "default C++ compile flags for debug build" FORCE)
+-set(CMAKE_C_FLAGS_RELEASE 
+-    "/MT /O2 /Ob2 /DNDEBUG" 
+-    CACHE STRING "default C++ compile flags for release build" FORCE)
+ 
+ message(STATUS "CMAKE_CXX_FLAGS=" ${CMAKE_CXX_FLAGS})
+ message(STATUS "CMAKE_CXX_FLAGS_DEBUG=" ${CMAKE_CXX_FLAGS_DEBUG})
--- a/ports/mmloader/portfile.cmake
+++ b/ports/mmloader/portfile.cmake
@ -1,4 +1,3 @@
-
 # fail early for unsupported triplets
 vcpkg_fail_port_install(
    MESSAGE "mmLoader supports only x86/x64-windows-static triplets"
@ -15,6 +14,8 @@ vcpkg_from_github(
    REF 1.0.0
    SHA512 ceb2ab4b60d74209fa46d198cde6fd87a97d911abb875ac35383288a67828d0420bb38ff8d2f17dd4a3f46ba3abf554152d1246eeb05215258e8af64ac4a39de
    HEAD_REF master
+    PATCHES
+        fix_c_and_cxx_flags.patch
 )

 # feature
@ -30,13 +31,13 @@ vcpkg_configure_cmake(
 )

 # pre-clean
-file(REMOVE_RECURSE ${SOURCE_PATH}/output)
+file(REMOVE_RECURSE "${SOURCE_PATH}/output")

 # build and install
 vcpkg_install_cmake(DISABLE_PARALLEL)

 # remove the debug/include directory
-file(REMOVE_RECURSE ${CURRENT_PACKAGES_DIR}/debug/include)
+file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/include")

 # collect license files
-file(INSTALL ${SOURCE_PATH}/License DESTINATION ${CURRENT_PACKAGES_DIR}/share/${PORT} RENAME copyright)
+file(INSTALL "${SOURCE_PATH}/License" DESTINATION "${CURRENT_PACKAGES_DIR}/share/${PORT}" RENAME copyright)
--- a/ports/mmloader/vcpkg.json
+++ b/ports/mmloader/vcpkg.json
@ -0,0 +1,13 @@
+{
+  "name": "mmloader",
+  "version-string": "2020-05-15",
+  "port-version": 1,
+  "description": "A library for loading dll module bypassing windows PE loader from memory (x86/x64)",
+  "homepage": "http://tishion.github.io/mmLoader/",
+  "supports": "(x86 | x64) & windows & static",
+  "features": {
+    "shellcode": {
+      "description": "Generate mmLoader shell code headers"
+    }
+  }
+}
--- a/ports/opentracing/CONTROL
+++ b/ports/opentracing/CONTROL
@ -1,5 +0,0 @@
-Source: opentracing
-Version: 1.6.0
-Description: C++ implementation of the OpenTracing API
-Homepage: https://opentracing.io
-Supports: !uwp
--- a/ports/opentracing/portfile.cmake
+++ b/ports/opentracing/portfile.cmake
@ -1,9 +1,7 @@
-if(VCPKG_CMAKE_SYSTEM_NAME STREQUAL WindowsStore)
-    message(FATAL_ERROR "Error: UWP build is not supported.")
-endif()
+vcpkg_fail_port_install(ON_TARGET uwp)

 if(VCPKG_LIBRARY_LINKAGE STREQUAL "dynamic")
-    set( LOCAL_OPTIONS
+    set(LOCAL_OPTIONS
        -DBUILD_STATIC_LIBS=OFF
    )
 else()
@ -16,13 +14,15 @@ vcpkg_from_github(
    REPO opentracing/opentracing-cpp
    REF 4bb431f7728eaf383a07e86f9754a5b67575dab0 # v1.6.0
    SHA512 1c69ff4cfd5f6037a48815367d3026c1bf06c3c49ebf232a64c43167385fb62e444c3b3224fc38f68ef0fdb378e3736db6ee6ba57160e6e578c87c09e92e527e
+    PATCHES
+        repair_mojibake.patch
 )

 vcpkg_configure_cmake(
-    SOURCE_PATH ${SOURCE_PATH}
+    SOURCE_PATH "${SOURCE_PATH}"
    PREFER_NINJA
    OPTIONS
-        ${OPTIONS}
+        -DENABLE_LINTING=OFF
        ${LOCAL_OPTIONS}
 )

@ -54,7 +54,7 @@ if(VCPKG_CMAKE_SYSTEM_NAME STREQUAL "WindowsStore" OR NOT VCPKG_CMAKE_SYSTEM_NAM
 endif()

 # Handle copyright
-file(INSTALL ${SOURCE_PATH}/LICENSE DESTINATION ${CURRENT_PACKAGES_DIR}/share/opentracing RENAME copyright)
+file(INSTALL "${SOURCE_PATH}/LICENSE" DESTINATION "${CURRENT_PACKAGES_DIR}/share/${PORT}" RENAME copyright)

 # Remove duplicate headers
-file(REMOVE_RECURSE ${CURRENT_PACKAGES_DIR}/debug/include)
+file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/include")
--- a/ports/opentracing/repair_mojibake.patch
+++ b/ports/opentracing/repair_mojibake.patch
@ -0,0 +1,59 @@
+From 31c92781ac953cb83963cfe5ac30324579fa7b2d Mon Sep 17 00:00:00 2001
+From: Billy Robert O'Neal III <bion@microsoft.com>
+Date: Fri, 18 Dec 2020 10:07:02 -0800
+Subject: [PATCH] Repair mojibake in expected.hpp
+
+The affected characters herein triggered warnings
+C:\Dev\vcpkg\buildtrees\opentracing\src\b67575dab0-0250653c81.clean\3rd_party\include\opentracing/expected/expected.hpp(1): warning C4828: The file contains a character starting at offset 0x4a77 that is illegal in the current source character set (codepage 65001).
+---
+ 3rd_party/include/opentracing/expected/expected.hpp | 12 ++++++------
+ 1 file changed, 6 insertions(+), 6 deletions(-)
+
+diff --git a/3rd_party/include/opentracing/expected/expected.hpp b/3rd_party/include/opentracing/expected/expected.hpp
+index 8283a77..0d21d43 100644
+--- a/3rd_party/include/opentracing/expected/expected.hpp
+++ b/3rd_party/include/opentracing/expected/expected.hpp
+@@ -778,13 +778,13 @@ class expected
+ //  expected<decltype(func(declval<T>())),E> map(F&& func) ;
+ 
+ //  template <typename F>
+-//  ’see below’ bind(F&& func);
+//  'see below' bind(F&& func);
+ 
+ //  template <typename F>
+ //  expected<T,E> catch_error(F&& f);
+ 
+ //  template <typename F>
+-//  ’see below’ then(F&& func);
+//  'see below' then(F&& func);
+ 
+ private:
+     bool has_value_;
+@@ -955,9 +955,9 @@ class expected<void, E>
+         return ! has_value() && std::is_base_of< Ex, decltype( get_unexpected().value() ) >::value;
+     }
+ 
+-//  template constexpr ’see below’ unwrap() const&;
+//  template constexpr 'see below' unwrap() const&;
+ //
+-//  template ’see below’ unwrap() &&;
+//  template 'see below' unwrap() &&;
+ 
+     // factories
+ 
+@@ -968,13 +968,13 @@ class expected<void, E>
+ //  expected<decltype(func()), E> map(F&& func) ;
+ //
+ //  template <typename F>
+-//  ’see below’ bind(F&& func) ;
+//  'see below' bind(F&& func) ;
+ //
+ //  template <typename F>
+ //  expected<void,E> catch_error(F&& f);
+ //
+ //  template <typename F>
+-//  ’see below’ then(F&& func);
+//  'see below' then(F&& func);
+ 
+ private:
+     bool has_value_;
--- a/ports/opentracing/vcpkg.json
+++ b/ports/opentracing/vcpkg.json
@ -0,0 +1,8 @@
+{
+  "name": "opentracing",
+  "version-string": "1.6.0",
+  "port-version": 1,
+  "description": "C++ implementation of the OpenTracing API",
+  "homepage": "https://opentracing.io",
+  "supports": "!uwp"
+}
--- a/scripts/azure-pipelines/azure-pipelines.yml
+++ b/scripts/azure-pipelines/azure-pipelines.yml
@ -2,8 +2,8 @@
 # SPDX-License-Identifier: MIT
 #
 variables:
-  windows-pool: 'PrWin-2020-08-12'
-  linux-pool: 'PrLin-2020-08-12'
+  windows-pool: 'PrWin-2020-12-17'
+  linux-pool: 'PrLin-2020-12-16'
  osx-pool: 'PrOsx-2020-09-28'

 stages:
--- a/scripts/azure-pipelines/linux/azure-pipelines.yml
+++ b/scripts/azure-pipelines/linux/azure-pipelines.yml
@ -41,7 +41,7 @@ jobs:
    inputs:
      failOnStderr: true
      filePath: 'scripts/azure-pipelines/test-modified-ports.ps1'
-      arguments: '-Triplet x64-linux -BuildReason $(Build.Reason) -ArchivesRoot /archives -WorkingRoot ${{ variables.WORKING_ROOT }} -ArtifactStagingDirectory $(Build.ArtifactStagingDirectory)'
+      arguments: '-Triplet x64-linux -BuildReason $(Build.Reason) -UseEnvironmentSasToken -WorkingRoot ${{ variables.WORKING_ROOT }} -ArtifactStagingDirectory $(Build.ArtifactStagingDirectory)'
  - bash: |
      df -h
    displayName: 'Report on Disk Space After Build'
--- a/scripts/azure-pipelines/linux/create-vmss.ps1
+++ b/scripts/azure-pipelines/linux/create-vmss.ps1
@ -64,7 +64,9 @@ Write-Progress `
  -Status 'Creating virtual network' `
  -PercentComplete (100 / $TotalProgress * $CurrentProgress++)

-$allowHttp = New-AzNetworkSecurityRuleConfig `
+$allFirewallRules = @()
+
+$allFirewallRules += New-AzNetworkSecurityRuleConfig `
  -Name AllowHTTP `
  -Description 'Allow HTTP(S)' `
  -Access Allow `
@ -76,49 +78,49 @@ $allowHttp = New-AzNetworkSecurityRuleConfig `
  -DestinationAddressPrefix * `
  -DestinationPortRange @(80, 443)

-$allowDns = New-AzNetworkSecurityRuleConfig `
-  -Name AllowDNS `
-  -Description 'Allow DNS' `
+$allFirewallRules += New-AzNetworkSecurityRuleConfig `
+  -Name AllowSFTP `
+  -Description 'Allow (S)FTP' `
  -Access Allow `
-  -Protocol * `
+  -Protocol Tcp `
  -Direction Outbound `
  -Priority 1009 `
  -SourceAddressPrefix * `
  -SourcePortRange * `
  -DestinationAddressPrefix * `
-  -DestinationPortRange 53
+  -DestinationPortRange @(21, 22)

-$allowGit = New-AzNetworkSecurityRuleConfig `
-  -Name AllowGit `
-  -Description 'Allow git' `
+$allFirewallRules += New-AzNetworkSecurityRuleConfig `
+  -Name AllowDNS `
+  -Description 'Allow DNS' `
  -Access Allow `
-  -Protocol Tcp `
+  -Protocol * `
  -Direction Outbound `
  -Priority 1010 `
  -SourceAddressPrefix * `
  -SourcePortRange * `
  -DestinationAddressPrefix * `
-  -DestinationPortRange 9418
+  -DestinationPortRange 53

-$allowStorage = New-AzNetworkSecurityRuleConfig `
-  -Name AllowStorage `
-  -Description 'Allow Storage' `
+$allFirewallRules += New-AzNetworkSecurityRuleConfig `
+  -Name AllowGit `
+  -Description 'Allow git' `
  -Access Allow `
-  -Protocol * `
+  -Protocol Tcp `
  -Direction Outbound `
  -Priority 1011 `
-  -SourceAddressPrefix VirtualNetwork `
+  -SourceAddressPrefix * `
  -SourcePortRange * `
-  -DestinationAddressPrefix Storage `
-  -DestinationPortRange *
+  -DestinationAddressPrefix * `
+  -DestinationPortRange 9418

-$denyEverythingElse = New-AzNetworkSecurityRuleConfig `
+$allFirewallRules += New-AzNetworkSecurityRuleConfig `
  -Name DenyElse `
  -Description 'Deny everything else' `
  -Access Deny `
  -Protocol * `
  -Direction Outbound `
-  -Priority 1012 `
+  -Priority 1013 `
  -SourceAddressPrefix * `
  -SourcePortRange * `
  -DestinationAddressPrefix * `
@ -129,13 +131,14 @@ $NetworkSecurityGroup = New-AzNetworkSecurityGroup `
  -Name $NetworkSecurityGroupName `
  -ResourceGroupName $ResourceGroupName `
  -Location $Location `
-  -SecurityRules @($allowHttp, $allowDns, $allowGit, $allowStorage, $denyEverythingElse)
+  -SecurityRules $allFirewallRules

 $SubnetName = $ResourceGroupName + 'Subnet'
 $Subnet = New-AzVirtualNetworkSubnetConfig `
  -Name $SubnetName `
  -AddressPrefix "10.0.0.0/16" `
-  -NetworkSecurityGroup $NetworkSecurityGroup
+  -NetworkSecurityGroup $NetworkSecurityGroup `
+  -ServiceEndpoint "Microsoft.Storage"

 $VirtualNetworkName = $ResourceGroupName + 'Network'
 $VirtualNetwork = New-AzVirtualNetwork `
@ -170,8 +173,31 @@ $StorageContext = New-AzStorageContext `
  -StorageAccountName $StorageAccountName `
  -StorageAccountKey $StorageAccountKey

-New-AzStorageShare -Name 'archives' -Context $StorageContext
-Set-AzStorageShareQuota -ShareName 'archives' -Context $StorageContext -Quota 1024
+New-AzStorageContainer -Name archives -Context $StorageContext -Permission Off
+$StartTime = [DateTime]::Now
+$ExpiryTime = $StartTime.AddMonths(6)
+
+$SasToken = New-AzStorageAccountSASToken `
+  -Service Blob `
+  -Permission "racwdlup" `
+  -Context $StorageContext `
+  -StartTime $StartTime `
+  -ExpiryTime $ExpiryTime `
+  -ResourceType Service,Container,Object `
+  -Protocol HttpsOnly
+
+$SasToken = $SasToken.Substring(1) # strip leading ?
+
+# Note that we put the storage account into the firewall after creating the above SAS token or we
+# would be denied since the person running this script isn't one of the VMs we're creating here.
+Set-AzStorageAccount `
+  -ResourceGroupName $ResourceGroupName `
+  -AccountName $StorageAccountName `
+  -NetworkRuleSet ( `
+    @{bypass="AzureServices"; `
+    virtualNetworkRules=( `
+      @{VirtualNetworkResourceId=$VirtualNetwork.Subnets[0].Id;Action="allow"}); `
+    defaultAction="Deny"})

 ####################################################################################################
 Write-Progress `
@ -220,15 +246,23 @@ Write-Progress `
  -Status 'Running provisioning script provision-image.sh in VM' `
  -PercentComplete (100 / $TotalProgress * $CurrentProgress++)

-$ProvisionImageResult = Invoke-AzVMRunCommand `
-  -ResourceGroupName $ResourceGroupName `
-  -VMName $ProtoVMName `
-  -CommandId 'RunShellScript' `
-  -ScriptPath "$PSScriptRoot\provision-image.sh" `
-  -Parameter @{StorageAccountName=$StorageAccountName; `
-    StorageAccountKey=$StorageAccountKey;}
+$tempScript = [System.IO.Path]::GetTempPath() + [System.IO.Path]::GetRandomFileName() + ".sh"
+try {
+  $script = Get-Content "$PSScriptRoot\provision-image.sh" -Encoding utf8NoBOM
+  $script += "echo `"PROVISIONED_AZURE_STORAGE_NAME=\`"$StorageAccountName\`"`" | sudo tee -a /etc/environment"
+  $script += "echo `"PROVISIONED_AZURE_STORAGE_SAS_TOKEN=\`"$SasToken\`"`" | sudo tee -a /etc/environment"
+  Set-Content -Path $tempScript -Value $script -Encoding utf8NoBOM

-Write-Host "provision-image.sh output: $($ProvisionImageResult.value.Message)"
+  $ProvisionImageResult = Invoke-AzVMRunCommand `
+    -ResourceGroupName $ResourceGroupName `
+    -VMName $ProtoVMName `
+    -CommandId 'RunShellScript' `
+    -ScriptPath $tempScript
+
+  Write-Host "provision-image.sh output: $($ProvisionImageResult.value.Message)"
+} finally {
+  Remove-Item $tempScript -Recurse -Force
+}

 ####################################################################################################
 Write-Progress `
--- a/scripts/azure-pipelines/linux/provision-image.sh
+++ b/scripts/azure-pipelines/linux/provision-image.sh
@ -11,11 +11,12 @@ APT_PACKAGES="at curl unzip tar libxt-dev gperf libxaw7-dev cifs-utils \
  libgl1-mesa-dev libglu1-mesa-dev mesa-common-dev libxinerama-dev \
  libxcursor-dev yasm libnuma1 libnuma-dev python-six python3-six python-yaml \
  flex libbison-dev autoconf libudev-dev libncurses5-dev libtool libxrandr-dev \
-  xutils-dev dh-autoreconf libgles2-mesa-dev ruby-full pkg-config"
+  xutils-dev dh-autoreconf autoconf-archive libgles2-mesa-dev ruby-full \
+  pkg-config meson"

 # Additionally required by qt5-base
 APT_PACKAGES="$APT_PACKAGES libxext-dev libxfixes-dev libxrender-dev \
-  libxcb1-dev libx11-xcb-dev libxcb-glx0-dev"
+  libxcb1-dev libx11-xcb-dev libxcb-glx0-dev libxcb-util0-dev"

 # Additionally required by qt5-base for qt5-x11extras
 APT_PACKAGES="$APT_PACKAGES libxkbcommon-dev libxcb-keysyms1-dev \
@ -26,31 +27,29 @@ APT_PACKAGES="$APT_PACKAGES libxkbcommon-dev libxcb-keysyms1-dev \
 # Additionally required by libhdfs3
 APT_PACKAGES="$APT_PACKAGES libkrb5-dev"

+# Additionally required by kf5windowsystem
+APT_PACKAGES="$APT_PACKAGES libxcb-res0-dev"
+
 # Additionally required by mesa
 APT_PACKAGES="$APT_PACKAGES python3-setuptools python3-mako"

 # Additionally required by some packages to install additional python packages
 APT_PACKAGES="$APT_PACKAGES python3-pip"

+# Additionally required by rtaudio
+APT_PACKAGES="$APT_PACKAGES libasound2-dev"
+
 # Additionally required/installed by Azure DevOps Scale Set Agents
 APT_PACKAGES="$APT_PACKAGES liblttng-ust0 libkrb5-3 zlib1g libicu60"

 sudo apt -y install $APT_PACKAGES

-# Delete /etc/debian_version to prevent Azure Pipelines Scale Set Agents from
-# removing some of the above
-sudo apt-mark hold libcurl4
-sudo apt-mark hold liblttng-ust0
-sudo apt-mark hold libkrb5-3
-sudo apt-mark hold zlib1g
-sudo apt-mark hold libicu60
-
 # Install newer version of nasm than the apt package, required by intel-ipsec
 mkdir /tmp/nasm
 cd /tmp/nasm
-curl -O https://www.nasm.us/pub/nasm/releasebuilds/2.14.02/nasm-2.14.02.tar.gz
-tar -xf nasm-2.14.02.tar.gz
-cd nasm-2.14.02/
+curl -O https://www.nasm.us/pub/nasm/releasebuilds/2.15.05/nasm-2.15.05.tar.gz
+tar -xf nasm-2.15.05.tar.gz
+cd nasm-2.15.05/
 ./configure --prefix=/usr && make -j
 sudo make install
 cd ~
@ -75,19 +74,4 @@ sudo apt update
 sudo add-apt-repository universe
 sudo apt install -y powershell

-if [ -z "$StorageAccountName" ]; then
-echo "No storage account supplied, skipping."
-else
-echo "Mapping storage account"
-
-# Write SMB credentials
-sudo mkdir /etc/smbcredentials
-smbCredentialFile=/etc/smbcredentials/$StorageAccountName.cred
-echo "username=$StorageAccountName" | sudo tee $smbCredentialFile > /dev/null
-echo "password=$StorageAccountKey" | sudo tee -a $smbCredentialFile > /dev/null
-sudo chmod 600 $smbCredentialFile
-
-# Mount the archives SMB share to /archives
-sudo mkdir /archives -m=777
-echo "//$StorageAccountName.file.core.windows.net/archives /archives cifs nofail,vers=3.0,credentials=$smbCredentialFile,serverino,dir_mode=0777,file_mode=0777 0 0" | sudo tee -a /etc/fstab
-fi
+# provision-image.ps1 will append installation of the SAS token here
--- a/scripts/azure-pipelines/test-modified-ports.ps1
+++ b/scripts/azure-pipelines/test-modified-ports.ps1
@ -18,14 +18,17 @@ The Azure Pipelines artifacts directory. If not supplied, defaults to the curren
 .PARAMETER ArchivesRoot
 Equivalent to '-BinarySourceStub "files,$ArchivesRoot"'

+.PARAMETER UseEnvironmentSasToken
+Equivalent to '-BinarySourceStub "x-azblob,https://$($env:PROVISIONED_AZURE_STORAGE_NAME).blob.core.windows.net/archives,$($env:PROVISIONED_AZURE_STORAGE_SAS_TOKEN)"'
+
 .PARAMETER BinarySourceStub
 The type and parameters of the binary source. Shared across runs of this script. If
 this parameter is not set, binary caching will not be used. Example: "files,W:\"

 .PARAMETER BuildReason
 The reason Azure Pipelines is running this script (controls in which mode Binary Caching is used).
-If ArchivesRoot is not set, this parameter has no effect. If ArchivesRoot is set and this is not,
-binary caching will default to read-write mode.
+If BinarySourceStub is not set, this parameter has no effect. If BinarySourceStub is set and this is
+not, binary caching will default to read-write mode.
 #>

 [CmdletBinding(DefaultParameterSetName="ArchivesRoot")]
@ -40,6 +43,8 @@ Param(
    $ArtifactStagingDirectory = '.',
    [Parameter(ParameterSetName='ArchivesRoot')]
    $ArchivesRoot = $null,
+    [switch]
+    $UseEnvironmentSasToken = $false,
    [Parameter(ParameterSetName='BinarySourceStub')]
    $BinarySourceStub = $null,
    $BuildReason = $null
@ -50,12 +55,32 @@ if (-Not (Test-Path "triplets/$Triplet.cmake")) {
    throw
 }

+$usingBinaryCaching = $true
+if ([string]::IsNullOrWhiteSpace($BinarySourceStub)) {
+    if ([string]::IsNullOrWhiteSpace($ArchivesRoot)) {
+        if ($UseEnvironmentSasToken) {
+            $BinarySourceStub = "x-azblob,https://$($env:PROVISIONED_AZURE_STORAGE_NAME).blob.core.windows.net/archives,$($env:PROVISIONED_AZURE_STORAGE_SAS_TOKEN)"
+        } else {
+            $usingBinaryCaching = $false
+        }
+    } else {
+        if ($UseEnvironmentSasToken) {
+            Write-Error "Only one binary caching setting may be used."
+            throw
+        } else {
+            $BinarySourceStub = "files,$ArchivesRoot"
+        }
+    }
+} elseif ((-Not [string]::IsNullOrWhiteSpace($ArchivesRoot)) -Or $UseEnvironmentSasToken) {
+    Write-Error "Only one binary caching setting may be used."
+    throw
+}
+
 $env:VCPKG_DOWNLOADS = Join-Path $WorkingRoot 'downloads'
 $buildtreesRoot = Join-Path $WorkingRoot 'buildtrees'
 $installRoot = Join-Path $WorkingRoot 'installed'
 $packagesRoot = Join-Path $WorkingRoot 'packages'

-$usingBinaryCaching = -Not ([string]::IsNullOrWhiteSpace($ArchivesRoot)) -Or -Not ([string]::IsNullOrWhiteSpace($BinarySourceStub))
 $commonArgs = @()
 if ($usingBinaryCaching) {
    $commonArgs += @('--binarycaching')
@ -84,12 +109,8 @@ if ($usingBinaryCaching) {
        Write-Host "Build reason was $BuildReason, using binary caching in write only mode."
        $binaryCachingMode = 'write'
    }
-    if ([string]::IsNullOrWhiteSpace($ArchivesRoot)) {
-        $commonArgs += @("--binarysource=clear;$BinarySourceStub,$binaryCachingMode")
-    }
-    else {
-        $commonArgs += @("--binarysource=clear;files,$ArchivesRoot,$binaryCachingMode")
-    }
+
+    $commonArgs += @("--binarysource=clear;$BinarySourceStub,$binaryCachingMode")
 }

 if ($Triplet -eq 'x64-linux') {
--- a/scripts/azure-pipelines/windows-unstable/job.yml
+++ b/scripts/azure-pipelines/windows-unstable/job.yml
@ -56,11 +56,6 @@ jobs:
      filePath: 'scripts/azure-pipelines/windows-unstable/rearrange-msvc-drop-layout.ps1'
      arguments: '-DropRoot "D:\msvc-drops\$(DropBuildNumber)" -BuildType ret'
      pwsh: true
-  - task: PowerShell@2
-    displayName: 'Initialize Environment'
-    inputs:
-      filePath: 'scripts/azure-pipelines/windows/initialize-environment.ps1'
-      pwsh: true
  - task: PowerShell@2
    displayName: 'Report on Disk Space'
    condition: always()
--- a/scripts/azure-pipelines/windows/azure-pipelines.yml
+++ b/scripts/azure-pipelines/windows/azure-pipelines.yml
@ -16,11 +16,6 @@ jobs:
    value: D:\downloads

  steps:
-  - task: PowerShell@2
-    displayName: 'Initialize Environment'
-    inputs:
-      filePath: 'scripts/azure-pipelines/windows/initialize-environment.ps1'
-      pwsh: true
  - task: PowerShell@2
    displayName: 'Report on Disk Space'
    condition: always()
@ -57,7 +52,7 @@ jobs:
    inputs:
      failOnStderr: true
      filePath: 'scripts/azure-pipelines/test-modified-ports.ps1'
-      arguments: '-Triplet ${{ parameters.triplet }} -BuildReason $(Build.Reason) -ArchivesRoot W:\ -WorkingRoot ${{ variables.WORKING_ROOT }} -ArtifactStagingDirectory $(Build.ArtifactStagingDirectory)'
+      arguments: '-Triplet ${{ parameters.triplet }} -BuildReason $(Build.Reason) -UseEnvironmentSasToken -WorkingRoot ${{ variables.WORKING_ROOT }} -ArtifactStagingDirectory $(Build.ArtifactStagingDirectory)'
      pwsh: true
  - task: PowerShell@2
    displayName: 'Report on Disk Space After Build'
--- a/scripts/azure-pipelines/windows/create-vmss.ps1
+++ b/scripts/azure-pipelines/windows/create-vmss.ps1
@ -70,7 +70,9 @@ Write-Progress `
  -Status 'Creating virtual network' `
  -PercentComplete (100 / $TotalProgress * $CurrentProgress++)

-$allowHttp = New-AzNetworkSecurityRuleConfig `
+$allFirewallRules = @()
+
+$allFirewallRules += New-AzNetworkSecurityRuleConfig `
  -Name AllowHTTP `
  -Description 'Allow HTTP(S)' `
  -Access Allow `
@ -82,75 +84,67 @@ $allowHttp = New-AzNetworkSecurityRuleConfig `
  -DestinationAddressPrefix * `
  -DestinationPortRange @(80, 443)

-$allowDns = New-AzNetworkSecurityRuleConfig `
-  -Name AllowDNS `
-  -Description 'Allow DNS' `
+$allFirewallRules += New-AzNetworkSecurityRuleConfig `
+  -Name AllowSFTP `
+  -Description 'Allow (S)FTP' `
  -Access Allow `
-  -Protocol * `
+  -Protocol Tcp `
  -Direction Outbound `
  -Priority 1009 `
  -SourceAddressPrefix * `
  -SourcePortRange * `
  -DestinationAddressPrefix * `
-  -DestinationPortRange 53
+  -DestinationPortRange @(21, 22)

-$allowGit = New-AzNetworkSecurityRuleConfig `
-  -Name AllowGit `
-  -Description 'Allow git' `
+$allFirewallRules += New-AzNetworkSecurityRuleConfig `
+  -Name AllowDNS `
+  -Description 'Allow DNS' `
  -Access Allow `
-  -Protocol Tcp `
+  -Protocol * `
  -Direction Outbound `
  -Priority 1010 `
  -SourceAddressPrefix * `
  -SourcePortRange * `
  -DestinationAddressPrefix * `
+  -DestinationPortRange 53
+
+$allFirewallRules += New-AzNetworkSecurityRuleConfig `
+  -Name AllowGit `
+  -Description 'Allow git' `
+  -Access Allow `
+  -Protocol Tcp `
+  -Direction Outbound `
+  -Priority 1011 `
+  -SourceAddressPrefix * `
+  -SourcePortRange * `
+  -DestinationAddressPrefix * `
  -DestinationPortRange 9418

-if (-Not $Unstable) {
-  $allowStorage = New-AzNetworkSecurityRuleConfig `
-    -Name AllowStorage `
-    -Description 'Allow Storage' `
-    -Access Allow `
-    -Protocol * `
-    -Direction Outbound `
-    -Priority 1011 `
-    -SourceAddressPrefix VirtualNetwork `
-    -SourcePortRange * `
-    -DestinationAddressPrefix Storage `
-    -DestinationPortRange *
-}
-
-$denyEverythingElse = New-AzNetworkSecurityRuleConfig `
+$allFirewallRules += New-AzNetworkSecurityRuleConfig `
  -Name DenyElse `
  -Description 'Deny everything else' `
  -Access Deny `
  -Protocol * `
  -Direction Outbound `
-  -Priority 1012 `
+  -Priority 1013 `
  -SourceAddressPrefix * `
  -SourcePortRange * `
  -DestinationAddressPrefix * `
  -DestinationPortRange *

 $NetworkSecurityGroupName = $ResourceGroupName + 'NetworkSecurity'
-$securityRules = @($allowHttp, $allowDns, $allowGit);
-if (-Not $Unstable) {
-  $securityRules += @($allowStorage)
-}
-
-$securityRules += @($denyEverythingElse)
-
 $NetworkSecurityGroup = New-AzNetworkSecurityGroup `
  -Name $NetworkSecurityGroupName `
  -ResourceGroupName $ResourceGroupName `
  -Location $Location `
-  -SecurityRules $securityRules
+  -SecurityRules $allFirewallRules

 $SubnetName = $ResourceGroupName + 'Subnet'
 $Subnet = New-AzVirtualNetworkSubnetConfig `
  -Name $SubnetName `
  -AddressPrefix "10.0.0.0/16" `
-  -NetworkSecurityGroup $NetworkSecurityGroup
+  -NetworkSecurityGroup $NetworkSecurityGroup `
+  -ServiceEndpoint "Microsoft.Storage"

 $VirtualNetworkName = $ResourceGroupName + 'Network'
 $VirtualNetwork = New-AzVirtualNetwork `
@ -186,8 +180,31 @@ if (-Not $Unstable) {
    -StorageAccountName $StorageAccountName `
    -StorageAccountKey $StorageAccountKey

-  New-AzStorageShare -Name 'archives' -Context $StorageContext
-  Set-AzStorageShareQuota -ShareName 'archives' -Context $StorageContext -Quota 2048
+  New-AzStorageContainer -Name archives -Context $StorageContext -Permission Off
+  $StartTime = [DateTime]::Now
+  $ExpiryTime = $StartTime.AddMonths(6)
+
+  $SasToken = New-AzStorageAccountSASToken `
+    -Service Blob `
+    -Permission "racwdlup" `
+    -Context $StorageContext `
+    -StartTime $StartTime `
+    -ExpiryTime $ExpiryTime `
+    -ResourceType Service,Container,Object `
+    -Protocol HttpsOnly
+
+  $SasToken = $SasToken.Substring(1) # strip leading ?
+
+  # Note that we put the storage account into the firewall after creating the above SAS token or we
+  # would be denied since the person running this script isn't one of the VMs we're creating here.
+  Set-AzStorageAccount `
+    -ResourceGroupName $ResourceGroupName `
+    -AccountName $StorageAccountName `
+    -NetworkRuleSet ( `
+      @{bypass="AzureServices"; `
+      virtualNetworkRules=( `
+        @{VirtualNetworkResourceId=$VirtualNetwork.Subnets[0].Id;Action="allow"}); `
+      defaultAction="Deny"})
 }

 ####################################################################################################
@ -234,7 +251,7 @@ Write-Progress `
 $provisionParameters = @{AdminUserPassword = $AdminPW;}
 if (-Not $Unstable) {
  $provisionParameters['StorageAccountName'] = $StorageAccountName
-  $provisionParameters['StorageAccountKey'] = $StorageAccountKey
+  $provisionParameters['StorageAccountSasToken'] = $SasToken
 }

 $ProvisionImageResult = Invoke-AzVMRunCommand `
--- a/scripts/azure-pipelines/windows/initialize-environment.ps1
+++ b/scripts/azure-pipelines/windows/initialize-environment.ps1
@ -1,42 +0,0 @@
-# Copyright (c) Microsoft Corporation.
-# SPDX-License-Identifier: MIT
-#
-<#
-.SYNOPSIS
-Sets up the environment to run other vcpkg CI steps in an Azure Pipelines job.
-
-.DESCRIPTION
-This script maps network drives from infrastructure and cleans out anything that
-might have been leftover from a previous run.
-#>
-
-if ([string]::IsNullOrWhiteSpace($env:StorageAccountName) -or  [string]::IsNullOrWhiteSpace($env:StorageAccountKey)) {
-    Write-Host 'No storage account and/or key set, skipping mount of W:\'
-} else {
-    $StorageAccountName = $env:StorageAccountName
-    $StorageAccountKey = $env:StorageAccountKey
-
-    Write-Host 'Setting up archives mount'
-    if (-Not (Test-Path W:)) {
-        net use W: "\\$StorageAccountName.file.core.windows.net\archives" /u:"AZURE\$StorageAccountName" $StorageAccountKey
-    }
-}
-
-Write-Host 'Creating downloads directory'
-mkdir D:\downloads -ErrorAction SilentlyContinue
-
-# Delete entries in the downloads folder, except:
-#   those in the 'tools' folder
-#   those last accessed in the last 30 days
-Get-ChildItem -Path D:\downloads -Exclude "tools" `
-    | Where-Object{ $_.LastAccessTime -lt (get-Date).AddDays(-30) } `
-    | ForEach-Object{Remove-Item -Path $_ -Recurse -Force}
-
-# Msys sometimes leaves a database lock file laying around, especially if there was a failed job
-# which causes unrelated failures in jobs that run later on the machine.
-# work around this by just removing the vcpkg installed msys2 if it exists
-if( Test-Path D:\downloads\tools\msys2 )
-{
-    Write-Host "removing previously installed msys2"
-    Remove-Item D:\downloads\tools\msys2 -Recurse -Force
-}
--- a/scripts/azure-pipelines/windows/provision-image.txt
+++ b/scripts/azure-pipelines/windows/provision-image.txt
@ -20,17 +20,18 @@ The administrator user's password; if this is $null, or not passed, then the
 script assumes it's running on an administrator account.

 .PARAMETER StorageAccountName
-The name of the storage account. Stored in the environment variable %StorageAccountName%.
+The name of the storage account. Stored in the environment variable %PROVISIONED_AZURE_STORAGE_NAME%.
 Used by the CI system to access the global storage.

-.PARAMETER StorageAccountKey
-The key of the storage account. Stored in the environment variable %StorageAccountKey%.
+.PARAMETER StorageAccountSasToken
+The SAS token to access the storage account. Stored in the environment variable
+%PROVISIONED_AZURE_STORAGE_SAS_TOKEN%.
 Used by the CI system to access the global storage.
 #>
 param(
  [string]$AdminUserPassword = $null,
  [string]$StorageAccountName = $null,
-  [string]$StorageAccountKey = $null
+  [string]$StorageAccountSasToken = $null
 )

 $ErrorActionPreference = 'Stop'
@ -88,9 +89,9 @@ if ([string]::IsNullOrEmpty($AdminUserPassword)) {
    $PsExecArgs += $StorageAccountName
  }

-  if (-Not ([string]::IsNullOrWhiteSpace($StorageAccountKey))) {
-    $PsExecArgs += '-StorageAccountKey'
-    $PsExecArgs += $StorageAccountKey
+  if (-Not ([string]::IsNullOrWhiteSpace($StorageAccountSasToken))) {
+    $PsExecArgs += '-StorageAccountSasToken'
+    $PsExecArgs += $StorageAccountSasToken
  }

  Write-Host "Executing: $PsExecPath $PsExecArgs"
@ -114,15 +115,16 @@ $Workloads = @(
  'Microsoft.VisualStudio.Component.VC.ATLMFC',
  'Microsoft.VisualStudio.Component.VC.v141.x86.x64.Spectre',
  'Microsoft.VisualStudio.Component.Windows10SDK.18362',
+  'Microsoft.VisualStudio.Component.Windows10SDK.19041',
  'Microsoft.Net.Component.4.8.SDK',
  'Microsoft.Component.NetFX.Native',
  'Microsoft.VisualStudio.Component.VC.Llvm.ClangToolset',
-  'Microsoft.VisualStudio.Component.VC.Llvm.Clang'
+  'Microsoft.VisualStudio.Component.VC.Llvm.Clang',
+  'Microsoft.VisualStudio.Component.VC.v141.x86.x64',
+  'Microsoft.VisualStudio.Component.VC.140'
 )

-$WindowsSDKUrl = 'https://download.microsoft.com/download/1/c/3/1c3d5161-d9e9-4e4b-9b43-b70fe8be268c/windowssdk/winsdksetup.exe'
-
-$WindowsWDKUrl = 'https://download.microsoft.com/download/1/a/7/1a730121-7aa7-46f7-8978-7db729aa413d/wdk/wdksetup.exe'
+$WindowsWDKUrl = 'https://go.microsoft.com/fwlink/?linkid=2128854'

 $MpiUrl = 'https://download.microsoft.com/download/a/5/2/a5207ca5-1203-491a-8fb8-906fd68ae623/msmpisetup.exe'

@ -133,7 +135,7 @@ $CudaFeatures = 'nvcc_10.1 cuobjdump_10.1 nvprune_10.1 cupti_10.1 gpu_library_ad
  'cusparse_dev_10.1 nvgraph_10.1 nvgraph_dev_10.1 npp_10.1 npp_dev_10.1 nvrtc_10.1 nvrtc_dev_10.1 nvml_dev_10.1 ' + `
  'occupancy_calculator_10.1 fortran_examples_10.1'

-$PwshUrl = 'https://github.com/PowerShell/PowerShell/releases/download/v7.0.3/PowerShell-7.0.3-win-x64.msi'
+$PwshUrl = 'https://github.com/PowerShell/PowerShell/releases/download/v7.1.0/PowerShell-7.1.0-win-x64.msi'

 $ErrorActionPreference = 'Stop'
 $ProgressPreference = 'SilentlyContinue'
@ -294,43 +296,6 @@ Function InstallZip {
  }
 }

-<#
-.SYNOPSIS
-Installs Windows SDK version 2004
-
-.DESCRIPTION
-Downloads the Windows SDK installer located at $Url, and installs it with the
-correct flags.
-
-.PARAMETER Url
-The URL of the installer.
-#>
-Function InstallWindowsSDK {
-  Param(
-    [String]$Url
-  )
-
-  try {
-    Write-Host 'Downloading Windows SDK...'
-    [string]$installerPath = Get-TempFilePath -Extension 'exe'
-    curl.exe -L -o $installerPath -s -S $Url
-    Write-Host 'Installing Windows SDK...'
-    $proc = Start-Process -FilePath $installerPath -ArgumentList @('/features', '+', '/q') -Wait -PassThru
-    $exitCode = $proc.ExitCode
-    if ($exitCode -eq 0) {
-      Write-Host 'Installation successful!'
-    }
-    else {
-      Write-Error "Installation failed! Exited with $exitCode."
-      throw
-    }
-  }
-  catch {
-    Write-Error "Failed to install Windows SDK! $($_.Exception.Message)"
-    throw
-  }
-}
-
 <#
 .SYNOPSIS
 Installs Windows WDK version 2004
@ -467,7 +432,6 @@ if ($null -eq $av) {
 }

 InstallVisualStudio -Workloads $Workloads -BootstrapperUrl $VisualStudioBootstrapperUrl -Nickname 'Stable'
-InstallWindowsSDK -Url $WindowsSDKUrl
 InstallWindowsWDK -Url $WindowsWDKUrl
 InstallMpi -Url $MpiUrl
 InstallCuda -Url $CudaUrl -Features $CudaFeatures
@ -477,14 +441,14 @@ if ([string]::IsNullOrWhiteSpace($StorageAccountName)) {
 } else {
  Write-Host 'Storing storage account name to environment'
  Set-ItemProperty -Path 'HKLM:\SYSTEM\CurrentControlSet\Control\Session Manager\Environment' `
-    -Name StorageAccountName `
+    -Name PROVISIONED_AZURE_STORAGE_NAME `
    -Value $StorageAccountName
 }
-if ([string]::IsNullOrWhiteSpace($StorageAccountKey)) {
+if ([string]::IsNullOrWhiteSpace($StorageAccountSasToken)) {
  Write-Host 'No storage account key configured.'
 } else {
  Write-Host 'Storing storage account key to environment'
  Set-ItemProperty -Path 'HKLM:\SYSTEM\CurrentControlSet\Control\Session Manager\Environment' `
-    -Name StorageAccountKey `
-    -Value $StorageAccountKey
+    -Name PROVISIONED_AZURE_STORAGE_SAS_TOKEN `
+    -Value $StorageAccountSasToken
 }
--- a/scripts/ci.baseline.txt
+++ b/scripts/ci.baseline.txt
@ -1315,7 +1315,6 @@ rpclib:arm-uwp=fail
 rpclib:x64-uwp=fail
 rsocket:x64-windows=fail
 rsocket:x64-windows-static=fail
-rtaudio:x64-linux=fail
 rtlsdr:x64-uwp=fail
 rtlsdr:arm64-windows=fail
 rtlsdr:arm-uwp=fail
--- a/scripts/vcpkgTools.xml
+++ b/scripts/vcpkgTools.xml
@ -134,10 +134,10 @@
        <archiveName>ninja-freebsd-1.8.2.zip</archiveName>
    </tool>
    <tool name="powershell-core" os="windows">
-        <version>7.0.3</version>
+        <version>7.1.0</version>
        <exeRelativePath>pwsh.exe</exeRelativePath>
-        <url>https://github.com/PowerShell/PowerShell/releases/download/v7.0.3/PowerShell-7.0.3-win-x86.zip</url>
-        <sha512>2e410fc5e429fc6ad478976eff3efcffe9d53a0620c7c31c2b8bfe1f0976cade74632fc202bf58959cec111f03b824ba42ad4d74c8a66d67ec21bbfe9b9c626d</sha512>
-        <archiveName>PowerShell-7.0.3-win-x86.zip</archiveName>
+        <url>https://github.com/PowerShell/PowerShell/releases/download/v7.1.0/PowerShell-7.1.0-win-x86.zip</url>
+        <sha512>ea6ed619c784af65b09a5e98cdfc0d41333ba7373b7409e488ec70aaf1e0c4e5568e846d2b1748d72c3e3a9c1a4827f0ba29e9f6e93e79b4f49312ca40af6d2a</sha512>
+        <archiveName>PowerShell-7.1.0-win-x86.zip</archiveName>
    </tool>
 </tools>
--- a/toolsrc/src/vcpkg/base/downloads.cpp
+++ b/toolsrc/src/vcpkg/base/downloads.cpp
@ -303,6 +303,7 @@ namespace vcpkg::Downloads
        cmd.string_arg("-w").string_arg(Strings::concat("\\n", guid_marker, "%{http_code}"));
        cmd.string_arg(url);
        cmd.string_arg("-T").path_arg(file);
+        cmd.string_arg("-H").string_arg("x-ms-version: 2020-04-08");
        cmd.string_arg("-H").string_arg("x-ms-blob-type: BlockBlob");
        int code = 0;
        auto res = System::cmd_execute_and_stream_lines(cmd, [&code](const std::string& line) {