diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index b817b06..46e3cf6 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -1,9 +1,9 @@ - + https://github.com/dotnet/arcade - 0558f85d950fee2838bf02b9ba1f20d67f00b504 + 1a66526b0c1eb068cab89909b7d52fe6f57d64df diff --git a/eng/common/build.sh b/eng/common/build.sh index bc07a1c..55b298f 100755 --- a/eng/common/build.sh +++ b/eng/common/build.sh @@ -187,10 +187,6 @@ function InitializeCustomToolset { } function Build { - - if [[ "$ci" == true ]]; then - TryLogClientIpAddress - fi InitializeToolset InitializeCustomToolset diff --git a/eng/common/cross/arm/tizen-build-rootfs.sh b/eng/common/cross/arm/tizen-build-rootfs.sh new file mode 100644 index 0000000..9fdb32e --- /dev/null +++ b/eng/common/cross/arm/tizen-build-rootfs.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +set -e + +__ARM_HARDFP_CrossDir=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) +__TIZEN_CROSSDIR="$__ARM_HARDFP_CrossDir/tizen" + +if [[ -z "$ROOTFS_DIR" ]]; then + echo "ROOTFS_DIR is not defined." + exit 1; +fi + +TIZEN_TMP_DIR=$ROOTFS_DIR/tizen_tmp +mkdir -p $TIZEN_TMP_DIR + +# Download files +echo ">>Start downloading files" +VERBOSE=1 $__ARM_HARDFP_CrossDir/tizen-fetch.sh $TIZEN_TMP_DIR +echo "<>Start constructing Tizen rootfs" +TIZEN_RPM_FILES=`ls $TIZEN_TMP_DIR/*.rpm` +cd $ROOTFS_DIR +for f in $TIZEN_RPM_FILES; do + rpm2cpio $f | cpio -idm --quiet +done +echo "<>Start configuring Tizen rootfs" +ln -sfn asm-arm ./usr/include/asm +patch -p1 < $__TIZEN_CROSSDIR/tizen.patch +echo "</dev/null; then + VERBOSE=0 +fi + +Log() +{ + if [ $VERBOSE -ge $1 ]; then + echo ${@:2} + fi +} + +Inform() +{ + Log 1 -e "\x1B[0;34m$@\x1B[m" +} + +Debug() +{ + Log 2 -e "\x1B[0;32m$@\x1B[m" +} + +Error() +{ + >&2 Log 0 -e "\x1B[0;31m$@\x1B[m" +} + +Fetch() +{ + URL=$1 + FILE=$2 + PROGRESS=$3 + if [ $VERBOSE -ge 1 ] && [ $PROGRESS ]; then + CURL_OPT="--progress-bar" + else + CURL_OPT="--silent" + fi + curl $CURL_OPT $URL > $FILE +} + +hash curl 2> /dev/null || { Error "Require 'curl' Aborting."; exit 1; } +hash xmllint 2> /dev/null || { Error "Require 'xmllint' Aborting."; exit 1; } +hash sha256sum 2> /dev/null || { Error "Require 'sha256sum' Aborting."; exit 1; } + +TMPDIR=$1 +if [ ! -d $TMPDIR ]; then + TMPDIR=./tizen_tmp + Debug "Create temporary directory : $TMPDIR" + mkdir -p $TMPDIR +fi + +TIZEN_URL=http://download.tizen.org/snapshots/tizen +BUILD_XML=build.xml +REPOMD_XML=repomd.xml +PRIMARY_XML=primary.xml +TARGET_URL="http://__not_initialized" + +Xpath_get() +{ + XPATH_RESULT='' + XPATH=$1 + XML_FILE=$2 + RESULT=$(xmllint --xpath $XPATH $XML_FILE) + if [[ -z ${RESULT// } ]]; then + Error "Can not find target from $XML_FILE" + Debug "Xpath = $XPATH" + exit 1 + fi + XPATH_RESULT=$RESULT +} + +fetch_tizen_pkgs_init() +{ + TARGET=$1 + PROFILE=$2 + Debug "Initialize TARGET=$TARGET, PROFILE=$PROFILE" + + TMP_PKG_DIR=$TMPDIR/tizen_${PROFILE}_pkgs + if [ -d $TMP_PKG_DIR ]; then rm -rf $TMP_PKG_DIR; fi + mkdir -p $TMP_PKG_DIR + + PKG_URL=$TIZEN_URL/$PROFILE/latest + + BUILD_XML_URL=$PKG_URL/$BUILD_XML + TMP_BUILD=$TMP_PKG_DIR/$BUILD_XML + TMP_REPOMD=$TMP_PKG_DIR/$REPOMD_XML + TMP_PRIMARY=$TMP_PKG_DIR/$PRIMARY_XML + TMP_PRIMARYGZ=${TMP_PRIMARY}.gz + + Fetch $BUILD_XML_URL $TMP_BUILD + + Debug "fetch $BUILD_XML_URL to $TMP_BUILD" + + TARGET_XPATH="//build/buildtargets/buildtarget[@name=\"$TARGET\"]/repo[@type=\"binary\"]/text()" + Xpath_get $TARGET_XPATH $TMP_BUILD + TARGET_PATH=$XPATH_RESULT + TARGET_URL=$PKG_URL/$TARGET_PATH + + REPOMD_URL=$TARGET_URL/repodata/repomd.xml + PRIMARY_XPATH='string(//*[local-name()="data"][@type="primary"]/*[local-name()="location"]/@href)' + + Fetch $REPOMD_URL $TMP_REPOMD + + Debug "fetch $REPOMD_URL to $TMP_REPOMD" + + Xpath_get $PRIMARY_XPATH $TMP_REPOMD + PRIMARY_XML_PATH=$XPATH_RESULT + PRIMARY_URL=$TARGET_URL/$PRIMARY_XML_PATH + + Fetch $PRIMARY_URL $TMP_PRIMARYGZ + + Debug "fetch $PRIMARY_URL to $TMP_PRIMARYGZ" + + gunzip $TMP_PRIMARYGZ + + Debug "unzip $TMP_PRIMARYGZ to $TMP_PRIMARY" +} + +fetch_tizen_pkgs() +{ + ARCH=$1 + PACKAGE_XPATH_TPL='string(//*[local-name()="metadata"]/*[local-name()="package"][*[local-name()="name"][text()="_PKG_"]][*[local-name()="arch"][text()="_ARCH_"]]/*[local-name()="location"]/@href)' + + PACKAGE_CHECKSUM_XPATH_TPL='string(//*[local-name()="metadata"]/*[local-name()="package"][*[local-name()="name"][text()="_PKG_"]][*[local-name()="arch"][text()="_ARCH_"]]/*[local-name()="checksum"]/text())' + + for pkg in ${@:2} + do + Inform "Fetching... $pkg" + XPATH=${PACKAGE_XPATH_TPL/_PKG_/$pkg} + XPATH=${XPATH/_ARCH_/$ARCH} + Xpath_get $XPATH $TMP_PRIMARY + PKG_PATH=$XPATH_RESULT + + XPATH=${PACKAGE_CHECKSUM_XPATH_TPL/_PKG_/$pkg} + XPATH=${XPATH/_ARCH_/$ARCH} + Xpath_get $XPATH $TMP_PRIMARY + CHECKSUM=$XPATH_RESULT + + PKG_URL=$TARGET_URL/$PKG_PATH + PKG_FILE=$(basename $PKG_PATH) + PKG_PATH=$TMPDIR/$PKG_FILE + + Debug "Download $PKG_URL to $PKG_PATH" + Fetch $PKG_URL $PKG_PATH true + + echo "$CHECKSUM $PKG_PATH" | sha256sum -c - > /dev/null + if [ $? -ne 0 ]; then + Error "Fail to fetch $PKG_URL to $PKG_PATH" + Debug "Checksum = $CHECKSUM" + exit 1 + fi + done +} + +Inform "Initialize arm base" +fetch_tizen_pkgs_init standard base +Inform "fetch common packages" +fetch_tizen_pkgs armv7hl gcc gcc-devel-static glibc glibc-devel libicu libicu-devel libatomic linux-glibc-devel keyutils keyutils-devel libkeyutils +Inform "fetch coreclr packages" +fetch_tizen_pkgs armv7hl lldb lldb-devel libgcc libstdc++ libstdc++-devel libunwind libunwind-devel lttng-ust-devel lttng-ust userspace-rcu-devel userspace-rcu +Inform "fetch corefx packages" +fetch_tizen_pkgs armv7hl libcom_err libcom_err-devel zlib zlib-devel libopenssl11 libopenssl1.1-devel krb5 krb5-devel + +Inform "Initialize standard unified" +fetch_tizen_pkgs_init standard unified +Inform "fetch corefx packages" +fetch_tizen_pkgs armv7hl gssdp gssdp-devel tizen-release + diff --git a/eng/common/cross/arm/tizen/tizen.patch b/eng/common/cross/arm/tizen/tizen.patch new file mode 100644 index 0000000..fb12ade --- /dev/null +++ b/eng/common/cross/arm/tizen/tizen.patch @@ -0,0 +1,9 @@ +diff -u -r a/usr/lib/libc.so b/usr/lib/libc.so +--- a/usr/lib/libc.so 2016-12-30 23:00:08.284951863 +0900 ++++ b/usr/lib/libc.so 2016-12-30 23:00:32.140951815 +0900 +@@ -2,4 +2,4 @@ + Use the shared library, but some functions are only in + the static library, so try that secondarily. */ + OUTPUT_FORMAT(elf32-littlearm) +-GROUP ( /lib/libc.so.6 /usr/lib/libc_nonshared.a AS_NEEDED ( /lib/ld-linux-armhf.so.3 ) ) ++GROUP ( libc.so.6 libc_nonshared.a AS_NEEDED ( ld-linux-armhf.so.3 ) ) diff --git a/eng/common/cross/armv6/sources.list.buster b/eng/common/cross/armv6/sources.list.buster new file mode 100644 index 0000000..f27fc4f --- /dev/null +++ b/eng/common/cross/armv6/sources.list.buster @@ -0,0 +1,2 @@ +deb http://raspbian.raspberrypi.org/raspbian/ buster main contrib non-free rpi +deb-src http://raspbian.raspberrypi.org/raspbian/ buster main contrib non-free rpi diff --git a/eng/common/cross/build-rootfs.sh b/eng/common/cross/build-rootfs.sh index 6fa2c8a..e94d13d 100755 --- a/eng/common/cross/build-rootfs.sh +++ b/eng/common/cross/build-rootfs.sh @@ -99,6 +99,15 @@ while :; do __AlpineArch=armv7 __QEMUArch=arm ;; + armv6) + __BuildArch=armv6 + __UbuntuArch=armhf + __QEMUArch=arm + __UbuntuRepo="http://raspbian.raspberrypi.org/raspbian/" + __CodeName=buster + __LLDB_Package="liblldb-6.0-dev" + __Keyring="/usr/share/keyrings/raspbian-archive-keyring.gpg" + ;; arm64) __BuildArch=arm64 __UbuntuArch=arm64 @@ -176,8 +185,8 @@ while :; do __LLDB_Package="liblldb-6.0-dev" ;; tizen) - if [ "$__BuildArch" != "armel" ] && [ "$__BuildArch" != "arm64" ]; then - echo "Tizen is available only for armel and arm64." + if [ "$__BuildArch" != "arm" ] && [ "$__BuildArch" != "armel" ] && [ "$__BuildArch" != "arm64" ]; then + echo "Tizen is available only for arm, armel and arm64." usage; exit 1; fi @@ -236,6 +245,12 @@ while :; do shift done +if [ -e "$__Keyring" ]; then + __Keyring="--keyring=$__Keyring" +else + __Keyring="" +fi + if [ "$__BuildArch" == "armel" ]; then __LLDB_Package="lldb-3.5-dev" fi @@ -337,7 +352,7 @@ elif [[ "$__CodeName" == "illumos" ]]; then wget -P "$__RootfsDir"/usr/include/netpacket https://raw.githubusercontent.com/illumos/illumos-gate/master/usr/src/uts/common/inet/sockmods/netpacket/packet.h wget -P "$__RootfsDir"/usr/include/sys https://raw.githubusercontent.com/illumos/illumos-gate/master/usr/src/uts/common/sys/sdt.h elif [[ -n $__CodeName ]]; then - qemu-debootstrap --arch $__UbuntuArch $__CodeName $__RootfsDir $__UbuntuRepo + qemu-debootstrap $__Keyring --arch $__UbuntuArch $__CodeName $__RootfsDir $__UbuntuRepo cp $__CrossDir/$__BuildArch/sources.list.$__CodeName $__RootfsDir/etc/apt/sources.list chroot $__RootfsDir apt-get update chroot $__RootfsDir apt-get -f -y install diff --git a/eng/common/cross/toolchain.cmake b/eng/common/cross/toolchain.cmake index 6501c3a..8369ae0 100644 --- a/eng/common/cross/toolchain.cmake +++ b/eng/common/cross/toolchain.cmake @@ -26,6 +26,9 @@ elseif(TARGET_ARCH_NAME STREQUAL "arm") else() set(TOOLCHAIN "arm-linux-gnueabihf") endif() + if("$ENV{__DistroRid}" MATCHES "tizen.*") + set(TIZEN_TOOLCHAIN "armv7hl-tizen-linux-gnueabihf/9.2.0") + endif() elseif(TARGET_ARCH_NAME STREQUAL "arm64") set(CMAKE_SYSTEM_PROCESSOR aarch64) if(EXISTS ${CROSS_ROOTFS}/usr/lib/gcc/aarch64-alpine-linux-musl) @@ -58,6 +61,10 @@ endif() # Specify include paths if(DEFINED TIZEN_TOOLCHAIN) + if(TARGET_ARCH_NAME STREQUAL "arm") + include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}/include/c++/) + include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}/include/c++/armv7hl-tizen-linux-gnueabihf) + endif() if(TARGET_ARCH_NAME STREQUAL "armel") include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}/include/c++/) include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}/include/c++/armv7l-tizen-linux-gnueabi) @@ -150,7 +157,7 @@ if(CMAKE_SYSTEM_NAME STREQUAL "Linux") add_toolchain_linker_flag("-Wl,--rpath-link=${CROSS_ROOTFS}/usr/lib/${TOOLCHAIN}") endif() -if(TARGET_ARCH_NAME STREQUAL "armel") +if(TARGET_ARCH_NAME STREQUAL "arm" OR TARGET_ARCH_NAME STREQUAL "armel") if(DEFINED TIZEN_TOOLCHAIN) # For Tizen only add_toolchain_linker_flag("-B${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}") add_toolchain_linker_flag("-L${CROSS_ROOTFS}/lib") @@ -205,7 +212,7 @@ elseif(TARGET_ARCH_NAME STREQUAL "x86") endif() if(DEFINED TIZEN_TOOLCHAIN) - if(TARGET_ARCH_NAME MATCHES "^(armel|arm64)$") + if(TARGET_ARCH_NAME MATCHES "^(arm|armel|arm64)$") add_compile_options(-Wno-deprecated-declarations) # compile-time option add_compile_options(-D__extern_always_inline=inline) # compile-time option endif() diff --git a/eng/common/darc-init.sh b/eng/common/darc-init.sh index 39abdbe..84c1d0c 100755 --- a/eng/common/darc-init.sh +++ b/eng/common/darc-init.sh @@ -53,7 +53,7 @@ fi function InstallDarcCli { local darc_cli_package_name="microsoft.dotnet.darc" - InitializeDotNetCli + InitializeDotNetCli true local dotnet_root=$_InitializeDotNetCli if [ -z "$toolpath" ]; then diff --git a/eng/common/dotnet-install.sh b/eng/common/dotnet-install.sh index fdfeea6..5c94e98 100755 --- a/eng/common/dotnet-install.sh +++ b/eng/common/dotnet-install.sh @@ -55,6 +55,9 @@ case $cpuname in aarch64) buildarch=arm64 ;; + loongarch64) + buildarch=loongarch64 + ;; amd64|x86_64) buildarch=x64 ;; diff --git a/eng/common/msbuild.ps1 b/eng/common/msbuild.ps1 index eea19cd..f041e5d 100644 --- a/eng/common/msbuild.ps1 +++ b/eng/common/msbuild.ps1 @@ -6,6 +6,7 @@ Param( [switch] $ci, [switch] $prepareMachine, [switch] $excludePrereleaseVS, + [string] $msbuildEngine = $null, [Parameter(ValueFromRemainingArguments=$true)][String[]]$extraArgs ) diff --git a/eng/common/native/CommonLibrary.psm1 b/eng/common/native/CommonLibrary.psm1 index adf707c..ca38268 100644 --- a/eng/common/native/CommonLibrary.psm1 +++ b/eng/common/native/CommonLibrary.psm1 @@ -276,7 +276,8 @@ function Get-MachineArchitecture { } if (($ProcessorArchitecture -Eq "AMD64") -Or ($ProcessorArchitecture -Eq "IA64") -Or - ($ProcessorArchitecture -Eq "ARM64")) { + ($ProcessorArchitecture -Eq "ARM64") -Or + ($ProcessorArchitecture -Eq "LOONGARCH64")) { return "x64" } return "x86" diff --git a/eng/common/native/init-compiler.sh b/eng/common/native/init-compiler.sh index 8c944f3..e361e03 100644 --- a/eng/common/native/init-compiler.sh +++ b/eng/common/native/init-compiler.sh @@ -2,6 +2,7 @@ # # This file detects the C/C++ compiler and exports it to the CC/CXX environment variables # +# NOTE: some scripts source this file and rely on stdout being empty, make sure to not output anything here! if [[ "$#" -lt 3 ]]; then echo "Usage..." @@ -111,12 +112,10 @@ if [[ -z "$CC" ]]; then exit 1 fi -if [[ "$compiler" == "clang" ]]; then - if command -v "lld$desired_version" > /dev/null; then - # Only lld version >= 9 can be considered stable - if [[ "$majorVersion" -ge 9 ]]; then - LDFLAGS="-fuse-ld=lld" - fi +# Only lld version >= 9 can be considered stable +if [[ "$compiler" == "clang" && "$majorVersion" -ge 9 ]]; then + if "$CC" -fuse-ld=lld -Wl,--version >/dev/null 2>&1; then + LDFLAGS="-fuse-ld=lld" fi fi diff --git a/eng/common/post-build/symbols-validation.ps1 b/eng/common/post-build/symbols-validation.ps1 index a4a92ef..cd2181b 100644 --- a/eng/common/post-build/symbols-validation.ps1 +++ b/eng/common/post-build/symbols-validation.ps1 @@ -134,17 +134,17 @@ $CountMissingSymbols = { # Save the output and get diagnostic output $output = & $dotnetSymbolExe --symbols --modules $WindowsPdbVerificationParam $TargetServerParam $FullPath -o $SymbolsPath --diagnostics | Out-String - if (Test-Path $PdbPath) { - return 'PDB' + if ((Test-Path $PdbPath) -and (Test-path $SymbolPath)) { + return 'Module and PDB for Module' } - elseif (Test-Path $NGenPdb) { - return 'NGen PDB' + elseif ((Test-Path $NGenPdb) -and (Test-Path $PdbPath) -and (Test-Path $SymbolPath)) { + return 'Dll, PDB and NGen PDB' } - elseif (Test-Path $SODbg) { - return 'DBG for SO' + elseif ((Test-Path $SODbg) -and (Test-Path $SymbolPath)) { + return 'So and DBG for SO' } - elseif (Test-Path $DylibDwarf) { - return 'Dwarf for Dylib' + elseif ((Test-Path $DylibDwarf) -and (Test-Path $SymbolPath)) { + return 'Dylib and Dwarf for Dylib' } elseif (Test-Path $SymbolPath) { return 'Module' diff --git a/eng/common/sdk-task.ps1 b/eng/common/sdk-task.ps1 index 7ab9baa..b1bca63 100644 --- a/eng/common/sdk-task.ps1 +++ b/eng/common/sdk-task.ps1 @@ -83,9 +83,6 @@ try { } if ($restore) { - if ($ci) { - Try-LogClientIpAddress - } Build 'Restore' } diff --git a/eng/common/sdl/execute-all-sdl-tools.ps1 b/eng/common/sdl/execute-all-sdl-tools.ps1 index 1157151..e5bef8e 100644 --- a/eng/common/sdl/execute-all-sdl-tools.ps1 +++ b/eng/common/sdl/execute-all-sdl-tools.ps1 @@ -124,7 +124,7 @@ try { Exec-BlockVerbosely { & $(Join-Path $PSScriptRoot 'run-sdl.ps1') ` -GuardianCliLocation $guardianCliLocation ` - -WorkingDirectory $workingDirectory ` + -WorkingDirectory $SourceDirectory ` -UpdateBaseline $UpdateBaseline ` -GdnFolder $gdnFolder } diff --git a/eng/common/sdl/packages.config b/eng/common/sdl/packages.config index 3bd8b29..4585cfd 100644 --- a/eng/common/sdl/packages.config +++ b/eng/common/sdl/packages.config @@ -1,4 +1,4 @@ - + diff --git a/eng/common/templates/job/execute-sdl.yml b/eng/common/templates/job/execute-sdl.yml index 3aafc82..8cf772b 100644 --- a/eng/common/templates/job/execute-sdl.yml +++ b/eng/common/templates/job/execute-sdl.yml @@ -51,14 +51,9 @@ jobs: value: ${{ parameters.AzDOPipelineId }} - name: AzDOBuildId value: ${{ parameters.AzDOBuildId }} - # The Guardian version specified in 'eng/common/sdl/packages.config'. This value must be kept in - # sync with the packages.config file. - - name: DefaultGuardianVersion - value: 0.53.3 + - template: /eng/common/templates/variables/sdl-variables.yml - name: GuardianVersion value: ${{ coalesce(parameters.overrideGuardianVersion, '$(DefaultGuardianVersion)') }} - - name: GuardianPackagesConfigFile - value: $(Build.SourcesDirectory)\eng\common\sdl\packages.config pool: vmImage: windows-2019 steps: @@ -125,57 +120,11 @@ jobs: displayName: Extract Archive Artifacts continueOnError: ${{ parameters.sdlContinueOnError }} - - ${{ if ne(parameters.overrideGuardianVersion, '') }}: - - powershell: | - $content = Get-Content $(GuardianPackagesConfigFile) - - Write-Host "packages.config content was:`n$content" - - $content = $content.Replace('$(DefaultGuardianVersion)', '$(GuardianVersion)') - $content | Set-Content $(GuardianPackagesConfigFile) - - Write-Host "packages.config content updated to:`n$content" - displayName: Use overridden Guardian version ${{ parameters.overrideGuardianVersion }} - - - task: NuGetToolInstaller@1 - displayName: 'Install NuGet.exe' - - task: NuGetCommand@2 - displayName: 'Install Guardian' - inputs: - restoreSolution: $(Build.SourcesDirectory)\eng\common\sdl\packages.config - feedsToUse: config - nugetConfigPath: $(Build.SourcesDirectory)\eng\common\sdl\NuGet.config - externalFeedCredentials: GuardianConnect - restoreDirectory: $(Build.SourcesDirectory)\.packages - - - ${{ if ne(parameters.overrideParameters, '') }}: - - powershell: ${{ parameters.executeAllSdlToolsScript }} ${{ parameters.overrideParameters }} - displayName: Execute SDL - continueOnError: ${{ parameters.sdlContinueOnError }} - - ${{ if eq(parameters.overrideParameters, '') }}: - - powershell: ${{ parameters.executeAllSdlToolsScript }} - -GuardianPackageName Microsoft.Guardian.Cli.$(GuardianVersion) - -NugetPackageDirectory $(Build.SourcesDirectory)\.packages - -AzureDevOpsAccessToken $(dn-bot-dotnet-build-rw-code-rw) - ${{ parameters.additionalParameters }} - displayName: Execute SDL - continueOnError: ${{ parameters.sdlContinueOnError }} - - - ${{ if ne(parameters.publishGuardianDirectoryToPipeline, 'false') }}: - # We want to publish the Guardian results and configuration for easy diagnosis. However, the - # '.gdn' dir is a mix of configuration, results, extracted dependencies, and Guardian default - # tooling files. Some of these files are large and aren't useful during an investigation, so - # exclude them by simply deleting them before publishing. (As of writing, there is no documented - # way to selectively exclude a dir from the pipeline artifact publish task.) - - task: DeleteFiles@1 - displayName: Delete Guardian dependencies to avoid uploading - inputs: - SourceFolder: $(Agent.BuildDirectory)/.gdn - Contents: | - c - i - condition: succeededOrFailed() - - publish: $(Agent.BuildDirectory)/.gdn - artifact: GuardianConfiguration - displayName: Publish GuardianConfiguration - condition: succeededOrFailed() + - template: /eng/common/templates/steps/execute-sdl.yml + parameters: + overrideGuardianVersion: ${{ parameters.overrideGuardianVersion }} + executeAllSdlToolsScript: ${{ parameters.executeAllSdlToolsScript }} + overrideParameters: ${{ parameters.overrideParameters }} + additionalParameters: ${{ parameters.additionalParameters }} + publishGuardianDirectoryToPipeline: ${{ parameters.publishGuardianDirectoryToPipeline }} + sdlContinueOnError: ${{ parameters.sdlContinueOnError }} diff --git a/eng/common/templates/job/job.yml b/eng/common/templates/job/job.yml index 37dceb1..7678b94 100644 --- a/eng/common/templates/job/job.yml +++ b/eng/common/templates/job/job.yml @@ -114,6 +114,7 @@ jobs: continueOnError: ${{ parameters.continueOnError }} condition: and(succeeded(), in(variables['_SignType'], 'real', 'test'), eq(variables['Agent.Os'], 'Windows_NT')) + - ${{ if and(eq(parameters.runAsPublic, 'false'), eq(variables['System.TeamProject'], 'internal')) }}: - task: NuGetAuthenticate@0 - ${{ if or(eq(parameters.artifacts.download, 'true'), ne(parameters.artifacts.download, '')) }}: diff --git a/eng/common/templates/job/source-build.yml b/eng/common/templates/job/source-build.yml index 5023d36..5cd5325 100644 --- a/eng/common/templates/job/source-build.yml +++ b/eng/common/templates/job/source-build.yml @@ -31,11 +31,6 @@ parameters: # container and pool. platform: {} - # The default VM host AzDO pool. This should be capable of running Docker containers: almost all - # source-build builds run in Docker, including the default managed platform. - defaultContainerHostPool: - vmImage: ubuntu-20.04 - jobs: - job: ${{ parameters.jobNamePrefix }}_${{ parameters.platform.name }} displayName: Source-Build (${{ parameters.platform.name }}) @@ -47,7 +42,15 @@ jobs: container: ${{ parameters.platform.container }} ${{ if eq(parameters.platform.pool, '') }}: - pool: ${{ parameters.defaultContainerHostPool }} + # The default VM host AzDO pool. This should be capable of running Docker containers: almost all + # source-build builds run in Docker, including the default managed platform. + pool: + ${{ if eq(variables['System.TeamProject'], 'public') }}: + name: NetCore1ESPool-Public + demands: ImageOverride -equals Build.Ubuntu.1804.Amd64.Open + ${{ if eq(variables['System.TeamProject'], 'internal') }}: + name: NetCore1ESPool-Internal + demands: ImageOverride -equals Build.Ubuntu.1804.Amd64 ${{ if ne(parameters.platform.pool, '') }}: pool: ${{ parameters.platform.pool }} diff --git a/eng/common/templates/job/source-index-stage1.yml b/eng/common/templates/job/source-index-stage1.yml index ae85a99..4af724e 100644 --- a/eng/common/templates/job/source-index-stage1.yml +++ b/eng/common/templates/job/source-index-stage1.yml @@ -5,8 +5,6 @@ parameters: sourceIndexBuildCommand: powershell -NoLogo -NoProfile -ExecutionPolicy Bypass -Command "eng/common/build.ps1 -restore -build -binarylog -ci" preSteps: [] binlogPath: artifacts/log/Debug/Build.binlog - pool: - vmImage: 'windows-2019' condition: '' dependsOn: '' @@ -24,7 +22,13 @@ jobs: - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - group: source-dot-net stage1 variables - pool: ${{ parameters.pool }} + pool: + ${{ if eq(variables['System.TeamProject'], 'public') }}: + name: NetCore1ESPool-Public + demands: ImageOverride -equals Build.Server.Amd64.VS2019.Open + ${{ if eq(variables['System.TeamProject'], 'internal') }}: + name: NetCore1ESPool-Internal + demands: ImageOverride -equals Build.Server.Amd64.VS2019 steps: - ${{ each preStep in parameters.preSteps }}: - ${{ preStep }} diff --git a/eng/common/templates/steps/execute-sdl.yml b/eng/common/templates/steps/execute-sdl.yml new file mode 100644 index 0000000..7b8ee18 --- /dev/null +++ b/eng/common/templates/steps/execute-sdl.yml @@ -0,0 +1,68 @@ +parameters: + overrideGuardianVersion: '' + executeAllSdlToolsScript: '' + overrideParameters: '' + additionalParameters: '' + publishGuardianDirectoryToPipeline: false + sdlContinueOnError: false + condition: '' + +steps: +- ${{ if ne(parameters.overrideGuardianVersion, '') }}: + - powershell: | + $content = Get-Content $(GuardianPackagesConfigFile) + + Write-Host "packages.config content was:`n$content" + + $content = $content.Replace('$(DefaultGuardianVersion)', '$(GuardianVersion)') + $content | Set-Content $(GuardianPackagesConfigFile) + + Write-Host "packages.config content updated to:`n$content" + displayName: Use overridden Guardian version ${{ parameters.overrideGuardianVersion }} + +- task: NuGetToolInstaller@1 + displayName: 'Install NuGet.exe' + +- task: NuGetCommand@2 + displayName: 'Install Guardian' + inputs: + restoreSolution: $(Build.SourcesDirectory)\eng\common\sdl\packages.config + feedsToUse: config + nugetConfigPath: $(Build.SourcesDirectory)\eng\common\sdl\NuGet.config + externalFeedCredentials: GuardianConnect + restoreDirectory: $(Build.SourcesDirectory)\.packages + +- ${{ if ne(parameters.overrideParameters, '') }}: + - powershell: ${{ parameters.executeAllSdlToolsScript }} ${{ parameters.overrideParameters }} + displayName: Execute SDL + continueOnError: ${{ parameters.sdlContinueOnError }} + condition: ${{ parameters.condition }} + +- ${{ if eq(parameters.overrideParameters, '') }}: + - powershell: ${{ parameters.executeAllSdlToolsScript }} + -GuardianPackageName Microsoft.Guardian.Cli.$(GuardianVersion) + -NugetPackageDirectory $(Build.SourcesDirectory)\.packages + -AzureDevOpsAccessToken $(dn-bot-dotnet-build-rw-code-rw) + ${{ parameters.additionalParameters }} + displayName: Execute SDL + continueOnError: ${{ parameters.sdlContinueOnError }} + condition: ${{ parameters.condition }} + +- ${{ if ne(parameters.publishGuardianDirectoryToPipeline, 'false') }}: + # We want to publish the Guardian results and configuration for easy diagnosis. However, the + # '.gdn' dir is a mix of configuration, results, extracted dependencies, and Guardian default + # tooling files. Some of these files are large and aren't useful during an investigation, so + # exclude them by simply deleting them before publishing. (As of writing, there is no documented + # way to selectively exclude a dir from the pipeline artifact publish task.) + - task: DeleteFiles@1 + displayName: Delete Guardian dependencies to avoid uploading + inputs: + SourceFolder: $(Agent.BuildDirectory)/.gdn + Contents: | + c + i + condition: succeededOrFailed() + - publish: $(Agent.BuildDirectory)/.gdn + artifact: GuardianConfiguration + displayName: Publish GuardianConfiguration + condition: succeededOrFailed() \ No newline at end of file diff --git a/eng/common/templates/variables/sdl-variables.yml b/eng/common/templates/variables/sdl-variables.yml new file mode 100644 index 0000000..dbdd66d --- /dev/null +++ b/eng/common/templates/variables/sdl-variables.yml @@ -0,0 +1,7 @@ +variables: +# The Guardian version specified in 'eng/common/sdl/packages.config'. This value must be kept in +# sync with the packages.config file. +- name: DefaultGuardianVersion + value: 0.109.0 +- name: GuardianPackagesConfigFile + value: $(Build.SourcesDirectory)\eng\common\sdl\packages.config \ No newline at end of file diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index 4448428..f1e1cb5 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -163,9 +163,6 @@ function InitializeDotNetCli([bool]$install, [bool]$createSdkLocationFile) { # Disable telemetry on CI. if ($ci) { $env:DOTNET_CLI_TELEMETRY_OPTOUT=1 - - # In case of network error, try to log the current IP for reference - Try-LogClientIpAddress } # Source Build uses DotNetCoreSdkDir variable @@ -301,31 +298,44 @@ function InstallDotNet([string] $dotnetRoot, if ($skipNonVersionedFiles) { $installParameters.SkipNonVersionedFiles = $skipNonVersionedFiles } if ($noPath) { $installParameters.NoPath = $True } - try { - & $installScript @installParameters - } - catch { - if ($runtimeSourceFeed -or $runtimeSourceFeedKey) { - Write-Host "Failed to install dotnet from public location. Trying from '$runtimeSourceFeed'" - if ($runtimeSourceFeed) { $installParameters.AzureFeed = $runtimeSourceFeed } + $variations = @() + $variations += @($installParameters) - if ($runtimeSourceFeedKey) { - $decodedBytes = [System.Convert]::FromBase64String($runtimeSourceFeedKey) - $decodedString = [System.Text.Encoding]::UTF8.GetString($decodedBytes) - $installParameters.FeedCredential = $decodedString - } + $dotnetBuilds = $installParameters.Clone() + $dotnetbuilds.AzureFeed = "https://dotnetbuilds.azureedge.net/public" + $variations += @($dotnetBuilds) - try { - & $installScript @installParameters - } - catch { - Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "Failed to install dotnet from custom location '$runtimeSourceFeed'." - ExitWithExitCode 1 - } - } else { - Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "Failed to install dotnet from public location." - ExitWithExitCode 1 + if ($runtimeSourceFeed) { + $runtimeSource = $installParameters.Clone() + $runtimeSource.AzureFeed = $runtimeSourceFeed + if ($runtimeSourceFeedKey) { + $decodedBytes = [System.Convert]::FromBase64String($runtimeSourceFeedKey) + $decodedString = [System.Text.Encoding]::UTF8.GetString($decodedBytes) + $runtimeSource.FeedCredential = $decodedString } + $variations += @($runtimeSource) + } + + $installSuccess = $false + foreach ($variation in $variations) { + if ($variation | Get-Member AzureFeed) { + $location = $variation.AzureFeed + } else { + $location = "public location"; + } + Write-Host "Attempting to install dotnet from $location." + try { + & $installScript @variation + $installSuccess = $true + break + } + catch { + Write-Host "Failed to install dotnet from $location." + } + } + if (-not $installSuccess) { + Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "Failed to install dotnet from any of the specified locations." + ExitWithExitCode 1 } } @@ -882,24 +892,6 @@ if (!$disableConfigureToolsetImport) { } } -function Try-LogClientIpAddress() -{ - Write-Host "Attempting to log this client's IP for Azure Package feed telemetry purposes" - try - { - $result = Invoke-WebRequest -Uri "http://co1.msedge.net/fdv2/diagnostics.aspx" -UseBasicParsing - $lines = $result.Content.Split([Environment]::NewLine) - $socketIp = $lines | Select-String -Pattern "^Socket IP:.*" - Write-Host $socketIp - $clientIp = $lines | Select-String -Pattern "^Client IP:.*" - Write-Host $clientIp - } - catch - { - Write-Host "Unable to get this machine's effective IP address for logging: $_" - } -} - # # If $ci flag is set, turn on (and log that we did) special environment variables for improved Nuget client retry logic. # diff --git a/eng/common/tools.sh b/eng/common/tools.sh index 6a4871e..17f0a36 100755 --- a/eng/common/tools.sh +++ b/eng/common/tools.sh @@ -178,7 +178,7 @@ function InstallDotNetSdk { if [[ $# -ge 3 ]]; then architecture=$3 fi - InstallDotNet "$root" "$version" $architecture 'sdk' 'false' $runtime_source_feed $runtime_source_feed_key + InstallDotNet "$root" "$version" $architecture 'sdk' 'true' $runtime_source_feed $runtime_source_feed_key } function InstallDotNet { @@ -188,28 +188,29 @@ function InstallDotNet { GetDotNetInstallScript "$root" local install_script=$_GetDotNetInstallScript - local archArg='' + local installParameters=(--version $version --install-dir "$root") + if [[ -n "${3:-}" ]] && [ "$3" != 'unset' ]; then - archArg="--architecture $3" + installParameters+=(--architecture $3) fi - local runtimeArg='' if [[ -n "${4:-}" ]] && [ "$4" != 'sdk' ]; then - runtimeArg="--runtime $4" + installParameters+=(--runtime $4) fi - local skipNonVersionedFilesArg="" if [[ "$#" -ge "5" ]] && [[ "$5" != 'false' ]]; then - skipNonVersionedFilesArg="--skip-non-versioned-files" + installParameters+=(--skip-non-versioned-files) fi - bash "$install_script" --version $version --install-dir "$root" $archArg $runtimeArg $skipNonVersionedFilesArg || { - local exit_code=$? - echo "Failed to install dotnet SDK from public location (exit code '$exit_code')." - local runtimeSourceFeed='' - if [[ -n "${6:-}" ]]; then - runtimeSourceFeed="--azure-feed $6" - fi + local variations=() # list of variable names with parameter arrays in them - local runtimeSourceFeedKey='' + local public_location=("${installParameters[@]}") + variations+=(public_location) + + local dotnetbuilds=("${installParameters[@]}" --azure-feed "https://dotnetbuilds.azureedge.net/public") + variations+=(dotnetbuilds) + + if [[ -n "${6:-}" ]]; then + variations+=(private_feed) + local private_feed=("${installParameters[@]}" --azure-feed $6) if [[ -n "${7:-}" ]]; then # The 'base64' binary on alpine uses '-d' and doesn't support '--decode' # '-d'. To work around this, do a simple detection and switch the parameter @@ -219,22 +220,27 @@ function InstallDotNet { decodeArg="-d" fi decodedFeedKey=`echo $7 | base64 $decodeArg` - runtimeSourceFeedKey="--feed-credential $decodedFeedKey" + private_feed+=(--feed-credential $decodedFeedKey) + fi + fi + + local installSuccess=0 + for variationName in "${variations[@]}"; do + local name="$variationName[@]" + local variation=("${!name}") + echo "Attempting to install dotnet from $variationName." + bash "$install_script" "${variation[@]}" && installSuccess=1 + if [[ "$installSuccess" -eq 1 ]]; then + break fi - if [[ -n "$runtimeSourceFeed" || -n "$runtimeSourceFeedKey" ]]; then - bash "$install_script" --version $version --install-dir "$root" $archArg $runtimeArg $skipNonVersionedFilesArg $runtimeSourceFeed $runtimeSourceFeedKey || { - local exit_code=$? - Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to install dotnet SDK from custom location '$runtimeSourceFeed' (exit code '$exit_code')." - ExitWithExitCode $exit_code - } - else - if [[ $exit_code != 0 ]]; then - Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to install dotnet SDK from public location (exit code '$exit_code')." - fi - ExitWithExitCode $exit_code - fi - } + echo "Failed to install dotnet from $variationName." + done + + if [[ "$installSuccess" -eq 0 ]]; then + Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to install dotnet SDK from any of the specified locations." + ExitWithExitCode 1 + fi } function with_retries { @@ -399,13 +405,6 @@ function StopProcesses { return 0 } -function TryLogClientIpAddress () { - echo 'Attempting to log this client''s IP for Azure Package feed telemetry purposes' - if command -v curl > /dev/null; then - curl -s 'http://co1.msedge.net/fdv2/diagnostics.aspx' | grep ' IP: ' || true - fi -} - function MSBuild { local args=$@ if [[ "$pipelines_log" == true ]]; then diff --git a/global.json b/global.json index b7fccfa..1d7fe5b 100644 --- a/global.json +++ b/global.json @@ -1,8 +1,8 @@ { "tools": { - "dotnet": "6.0.100-rc.1.21430.12" + "dotnet": "6.0.100" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "7.0.0-beta.21529.1" + "Microsoft.DotNet.Arcade.Sdk": "7.0.0-beta.21623.2" } }