From 7966421cd1474528e070b5e6b46b139a26763bd4 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Tue, 27 Jun 2023 18:29:41 +0200 Subject: [PATCH] [release/dev17.7] Update dependencies from dotnet/arcade (#15459) --- eng/SourceBuildPrebuiltBaseline.xml | 6 +- eng/Version.Details.xml | 18 ++--- eng/common/cross/build-rootfs.sh | 86 ++++++++++----------- eng/common/cross/toolchain.cmake | 13 ++-- eng/common/cross/x64/sources.list.bionic | 11 +++ eng/common/dotnet-install.sh | 4 + eng/common/sdk-task.ps1 | 2 +- eng/common/templates/job/job.yml | 12 ++- eng/common/templates/steps/source-build.yml | 6 ++ eng/common/tools.ps1 | 6 +- global.json | 10 +-- tests/AheadOfTime/Trimming/check.ps1 | 2 +- tests/scripts/scriptlib.fsx | 2 +- 13 files changed, 101 insertions(+), 77 deletions(-) create mode 100644 eng/common/cross/x64/sources.list.bionic diff --git a/eng/SourceBuildPrebuiltBaseline.xml b/eng/SourceBuildPrebuiltBaseline.xml index 426cdeb0d5..0ef8d79d1d 100644 --- a/eng/SourceBuildPrebuiltBaseline.xml +++ b/eng/SourceBuildPrebuiltBaseline.xml @@ -11,9 +11,9 @@ These will go away when repo updates targeting to net8.0 Tracked with https://github.com/dotnet/fsharp/issues/14765 --> - - - + + + diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index ae127043f1..14c20d4415 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -1,6 +1,11 @@ + + https://github.com/dotnet/xliff-tasks + 4234ffac87e305da80597cb46dc0d4a87fc4f7c2 + + https://github.com/dotnet/source-build-reference-packages e910f7bdd090fb8f4a9122a1054a8b43a006b0a7 @@ -29,20 +34,15 @@ - + https://github.com/dotnet/arcade - 5dff25b814b90abb85c847ed274cf4afb2c1b276 + 06d73ccc1f421af06c2794741a9913f474f66b3d - + https://github.com/dotnet/sourcelink - 47c52dd2ebf9edfd40abdcff999c13eb461f6ce2 + 4d2c8bf58e8cb7900ec2d9077c155572e2d3cd88 - - https://github.com/dotnet/xliff-tasks - 513000e77df5ec1f9b86397d8df6e3254703c636 - - diff --git a/eng/common/cross/build-rootfs.sh b/eng/common/cross/build-rootfs.sh index 6a59f753b5..9caf9b021d 100755 --- a/eng/common/cross/build-rootfs.sh +++ b/eng/common/cross/build-rootfs.sh @@ -27,6 +27,7 @@ __AlpineArch=armv7 __FreeBSDArch=arm __FreeBSDMachineArch=armv7 __IllumosArch=arm7 +__HaikuArch=arm __QEMUArch=arm __UbuntuArch=armhf __UbuntuRepo="http://ports.ubuntu.com/" @@ -85,8 +86,12 @@ __IllumosPackages+=" mit-krb5" __IllumosPackages+=" openssl" __IllumosPackages+=" zlib" -__HaikuPackages="gmp" +__HaikuPackages="gcc_syslibs" +__HaikuPackages+=" gcc_syslibs_devel" +__HaikuPackages+=" gmp" __HaikuPackages+=" gmp_devel" +__HaikuPackages+=" icu66" +__HaikuPackages+=" icu66_devel" __HaikuPackages+=" krb5" __HaikuPackages+=" krb5_devel" __HaikuPackages+=" libiconv" @@ -95,6 +100,10 @@ __HaikuPackages+=" llvm12_libunwind" __HaikuPackages+=" llvm12_libunwind_devel" __HaikuPackages+=" mpfr" __HaikuPackages+=" mpfr_devel" +__HaikuPackages+=" openssl" +__HaikuPackages+=" openssl_devel" +__HaikuPackages+=" zlib" +__HaikuPackages+=" zlib_devel" # ML.NET dependencies __UbuntuPackages+=" libomp5" @@ -210,6 +219,7 @@ while :; do __FreeBSDArch=amd64 __FreeBSDMachineArch=amd64 __illumosArch=x86_64 + __HaikuArch=x86_64 __UbuntuRepo="http://archive.ubuntu.com/ubuntu/" ;; x86) @@ -340,7 +350,6 @@ while :; do ;; haiku) __CodeName=haiku - __BuildArch=x64 __SkipUnmount=1 ;; --skipunmount) @@ -559,67 +568,54 @@ elif [[ "$__CodeName" == "illumos" ]]; then elif [[ "$__CodeName" == "haiku" ]]; then JOBS=${MAXJOBS:="$(getconf _NPROCESSORS_ONLN)"} - echo "Building Haiku sysroot for x86_64" + echo "Building Haiku sysroot for $__HaikuArch" mkdir -p "$__RootfsDir/tmp" - cd "$__RootfsDir/tmp" - git clone -b hrev56235 https://review.haiku-os.org/haiku - git clone -b btrev43195 https://review.haiku-os.org/buildtools - cd "$__RootfsDir/tmp/buildtools" && git checkout 7487388f5110021d400b9f3b88e1a7f310dc066d + pushd "$__RootfsDir/tmp" - # Fetch some unmerged patches - cd "$__RootfsDir/tmp/haiku" - ## Add development build profile (slimmer than nightly) - git fetch origin refs/changes/64/4164/1 && git -c commit.gpgsign=false cherry-pick FETCH_HEAD + mkdir "$__RootfsDir/tmp/download" - # Build jam - cd "$__RootfsDir/tmp/buildtools/jam" - make + echo "Downloading Haiku package tool" + git clone https://github.com/haiku/haiku-toolchains-ubuntu --depth 1 $__RootfsDir/tmp/script + wget -O "$__RootfsDir/tmp/download/hosttools.zip" $($__RootfsDir/tmp/script/fetch.sh --hosttools) + unzip -o "$__RootfsDir/tmp/download/hosttools.zip" -d "$__RootfsDir/tmp/bin" - # Configure cross tools - echo "Building cross-compiler" - mkdir -p "$__RootfsDir/generated" - cd "$__RootfsDir/generated" - "$__RootfsDir/tmp/haiku/configure" -j"$JOBS" --sysroot "$__RootfsDir" --cross-tools-source "$__RootfsDir/tmp/buildtools" --build-cross-tools x86_64 + DepotBaseUrl="https://depot.haiku-os.org/__api/v2/pkg/get-pkg" + HpkgBaseUrl="https://eu.hpkg.haiku-os.org/haiku/master/$__HaikuArch/current" - # Build Haiku packages - echo "Building Haiku" - echo 'HAIKU_BUILD_PROFILE = "development-raw" ;' > UserProfileConfig - "$__RootfsDir/tmp/buildtools/jam/jam0" -j"$JOBS" -q 'package' 'Haiku' - - BaseUrl="https://depot.haiku-os.org/__api/v2/pkg/get-pkg" - - # Download additional packages - echo "Downloading additional required packages" + # Download Haiku packages + echo "Downloading Haiku packages" read -ra array <<<"$__HaikuPackages" for package in "${array[@]}"; do echo "Downloading $package..." # API documented here: https://github.com/haiku/haikudepotserver/blob/master/haikudepotserver-api2/src/main/resources/api2/pkg.yaml#L60 # The schema here: https://github.com/haiku/haikudepotserver/blob/master/haikudepotserver-api2/src/main/resources/api2/pkg.yaml#L598 - hpkgDownloadUrl="$(wget -qO- --post-data='{"name":"'"$package"'","repositorySourceCode":"haikuports_x86_64","versionType":"LATEST","naturalLanguageCode":"en"}' \ - --header='Content-Type:application/json' "$BaseUrl" | jq -r '.result.versions[].hpkgDownloadURL')" - wget -P "$__RootfsDir/generated/download" "$hpkgDownloadUrl" + hpkgDownloadUrl="$(wget -qO- --post-data='{"name":"'"$package"'","repositorySourceCode":"haikuports_'$__HaikuArch'","versionType":"LATEST","naturalLanguageCode":"en"}' \ + --header='Content-Type:application/json' "$DepotBaseUrl" | jq -r '.result.versions[].hpkgDownloadURL')" + wget -P "$__RootfsDir/tmp/download" "$hpkgDownloadUrl" + done + for package in haiku haiku_devel; do + echo "Downloading $package..." + hpkgVersion="$(wget -qO- $HpkgBaseUrl | sed -n 's/^.*version: "\([^"]*\)".*$/\1/p')" + wget -P "$__RootfsDir/tmp/download" "$HpkgBaseUrl/packages/$package-$hpkgVersion-1-$__HaikuArch.hpkg" done - # Setup the sysroot - echo "Setting up sysroot and extracting needed packages" + # Set up the sysroot + echo "Setting up sysroot and extracting required packages" mkdir -p "$__RootfsDir/boot/system" - for file in "$__RootfsDir/generated/objects/haiku/x86_64/packaging/packages/"*.hpkg; do - "$__RootfsDir/generated/objects/linux/x86_64/release/tools/package/package" extract -C "$__RootfsDir/boot/system" "$file" - done - for file in "$__RootfsDir/generated/download/"*.hpkg; do - "$__RootfsDir/generated/objects/linux/x86_64/release/tools/package/package" extract -C "$__RootfsDir/boot/system" "$file" + for file in "$__RootfsDir/tmp/download/"*.hpkg; do + echo "Extracting $file..." + LD_LIBRARY_PATH="$__RootfsDir/tmp/bin" "$__RootfsDir/tmp/bin/package" extract -C "$__RootfsDir/boot/system" "$file" done + # Download buildtools + echo "Downloading Haiku buildtools" + wget -O "$__RootfsDir/tmp/download/buildtools.zip" $($__RootfsDir/tmp/script/fetch.sh --buildtools --arch=$__HaikuArch) + unzip -o "$__RootfsDir/tmp/download/buildtools.zip" -d "$__RootfsDir" + # Cleaning up temporary files echo "Cleaning up temporary files" + popd rm -rf "$__RootfsDir/tmp" - for name in "$__RootfsDir/generated/"*; do - if [[ "$name" =~ "cross-tools-" ]]; then - : # Keep the cross-compiler - else - rm -rf "$name" - fi - done elif [[ -n "$__CodeName" ]]; then if [[ "$__SkipSigCheck" == "0" ]]; then diff --git a/eng/common/cross/toolchain.cmake b/eng/common/cross/toolchain.cmake index ce01673c7a..a88d643c8a 100644 --- a/eng/common/cross/toolchain.cmake +++ b/eng/common/cross/toolchain.cmake @@ -6,6 +6,7 @@ unset(FREEBSD) unset(ILLUMOS) unset(ANDROID) unset(TIZEN) +unset(HAIKU) set(TARGET_ARCH_NAME $ENV{TARGET_BUILD_ARCH}) if(EXISTS ${CROSS_ROOTFS}/bin/freebsd-version) @@ -16,6 +17,7 @@ elseif(EXISTS ${CROSS_ROOTFS}/usr/platform/i86pc) set(ILLUMOS 1) elseif(EXISTS ${CROSS_ROOTFS}/boot/system/develop/headers/config/HaikuConfig.h) set(CMAKE_SYSTEM_NAME Haiku) + set(HAIKU 1) else() set(CMAKE_SYSTEM_NAME Linux) set(LINUX 1) @@ -76,7 +78,7 @@ elseif(TARGET_ARCH_NAME STREQUAL "riscv64") set(CMAKE_SYSTEM_PROCESSOR riscv64) if(EXISTS ${CROSS_ROOTFS}/usr/lib/gcc/riscv64-alpine-linux-musl) set(TOOLCHAIN "riscv64-alpine-linux-musl") - else() + else() set(TOOLCHAIN "riscv64-linux-gnu") endif() elseif(TARGET_ARCH_NAME STREQUAL "s390x") @@ -100,7 +102,7 @@ elseif(TARGET_ARCH_NAME STREQUAL "x64") elseif(ILLUMOS) set(TOOLCHAIN "x86_64-illumos") elseif(HAIKU) - set(TOOLCHAIN "x64_64-unknown-haiku") + set(TOOLCHAIN "x86_64-unknown-haiku") endif() elseif(TARGET_ARCH_NAME STREQUAL "x86") set(CMAKE_SYSTEM_PROCESSOR i686) @@ -214,10 +216,8 @@ elseif(HAIKU) return() endif() - set(SEARCH_PATH "${CROSS_ROOTFS}/generated/cross-tools-x86_64/bin") - find_program(EXEC_LOCATION_${exec} - PATHS ${SEARCH_PATH} + PATHS "${CROSS_ROOTFS}/cross-tools-x86_64/bin" NAMES "${TOOLSET_PREFIX}${exec}${CLR_CMAKE_COMPILER_FILE_NAME_VERSION}" "${TOOLSET_PREFIX}${exec}") @@ -296,6 +296,9 @@ elseif(TARGET_ARCH_NAME STREQUAL "x86") elseif(ILLUMOS) add_toolchain_linker_flag("-L${CROSS_ROOTFS}/lib/amd64") add_toolchain_linker_flag("-L${CROSS_ROOTFS}/usr/amd64/lib") +elseif(HAIKU) + add_toolchain_linker_flag("-lnetwork") + add_toolchain_linker_flag("-lroot") endif() # Specify compile options diff --git a/eng/common/cross/x64/sources.list.bionic b/eng/common/cross/x64/sources.list.bionic new file mode 100644 index 0000000000..a71ccadcff --- /dev/null +++ b/eng/common/cross/x64/sources.list.bionic @@ -0,0 +1,11 @@ +deb http://archive.ubuntu.com/ubuntu/ bionic main restricted universe +deb-src http://archive.ubuntu.com/ubuntu/ bionic main restricted universe + +deb http://archive.ubuntu.com/ubuntu/ bionic-updates main restricted universe +deb-src http://archive.ubuntu.com/ubuntu/ bionic-updates main restricted universe + +deb http://archive.ubuntu.com/ubuntu/ bionic-backports main restricted +deb-src http://archive.ubuntu.com/ubuntu/ bionic-backports main restricted + +deb http://archive.ubuntu.com/ubuntu/ bionic-security main restricted universe multiverse +deb-src http://archive.ubuntu.com/ubuntu/ bionic-security main restricted universe multiverse diff --git a/eng/common/dotnet-install.sh b/eng/common/dotnet-install.sh index abd045a324..7e69e3a9e2 100755 --- a/eng/common/dotnet-install.sh +++ b/eng/common/dotnet-install.sh @@ -54,6 +54,10 @@ cpuname=$(uname -m) case $cpuname in arm64|aarch64) buildarch=arm64 + if [ "$(getconf LONG_BIT)" -lt 64 ]; then + # This is 32-bit OS running on 64-bit CPU (for example Raspberry Pi OS) + buildarch=arm + fi ;; loongarch64) buildarch=loongarch64 diff --git a/eng/common/sdk-task.ps1 b/eng/common/sdk-task.ps1 index e10a596879..6c4ac6fec1 100644 --- a/eng/common/sdk-task.ps1 +++ b/eng/common/sdk-task.ps1 @@ -64,7 +64,7 @@ try { $GlobalJson.tools | Add-Member -Name "vs" -Value (ConvertFrom-Json "{ `"version`": `"16.5`" }") -MemberType NoteProperty } if( -not ($GlobalJson.tools.PSObject.Properties.Name -match "xcopy-msbuild" )) { - $GlobalJson.tools | Add-Member -Name "xcopy-msbuild" -Value "17.4.1" -MemberType NoteProperty + $GlobalJson.tools | Add-Member -Name "xcopy-msbuild" -Value "17.6.0-2" -MemberType NoteProperty } if ($GlobalJson.tools."xcopy-msbuild".Trim() -ine "none") { $xcopyMSBuildToolsFolder = InitializeXCopyMSBuild $GlobalJson.tools."xcopy-msbuild" -install $true diff --git a/eng/common/templates/job/job.yml b/eng/common/templates/job/job.yml index 44ad26abf5..e20ee3a983 100644 --- a/eng/common/templates/job/job.yml +++ b/eng/common/templates/job/job.yml @@ -75,6 +75,10 @@ jobs: - ${{ if eq(parameters.enableRichCodeNavigation, 'true') }}: - name: EnableRichCodeNavigation value: 'true' + # Retry signature validation up to three times, waiting 2 seconds between attempts. + # See https://learn.microsoft.com/en-us/nuget/reference/errors-and-warnings/nu3028#retry-untrusted-root-failures + - name: NUGET_EXPERIMENTAL_CHAIN_BUILD_RETRY_POLICY + value: 3,2000 - ${{ each variable in parameters.variables }}: # handle name-value variable syntax # example: @@ -83,7 +87,7 @@ jobs: - ${{ if ne(variable.name, '') }}: - name: ${{ variable.name }} value: ${{ variable.value }} - + # handle variable groups - ${{ if ne(variable.group, '') }}: - group: ${{ variable.group }} @@ -169,7 +173,7 @@ jobs: - ${{ if eq(parameters.enableMicrobuild, 'true') }}: - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - task: MicroBuildCleanup@1 - displayName: Execute Microbuild cleanup tasks + displayName: Execute Microbuild cleanup tasks condition: and(always(), in(variables['_SignType'], 'real', 'test'), eq(variables['Agent.Os'], 'Windows_NT')) continueOnError: ${{ parameters.continueOnError }} env: @@ -219,7 +223,7 @@ jobs: displayName: Publish XUnit Test Results inputs: testResultsFormat: 'xUnit' - testResultsFiles: '*.xml' + testResultsFiles: '*.xml' searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)' testRunTitle: ${{ coalesce(parameters.testRunTitle, parameters.name, '$(System.JobName)') }}-xunit mergeTestResults: ${{ parameters.mergeTestResults }} @@ -230,7 +234,7 @@ jobs: displayName: Publish TRX Test Results inputs: testResultsFormat: 'VSTest' - testResultsFiles: '*.trx' + testResultsFiles: '*.trx' searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)' testRunTitle: ${{ coalesce(parameters.testRunTitle, parameters.name, '$(System.JobName)') }}-trx mergeTestResults: ${{ parameters.mergeTestResults }} diff --git a/eng/common/templates/steps/source-build.yml b/eng/common/templates/steps/source-build.yml index a97a185a36..1100521834 100644 --- a/eng/common/templates/steps/source-build.yml +++ b/eng/common/templates/steps/source-build.yml @@ -68,6 +68,11 @@ steps: runtimeOsArgs='/p:RuntimeOS=${{ parameters.platform.runtimeOS }}' fi + baseOsArgs= + if [ '${{ parameters.platform.baseOS }}' != '' ]; then + baseOsArgs='/p:BaseOS=${{ parameters.platform.baseOS }}' + fi + publishArgs= if [ '${{ parameters.platform.skipPublishValidation }}' != 'true' ]; then publishArgs='--publish' @@ -86,6 +91,7 @@ steps: $internalRestoreArgs \ $targetRidArgs \ $runtimeOsArgs \ + $baseOsArgs \ /p:SourceBuildNonPortable=${{ parameters.platform.nonPortable }} \ /p:ArcadeBuildFromSource=true \ /p:AssetManifestFileName=$assetManifestFileName diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index 38cf94ff88..c9eced9f7d 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -379,13 +379,13 @@ function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements = } # Minimum VS version to require. - $vsMinVersionReqdStr = '16.8' + $vsMinVersionReqdStr = '17.6' $vsMinVersionReqd = [Version]::new($vsMinVersionReqdStr) # If the version of msbuild is going to be xcopied, # use this version. Version matches a package here: - # https://dev.azure.com/dnceng/public/_packaging?_a=package&feed=dotnet-eng&package=RoslynTools.MSBuild&protocolType=NuGet&version=17.4.1&view=overview - $defaultXCopyMSBuildVersion = '17.4.1' + # https://dev.azure.com/dnceng/public/_artifacts/feed/dotnet-eng/NuGet/RoslynTools.MSBuild/versions/17.6.0-2 + $defaultXCopyMSBuildVersion = '17.6.0-2' if (!$vsRequirements) { if (Get-Member -InputObject $GlobalJson.tools -Name 'vs') { diff --git a/global.json b/global.json index 7e61f258d8..69c511874c 100644 --- a/global.json +++ b/global.json @@ -1,24 +1,24 @@ { "sdk": { - "version": "8.0.100-preview.3.23178.7", + "version": "8.0.100-preview.5.23303.2", "allowPrerelease": true, "rollForward": "latestPatch" }, "tools": { - "dotnet": "8.0.100-preview.3.23178.7", + "dotnet": "8.0.100-preview.5.23303.2", "vs": { - "version": "17.5", + "version": "17.6", "components": [ "Microsoft.VisualStudio.Component.FSharp" ] }, - "xcopy-msbuild": "17.4.1" + "xcopy-msbuild": "17.6.0-2" }, "native-tools": { "perl": "5.32.1.1" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.23255.2", + "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.23320.3", "Microsoft.DotNet.Helix.Sdk": "8.0.0-beta.23255.2" } } diff --git a/tests/AheadOfTime/Trimming/check.ps1 b/tests/AheadOfTime/Trimming/check.ps1 index d8151aef14..f7e7da7438 100644 --- a/tests/AheadOfTime/Trimming/check.ps1 +++ b/tests/AheadOfTime/Trimming/check.ps1 @@ -47,4 +47,4 @@ CheckTrim -root "SelfContained_Trimming_Test" -tfm "net472" -outputfile "FSharp. CheckTrim -root "StaticLinkedFSharpCore_Trimming_Test" -tfm "net472" -outputfile "StaticLinkedFSharpCore_Trimming_Test.exe" -expected_len -1 # Check net7.0 trimmed assemblies -CheckTrim -root "StaticLinkedFSharpCore_Trimming_Test" -tfm "net7.0" -outputfile "StaticLinkedFSharpCore_Trimming_Test.dll" -expected_len 8820736 +CheckTrim -root "StaticLinkedFSharpCore_Trimming_Test" -tfm "net7.0" -outputfile "StaticLinkedFSharpCore_Trimming_Test.dll" -expected_len 8821248 diff --git a/tests/scripts/scriptlib.fsx b/tests/scripts/scriptlib.fsx index 071196cf87..b074b78ba2 100644 --- a/tests/scripts/scriptlib.fsx +++ b/tests/scripts/scriptlib.fsx @@ -99,7 +99,7 @@ module Scripting = | "" -> exe | _ -> Path.Combine(baseDir,exe) |> Path.GetFullPath - let exec cmdArgs (workDir: FilePath) envs (path: FilePath) arguments = + let exec cmdArgs (workDir: FilePath) envs (path: FilePath) (arguments: string) = let exePath = path |> processExePath workDir let processInfo = new ProcessStartInfo(exePath, arguments)