From 8cdba4f7ae89d2c4111d3591d1eb8ed85122b321 Mon Sep 17 00:00:00 2001 From: Ryan Brandenburg Date: Tue, 19 Sep 2017 13:47:09 -0700 Subject: [PATCH] Update bootstrappers --- build.cmd | 2 +- build.ps1 | 223 +++++++++++++++++++++++++++++++++++++------------ build.sh | 245 +++++++++++++++++++++++++++++++++++++++++++++--------- 3 files changed, 379 insertions(+), 91 deletions(-) diff --git a/build.cmd b/build.cmd index 7d4894c..b6c8d24 100644 --- a/build.cmd +++ b/build.cmd @@ -1,2 +1,2 @@ @ECHO OFF -PowerShell -NoProfile -NoLogo -ExecutionPolicy unrestricted -Command "[System.Threading.Thread]::CurrentThread.CurrentCulture = ''; [System.Threading.Thread]::CurrentThread.CurrentUICulture = '';& '%~dp0build.ps1' %*; exit $LASTEXITCODE" \ No newline at end of file +PowerShell -NoProfile -NoLogo -ExecutionPolicy unrestricted -Command "[System.Threading.Thread]::CurrentThread.CurrentCulture = ''; [System.Threading.Thread]::CurrentThread.CurrentUICulture = '';& '%~dp0build.ps1' %*; exit $LASTEXITCODE" diff --git a/build.ps1 b/build.ps1 index 1785334..b7081bc 100644 --- a/build.ps1 +++ b/build.ps1 @@ -1,67 +1,186 @@ -$ErrorActionPreference = "Stop" +#!/usr/bin/env powershell +#requires -version 4 -function DownloadWithRetry([string] $url, [string] $downloadLocation, [int] $retries) +<# +.SYNOPSIS +Build this repository + +.DESCRIPTION +Downloads korebuild if required. Then builds the repository. + +.PARAMETER Path +The folder to build. Defaults to the folder containing this script. + +.PARAMETER Channel +The channel of KoreBuild to download. Overrides the value from the config file. + +.PARAMETER DotNetHome +The directory where .NET Core tools will be stored. + +.PARAMETER ToolsSource +The base url where build tools can be downloaded. Overrides the value from the config file. + +.PARAMETER Update +Updates KoreBuild to the latest version even if a lock file is present. + +.PARAMETER ConfigFile +The path to the configuration file that stores values. Defaults to version.props. + +.PARAMETER MSBuildArgs +Arguments to be passed to MSBuild + +.NOTES +This function will create a file $PSScriptRoot/korebuild-lock.txt. This lock file can be committed to source, but does not have to be. +When the lockfile is not present, KoreBuild will create one using latest available version from $Channel. + +The $ConfigFile is expected to be an JSON file. It is optional, and the configuration values in it are optional as well. Any options set +in the file are overridden by command line parameters. + +.EXAMPLE +Example config file: +```json { - while($true) - { - try - { - Invoke-WebRequest $url -OutFile $downloadLocation - break - } - catch - { - $exceptionMessage = $_.Exception.Message - Write-Host "Failed to download '$url': $exceptionMessage" - if ($retries -gt 0) { - $retries-- - Write-Host "Waiting 10 seconds before retrying. Retries left: $retries" - Start-Sleep -Seconds 10 + "$schema": "https://raw.githubusercontent.com/aspnet/BuildTools/dev/tools/korebuild.schema.json", + "channel": "dev", + "toolsSource": "https://aspnetcore.blob.core.windows.net/buildtools" +} +``` +#> +[CmdletBinding(PositionalBinding = $false)] +param( + [string]$Path = $PSScriptRoot, + [Alias('c')] + [string]$Channel, + [Alias('d')] + [string]$DotNetHome, + [Alias('s')] + [string]$ToolsSource, + [Alias('u')] + [switch]$Update, + [string]$ConfigFile = $null, + [Parameter(ValueFromRemainingArguments = $true)] + [string[]]$MSBuildArgs +) +Set-StrictMode -Version 2 +$ErrorActionPreference = 'Stop' + +# +# Functions +# + +function Get-KoreBuild { + + $lockFile = Join-Path $Path 'korebuild-lock.txt' + + if (!(Test-Path $lockFile) -or $Update) { + Get-RemoteFile "$ToolsSource/korebuild/channels/$Channel/latest.txt" $lockFile + } + + $version = Get-Content $lockFile | Where-Object { $_ -like 'version:*' } | Select-Object -first 1 + if (!$version) { + Write-Error "Failed to parse version from $lockFile. Expected a line that begins with 'version:'" + } + $version = $version.TrimStart('version:').Trim() + $korebuildPath = Join-Paths $DotNetHome ('buildtools', 'korebuild', $version) + + if (!(Test-Path $korebuildPath)) { + Write-Host -ForegroundColor Magenta "Downloading KoreBuild $version" + New-Item -ItemType Directory -Path $korebuildPath | Out-Null + $remotePath = "$ToolsSource/korebuild/artifacts/$version/korebuild.$version.zip" + + try { + $tmpfile = Join-Path ([IO.Path]::GetTempPath()) "KoreBuild-$([guid]::NewGuid()).zip" + Get-RemoteFile $remotePath $tmpfile + if (Get-Command -Name 'Expand-Archive' -ErrorAction Ignore) { + # Use built-in commands where possible as they are cross-plat compatible + Expand-Archive -Path $tmpfile -DestinationPath $korebuildPath } - else - { - $exception = $_.Exception - throw $exception + else { + # Fallback to old approach for old installations of PowerShell + Add-Type -AssemblyName System.IO.Compression.FileSystem + [System.IO.Compression.ZipFile]::ExtractToDirectory($tmpfile, $korebuildPath) } } + catch { + remove-item -Recurse -Force $korebuildPath -ErrorAction Ignore + throw + } + finally { + remove-item $tmpfile -ErrorAction Ignore + } + } + + return $korebuildPath +} + +function Join-Paths([string]$path, [string[]]$childPaths) { + $childPaths | ForEach-Object { $path = Join-Path $path $_ } + return $path +} + +function Get-RemoteFile([string]$RemotePath, [string]$LocalPath) { + if ($RemotePath -notlike 'http*') { + Copy-Item $RemotePath $LocalPath + return + } + + $retries = 10 + while ($retries -gt 0) { + $retries -= 1 + try { + Invoke-WebRequest -UseBasicParsing -Uri $RemotePath -OutFile $LocalPath + return + } + catch { + Write-Verbose "Request failed. $retries retries remaining" + } + } + + Write-Error "Download failed: '$RemotePath'." +} + +# +# Main +# + +# Load configuration or set defaults + +$Path = Resolve-Path $Path +if (!$ConfigFile) { $ConfigFile = Join-Path $Path 'korebuild.json' } + +if (Test-Path $ConfigFile) { + try { + $config = Get-Content -Raw -Encoding UTF8 -Path $ConfigFile | ConvertFrom-Json + if ($config) { + if (!($Channel) -and (Get-Member -Name 'channel' -InputObject $config)) { [string] $Channel = $config.channel } + if (!($ToolsSource) -and (Get-Member -Name 'toolsSource' -InputObject $config)) { [string] $ToolsSource = $config.toolsSource} + } + } catch { + Write-Warning "$ConfigFile could not be read. Its settings will be ignored." + Write-Warning $Error[0] } } -cd $PSScriptRoot - -$repoFolder = $PSScriptRoot -$env:REPO_FOLDER = $repoFolder - -$koreBuildZip="https://github.com/aspnet/KoreBuild/archive/rel/2.0.0.zip" -if ($env:KOREBUILD_ZIP) -{ - $koreBuildZip=$env:KOREBUILD_ZIP +if (!$DotNetHome) { + $DotNetHome = if ($env:DOTNET_HOME) { $env:DOTNET_HOME } ` + elseif ($env:USERPROFILE) { Join-Path $env:USERPROFILE '.dotnet'} ` + elseif ($env:HOME) {Join-Path $env:HOME '.dotnet'}` + else { Join-Path $PSScriptRoot '.dotnet'} } -$buildFolder = ".build" -$buildFile="$buildFolder\KoreBuild.ps1" +if (!$Channel) { $Channel = 'dev' } +if (!$ToolsSource) { $ToolsSource = 'https://aspnetcore.blob.core.windows.net/buildtools' } -if (!(Test-Path $buildFolder)) { - Write-Host "Downloading KoreBuild from $koreBuildZip" +# Execute - $tempFolder=$env:TEMP + "\KoreBuild-" + [guid]::NewGuid() - New-Item -Path "$tempFolder" -Type directory | Out-Null +$korebuildPath = Get-KoreBuild +Import-Module -Force -Scope Local (Join-Path $korebuildPath 'KoreBuild.psd1') - $localZipFile="$tempFolder\korebuild.zip" - - DownloadWithRetry -url $koreBuildZip -downloadLocation $localZipFile -retries 6 - - Add-Type -AssemblyName System.IO.Compression.FileSystem - [System.IO.Compression.ZipFile]::ExtractToDirectory($localZipFile, $tempFolder) - - New-Item -Path "$buildFolder" -Type directory | Out-Null - copy-item "$tempFolder\**\build\*" $buildFolder -Recurse - - # Cleanup - if (Test-Path $tempFolder) { - Remove-Item -Recurse -Force $tempFolder - } +try { + Install-Tools $ToolsSource $DotNetHome + Invoke-RepositoryBuild $Path @MSBuildArgs +} +finally { + Remove-Module 'KoreBuild' -ErrorAction Ignore } - -&"$buildFile" @args diff --git a/build.sh b/build.sh index 08c555e..9e32358 100755 --- a/build.sh +++ b/build.sh @@ -1,47 +1,216 @@ #!/usr/bin/env bash -repoFolder="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -cd $repoFolder -koreBuildZip="https://github.com/aspnet/KoreBuild/archive/rel/2.0.0.zip" -if [ ! -z $KOREBUILD_ZIP ]; then - koreBuildZip=$KOREBUILD_ZIP -fi +set -euo pipefail -buildFolder=".build" -buildFile="$buildFolder/KoreBuild.sh" +# +# variables +# -if test ! -d $buildFolder; then - echo "Downloading KoreBuild from $koreBuildZip" +RESET="\033[0m" +RED="\033[0;31m" +YELLOW="\033[0;33m" +MAGENTA="\033[0;95m" +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +[ -z "${DOTNET_HOME:-}"] && DOTNET_HOME="$HOME/.dotnet" +config_file="$DIR/korebuild.json" +verbose=false +update=false +repo_path="$DIR" +channel='' +tools_source='' - tempFolder="/tmp/KoreBuild-$(uuidgen)" - mkdir $tempFolder +# +# Functions +# +__usage() { + echo "Usage: $(basename ${BASH_SOURCE[0]}) [options] [[--] ...]" + echo "" + echo "Arguments:" + echo " ... Arguments passed to MSBuild. Variable number of arguments allowed." + echo "" + echo "Options:" + echo " --verbose Show verbose output." + echo " -c|--channel The channel of KoreBuild to download. Overrides the value from the config file.." + echo " --config-file The path to the configuration file that stores values. Defaults to korebuild.json." + echo " -d|--dotnet-home The directory where .NET Core tools will be stored. Defaults to '\$DOTNET_HOME' or '\$HOME/.dotnet." + echo " --path The directory to build. Defaults to the directory containing the script." + echo " -s|--tools-source The base url where build tools can be downloaded. Overrides the value from the config file." + echo " -u|--update Update to the latest KoreBuild even if the lock file is present." + echo "" + echo "Description:" + echo " This function will create a file \$DIR/korebuild-lock.txt. This lock file can be committed to source, but does not have to be." + echo " When the lockfile is not present, KoreBuild will create one using latest available version from \$channel." - localZipFile="$tempFolder/korebuild.zip" - - retries=6 - until (wget -O $localZipFile $koreBuildZip 2>/dev/null || curl -o $localZipFile --location $koreBuildZip 2>/dev/null) - do - echo "Failed to download '$koreBuildZip'" - if [ "$retries" -le 0 ]; then - exit 1 - fi - retries=$((retries - 1)) - echo "Waiting 10 seconds before retrying. Retries left: $retries" - sleep 10s - done - - unzip -q -d $tempFolder $localZipFile - - mkdir $buildFolder - cp -r $tempFolder/**/build/** $buildFolder - - chmod +x $buildFile - - # Cleanup - if test -d $tempFolder; then - rm -rf $tempFolder + if [[ "${1:-}" != '--no-exit' ]]; then + exit 2 fi +} + +get_korebuild() { + local lock_file="$repo_path/korebuild-lock.txt" + if [ ! -f $lock_file ] || [ "$update" = true ]; then + __get_remote_file "$tools_source/korebuild/channels/$channel/latest.txt" $lock_file + fi + local version="$(grep 'version:*' -m 1 $lock_file)" + if [[ "$version" == '' ]]; then + __error "Failed to parse version from $lock_file. Expected a line that begins with 'version:'" + return 1 + fi + version="$(echo ${version#version:} | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')" + local korebuild_path="$DOTNET_HOME/buildtools/korebuild/$version" + + { + if [ ! -d "$korebuild_path" ]; then + mkdir -p "$korebuild_path" + local remote_path="$tools_source/korebuild/artifacts/$version/korebuild.$version.zip" + tmpfile="$(mktemp)" + echo -e "${MAGENTA}Downloading KoreBuild ${version}${RESET}" + if __get_remote_file $remote_path $tmpfile; then + unzip -q -d "$korebuild_path" $tmpfile + fi + rm $tmpfile || true + fi + + source "$korebuild_path/KoreBuild.sh" + } || { + if [ -d "$korebuild_path" ]; then + echo "Cleaning up after failed installation" + rm -rf "$korebuild_path" || true + fi + return 1 + } +} + +__error() { + echo -e "${RED}error: $*${RESET}" 1>&2 +} + +__warn() { + echo -e "${YELLOW}warning: $*${RESET}" +} + +__machine_has() { + hash "$1" > /dev/null 2>&1 + return $? +} + +__get_remote_file() { + local remote_path=$1 + local local_path=$2 + + if [[ "$remote_path" != 'http'* ]]; then + cp $remote_path $local_path + return 0 + fi + + failed=false + if __machine_has curl ; then + curl --retry 10 -sSL -f --create-dirs -o $local_path $remote_path || failed=true + elif __machine_has wget; then + wget --tries 10 -O $local_path $remote_path || failed=true + else + failed=true + fi + + if [ "$failed" = true ]; then + __error "Download failed: $remote_path" 1>&2 + return 1 + fi +} + +# +# main +# + +while [[ $# > 0 ]]; do + case $1 in + -\?|-h|--help) + __usage --no-exit + exit 0 + ;; + -c|--channel|-Channel) + shift + channel=${1:-} + [ -z "$channel" ] && __usage + ;; + --config-file|-ConfigFile) + shift + config_file="${1:-}" + [ -z "$config_file" ] && __usage + if [ ! -f "$config_file" ]; then + __error "Invalid value for --config-file. $config_file does not exist." + exit 1 + fi + ;; + -d|--dotnet-home|-DotNetHome) + shift + DOTNET_HOME=${1:-} + [ -z "$DOTNET_HOME" ] && __usage + ;; + --path|-Path) + shift + repo_path="${1:-}" + [ -z "$repo_path" ] && __usage + ;; + -s|--tools-source|-ToolsSource) + shift + tools_source="${1:-}" + [ -z "$tools_source" ] && __usage + ;; + -u|--update|-Update) + update=true + ;; + --verbose|-Verbose) + verbose=true + ;; + --) + shift + break + ;; + *) + break + ;; + esac + shift +done + +if ! __machine_has unzip; then + __error 'Missing required command: unzip' + exit 1 fi -chmod +x .build/dotnet/dotnet-install.sh -$buildFile -r $repoFolder "$@" +if ! __machine_has curl && ! __machine_has wget; then + __error 'Missing required command. Either wget or curl is required.' + exit 1 +fi + +[ -z "${config_file:-}" ] && config_file="$repo_path/korebuild.json" +if [ -f "$config_file" ]; then + if __machine_has jq ; then + if jq '.' "$config_file" >/dev/null ; then + config_channel="$(jq -r 'select(.channel!=null) | .channel' "$config_file")" + config_tools_source="$(jq -r 'select(.toolsSource!=null) | .toolsSource' "$config_file")" + else + __warn "$config_file is invalid JSON. Its settings will be ignored." + fi + elif __machine_has python ; then + if python -c "import json,codecs;obj=json.load(codecs.open('$config_file', 'r', 'utf-8-sig'))" >/dev/null ; then + config_channel="$(python -c "import json,codecs;obj=json.load(codecs.open('$config_file', 'r', 'utf-8-sig'));print(obj['channel'] if 'channel' in obj else '')")" + config_tools_source="$(python -c "import json,codecs;obj=json.load(codecs.open('$config_file', 'r', 'utf-8-sig'));print(obj['toolsSource'] if 'toolsSource' in obj else '')")" + else + __warn "$config_file is invalid JSON. Its settings will be ignored." + fi + else + __warn 'Missing required command: jq or pyton. Could not parse the JSON file. Its settings will be ignored.' + fi + + [ ! -z "${config_channel:-}" ] && channel="$config_channel" + [ ! -z "${config_tools_source:-}" ] && tools_source="$config_tools_source" +fi + +[ -z "$channel" ] && channel='dev' +[ -z "$tools_source" ] && tools_source='https://aspnetcore.blob.core.windows.net/buildtools' + +get_korebuild +install_tools "$tools_source" "$DOTNET_HOME" +invoke_repository_build "$repo_path" $@