diff --git a/.azdo/remoting/az-login.yml b/.azdo/remoting/az-login.yml new file mode 100644 index 000000000..ee7a4e7e9 --- /dev/null +++ b/.azdo/remoting/az-login.yml @@ -0,0 +1,16 @@ +parameters: +- name: SPAppId + type: string + default: $(BuildXLPipelinesAppId) +- name: SPPassword + type: string + default: $(AzureApp-BuildXL-Pipelines) +- name: SPTenant + type: string + default: $(BuildXLPipelinesTenantId) + +steps: +- script: | + call az login --service-principal --username ${{ parameters.SPAppId }} --password ${{ parameters.SPPassword }} --tenant ${{ parameters.SPTenant }} + if ERRORLEVEL 1 echo "az login failed" & exit /b 1 + displayName: 'Azure login' diff --git a/.azdo/remoting/install-client.yml b/.azdo/remoting/install-client.yml new file mode 100644 index 000000000..c493c6ae6 --- /dev/null +++ b/.azdo/remoting/install-client.yml @@ -0,0 +1,37 @@ +parameters: +- name: AnyBuildSource + type: string + default: '' +- name: Ring + type: string + default: 'Dogfood' + +steps: +- powershell: | + Remove-Item -Force -Recurse "$env:LOCALAPPDATA\Microsoft\AnyBuild" -ea SilentlyContinue + + # Make lowercase as AzStorage cannot handle uppercase in URL. + $source = "${{ parameters.AnyBuildSource }}".ToLowerInvariant() + + $bootstrapperArgs = @("$source", "${{ parameters.Ring }}") + + Write-Host "Bootstrapper args: '$bootstrapperArgs'"; + + while ($true) + { + $script = ((curl.exe -s -S --retry 10 --retry-connrefused "$source/bootstrapper.ps1") | Out-String) + Write-Host "Downloaded script: `r`n $script"; + if ($LASTEXITCODE -eq 0) + { + break; + } + + # We sometimes get a 404 error when downloading the bootstrapper + # while the bootstrapper script is updating. Needs an eventual fix instead of this workaround. + Write-Host "ERROR: Failed downloading the bootstrapper script. Trying again." + } + + Invoke-Command -ScriptBlock ([scriptblock]::Create($script)) -ArgumentList $bootstrapperArgs + failOnStderr: true + displayName: 'Install AnyBuild client' + continueOnError: false \ No newline at end of file diff --git a/.azdo/remoting/pipeline.yml b/.azdo/remoting/pipeline.yml new file mode 100644 index 000000000..7abb8c1e9 --- /dev/null +++ b/.azdo/remoting/pipeline.yml @@ -0,0 +1,85 @@ +trigger: none + +pool: + name: BuildXL-DevOpsAgents-PME + +variables: + NugetCredentialProviderPath: '$(Build.SourcesDirectory)\Public\Src\Tools\BuildXL.Tools.CredentialProvider\bin\Debug\netcoreapp2.1\win-x64' + Datacenter: 'westus2' + ClusterName: 'Bxl1' + RemoteUri: 'https://westus2.anybuild-test.microsoft.com/clusters/07F427C5-7979-415C-B6D9-01BAD5118191' + ClientContainerUrl: 'https://anybuild$(ClusterName)$(Datacenter).blob.core.windows.net/clientreleases' + BootstrapArgs: '-DeployDev /p:[Sdk.BuildXL]microsoftInternal=1' + RemotingArgs: '-UseDev -Minimal -EnableProcessRemoting -RemoteServiceUri $(RemoteUri) /p:BUILDXL_ANYBUILD_SERVICE_PRINCIPAL_APP_ID=$(BuildXLPipelinesAppId) /p:BUILDXL_ANYBUILD_SERVICE_PRINCIPAL_PWD_ENV=AnyBuildServicePrincipalPwd /numRemoteAgentLeases:24 /remotingThresholdMultiplier:1.25 /p:[Sdk.BuildXL]microsoftInternal=1 /p:BUILDXL_FINGERPRINT_SALT=*' + +steps: + +- checkout: self + fetchDepth: 1 + +- template: ./az-login.yml + +- script: 'C:\windows\system32\fsutil usn createjournal m=0x20000000 a=0x8000000 D:' + displayName: 'Enable Journaling for D drive' + +- script: | + if EXIST %AppData%\NuGet\NuGet.Config ( + del %AppData%\NuGet\NuGet.Config + ) + displayName: 'Nuget Hack: delete old nuget.config first' + +- task: UseDotNet@2 + displayName: 'Use .NET Core sdk 6.x' + inputs: + version: 6.x + +- task: DotNetCoreCLI@2 + displayName: 'Build BuildXL.Tools.CredentialProvider' + inputs: + projects: '$(Build.SourcesDirectory)\Public\Src\Tools\BuildXL.Tools.CredentialProvider\BuildXL.Tools.CredentialProvider.csproj' + arguments: '-r win-x64' + +- template: ./install-client.yml + parameters: + AnyBuildSource: $(ClientContainerUrl) + +- task: BatchScript@1 + displayName: 'Kill existing BuildXL processes' + inputs: + filename: Shared/Scripts/KillBxlInstancesInRepo.cmd + continueOnError: true + condition: always() + +- task: PowerShell@2 + displayName: 'Build BXL' + inputs: + targetType: filePath + filePath: '$(Build.SourcesDirectory)\RunBxlWithPAT.ps1' + arguments: '-OneEsPat $(PAT-TseBuild-AzureDevOps-1esSharedAssets-Package-Read) -CbPat $(PAT-TseBuild-AzureDevOps-CloudBuild-Packaging-Read) -NcPath $(NugetCredentialProviderPath) -MsEngGitPat $(PAT-TseBuild-AzureDevOps-MsEng-ReadCode) $(BootstrapArgs)' + +- task: PowerShell@2 + displayName: 'Run BXL with remoting' + inputs: + targetType: filePath + filePath: '$(Build.SourcesDirectory)\RunBxlWithPAT.ps1' + arguments: '-OneEsPat $(PAT-TseBuild-AzureDevOps-1esSharedAssets-Package-Read) -CbPat $(PAT-TseBuild-AzureDevOps-CloudBuild-Packaging-Read) -NcPath $(NugetCredentialProviderPath) -MsEngGitPat $(PAT-TseBuild-AzureDevOps-MsEng-ReadCode) $(RemotingArgs)' + env: + AnyBuildServicePrincipalPwd: $(AzureApp-BuildXL-Pipelines) + +- task: BatchScript@1 + displayName: 'Kill existing BuildXL processes' + inputs: + filename: Shared/Scripts/KillBxlInstancesInRepo.cmd + continueOnError: true + condition: always() + +- task: PublishPipelineArtifact@1 + displayName: 'Upload logs' + inputs: + targetPath: '$(Build.SourcesDirectory)\Out\Logs' + condition: always() + +- template: ./validate.yml + parameters: + LogDirectory: '$(Build.SourcesDirectory)\Out\Logs' + diff --git a/.azdo/remoting/validate.yml b/.azdo/remoting/validate.yml new file mode 100644 index 000000000..917eadf35 --- /dev/null +++ b/.azdo/remoting/validate.yml @@ -0,0 +1,32 @@ +parameters: +- name: LogDirectory + type: string + default: '$(Build.SourcesDirectory)\Out\Logs' + +steps: +- powershell: | + $f = Get-ChildItem -Path '${{ parameters.LogDirectory }}\*\AnyBuildDaemonManager.log' -Recurse | Select-Object -First 1 + $bxlWithRemotingLogDir = $f.DirectoryName + $statsContent = Get-Content -Path (Join-Path $bxlWithRemotingLogDir "BuildXL.stats") | Out-String + $stats = ConvertFrom-StringData -StringData $statsContent + $runRemoteCount = $stats.Get_Item("PipExecution.TotalRunRemoteProcesses") + $runLocalFallbackCount = $stats.Get_Item("PipExecution.TotalRemoteFallbackRetries") + + Write-Host "Run remote: $runRemoteCount" + Write-Host "Run local due to fallback: $runLocalFallbackCount" + + if ($runRemoteCount -eq 0) + { + Write-Host "No process pip ran remotely" + exit 1 + } + + if ($runLocalFallbackCount -ne 0) + { + Write-Host "$runLocalFallbackCount remoted process pip(s) ran locally" + exit 1 + } + + exit 0 + continueOnError: true + displayName: 'Validate remoting' \ No newline at end of file diff --git a/Public/Src/Engine/Processes/Remoting/AnyBuild/AnyBuildRemoteProcessManager.cs b/Public/Src/Engine/Processes/Remoting/AnyBuild/AnyBuildRemoteProcessManager.cs index c51d9bf42..8f4156009 100644 --- a/Public/Src/Engine/Processes/Remoting/AnyBuild/AnyBuildRemoteProcessManager.cs +++ b/Public/Src/Engine/Processes/Remoting/AnyBuild/AnyBuildRemoteProcessManager.cs @@ -118,7 +118,8 @@ namespace BuildXL.Processes.Remoting closeDaemonOnDispose: true, m_executionContext.CancellationToken, logDirectory: logDir, - additionalAnyBuildParameters: extraParams); + additionalAnyBuildParameters: extraParams, + inheritHandlesOnProcessCreation: false); } catch (Exception e) { @@ -165,6 +166,20 @@ namespace BuildXL.Processes.Remoting $"--CacheDir {localCacheDir}", }; + if (!string.IsNullOrEmpty(EngineEnvironmentSettings.AnyBuildServicePrincipalAppId)) + { + args.Add("--ClientApplicationId"); + args.Add(EngineEnvironmentSettings.AnyBuildServicePrincipalAppId); + + if (string.IsNullOrEmpty(EngineEnvironmentSettings.AnyBuildServicePrincipalPwdEnv)) + { + throw new BuildXLException($"Service principal password is required for starting AnyBuild daemon"); + } + + args.Add("--ClientSecretEnvironmentVariable"); + args.Add(EngineEnvironmentSettings.AnyBuildServicePrincipalPwdEnv); + } + if (!string.IsNullOrEmpty(m_configuration.Schedule.RemoteExecutionServiceUri)) { try diff --git a/Public/Src/Engine/Processes/Remoting/AnyBuild/AnyBuildRemoteProcessPipResult.cs b/Public/Src/Engine/Processes/Remoting/AnyBuild/AnyBuildRemoteProcessPipResult.cs index f44222573..3d63dc60f 100644 --- a/Public/Src/Engine/Processes/Remoting/AnyBuild/AnyBuildRemoteProcessPipResult.cs +++ b/Public/Src/Engine/Processes/Remoting/AnyBuild/AnyBuildRemoteProcessPipResult.cs @@ -41,9 +41,8 @@ namespace BuildXL.Processes.Remoting.AnyBuild ExitCode = result.ExitCode, StdOut = result.StdOut, StdErr = result.StdErr, - Disposition = result.Disposition switch + Disposition = result.ExecutionDisposition switch { - CommandExecutionDisposition.Unknown => RemoteResultDisposition.Unknown, CommandExecutionDisposition.CacheHit => RemoteResultDisposition.CacheHit, CommandExecutionDisposition.Remoted => RemoteResultDisposition.Remoted, CommandExecutionDisposition.RanLocally => RemoteResultDisposition.RanLocally, diff --git a/Public/Src/Engine/Processes/Remoting/RemoteResultDisposition.cs b/Public/Src/Engine/Processes/Remoting/RemoteResultDisposition.cs index 41bb9111e..7936a55a3 100644 --- a/Public/Src/Engine/Processes/Remoting/RemoteResultDisposition.cs +++ b/Public/Src/Engine/Processes/Remoting/RemoteResultDisposition.cs @@ -8,11 +8,6 @@ namespace BuildXL.Processes.Remoting /// public enum RemoteResultDisposition { - /// - /// Default zero value. - /// - Unknown, - /// /// Process completed with cache hit. /// diff --git a/Public/Src/Utilities/Configuration/EngineEnvironmentSettings.cs b/Public/Src/Utilities/Configuration/EngineEnvironmentSettings.cs index 04b9f4f24..db5b84ab8 100644 --- a/Public/Src/Utilities/Configuration/EngineEnvironmentSettings.cs +++ b/Public/Src/Utilities/Configuration/EngineEnvironmentSettings.cs @@ -88,6 +88,16 @@ namespace BuildXL.Utilities.Configuration /// public static readonly Setting AnyBuildExtraArgs = CreateSetting("BUILDXL_ANYBUILD_EXTRA_ARGS", value => value); + /// + /// Service principal app id for starting AnyBuild daemon. + /// + public static readonly Setting AnyBuildServicePrincipalAppId = CreateSetting("BUILDXL_ANYBUILD_SERVICE_PRINCIPAL_APP_ID", value => value); + + /// + /// Environment variable where the service principal password for starting AnyBuild daemon is stored. + /// + public static readonly Setting AnyBuildServicePrincipalPwdEnv = CreateSetting("BUILDXL_ANYBUILD_SERVICE_PRINCIPAL_PWD_ENV", value => value); + /// /// Indicates whether the application should fail fast on null reference exceptions /// diff --git a/RunBxlWithPAT.ps1 b/RunBxlWithPAT.ps1 new file mode 100644 index 000000000..10dc3805d --- /dev/null +++ b/RunBxlWithPAT.ps1 @@ -0,0 +1,82 @@ +[CmdletBinding(PositionalBinding=$false)] +Param( + [Parameter(mandatory=$true)] + [String]$OneEsPat, + [Parameter(mandatory=$true)] + [String]$CbPat, + [Parameter(mandatory=$true)] + [String]$NcPath, + [Parameter(mandatory=$true)] + [String]$MsEngGitPat, + + [Parameter(Mandatory=$false)] + [switch]$DeployDev = $false, + [Parameter(Mandatory=$false)] + [switch]$UseDev = $false, + [Parameter(Mandatory=$false)] + [switch]$Minimal = $false, + [Parameter(Mandatory=$false)] + [switch]$Release = $false, + + [Parameter(Mandatory=$false)] + [switch]$EnableProcessRemoting = $false, + [Parameter(Mandatory=$false)] + [string]$RemoteServiceUri = "https://westus2.anybuild-test.microsoft.com/clusters/07F427C5-7979-415C-B6D9-01BAD5118191", + [Parameter(Mandatory=$false)] + [string]$AnyBuildClientDir, + + [Parameter(mandatory=$false, ValueFromRemainingArguments=$true)] + [string[]]$BxlArgs +) + +[Environment]::SetEnvironmentVariable("1ESSHAREDASSETS_BUILDXL_FEED_PAT", $OneEsPat, "Process") +[Environment]::SetEnvironmentVariable("CLOUDBUILD_BUILDXL_SELFHOST_FEED_PAT", $CbPat, "Process") +[Environment]::SetEnvironmentVariable("MSENG_GIT_PAT", $MsEngGitPat, "Process") +[Environment]::SetEnvironmentVariable("NUGET_CREDENTIALPROVIDERS_PATH", $NcPath, "Process") + +[Environment]::SetEnvironmentVariable("VSS_NUGET_EXTERNAL_FEED_ENDPOINTS", " +{ + 'endpointCredentials': [ + {'endpoint':'https://pkgs.dev.azure.com/1essharedassets/_packaging/BuildXL/nuget/v3/index.json', 'password':'$OneEsPat'}, + {'endpoint':'https://pkgs.dev.azure.com/cloudbuild/_packaging/BuildXL.Selfhost/nuget/v3/index.json', 'password':'$CbPat'} + ] +}", "Process") + +$BxlCmdArgs = @() + +if ($UseDev) +{ + $BxlCmdArgs += @("-Use", "Dev") +} + +if ($DeployDev) +{ + $BxlCmdArgs += "-DeployDev" +} + +if ($Release) +{ + $BxlCmdArgs += @("-DeployConfig", "Release") +} + +if ($Minimal) +{ + $BxlCmdArgs += "-Minimal" +} + +if ($EnableProcessRemoting) +{ + $BxlCmdArgs += "-EnableProcessRemoting" + $BxlCmdArgs += @("-RemoteServiceUri", $RemoteServiceUri) +} + +if (-not [string]::IsNullOrEmpty($AnyBuildClientDir)) +{ + $BxlCmdArgs += @("-AnyBuildClientDir", "$AnyBuildClientDir") +} + +$BxlCmdArgs += $BxlArgs + +Write-Host "Call bxl.cmd $BxlCmdArgs" + +.\bxl.cmd $BxlCmdArgs \ No newline at end of file diff --git a/Shared/Scripts/bxl.ps1 b/Shared/Scripts/bxl.ps1 index e00140bad..c2cdc0b77 100644 --- a/Shared/Scripts/bxl.ps1 +++ b/Shared/Scripts/bxl.ps1 @@ -125,7 +125,7 @@ param( [switch]$DoNotUseDefaultCacheConfigFilePath = $false, [Parameter(Mandatory=$false)] - [switch]$UseDedupStore = $false, + [switch]$UseDedupStore = $false, [Parameter(Mandatory=$false)] [switch]$UseVfs = $false, @@ -139,17 +139,26 @@ param( [Parameter(Mandatory=$false)] [switch]$VsAll = $false, - + [Parameter(Mandatory=$false)] [switch]$UseManagedSharedCompilation = $true, - [Parameter(ValueFromRemainingArguments=$true)] - [string[]]$DominoArguments, - [Parameter(Mandatory=$false)] [switch]$NoQTest = $false, - [switch]$NoSubst = $false + [switch]$NoSubst = $false, + + [Parameter(Mandatory=$false)] + [switch]$EnableProcessRemoting = $false, + + [Parameter(Mandatory=$false)] + [string]$RemoteServiceUri = "https://westus2.anybuild-test.microsoft.com/clusters/07F427C5-7979-415C-B6D9-01BAD5118191", + + [Parameter(Mandatory=$false)] + [string]$AnyBuildClientDir, + + [Parameter(ValueFromRemainingArguments=$true)] + [string[]]$DominoArguments ) $ErrorActionPreference = "Stop"; @@ -211,7 +220,7 @@ $isMicrosoftInternal = [Environment]::GetEnvironmentVariable("[Sdk.BuildXL]micro # - this is not considered an internal build # We might decide to relax this once shared compilation gets enough mileage. if ($UseManagedSharedCompilation -and - (($DominoArguments -like '*/ado*') -or (-not $isMicrosoftInternal))) { + (($DominoArguments -like '*/ado*') -or (-not $isMicrosoftInternal) -or $EnableProcessRemoting)) { $UseManagedSharedCompilation = $false } @@ -312,6 +321,16 @@ if (($DominoArguments -match "logsDirectory:.*").Length -eq 0 -and ($DominoArgum $AdditionalBuildXLArguments += "/logsToRetain:20"; } +if ($EnableProcessRemoting) { + # Unit tests are not supported for remoting because change journal is not enabled on agents + # and all volumes in agents have the same serial. + $AdditionalBuildXLArguments += @("/server-", "/enableLazyOutputs-", "/exp:lazysodeletion-", "/enableProcessRemoting+", "/processCanRunRemoteTags:compile", "/processMustRunLocalTags:telemetry:xUnit;telemetry:xUnitUntracked;telemetry:QTest", "/remoteExecutionServiceUri:$RemoteServiceUri"); + + if (-not [string]::IsNullOrEmpty($AnyBuildClientDir)) { + $AdditionalBuildXLArguments += " /p:BUILDXL_ANYBUILD_CLIENT_INSTALL_DIR=$AnyBuildClientDir" + } +} + if ($Deploy -eq "LKG") { throw "The LKG deployment is special since it comes from a published NuGet package. It cannot be re-deployed in this selfhost wrapper."; } diff --git a/bxlremote.cmd b/bxlremote.cmd deleted file mode 100644 index 0bc0ab3a6..000000000 --- a/bxlremote.cmd +++ /dev/null @@ -1,31 +0,0 @@ -@ECHO OFF - -REM Builds using bxl.exe hooked up to AnyBuild for remoting -REM -REM Usage: -REM bxlremote.cmd -REM where -REM - : Additional options to pass through to bxl.cmd. -REM -REM Example: -REM bxlremote.cmd -deploy dev -minimal -REM -REM Prerequesites: -REM Requires a debug built AnyBuild repository as a sibling of the current repo root. - -SETLOCAL - -set _AnyBuildDistrib=%~dp0..\AnyBuild\distrib\Debug -set Passthrough_Args=%* -set _ClusterId=07F427C5-7979-415C-B6D9-01BAD5118191 - - -REM real cluster mode -REM To use loopback mode, replace --RemoveExecServiceUri for --loopback -%_AnyBuildDistrib%\AnyBuild\AnyBuild.exe --JsonConfigOverrides ProcessSubstitution.MaxParallelLocalExecutionsFactor=0 Run.DisableDirectoryMetadataDedup=true --remoteAll -v --DoNotUseMachineUtilizationForScheduling -w --NoSandboxingBuildEngine --CacheDir out\AnyBuildCache --LogDir out\AnyBuildLogs --DisableActionCache --RemoteExecServiceUri https://westus2.anybuild-test.microsoft.com/clusters/%_ClusterId% -- bxl.cmd /enableProcessRemoting+ /enableLazyOutputs- -nosubst /maxprocmultiplier:0.75 /numRemoteLeases:100 /p:[Sdk.BuildXL]useManagedSharedCompilation=false %Passthrough_Args% - -if %ERRORLEVEL% NEQ 0 ( - ENDLOCAL && EXIT /b 1 -) - -ENDLOCAL && EXIT /b 0 \ No newline at end of file diff --git a/cg/nuget/cgmanifest.json b/cg/nuget/cgmanifest.json index 66f0ad9a6..9d3da5c65 100644 --- a/cg/nuget/cgmanifest.json +++ b/cg/nuget/cgmanifest.json @@ -15,7 +15,7 @@ "Type": "NuGet", "NuGet": { "Name": "AnyBuild.SDK", - "Version": "0.1.5" + "Version": "0.1.10" } } }, diff --git a/config.microsoftInternal.dsc b/config.microsoftInternal.dsc index fb7fe485a..dce7bd406 100644 --- a/config.microsoftInternal.dsc +++ b/config.microsoftInternal.dsc @@ -84,7 +84,7 @@ export const pkgs = isMicrosoftInternal ? [ { id: "Microsoft.VisualStudio.Services.Governance.ComponentDetection.Contracts", version: "0.2010.80" }, // Process remoting - { id: "AnyBuild.SDK", version: "0.1.5" }, + { id: "AnyBuild.SDK", version: "0.1.10" }, ] : [ // Artifact packages and dependencies in OSS