зеркало из https://github.com/microsoft/BuildXL.git
Merged PR 772709: Use 1ESPT with BuildXL workflow throughout our Linux validations
Add a `bxl_ado.sh` script that can interpret the command line given by 1ESPT (notably, AdoBuildRunner arguments are separated by `--`, and we use `--runner-arg` in our bash scripts to process these), and migrate all our selfhost builds to use 1ESPT with automatic cache config generation.
This commit is contained in:
Marcelo Lynch 🧉
Родитель
c3b5c47dbc
Коммит
72ac0e3e30
|
|
@ -0,0 +1,24 @@
|
|||
#!/bin/bash
|
||||
# The arguments will be given separated by "--": the first set of arguments are AdoBuildRunner arguments
|
||||
runnerArgs=()
|
||||
args=()
|
||||
found_separator=false
|
||||
|
||||
for arg in "$@"; do
|
||||
if [ "$arg" == "--" ]; then
|
||||
found_separator=true
|
||||
elif [ "$found_separator" == false ]; then
|
||||
runnerArgs+=("$arg")
|
||||
else
|
||||
args+=("$arg")
|
||||
fi
|
||||
done
|
||||
|
||||
if [ "$found_separator" == false ]; then
|
||||
args=${runnerArgs[@]}
|
||||
runnerArgs=()
|
||||
fi
|
||||
|
||||
# Call bxl.sh specifying the runner arguments explicitly
|
||||
# Note this script should be called with the repo root as working directory
|
||||
./bxl.sh "--use-adobuildrunner" "${runnerArgs[@]/#/--runner-arg }" "${args[@]}"
|
||||
|
|
@ -1,68 +0,0 @@
|
|||
# Template for validations that run bxl.sh with L3 cache
|
||||
# and share common pre- and post- build steps
|
||||
|
||||
parameters:
|
||||
- name: cacheUniverse
|
||||
type: string
|
||||
- name: buildSteps
|
||||
type: stepList
|
||||
- name: writeCacheConfigFile
|
||||
type: boolean
|
||||
default: true
|
||||
|
||||
steps:
|
||||
- task: UseDotNet@2
|
||||
displayName: Use .NET Core sdk 7.x
|
||||
inputs:
|
||||
version: 7.x
|
||||
|
||||
- bash: |
|
||||
set -eu
|
||||
# install mono
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y mono-complete mono-devel
|
||||
mono --version
|
||||
displayName: Install Mono
|
||||
|
||||
- ${{ if eq(parameters.writeCacheConfigFile, 'true') }}:
|
||||
- bash: |
|
||||
set -euo pipefail
|
||||
mkdir -p Out
|
||||
tee Out/CacheConfig.json << EOF
|
||||
{
|
||||
"RemoteIsReadOnly": false,
|
||||
"SkipDeterminismRecovery": true,
|
||||
"RemoteConstructionTimeoutMilliseconds": 10000,
|
||||
"Assembly": "BuildXL.Cache.VerticalAggregator",
|
||||
"Type": "BuildXL.Cache.VerticalAggregator.VerticalCacheAggregatorFactory",
|
||||
"RemoteCache": {
|
||||
"Assembly": "BuildXL.Cache.MemoizationStoreAdapter",
|
||||
"CacheLogPath": "[BuildXLSelectedLogPath].Remote.log",
|
||||
"Type": "BuildXL.Cache.MemoizationStoreAdapter.BlobCacheFactory",
|
||||
"CacheId": "L3Cache",
|
||||
"Universe": "${{ parameters.cacheUniverse }}",
|
||||
"RetentionPolicyInDays": 6,
|
||||
"StorageAccountEndpoint": "https://l3bxlselfhost.blob.core.windows.net",
|
||||
"ManagedIdentityId": "eb694749-b1d6-45bc-b7af-2bd81603968a"
|
||||
},
|
||||
"LocalCache": {
|
||||
"MaxCacheSizeInMB": 20240,
|
||||
"Assembly": "BuildXL.Cache.MemoizationStoreAdapter",
|
||||
"UseStreamCAS": true,
|
||||
"Type": "BuildXL.Cache.MemoizationStoreAdapter.MemoizationStoreCacheFactory",
|
||||
"CacheLogPath": "[BuildXLSelectedLogPath]",
|
||||
"CacheRootPath": "[BuildXLSelectedRootPath]",
|
||||
"CacheId": "SelfhostCS2L1",
|
||||
"UseRocksDbMemoizationStore": true
|
||||
}
|
||||
}
|
||||
EOF
|
||||
displayName: Write cache config file
|
||||
|
||||
- bash: |
|
||||
sudo mkdir /home/subst
|
||||
sudo mount --verbose --bind $(Build.SourcesDirectory) /home/subst
|
||||
displayName: Bind /home/subst to sources directory
|
||||
|
||||
- ${{ each step in parameters.buildSteps }}:
|
||||
- ${{ step }}
|
||||
|
|
@ -3,51 +3,17 @@ parameters:
|
|||
type: string
|
||||
|
||||
jobs:
|
||||
- job: Selfhost_Bootstrap
|
||||
displayName: Bootstrap internal
|
||||
pool:
|
||||
name: BuildXL-DevOpsAgents-Linux-Stateless-PME
|
||||
os: linux
|
||||
|
||||
templateContext:
|
||||
inputs:
|
||||
- input: checkout
|
||||
repository: self
|
||||
fetchDepth: 1
|
||||
- template: job-build-selfhost-base.yml
|
||||
parameters:
|
||||
validationName: Bootstrap
|
||||
internal: true
|
||||
|
||||
outputs:
|
||||
- output: pipelineArtifact
|
||||
targetPath: $(Build.SourcesDirectory)/Out/Logs
|
||||
artifactName: BuildXL-Bootstrap-Logs.$(System.JobName).$(System.JobAttempt)
|
||||
condition: always()
|
||||
continueOnError: true
|
||||
displayName: Upload BuildXL Logs
|
||||
- output: pipelineArtifact
|
||||
targetPath: $(Build.SourcesDirectory)/Out/Selfhost/Dev
|
||||
artifactName: Dev
|
||||
condition: always()
|
||||
continueOnError: true
|
||||
displayName: Upload built engine for validations to consume
|
||||
sdl:
|
||||
credscan:
|
||||
enabled: false
|
||||
antimalwareScan:
|
||||
enabled: false
|
||||
publishLogs:
|
||||
enabled: false
|
||||
|
||||
timeoutInMinutes: 90
|
||||
|
||||
steps:
|
||||
- template: build-selfhost-base.yml
|
||||
parameters:
|
||||
cacheUniverse: blob3bxlselfhost
|
||||
buildSteps:
|
||||
- bash: |
|
||||
set -eu
|
||||
# Set a 60m timeout so we can catch hangs *and* get logs collected at the same time. Otherwise the whole job will timeout (check 'timeoutInMinutes' above).
|
||||
timeout --signal 9 60m bash bxl.sh /logsDirectory:"Out/Logs/Build" --minimal --internal --deploy-dev-net8 --release /cacheConfigFilePath:Out/CacheConfig.json ${{ parameters.BxlCommonArgs }}
|
||||
displayName: Build
|
||||
workingDirectory: /home/subst
|
||||
env:
|
||||
PAT1esSharedAssets: $(PAT-TseBuild-AzureDevOps-1esSharedAssets-Package-Read)
|
||||
PATCloudBuild: $(PAT-TseBuild-AzureDevOps-CloudBuild-Packaging-Read)
|
||||
buildXLArguments: --minimal --deploy-dev-net8 --release ${{ parameters.BxlCommonArgs }}
|
||||
|
|
@ -0,0 +1,119 @@
|
|||
# Template for all Linux selfhost builds, ran using the bxl_ado.sh wrapper script
|
||||
# These can be internal or external builds, distributed or single-machine: this is controlled
|
||||
# with the below parameters.
|
||||
parameters:
|
||||
- name: validationName
|
||||
type: string
|
||||
|
||||
- name: cacheUniverse
|
||||
type: string
|
||||
default: bxlselfhost
|
||||
|
||||
# Additional outputs to set for the job (apart from logs, which are automatically uploaded)
|
||||
- name: outputs
|
||||
type: object
|
||||
default: []
|
||||
|
||||
# Pre-build steps, including downloading BuildXL if needed
|
||||
- name: preBuildSteps
|
||||
type: stepList
|
||||
default: []
|
||||
|
||||
- name: buildXLArguments
|
||||
type: string
|
||||
|
||||
# Whether this is an internal build: if true, PATs will be added to the environment
|
||||
- name: internal
|
||||
type: boolean
|
||||
default: false
|
||||
|
||||
- name: publishTestResults
|
||||
type: boolean
|
||||
default: false
|
||||
|
||||
# distribution:
|
||||
# enabled: <bool>
|
||||
# workerCount: <int>
|
||||
- name: distribution
|
||||
type: object
|
||||
default: {}
|
||||
|
||||
- name: cacheType
|
||||
type: string
|
||||
default: EphemeralDatacenterWide
|
||||
|
||||
jobs:
|
||||
- job: Selfhost_${{ parameters.validationName }}
|
||||
displayName: Build and Validate Selfhost (${{ parameters.validationName }})
|
||||
|
||||
pool:
|
||||
name: BuildXL-DevOpsAgents-Linux-Stateless-PME
|
||||
os: linux
|
||||
timeoutInMinutes: 90
|
||||
templateContext:
|
||||
sdl:
|
||||
credscan:
|
||||
enabled: false
|
||||
antimalwareScan:
|
||||
enabled: false
|
||||
publishLogs:
|
||||
enabled: false
|
||||
|
||||
outputs:
|
||||
- ${{ each output in parameters.outputs }}:
|
||||
- ${{ output }}
|
||||
|
||||
# https://eng.ms/docs/cloud-ai-platform/devdiv/one-engineering-system-1es/1es-build/buildxl/ado-onboarding-steps/running-bxl
|
||||
workflow: BuildXL
|
||||
distribution: ${{ parameters.distribution }}
|
||||
cache:
|
||||
storageAccountEndpoint: https://l3bxlselfhost.blob.core.windows.net
|
||||
managedIdentityClientId: eb694749-b1d6-45bc-b7af-2bd81603968a
|
||||
universe: ${{ parameters.cacheUniverse }}
|
||||
cacheType: ${{ parameters.cacheType }}
|
||||
logGeneratedConfiguration: true
|
||||
# retentionPolicyInDays: {default}
|
||||
|
||||
preBuildSteps:
|
||||
- template: /.azdo/common/use-latest-dotnet-sdk.yml@self
|
||||
|
||||
- bash: |
|
||||
set -eu
|
||||
# install mono
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y mono-complete mono-devel
|
||||
mono --version
|
||||
displayName: Install Mono
|
||||
|
||||
- bash: |
|
||||
sudo mkdir /home/subst
|
||||
sudo mount --verbose --bind $(Build.SourcesDirectory) /home/subst
|
||||
displayName: Bind /home/subst to sources directory
|
||||
|
||||
- ${{ each step in parameters.preBuildSteps }}:
|
||||
- ${{ step }}
|
||||
|
||||
buildXL:
|
||||
invoker: custom
|
||||
# Set a 60m timeout so we can catch hangs *and* get logs collected at the same time. Otherwise the whole job will timeout (check 'timeoutInMinutes' above).
|
||||
invocationPrefix: timeout --signal 9 60m bash .azdo/bxl_ado.sh
|
||||
${{ if eq(parameters.internal, true) }}:
|
||||
arguments: --internal ${{ parameters.buildXLArguments }}
|
||||
${{ else }}:
|
||||
arguments: ${{ parameters.buildXLArguments }}
|
||||
displayName: ${{ parameters.validationName }}
|
||||
workingDirectory: /home/subst
|
||||
${{ if eq(parameters.internal, true) }}:
|
||||
env:
|
||||
PAT1esSharedAssets: $(PAT-TseBuild-AzureDevOps-1esSharedAssets-Package-Read)
|
||||
PATCloudBuild: $(PAT-TseBuild-AzureDevOps-CloudBuild-Packaging-Read)
|
||||
|
||||
${{ if parameters.publishTestResults }}:
|
||||
postBuildSteps:
|
||||
- task: PublishTestResults@2
|
||||
displayName: Publish Test Results
|
||||
inputs:
|
||||
testResultsFormat: XUnit
|
||||
testResultsFiles: 'Out/Objects/**/xunit-logs/**/xunit*.xml'
|
||||
condition: always()
|
||||
continueOnError: true
|
||||
|
|
@ -53,8 +53,6 @@ jobs:
|
|||
|
||||
# Build parameters
|
||||
buildXL:
|
||||
invocationKey: DistributedTest
|
||||
|
||||
# invoker: exe means we need to provide the directory where AdoBuildRunner is located
|
||||
invoker: exe
|
||||
engineLocation: $(BootstrapLocation)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,52 @@
|
|||
parameters:
|
||||
- name: BxlCommonArgs # We reuse these across jobs like a constant
|
||||
type: string
|
||||
|
||||
jobs:
|
||||
- template: job-build-selfhost-base.yml
|
||||
parameters:
|
||||
validationName: ExternalRelease
|
||||
cacheUniverse: bxlselfhostpublic
|
||||
internal: false
|
||||
publishTestResults: true
|
||||
|
||||
outputs:
|
||||
- output: pipelineArtifact
|
||||
targetPath: $(Build.SourcesDirectory)/Out/Logs/Bootstrap
|
||||
artifactName: BuildXL-Logs.External.Bootstrap.$(System.JobAttempt)
|
||||
condition: always()
|
||||
continueOnError: true
|
||||
displayName: Upload bootstrap Logs
|
||||
|
||||
buildXLArguments: >-
|
||||
--use-dev
|
||||
${{ parameters.BxlCommonArgs }}
|
||||
/q:ReleaseLinux
|
||||
"/f:tag='test'"
|
||||
/logToKusto
|
||||
/cacheLogToKusto
|
||||
/logToKustoBlobUri:https://adomessages.blob.core.windows.net/adomessages
|
||||
/logToKustoIdentityId:6e0959cf-a9ba-4988-bbf1-7facd9deda51
|
||||
|
||||
preBuildSteps:
|
||||
# 1. Bootstrap external bits manually - note this is a pre-build step for the actual "build" (which is running tests)
|
||||
- bash: >
|
||||
set -eu
|
||||
|
||||
# Set a 60m timeout so we can catch hangs *and* get logs collected at the same time. Otherwise the whole job will timeout (check 'timeoutInMinutes' above).
|
||||
|
||||
timeout --signal 9 60m bash bxl.sh
|
||||
--use-adobuildrunner
|
||||
--minimal
|
||||
--deploy-dev --release
|
||||
--runner-arg /cacheConfigLogGeneratedConfiguration:true
|
||||
--runner-arg /cacheConfigStorageAccountEndpoint:https://l3bxlselfhost.blob.core.windows.net
|
||||
--runner-arg /cacheConfigManagedIdentityId:eb694749-b1d6-45bc-b7af-2bd81603968a
|
||||
/logsDirectory:"Out/Logs/Bootstrap"
|
||||
/forceAddExecutionPermission-
|
||||
${{ parameters.BxlCommonArgs }}
|
||||
displayName: Build
|
||||
env:
|
||||
AdoBuildRunnerInvocationKey: BuildExternal
|
||||
SYSTEM_ACCESSTOKEN: $(System.AccessToken)
|
||||
workingDirectory: /home/subst
|
||||
|
|
@ -1,49 +1,27 @@
|
|||
parameters:
|
||||
- name: BxlCommonArgs # We reuse these across jobs like a constant
|
||||
type: string
|
||||
|
||||
jobs:
|
||||
- job: Selfhost_InternalPTrace
|
||||
displayName: Build and Validate Selfhost with PTrace
|
||||
pool:
|
||||
name: BuildXL-DevOpsAgents-Linux-Stateless-PME
|
||||
os: linux
|
||||
|
||||
templateContext:
|
||||
inputs:
|
||||
- input: checkout
|
||||
repository: self
|
||||
fetchDepth: 1
|
||||
- input: pipelineArtifact
|
||||
artifactName: 'Dev'
|
||||
# Use the path that --deploy-dev would use, because we --use-dev in the validation
|
||||
targetPath: "$(Build.SourcesDirectory)/Out/Selfhost/Dev"
|
||||
|
||||
outputs:
|
||||
- output: pipelineArtifact
|
||||
targetPath: $(Build.SourcesDirectory)/Out/Logs
|
||||
artifactName: BuildXL-Logs.InternalPTrace.$(System.JobName).$(System.JobAttempt)
|
||||
condition: always()
|
||||
continueOnError: true
|
||||
displayName: Upload BuildXL Logs
|
||||
sdl:
|
||||
credscan:
|
||||
enabled: false
|
||||
antimalwareScan:
|
||||
enabled: false
|
||||
publishLogs:
|
||||
enabled: false
|
||||
|
||||
timeoutInMinutes: 90
|
||||
|
||||
steps:
|
||||
- template: build-selfhost-base.yml
|
||||
parameters:
|
||||
cacheUniverse: bxlselfhostptrace
|
||||
buildSteps:
|
||||
# This step currently only builds selfhost with the --minimal flag, but will be extended in the future to run more unit tests with ptrace
|
||||
- bash: |
|
||||
set -eu
|
||||
bash bxl.sh --minimal --internal --use-dev /ado /cacheMiss:"[Bxl.Selfhost.Linux.PTrace]" /logObservedFileAccesses /cacheConfigFilePath:Out/CacheConfig.json /logoutput:FullOutputOnError /logsToRetain:10 /logsDirectory:"Out/Logs/Build" /forceEnableLinuxPTraceSandbox+ /injectCacheMisses:0.3
|
||||
displayName: Build BXL with LKG and PTrace
|
||||
workingDirectory: /home/subst
|
||||
env:
|
||||
PAT1esSharedAssets: $(PAT-TseBuild-AzureDevOps-1esSharedAssets-Package-Read)
|
||||
PATCloudBuild: $(PAT-TseBuild-AzureDevOps-CloudBuild-Packaging-Read)
|
||||
|
||||
- template: job-build-selfhost-base.yml
|
||||
parameters:
|
||||
validationName: PTrace
|
||||
cacheUniverse: bxlselfhostptrace
|
||||
internal: true
|
||||
publishTestResults: true
|
||||
|
||||
buildXLArguments: >-
|
||||
--minimal
|
||||
--use-dev
|
||||
${{ parameters.BxlCommonArgs }}
|
||||
/cacheMiss:"[Bxl.Selfhost.Linux.PTrace]"
|
||||
/forceEnableLinuxPTraceSandbox+
|
||||
/injectCacheMisses:0.3
|
||||
preBuildSteps:
|
||||
- task: DownloadPipelineArtifact@2
|
||||
inputs:
|
||||
buildType: current
|
||||
artifactName: Dev
|
||||
# Use the path that --deploy-dev would use, because we --use-dev in the validation
|
||||
targetPath: "$(Build.SourcesDirectory)/Out/Selfhost/Dev"
|
||||
displayName: Download BuildXL
|
||||
|
|
@ -1,78 +0,0 @@
|
|||
parameters:
|
||||
- name: BxlCommonArgs # We reuse these across jobs like a constant
|
||||
type: string
|
||||
- name: Role
|
||||
type: string
|
||||
default: SingleMachine
|
||||
values:
|
||||
- Orchestrator
|
||||
- Worker
|
||||
- SingleMachine
|
||||
- name: ValidationName
|
||||
type: string
|
||||
|
||||
jobs:
|
||||
- job: Selfhost_${{ parameters.validationName }}
|
||||
displayName: Build and Validate Selfhost (${{ parameters.validationName }})
|
||||
|
||||
pool:
|
||||
name: BuildXL-DevOpsAgents-Linux-Stateless-PME
|
||||
os: linux
|
||||
|
||||
templateContext:
|
||||
inputs:
|
||||
- input: checkout
|
||||
repository: self
|
||||
fetchDepth: 1
|
||||
|
||||
outputs:
|
||||
- output: pipelineArtifact
|
||||
targetPath: $(Build.SourcesDirectory)/Out/Logs
|
||||
artifactName: BuildXL-Logs.${{ parameters.validationName }}.${{ parameters.Role }}.$(System.JobName).$(System.JobAttempt)
|
||||
condition: always()
|
||||
continueOnError: true
|
||||
displayName: Upload BuildXL Logs
|
||||
sdl:
|
||||
credscan:
|
||||
enabled: false
|
||||
antimalwareScan:
|
||||
enabled: false
|
||||
publishLogs:
|
||||
enabled: false
|
||||
|
||||
timeoutInMinutes: 90
|
||||
|
||||
steps:
|
||||
- template: build-selfhost-base.yml
|
||||
parameters:
|
||||
cacheUniverse: bxlselfhostpublic
|
||||
writeCacheConfigFile: true
|
||||
buildSteps:
|
||||
# 1. Bootstrap public bits
|
||||
- bash: |
|
||||
set -eu
|
||||
# Set a 60m timeout so we can catch hangs *and* get logs collected at the same time. Otherwise the whole job will timeout (check 'timeoutInMinutes' above).
|
||||
timeout --signal 9 60m bash bxl.sh /logsDirectory:"Out/Logs/Build" --minimal --deploy-dev --release /forceAddExecutionPermission- /cacheConfigFilePath:Out/CacheConfig.json ${{ parameters.BxlCommonArgs }}
|
||||
displayName: Build
|
||||
workingDirectory: /home/subst
|
||||
|
||||
# 2. Run tests with the public bits
|
||||
- bash: |
|
||||
set -eu
|
||||
# - the disks on Azure Pipeline VMs are too small to build everything, so let's instead run tests
|
||||
# Set a 60m timeout so we can catch hangs *and* get logs collected at the same time. Otherwise the whole job will timeout (check 'timeoutInMinutes' above).
|
||||
timeout --signal 9 60m ./bxl.sh --use-dev --use-adobuildrunner --runner-arg /cacheConfigUniverse:bxlselfhostpublic ${{ parameters.BxlCommonArgs }} /logsDirectory:"Out/Logs/${{ parameters.validationName }}" /q:ReleaseLinux "/f:tag='test'" /logToKusto /cacheLogToKusto /logToKustoBlobUri:https://adomessages.blob.core.windows.net/adomessages /logToKustoIdentityId:6e0959cf-a9ba-4988-bbf1-7facd9deda51 /logToKustoTenantId:975f013f-7f24-47e8-a7d3-abc4752bf346 /dynamicBuildWorkerSlots:0
|
||||
displayName: Test (${{ parameters.validationName }})
|
||||
workingDirectory: /home/subst
|
||||
env:
|
||||
SYSTEM_ACCESSTOKEN: $(System.AccessToken)
|
||||
AdoBuildRunnerInvocationKey: LinuxSelfhostPublicValidation
|
||||
AdoBuildRunnerWorkerPipelineRole: Orchestrator
|
||||
|
||||
- task: PublishTestResults@2
|
||||
displayName: Publish Test Results
|
||||
inputs:
|
||||
testResultsFormat: XUnit
|
||||
testResultsFiles: 'Out/Objects/**/xunit-logs/**/xunit*.xml'
|
||||
condition: always()
|
||||
continueOnError: true
|
||||
|
|
@ -1,93 +1,32 @@
|
|||
parameters:
|
||||
- name: BxlCommonArgs # We reuse these across jobs like a constant
|
||||
type: string
|
||||
- name: Role
|
||||
type: string
|
||||
default: SingleMachine
|
||||
values:
|
||||
- Orchestrator
|
||||
- Worker
|
||||
- SingleMachine
|
||||
# The following parameters change
|
||||
- name: BxlExtraArgs
|
||||
type: string
|
||||
- name: ValidationName
|
||||
type: string
|
||||
|
||||
jobs:
|
||||
- job: Selfhost_${{ parameters.validationName }}
|
||||
displayName: Build and Validate Selfhost (${{ parameters.validationName }})
|
||||
|
||||
${{ if eq(parameters.Role, 'Worker') }}:
|
||||
continueOnError: true
|
||||
|
||||
pool:
|
||||
name: BuildXL-DevOpsAgents-Linux-Stateless-PME
|
||||
os: linux
|
||||
|
||||
templateContext:
|
||||
inputs:
|
||||
- input: checkout
|
||||
repository: self
|
||||
fetchDepth: 1
|
||||
- input: pipelineArtifact
|
||||
artifactName: 'Dev'
|
||||
# Use the path that --deploy-dev would use, because we --use-dev in the validation
|
||||
targetPath: "$(Build.SourcesDirectory)/Out/Selfhost/Dev"
|
||||
|
||||
outputs:
|
||||
- output: pipelineArtifact
|
||||
targetPath: $(Build.SourcesDirectory)/Out/Logs
|
||||
artifactName: BuildXL-Logs.${{ parameters.validationName }}.${{ parameters.Role }}.$(System.JobName).$(System.JobAttempt)
|
||||
condition: always()
|
||||
continueOnError: true
|
||||
displayName: Upload BuildXL Logs
|
||||
sdl:
|
||||
credscan:
|
||||
enabled: false
|
||||
antimalwareScan:
|
||||
enabled: false
|
||||
publishLogs:
|
||||
enabled: false
|
||||
|
||||
timeoutInMinutes: 90
|
||||
|
||||
steps:
|
||||
- template: build-selfhost-base.yml
|
||||
parameters:
|
||||
cacheUniverse: bxlselfhost
|
||||
# if we are running single machine, write down a cache config file. Otherwise the adobuildrunner will generate one
|
||||
writeCacheConfigFile: ${{ eq(parameters.Role, 'SingleMachine') }}
|
||||
buildSteps:
|
||||
- bash: |
|
||||
set -eu
|
||||
# - the disks on Azure Pipeline VMs are too small to build everything, so let's instead run tests
|
||||
# Set a 60m timeout so we can catch hangs *and* get logs collected at the same time. Otherwise the whole job will timeout (check 'timeoutInMinutes' above).
|
||||
|
||||
USE_ADOBUILDRUNNER=""
|
||||
DYNAMIC_WORKER_SLOTS=""
|
||||
|
||||
# if we are doing a distributed build, use the ado build runner and configure
|
||||
# an extra worker
|
||||
if [ "${{ parameters.Role }}" != "SingleMachine" ]; then
|
||||
USE_ADOBUILDRUNNER="--use-adobuildrunner --runner-arg /cacheConfigUniverse:bxlselfhost"
|
||||
DYNAMIC_WORKER_SLOTS="/dynamicBuildWorkerSlots:1"
|
||||
fi
|
||||
|
||||
timeout --signal 9 60m ./bxl.sh $USE_ADOBUILDRUNNER --use-dev ${{ parameters.BxlCommonArgs }} /logsDirectory:"Out/Logs/${{ parameters.validationName }}" ${{ parameters.bxlExtraArgs }} "/f:tag='test'" /logToKusto /cacheLogToKusto /logToKustoBlobUri:https://adomessages.blob.core.windows.net/adomessages /logToKustoIdentityId:6e0959cf-a9ba-4988-bbf1-7facd9deda51 /logToKustoTenantId:975f013f-7f24-47e8-a7d3-abc4752bf346 $DYNAMIC_WORKER_SLOTS
|
||||
displayName: Test (${{ parameters.validationName }})
|
||||
workingDirectory: /home/subst
|
||||
env:
|
||||
PAT1esSharedAssets: $(PAT-TseBuild-AzureDevOps-1esSharedAssets-Package-Read)
|
||||
PATCloudBuild: $(PAT-TseBuild-AzureDevOps-CloudBuild-Packaging-Read)
|
||||
SYSTEM_ACCESSTOKEN: $(System.AccessToken)
|
||||
AdoBuildRunnerInvocationKey: LinuxSelfhostDistributedValidation
|
||||
AdoBuildRunnerWorkerPipelineRole: ${{ parameters.Role }}
|
||||
|
||||
- task: PublishTestResults@2
|
||||
displayName: Publish Test Results
|
||||
inputs:
|
||||
testResultsFormat: XUnit
|
||||
testResultsFiles: 'Out/Objects/**/xunit-logs/**/xunit*.xml'
|
||||
condition: always()
|
||||
continueOnError: true
|
||||
- template: job-build-selfhost-base.yml
|
||||
parameters:
|
||||
validationName: InternalRelease
|
||||
internal: true
|
||||
publishTestResults: true
|
||||
distribution:
|
||||
enabled: true
|
||||
workerCount: 1
|
||||
buildXLArguments: >-
|
||||
--use-dev
|
||||
/q:ReleaseLinux
|
||||
/q:ReleaseLinuxNet8
|
||||
${{ parameters.BxlCommonArgs }}
|
||||
"/f:tag='test'"
|
||||
/logToKusto
|
||||
/cacheLogToKusto
|
||||
/logToKustoBlobUri:https://adomessages.blob.core.windows.net/adomessages
|
||||
/logToKustoIdentityId:6e0959cf-a9ba-4988-bbf1-7facd9deda51
|
||||
|
||||
preBuildSteps:
|
||||
# Get bootstrap
|
||||
- task: DownloadPipelineArtifact@2
|
||||
displayName: 'Download bootstrap'
|
||||
inputs:
|
||||
artifactName: Dev
|
||||
# Use the path that --deploy-dev would use, because we --use-dev in the validation
|
||||
targetPath: "$(Build.SourcesDirectory)/Out/Selfhost/Dev"
|
||||
|
|
|
|||
|
|
@ -21,12 +21,8 @@ parameters:
|
|||
# We pass xunit semaphore `/p:[Sdk.BuildXL]xunitSemaphoreCount=8` to limit the number of parallel xunit pips.
|
||||
# Too many xunit pips running in parallel can cause the long running ones to hang.
|
||||
default: '
|
||||
--shared-comp
|
||||
/ado
|
||||
/cacheMiss:"[Bxl.Selfhost.Linux]"
|
||||
/logObservedFileAccesses
|
||||
/logoutput:FullOutputOnError
|
||||
/logsToRetain:10
|
||||
/p:[Sdk.BuildXL]xunitSemaphoreCount=8
|
||||
/forceAddExecutionPermission-
|
||||
'
|
||||
|
|
@ -50,16 +46,15 @@ extends:
|
|||
# Build and test selfhost with BuildXL
|
||||
- template: /.azdo/linux/job-bootstrap.yml@self
|
||||
parameters:
|
||||
BxlCommonArgs: ${{ parameters.BxlCommonArgs }}
|
||||
BxlCommonArgs: --shared-comp ${{ parameters.BxlCommonArgs }}
|
||||
|
||||
- stage: Build_Public
|
||||
displayName: Public validation
|
||||
- stage: Build_External
|
||||
displayName: External validation
|
||||
dependsOn: []
|
||||
jobs:
|
||||
- template: /.azdo/linux/job-public.yml@self
|
||||
- template: /.azdo/linux/job-external.yml@self
|
||||
parameters:
|
||||
BxlCommonArgs: ${{ parameters.BxlCommonArgs }}
|
||||
ValidationName: PublicRelease
|
||||
BxlCommonArgs: --shared-comp ${{ parameters.BxlCommonArgs }}
|
||||
|
||||
- stage: Build_Internal
|
||||
displayName: Internal validation
|
||||
|
|
@ -67,22 +62,8 @@ extends:
|
|||
jobs:
|
||||
- template: /.azdo/linux/job-selfhost.yml@self
|
||||
parameters:
|
||||
BxlCommonArgs: ${{ parameters.BxlCommonArgs }}
|
||||
ValidationName: InternalRelease
|
||||
BxlExtraArgs: --internal /q:ReleaseLinux /q:ReleaseLinuxNet8
|
||||
Role: Orchestrator
|
||||
BxlCommonArgs: --shared-comp ${{ parameters.BxlCommonArgs }}
|
||||
|
||||
- stage: Build_Internal_Workers
|
||||
displayName: "[Workers] Internal validation"
|
||||
dependsOn: Bootstrap_Internal
|
||||
jobs:
|
||||
- template: /.azdo/linux/job-selfhost.yml@self
|
||||
parameters:
|
||||
BxlCommonArgs: ${{ parameters.BxlCommonArgs }}
|
||||
ValidationName: InternalRelease
|
||||
BxlExtraArgs: --internal /q:ReleaseLinux /q:ReleaseLinuxNet8
|
||||
Role: Worker
|
||||
|
||||
- stage: Verify_PTrace
|
||||
displayName: PTrace validation
|
||||
dependsOn: Bootstrap_Internal
|
||||
|
|
|
|||
|
|
@ -60,10 +60,6 @@ function setBxlCmdArgs {
|
|||
)
|
||||
else
|
||||
g_adoBuildRunnerCmdArgs+=(
|
||||
"/cacheConfigLogGeneratedConfiguration"
|
||||
"/cacheConfigStorageAccountEndpoint:https://l3bxlselfhost.blob.core.windows.net"
|
||||
"/cacheConfigManagedIdentityId:eb694749-b1d6-45bc-b7af-2bd81603968a"
|
||||
"/cacheConfigCacheType:EphemeralDatacenterWide"
|
||||
"${arg_Runner[@]}"
|
||||
)
|
||||
fi
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче