Merged PR 753269: [Linux PR validation] Make public validation fully public and use 1ESPT for distributed build

- Make the public validation public end-to-end, by building a public engine and then using that to run tests (also without passing `--internal`) - Use the 1ESPT build job for the distributed build test. We're still keeping the old YML for backup, but it becomes unused. - Note that using 1ESPT for our own selfhost is not possible due to how we invoke `bxl.sh`, that's why that one still adds the worker stage manually.
2023-11-28 23:20:17 +00:00 · 2023-11-28 23:20:17 +00:00 · 84a8ae1dea
--- a/.azdo/linux/build-selfhost-base.yml
+++ b/.azdo/linux/build-selfhost-base.yml
@ -65,8 +65,4 @@ steps:
  displayName: Bind /home/subst to sources directory  

 - ${{ each step in parameters.buildSteps }}:
-  - ${{ step }}
-
- bash: sudo umount -v --lazy /home/subst
-  condition: always()
-  displayName: Unmount /home/subst
+  - ${{ step }}
--- a/.azdo/linux/job-bootstrap.yml
+++ b/.azdo/linux/job-bootstrap.yml
@ -45,7 +45,7 @@ jobs:
      - bash: |
          set -eu
          # Set a 60m timeout so we can catch hangs *and* get logs collected at the same time. Otherwise the whole job will timeout (check 'timeoutInMinutes' above).
-          timeout --signal 9 60m bash bxl.sh /logsDirectory:"Out/Logs/Build" --minimal --internal --deploy-dev --release /forceAddExecutionPermission- /cacheConfigFilePath:Out/CacheConfig.json ${{ parameters.BxlCommonArgs }}
+          timeout --signal 9 60m bash bxl.sh /logsDirectory:"Out/Logs/Build" --minimal --internal --deploy-dev --release /cacheConfigFilePath:Out/CacheConfig.json ${{ parameters.BxlCommonArgs }}
        displayName: Build
        workingDirectory: /home/subst
        env:
--- a/.azdo/linux/job-distributed-1espt.yml
+++ b/.azdo/linux/job-distributed-1espt.yml
@ -0,0 +1,81 @@
+jobs:
+- job: TestDistributed
+  displayName: Test distributed build
+
+  pool:
+    name: BuildXL-DevOpsAgents-Linux-Stateless-PME
+    os: linux
+
+  variables:
+    - name: BootstrapLocation
+      value: $(System.DefaultWorkingDirectory)/Bootstrap
+
+  timeoutInMinutes: 20
+
+  templateContext:
+    type: buildJob
+    workflow: BuildXL
+    distribution:
+      enabled: true
+      workerCount: 1
+    outputs:
+    - output: pipelineArtifact
+      targetPath: $(Build.SourcesDirectory)/Out/Logs
+      artifactName: BuildXL-Logs.DistributedTest.$(System.JobName).$(System.StageName)
+      condition: always()
+      continueOnError: true
+      displayName: Upload Distributed test Logs
+    sdl:
+      credscan:
+        enabled: false
+      antimalwareScan:
+        enabled: false
+      publishLogs:
+        enabled: false
+    
+    # Pre-build steps: get the engine & prepare workspace
+    restoreSteps:
+    - task: DownloadPipelineArtifact@2
+      inputs:
+        buildType: current
+        artifactName: Dev
+        targetPath: $(BootstrapLocation)
+      displayName: Download BuildXL
+
+    - bash: |
+        sudo mkdir /home/subst
+        sudo mount --verbose --bind $(Build.SourcesDirectory) /home/subst
+      displayName: Bind /home/subst to sources directory  
+
+    - bash: |
+        set -eu
+        chmod +x $(BootstrapLocation)/AdoBuildRunner
+        chmod +x $(BootstrapLocation)/bxl
+      displayName: Set execution bits
+    
+    # Build parameters
+    buildXL:
+      invocationKey: DistributedTest
+
+      # invoker: exe means we need to provide the directory where AdoBuildRunner is located
+      invoker: exe
+      engineLocation: $(BootstrapLocation)
+      
+      arguments: '
+        /c:/home/subst/Private/AdoDistributionTests/config.dsc
+        /logsDirectory:$(Build.SourcesDirectory)/Out/Logs
+        /p:BUILDXL_FINGERPRINT_SALT=*
+        /p:BUILDXL_GRAPH_FINGERPRINT_SALT=*
+        /replicateOutputsToWorkers+ 
+        /p:BuildXLMinimumWaitForRemoteWorkerMin=5
+        '
+      env:
+        TRANSFORMERS_SDK_DIR: /home/subst/Public/Sdk/Public/Transformers
+        OUTPUT_DIR: /home/subst/Out
+
+      cache:
+        storageAccountEndpoint: https://l3bxlselfhost.blob.core.windows.net
+        managedIdentityClientId: eb694749-b1d6-45bc-b7af-2bd81603968a
+        cacheType: EphemeralDatacenterWide
+        logGeneratedConfiguration: true
+        # retentionPolicyInDays: {default}
--- a/.azdo/linux/job-public.yml
+++ b/.azdo/linux/job-public.yml
@ -0,0 +1,78 @@
+parameters:
+- name: BxlCommonArgs # We reuse these across jobs like a constant
+  type: string
+- name: Role
+  type: string
+  default: SingleMachine
+  values:
+  - Orchestrator
+  - Worker
+  - SingleMachine
+- name: ValidationName
+  type: string
+
+jobs:
+- job: Selfhost_${{ parameters.validationName }}
+  displayName: Build and Validate Selfhost (${{ parameters.validationName }})
+
+  pool:
+    name: BuildXL-DevOpsAgents-Linux-Stateless-PME
+    os: linux
+
+  templateContext:
+    inputs:
+    - input: checkout
+      repository: self
+      fetchDepth: 1
+      
+    outputs:
+    - output: pipelineArtifact
+      targetPath: $(Build.SourcesDirectory)/Out/Logs
+      artifactName: BuildXL-Logs.${{ parameters.validationName }}.${{ parameters.Role }}.$(System.JobName).$(System.JobAttempt)
+      condition: always()
+      continueOnError: true
+      displayName: Upload BuildXL Logs
+    sdl:
+      credscan:
+        enabled: false
+      antimalwareScan:
+        enabled: false
+      publishLogs:
+        enabled: false
+
+  timeoutInMinutes: 90
+
+  steps:
+  - template: build-selfhost-base.yml
+    parameters:
+      cacheUniverse: bxlselfhostpublic
+      writeCacheConfigFile: true
+      buildSteps:
+      # 1. Bootstrap public bits
+      - bash: |
+          set -eu
+          # Set a 60m timeout so we can catch hangs *and* get logs collected at the same time. Otherwise the whole job will timeout (check 'timeoutInMinutes' above).
+          timeout --signal 9 60m bash bxl.sh /logsDirectory:"Out/Logs/Build" --minimal --deploy-dev --release /forceAddExecutionPermission- /cacheConfigFilePath:Out/CacheConfig.json ${{ parameters.BxlCommonArgs }}
+        displayName: Build
+        workingDirectory: /home/subst
+
+      # 2. Run tests with the public bits
+      - bash: |
+          set -eu
+          # - the disks on Azure Pipeline VMs are too small to build everything, so let's instead run tests
+          # Set a 60m timeout so we can catch hangs *and* get logs collected at the same time. Otherwise the whole job will timeout (check 'timeoutInMinutes' above).
+          timeout --signal 9 60m ./bxl.sh --use-dev --use-adobuildrunner --runner-arg /cacheConfigUniverse:bxlselfhostpublic ${{ parameters.BxlCommonArgs }} /logsDirectory:"Out/Logs/${{ parameters.validationName }}" /q:ReleaseLinux "/f:tag='test'" /logToKusto /logToKustoBlobUri:https://adomessages.blob.core.windows.net/adomessages /logToKustoIdentityId:6e0959cf-a9ba-4988-bbf1-7facd9deda51 /logToKustoTenantId:975f013f-7f24-47e8-a7d3-abc4752bf346 /dynamicBuildWorkerSlots:0
+        displayName: Test (${{ parameters.validationName }})
+        workingDirectory: /home/subst
+        env:
+          SYSTEM_ACCESSTOKEN: $(System.AccessToken)
+          AdoBuildRunnerInvocationKey: LinuxSelfhostPublicValidation
+          AdoBuildRunnerWorkerPipelineRole: Orchestrator
+
+      - task: PublishTestResults@2
+        displayName: Publish Test Results
+        inputs:
+          testResultsFormat: XUnit
+          testResultsFiles: 'Out/Objects/**/xunit-logs/**/xunit*.xml'
+        condition: always()
+        continueOnError: true
--- a/.azdo/linux/job-selfhost.yml
+++ b/.azdo/linux/job-selfhost.yml
@ -67,11 +67,11 @@ jobs:
          # if we are doing a distributed build, use the ado build runner and configure
          # an extra worker
          if [ "${{ parameters.Role }}" != "SingleMachine" ]; then
-            USE_ADOBUILDRUNNER="--use-adobuildrunner"
+            USE_ADOBUILDRUNNER="--use-adobuildrunner --runner-arg /cacheConfigUniverse:bxlselfhost"
            DYNAMIC_WORKER_SLOTS="/dynamicBuildWorkerSlots:1"
          fi

-          timeout --signal 9 60m ./bxl.sh $USE_ADOBUILDRUNNER --use-dev ${{ parameters.BxlCommonArgs }} /logsDirectory:"Out/Logs/${{ parameters.validationName }}" ${{ parameters.bxlExtraArgs }} "/f:tag='test'" /logToKusto /logToKustoBlobUri:https://adomessages.blob.core.windows.net/adomessages /logToKustoIdentityId:6e0959cf-a9ba-4988-bbf1-7facd9deda51 /logToKustoTenantId:975f013f-7f24-47e8-a7d3-abc4752bf346 /historicMetadataCache- $DYNAMIC_WORKER_SLOTS
+          timeout --signal 9 60m ./bxl.sh $USE_ADOBUILDRUNNER --use-dev ${{ parameters.BxlCommonArgs }} /logsDirectory:"Out/Logs/${{ parameters.validationName }}" ${{ parameters.bxlExtraArgs }} "/f:tag='test'" /logToKusto /logToKustoBlobUri:https://adomessages.blob.core.windows.net/adomessages /logToKustoIdentityId:6e0959cf-a9ba-4988-bbf1-7facd9deda51 /logToKustoTenantId:975f013f-7f24-47e8-a7d3-abc4752bf346 $DYNAMIC_WORKER_SLOTS
        displayName: Test (${{ parameters.validationName }})
        workingDirectory: /home/subst
        env:
--- a/.azdo/linux/pipeline.yml
+++ b/.azdo/linux/pipeline.yml
@ -2,10 +2,15 @@ trigger: none  # Explicitly scheduled for PRs

 resources:
  repositories:
-  - repository: 1esPipelines
+  # Temporarily use the private repo until buildJob with BuildXL gets released 
+  # - repository: 1esPipelines
+  #   type: git
+  #   name: 1ESPipelineTemplates/1ESPipelineTemplates
+  #   ref: refs/tags/release
+  - repository: 1ESPipelines
    type: git
-    name: 1ESPipelineTemplates/1ESPipelineTemplates
-    ref: refs/tags/release
+    name: Domino/1ESPipelines
+    ref: main

 parameters:
 - name: BuildSelfhost
@ -15,7 +20,16 @@ parameters:
  type: string
  # We pass xunit semaphore `/p:[Sdk.BuildXL]xunitSemaphoreCount=8` to limit the number of parallel xunit pips.
  # Too many xunit pips running in parallel can cause the long running ones to hang. 
-  default: --shared-comp /ado /cacheMiss:"[Bxl.Selfhost.Linux]" /logObservedFileAccesses /logoutput:FullOutputOnError /logsToRetain:10 /p:[Sdk.BuildXL]xunitSemaphoreCount=8
+  default: '
+    --shared-comp 
+    /ado 
+    /cacheMiss:"[Bxl.Selfhost.Linux]" 
+    /logObservedFileAccesses 
+    /logoutput:FullOutputOnError 
+    /logsToRetain:10 
+    /p:[Sdk.BuildXL]xunitSemaphoreCount=8 
+    /forceAddExecutionPermission-
+  '
 extends:
  template: v1/1ES.Unofficial.PipelineTemplate.yml@1esPipelines
  parameters:
@ -40,13 +54,12 @@ extends:
    
    - stage: Build_Public
      displayName: Public validation
-      dependsOn: Bootstrap_Internal
+      dependsOn: []
      jobs:
-        - template: /.azdo/linux/job-selfhost.yml@self
+        - template: /.azdo/linux/job-public.yml@self
          parameters:
            BxlCommonArgs: ${{ parameters.BxlCommonArgs }}
            ValidationName: PublicRelease
-            BxlExtraArgs: /q:ReleaseLinux /forceAddExecutionPermission-

    - stage: Build_Internal
      displayName: Internal validation
@ -56,18 +69,18 @@ extends:
        parameters:
          BxlCommonArgs: ${{ parameters.BxlCommonArgs }}
          ValidationName: InternalRelease
-          BxlExtraArgs: --internal /q:ReleaseLinux /forceAddExecutionPermission-
+          BxlExtraArgs: --internal /q:ReleaseLinux
          Role: Orchestrator
    
    - stage: Build_Internal_Workers
-      displayName: Internal validation [Workers]
+      displayName: "[Workers] Internal validation"
      dependsOn: Bootstrap_Internal
      jobs:
      - template: /.azdo/linux/job-selfhost.yml@self
        parameters:
          BxlCommonArgs: ${{ parameters.BxlCommonArgs }}
          ValidationName: InternalRelease
-          BxlExtraArgs: --internal /q:ReleaseLinux /forceAddExecutionPermission-
+          BxlExtraArgs: --internal /q:ReleaseLinux 
          Role: Worker

    - stage: Verify_PTrace
@ -79,19 +92,7 @@ extends:
          BxlCommonArgs: ${{ parameters.BxlCommonArgs }}
    
    - stage: Build_Distributed
-      displayName: Distributed test [Orchestrator]
-      dependsOn: [Build_Public, Build_Internal, Verify_PTrace] 
+      displayName: Distributed test
+      dependsOn: [Build_Internal, Verify_PTrace] 
      jobs:
-      # Build and test selfhost with BuildXL
-      - template: /.azdo/linux/job-distributed.yml@self
-        parameters:
-          Role: Orchestrator
-    
-    - stage: Build_Distributed_Workers
-      displayName: Distributed test [Workers]
-      dependsOn: [Build_Public, Build_Internal, Verify_PTrace]
-      jobs:
-      # Build and test selfhost with BuildXL
-      - template: /.azdo/linux/job-distributed.yml@self
-        parameters:
-          Role: Worker
+      - template: /.azdo/linux/job-distributed-1espt.yml@self
--- a/Public/Src/Sandbox/MacOs/scripts/bxl.sh
+++ b/Public/Src/Sandbox/MacOs/scripts/bxl.sh
@ -15,6 +15,7 @@ declare DEFAULT_CACHE_CONFIG_FILE_NAME=DefaultCacheConfig.json

 declare arg_CacheConfigFile=""
 declare arg_Positional=()
+declare arg_Runner=()
 declare arg_BuildXLBin=""
 declare arg_MainConfig=""
 declare arg_SymlinkSdksInto=""
@ -24,6 +25,7 @@ declare arg_installDaemon=""
 declare arg_useAdoBuildRunner=""

 declare g_bxlCmdArgs=()
+declare g_adoBuildRunnerCmdArgs=()

 # Allow for up to 2MB of thread stack size, frontend evaluation stack frames can easily grow beyond the default stack size,
 # which is PTHREAD_STACK_MIN for the CLR running on Unix systems
@ -56,6 +58,14 @@ function setBxlCmdArgs {
        g_bxlCmdArgs+=(
            "/cacheConfigFilePath:$arg_CacheConfigFile"
        )
+    else
+        g_adoBuildRunnerCmdArgs+=(
+            "/cacheConfigLogGeneratedConfiguration" 
+            "/cacheConfigStorageAccountEndpoint:https://l3bxlselfhost.blob.core.windows.net" 
+            "/cacheConfigManagedIdentityId:eb694749-b1d6-45bc-b7af-2bd81603968a" 
+            "/cacheConfigCacheType:EphemeralDatacenterWide" 
+            "${arg_Runner[@]}"
+        )
    fi

    # all other user-specified args
@ -215,15 +225,8 @@ function build {
    if [[ -n "$arg_useAdoBuildRunner" ]]; then
        local adoBuildRunnerExe="$arg_BuildXLBin/AdoBuildRunner"
        chmod u=rx "$adoBuildRunnerExe" || true
-        print_info "${tputBold}Running AdoBuildRunner:${tputReset} '$adoBuildRunnerExe' ${g_bxlCmdArgs[@]}"
-
-        "$adoBuildRunnerExe" \
-        "/cacheConfigLogGeneratedConfiguration" \
-        "/cacheConfigStorageAccountEndpoint:https://l3bxlselfhost.blob.core.windows.net" \
-        "/cacheConfigManagedIdentityId:eb694749-b1d6-45bc-b7af-2bd81603968a" \
-        "/cacheConfigCacheType:EphemeralDatacenterWide"\
-        "--" \
-        "${g_bxlCmdArgs[@]}"
+        print_info "${tputBold}Running AdoBuildRunner:${tputReset} '$adoBuildRunnerExe' ${g_adoBuildRunnerCmdArgs[@]} -- ${g_bxlCmdArgs[@]}"
+        "$adoBuildRunnerExe" "${g_adoBuildRunnerCmdArgs[@]}" "--" "${g_bxlCmdArgs[@]}"
    else
        print_info "${tputBold}Running bxl:${tputReset} '$bxlExe' ${g_bxlCmdArgs[@]}"

@ -305,6 +308,11 @@ function parseArgs {
                arg_useAdoBuildRunner="1"
                shift
                ;;
+            --runner-arg)
+                arg_Runner+=("$2")
+                shift
+                shift
+                ;;
            *)
                arg_Positional+=("$1")
                shift
--- a/Public/Src/Tools/AdoBuildRunner/Args.cs
+++ b/Public/Src/Tools/AdoBuildRunner/Args.cs
@ -64,6 +64,9 @@ namespace BuildXL.AdoBuildRunner
                    OptionHandlerFactory.CreateOption(
                            "cacheConfigStorageAccountEndpoint",
                            opt => cacheConfigGenerationConfiguration.StorageAccountEndpoint = CommandLineUtilities.ParseUriOption(opt)),
+                    OptionHandlerFactory.CreateOption(
+                            "cacheConfigUniverse",
+                            opt => cacheConfigGenerationConfiguration.Universe = opt.Value),
                }.SelectMany(x => x)
                 .OrderBy(opt => opt.OptionName, StringComparer.OrdinalIgnoreCase)
                 .ToArray();
--- a/bxl.sh
+++ b/bxl.sh
@ -106,7 +106,7 @@ function compileWithBxl() {
 }

 function printHelp() {
-    echo "${BASH_SOURCE[0]} [--deploy-dev] [--use-dev] [--minimal] [--internal] [--release] [--shared-comp] [--vs] [--test-method <full-test-method-name>] [--test-class <full-test-class-name>] <other-arguments>"
+    echo "${BASH_SOURCE[0]} [--deploy-dev] [--use-dev] [--minimal] [--internal] [--release] [--shared-comp] [--vs] [--use-adobuildrunner] [--runner-arg <arg-to-buildrunner>] [--test-method <full-test-method-name>] [--test-class <full-test-class-name>] <other-arguments>"
 }

 function parseArgs() {
@ -158,6 +158,11 @@ function parseArgs() {
            arg_Positional+=("--use-adobuildrunner")
            shift
            ;;
+        --runner-arg)
+            arg_Positional+=("--runner-arg $2")
+            shift
+            shift
+            ;;
        --vs)
            arg_Positional+=(
                "/vs"