Updates for custom build agents (#10)

CI/CI.yaml

@@ -38,9 +38,8 @@ stages:
       # Official Builds
       - template: "templates/BuildAndTest.job_template.yaml"
         parameters:
-          # agent_pool: AI-Pytorch-Win-CPU
+          agent_pool: FeaturizersLibrary
-          # agent_pool_is_custom: true
+          agent_pool_is_custom: true
-          agent_pool: windows-2019
           configuration: x64
           esrp_connected_service_name: "ESRP CodeSigning Connection"
           operating_system: Windows
@@ -48,9 +47,8 @@ stages:
 
       - template: "templates/BuildAndTest.job_template.yaml"
         parameters:
-          # agent_pool: AI-Pytorch-Win-CPU
+          agent_pool: FeaturizersLibrary
-          # agent_pool_is_custom: true
+          agent_pool_is_custom: true
-          agent_pool: windows-2019
           configuration: x86
           esrp_connected_service_name: "ESRP CodeSigning Connection"
           operating_system: Windows
@@ -75,9 +73,8 @@ stages:
       # When code coverage is enabled, we only test debug builds. Execute release tests to ensure that everything is working.
       - template: "templates/BuildAndTest.job_template.yaml"
         parameters:
-          # agent_pool: AI-Pytorch-Win-CPU
+          agent_pool: FeaturizersLibrary
-          # agent_pool_is_custom: true
+          agent_pool_is_custom: true
-          agent_pool: windows-2019
           configuration: x64
           job_suffix: " (test only)"
           operating_system: Windows
@@ -86,17 +83,15 @@ stages:
 
       - template: "templates/BuildAndTest.job_template.yaml"
         parameters:
-          # agent_pool: AI-Pytorch-Win-CPU
+          agent_pool: FeaturizersLibrary
-          # agent_pool_is_custom: true
+          agent_pool_is_custom: true
-          agent_pool: windows-2019
           configuration: x64_MSVC
           operating_system: Windows
 
       - template: "templates/BuildAndTest.job_template.yaml"
         parameters:
-          # agent_pool: AI-Pytorch-Win-CPU
+          agent_pool: FeaturizersLibrary
-          # agent_pool_is_custom: true
+          agent_pool_is_custom: true
-          agent_pool: windows-2019
           configuration: x86_MSVC
           operating_system: Windows
 
@@ -124,11 +119,11 @@ stages:
     jobs:
       - template: "templates/_job_template.yaml"
         parameters:
-          # agent_pool: AI-Pytorch-Win-CPU
+          agent_pool: FeaturizersLibrary
-          # agent_pool_is_custom: true
+          agent_pool_is_custom: true
-          agent_pool: windows-2019
           job_name: Package_Job
           job_display_name: "Package"
+          operating_system: Windows
           steps:
             - template: "BuildAndTest.package_steps_template.yaml"
               parameters:
@@ -138,11 +133,11 @@ stages:
 
       - template: "templates/_job_template.yaml"
         parameters:
-          # agent_pool: AI-Pytorch-Win-CPU
+          agent_pool: FeaturizersLibrary
-          # agent_pool_is_custom: true
+          agent_pool_is_custom: true
-          agent_pool: windows-2019
           job_name: Documentation_Job
           job_display_name: "Generate Documentation"
+          operating_system: Windows
           steps:
             - template: "BuildAndTest.documentation_steps_template.yaml"
               parameters:

CI/PR.yaml

@@ -27,18 +27,16 @@ stages:
       # Official Builds
       - template: "templates/BuildAndTest.job_template.yaml"
         parameters:
-          # agent_pool: AI-Pytorch-Win-CPU
+          agent_pool: FeaturizersLibrary
-          # agent_pool_is_custom: true
+          agent_pool_is_custom: true
-          agent_pool: windows-2019
           configuration: x64
           operating_system: Windows
           tester_code_coverage: True
 
       - template: "templates/BuildAndTest.job_template.yaml"
         parameters:
-          # agent_pool: AI-Pytorch-Win-CPU
+          agent_pool: FeaturizersLibrary
-          # agent_pool_is_custom: true
+          agent_pool_is_custom: true
-          agent_pool: windows-2019
           configuration: x86
           operating_system: Windows
 
@@ -60,9 +58,8 @@ stages:
       # When code coverage is enabled, we only test debug builds. Execute release tests to ensure that everything is working.
       - template: "templates/BuildAndTest.job_template.yaml"
         parameters:
-          # agent_pool: AI-Pytorch-Win-CPU
+          agent_pool: FeaturizersLibrary
-          # agent_pool_is_custom: true
+          agent_pool_is_custom: true
-          agent_pool: windows-2019
           configuration: x64
           job_suffix: " (test only)"
           operating_system: Windows
@@ -71,17 +68,15 @@ stages:
 
       - template: "templates/BuildAndTest.job_template.yaml"
         parameters:
-          # agent_pool: AI-Pytorch-Win-CPU
+          agent_pool: FeaturizersLibrary
-          # agent_pool_is_custom: true
+          agent_pool_is_custom: true
-          agent_pool: windows-2019
           configuration: x64_MSVC
           operating_system: Windows
 
       - template: "templates/BuildAndTest.job_template.yaml"
         parameters:
-          # agent_pool: AI-Pytorch-Win-CPU
+          agent_pool: FeaturizersLibrary
-          # agent_pool_is_custom: true
+          agent_pool_is_custom: true
-          agent_pool: windows-2019
           configuration: x86_MSVC
           operating_system: Windows
 
@@ -109,11 +104,11 @@ stages:
     jobs:
       - template: "templates/_job_template.yaml"
         parameters:
-          # agent_pool: AI-Pytorch-Win-CPU
+          agent_pool: FeaturizersLibrary
-          # agent_pool_is_custom: true
+          agent_pool_is_custom: true
-          agent_pool: windows-2019
           job_name: Package_Job
           job_display_name: "Package"
+          operating_system: Windows
           steps:
             - template: "BuildAndTest.package_steps_template.yaml"
               parameters:

CI/templates/BuildAndTest.job_template.yaml

@@ -22,6 +22,7 @@ jobs:
       agent_pool_is_custom: ${{ parameters.agent_pool_is_custom }}
       job_name: BuildAndTest_${{ parameters.operating_system }}_${{ parameters.configuration }}_${{ parameters.agent_pool_container }}_${{ parameters.test_only }}_Job
       job_display_name: "${{ parameters.operating_system }} - ${{ parameters.configuration }}${{ parameters.job_suffix }}"
+      operating_system: ${{ parameters.operating_system }}
       steps:
         - template: BuildAndTest.steps_template.yaml
           parameters:

CI/templates/BuildAndTest.prologue_stage_template.yaml

@@ -16,14 +16,12 @@ stages:
     jobs:
       - template: "_job_template.yaml"
         parameters:
-          # agent_pool: AI-Pytorch-Win-CPU
+          agent_pool: FeaturizersLibrary
-          # agent_pool_is_custom: true
+          agent_pool_is_custom: true
-          agent_pool: windows-2019
           job_name: "Globals_Job"
           job_display_name: "Global Variables"
+          operating_system: Windows
           steps:
-            - checkout: none
-
             - task: PythonScript@0
               displayName: "[IMPL] Create JSON"
 
@@ -34,6 +32,7 @@ stages:
                 script: |-
                   import datetime
                   import json
+                  import subprocess
                   import sys
 
                   from collections import OrderedDict
@@ -42,6 +41,24 @@ stages:
 
                   values["is_release_build"] = "${{ parameters.release_build }}".lower() in ["true", "yes"]
 
+                  # Get the current git branch. This is a bit more complicated that normal, as the agent will
+                  # be in a detacted head state.
+                  git_output = subprocess.check_output(
+                      "git show -s --pretty=%d HEAD",
+                      shell=True,
+                  ).decode("utf-8").strip()
+
+                  # The output will be something like "(HEAD, origin/master)"
+                  git_output = git_output.split(",")
+                  assert len(git_output) == 2, git_output
+
+                  git_output = git_output[1]
+                  assert git_output.endswith(")"), git_output
+                  git_output = git_output[:-1]
+
+                  # Only take the last component of the branch name
+                  git_branch = git_output.split("/")[-1]
+
                   # the intention here is to compare if var == '$(var)', but as the right side of == will be a string of
                   # "$(var)" if var is empty, or the exact value which is equal to var if var is assigned the value.
                   # Therefore, a non-standard comparison is necessary in this case
@@ -72,6 +89,10 @@ stages:
                               now.second,
                           )
 
+                  if build_name and git_branch != "master":
+                      build_name = "%s-%s" % (git_branch, build_name)
+
+                  values["git_branch"] = git_branch
                   values["is_prerelease_build"] = is_prerelease_build
                   values["build_name"] = build_name
 

CI/templates/_job_template.yaml

@@ -7,6 +7,7 @@ parameters:
   agent_pool_is_custom: False # Set this value to True if the agent is part of a custom pool
   job_name: ""
   job_display_name: ""
+  operating_system: "" # Windows|Linux|MacOS
   steps: []
   timeout_in_minutes: 240
 
@@ -16,18 +17,39 @@ parameters:
 jobs:
   - ${{ if eq(parameters.agent_pool_container, '') }}:
       - ${{ if eq(parameters.agent_pool_is_custom, 'true') }}:
-          - job: ${{ parameters.job_name }}
+          - ${{ if eq(parameters.operating_system, 'Windows') }}:
-            displayName: ${{ parameters.job_display_name }}
+              - job: ${{ parameters.job_name }}
+                displayName: ${{ parameters.job_display_name }}
 
-            timeoutInMinutes: ${{ parameters.timeout_in_minutes }}
+                timeoutInMinutes: ${{ parameters.timeout_in_minutes }}
 
-            pool: "${{ parameters.agent_pool }}"
+                pool:
+                  name: "${{ parameters.agent_pool }}"
+                  demands:
+                      - Agent.OS -equals Windows_NT
 
-            workspace:
+                workspace:
-              clean: All
+                  clean: All
 
-            steps:
+                steps:
-              ${{ parameters.steps }}
+                  ${{ parameters.steps }}
+
+          - ${{ if eq(parameters.operating_system, 'Linux') }}:
+              - job: ${{ parameters.job_name }}
+                displayName: ${{ parameters.job_display_name }}
+
+                timeoutInMinutes: ${{ parameters.timeout_in_minutes }}
+
+                pool:
+                  name: "${{ parameters.agent_pool }}"
+                  demands:
+                      - Agent.OS -equals Linux
+
+                workspace:
+                  clean: All
+
+                steps:
+                  ${{ parameters.steps }}
 
       - ${{ if not(eq(parameters.agent_pool_is_custom, 'true')) }}:
           - job: ${{ parameters.job_name }}