Sync eng/common directory with azure-sdk-tools for PR 8929 (#37240)

* Refactor prepare-pipelines

* Convert pipeling generation single to use template

* Convert to pwsh for setting the variables

* Fix switch expression

* Add path separator in front of devopspath

* Continue on error for individual step failures

* Remove unnecessary parameters

---------

Co-authored-by: Wes Haggard <Wes.Haggard@microsoft.com>
Co-authored-by: Wes Haggard <weshaggard@users.noreply.github.com>

eng/common/pipelines/templates/jobs/prepare-pipelines.yml

@@ -2,29 +2,41 @@ parameters:
   - name: Repository
     type: string
     default: $(Build.Repository.Name)
-  - name: Prefix
+  - name: RepositoryPath
     type: string
-  - name: CIConventionOptions
-    type: string
-    default: ''
-  - name: UPConventionOptions
-    type: string
-    default: ''
-  - name: TestsConventionOptions
-    type: string
-    default: ''
-  - name: GenerateUnifiedWeekly
-    type: boolean
-    default: false
+    default: 'sdk'
 
 jobs:
-- job: PreparePipelines
+- job:
+  displayName: 'Prepare pipelines for ${{ parameters.Repository }}'
   pool:
-    name: azsdk-pool-mms-ubuntu-2204-general
-    vmImage: ubuntu-22.04
+    name: $(LINUXPOOL)
+    vmImage: $(LINUXVMIMAGE)
+
+  variables:
+    RepositoryPath: "$(System.DefaultWorkingDirectory)/${{ parameters.RepositoryPath }}"
+    ProjectForPRValidation: 'public'
+    Prefix: 'na'
+    DevOpsPath: 'na'
+    InternalVariableGroups: ''
+    GenerateUnifiedWeekly: ''
+    TestVariableGroups: ''
+    
+    # Standard set of variable groups for the Azure SDK repositories
+    AzureSDK_Maven_Release_Pipeline_Secrets: 1
+    AzureSDK_Nuget_Release_Pipeline_Secrets: 13
+    AzureSDK_CocoaPods_Release_Pipeline_Secrets: 98
+    NPM_Registry_Authentication: 24
+    Release_Secrets_for_GitHub: 58
+    Secrets_for_Resource_Provisioner: 64
+    APIReview_AutoCreate_Configurations: 93
+
   steps:
     - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml
       parameters:
+        Repositories:
+          - Name: ${{ parameters.Repository }}
+            WorkingDirectory: $(System.DefaultWorkingDirectory)
         Paths:
           - 'sdk/**/*.yml'
           - '!sdk/**/test-recordings/*'
@@ -33,183 +45,182 @@ jobs:
           - '!sdk/**/session-records/*'
     - template: /eng/common/pipelines/templates/steps/install-pipeline-generation.yml
     - template: /eng/common/pipelines/templates/steps/set-default-branch.yml
-    # This covers our public repos.
-    - ${{ if not(endsWith(parameters.Repository, '-pr'))}}:
-      - task: AzureCLI@2
-        inputs:
-          azureSubscription: 'opensource-api-connection'
-          scriptType: pscore
-          scriptLocation: inlineScript
-          inlineScript: >
-            $(Pipeline.Workspace)/pipeline-generator/pipeline-generator generate
-            --organization azure-sdk
-            --project public
-            --prefix ${{parameters.Prefix}}
-            --devopspath "\${{parameters.Prefix}}"
-            --path $(System.DefaultWorkingDirectory)/sdk
-            --endpoint Azure
-            --repository ${{parameters.Repository}}
-            --convention ci
-            --agentpool Hosted
-            --branch refs/heads/$(DefaultBranch)
-            --set-managed-variables
-            --debug
-            ${{parameters.CIConventionOptions}}
-        displayName: Create CI Pipelines for Public Repository
-      - task: AzureCLI@2
-        inputs:
-          azureSubscription: 'opensource-api-connection'
-          scriptType: pscore
-          scriptLocation: inlineScript
-          inlineScript: >
-            $(Pipeline.Workspace)/pipeline-generator/pipeline-generator generate
-            --organization azure-sdk
-            --project internal
-            --prefix ${{parameters.Prefix}}
-            --devopspath "\${{parameters.Prefix}}"
-            --path $(System.DefaultWorkingDirectory)/sdk
-            --endpoint Azure
-            --repository ${{parameters.Repository}}
-            --convention up
-            --agentpool Hosted
-            --branch refs/heads/$(DefaultBranch)
-            --set-managed-variables
-            --debug
-            ${{parameters.UPConventionOptions}}
-        displayName: Create UP Pipelines for Public Repository
-      - task: AzureCLI@2
-        inputs:
-          azureSubscription: 'opensource-api-connection'
-          scriptType: pscore
-          scriptLocation: inlineScript
-          inlineScript: >
-            $(Pipeline.Workspace)/pipeline-generator/pipeline-generator generate
-            --organization azure-sdk
-            --project internal
-            --prefix ${{parameters.Prefix}}
-            --devopspath "\${{parameters.Prefix}}"
-            --path $(System.DefaultWorkingDirectory)/sdk
-            --endpoint Azure
-            --repository ${{parameters.Repository}}
-            --convention tests
-            --agentpool Hosted
-            --branch refs/heads/$(DefaultBranch)
-            --set-managed-variables
-            --debug
-            ${{parameters.TestsConventionOptions}}
-        displayName: Create Live Test Pipelines for Public Repository
-        condition: and(succeeded(), ne('${{parameters.TestsConventionOptions}}',''))
-      - task: AzureCLI@2
-        inputs:
-          azureSubscription: 'opensource-api-connection'
-          scriptType: pscore
-          scriptLocation: inlineScript
-          inlineScript: >
-            $(Pipeline.Workspace)/pipeline-generator/pipeline-generator generate
-            --organization azure-sdk
-            --project internal
-            --prefix ${{parameters.Prefix}}
-            --devopspath "\${{parameters.Prefix}}"
-            --path $(System.DefaultWorkingDirectory)/sdk
-            --endpoint Azure
-            --repository ${{parameters.Repository}}
-            --convention testsweekly
-            --agentpool Hosted
-            --branch refs/heads/$(DefaultBranch)
-            --set-managed-variables
-            --debug
-            ${{parameters.TestsConventionOptions}}
-        displayName: Create Weekly (Multi-Cloud) Live Test Pipelines for Public Repository
-        condition: and(succeeded(), ne('${{parameters.TestsConventionOptions}}',''))
-      - task: AzureCLI@2
-        inputs:
-          azureSubscription: 'opensource-api-connection'
-          scriptType: pscore
-          scriptLocation: inlineScript
-          inlineScript: >
-            $(Pipeline.Workspace)/pipeline-generator/pipeline-generator generate
-            --organization azure-sdk
-            --project internal
-            --prefix ${{parameters.Prefix}}
-            --devopspath "\${{parameters.Prefix}}"
-            --path $(System.DefaultWorkingDirectory)/sdk
-            --endpoint Azure
-            --repository ${{parameters.Repository}}
-            --convention upweekly
-            --agentpool Hosted
-            --branch refs/heads/$(DefaultBranch)
-            --set-managed-variables
-            --debug
-            ${{parameters.UPConventionOptions}}
-        displayName: Create Weekly (Multi-Cloud) Unified Test Pipelines for Public Repository
-        condition: and(succeeded(), eq(${{parameters.GenerateUnifiedWeekly}},true))
 
-    # This covers our -pr repositories.
-    - ${{ if endsWith(parameters.Repository, '-pr')}}:
-      - task: AzureCLI@2
-        inputs:
-          azureSubscription: 'opensource-api-connection'
-          scriptType: pscore
-          scriptLocation: inlineScript
-          inlineScript: >
-            $(Pipeline.Workspace)/pipeline-generator/pipeline-generator generate
-            --organization azure-sdk
-            --project internal
-            --prefix ${{parameters.Prefix}}-pr
-            --devopspath "\${{parameters.Prefix}}\pr"
-            --path $(System.DefaultWorkingDirectory)/sdk
-            --endpoint Azure
-            --repository ${{parameters.Repository}}
-            --convention ci
-            --agentpool Hosted
-            --branch refs/heads/$(DefaultBranch)
-            --set-managed-variables
-            --debug
-            --no-schedule
-            ${{parameters.CIConventionOptions}}
-        displayName: Create CI Pipelines for Private Repository
-      - task: AzureCLI@2
-        inputs:
-          azureSubscription: 'opensource-api-connection'
-          scriptType: pscore
-          scriptLocation: inlineScript
-          inlineScript: >
-            $(Pipeline.Workspace)/pipeline-generator/pipeline-generator generate
-            --organization azure-sdk
-            --project internal
-            --prefix ${{parameters.Prefix}}-pr
-            --devopspath "\${{parameters.Prefix}}\pr"
-            --path $(System.DefaultWorkingDirectory)/sdk
-            --endpoint Azure
-            --repository ${{parameters.Repository}}
-            --convention up
-            --agentpool Hosted
-            --branch refs/heads/$(DefaultBranch)
-            --set-managed-variables
-            --debug
-            --no-schedule
-            ${{parameters.UPConventionOptions}}
-        displayName: Create UP Pipelines for Private Repository
-      - task: AzureCLI@2
-        inputs:
-          azureSubscription: 'opensource-api-connection'
-          scriptType: pscore
-          scriptLocation: inlineScript
-          inlineScript: >
-            $(Pipeline.Workspace)/pipeline-generator/pipeline-generator generate
-            --organization azure-sdk
-            --project internal
-            --prefix ${{parameters.Prefix}}-pr
-            --devopspath "\${{parameters.Prefix}}\pr"
-            --path $(System.DefaultWorkingDirectory)/sdk
-            --endpoint Azure
-            --repository ${{parameters.Repository}}
-            --convention tests
-            --agentpool Hosted
-            --branch refs/heads/$(DefaultBranch)
-            --set-managed-variables
-            --debug
-            --no-schedule
-            ${{parameters.TestsConventionOptions}}
-        displayName: Create Live Test Pipelines for Private Repository
-        condition: and(succeeded(), ne('${{parameters.TestsConventionOptions}}',''))
+    - pwsh: | 
+        Write-Host "Setting up pipeline variables"
+        if ("${{ parameters.Repository }}" -match "Azure/azure-sdk-for-(?<prefix>[^-]*)(?<pr>-pr)?") {
+          $prefix = $matches['prefix']
+          $devOpsPath = "\$prefix"
+          if ($matches['pr']) {
+            $devOpsPath = "${devOpsPath}\pr"
+            $prefix = "${prefix}-pr"
+            Write-Host "##vso[task.setvariable variable=ProjectForPRValidation]internal"
+          }
+          Write-Host "Prefix = $prefix"
+          Write-Host "DevOpsPath = $devOpsPath"
+          Write-Host "##vso[task.setvariable variable=Prefix]$prefix"
+          Write-Host "##vso[task.setvariable variable=DevOpsPath]$devOpsPath"
+        }
+        else {
+          Write-Error "Repository name '${{ parameters.Repository }}' is not in the expected format."
+          exit 1  
+        }
+
+        # Map the language prefix to the appropriate variable groups
+        switch ($prefix)
+        {
+          "java" {
+            $internalVariableGroups = '$(AzureSDK_Maven_Release_Pipeline_Secrets) $(Release_Secrets_for_GitHub) $(APIReview_AutoCreate_Configurations)'
+            $testVariableGroups = '$(Secrets_for_Resource_Provisioner)'
+          }
+          "android" {
+            $internalVariableGroups = '$(AzureSDK_Maven_Release_Pipeline_Secrets) $(Release_Secrets_for_GitHub) $(APIReview_AutoCreate_Configurations)'
+          }
+          "js" {
+            $internalVariableGroups = '$(NPM_Registry_Authentication) $(Release_Secrets_for_GitHub) $(APIReview_AutoCreate_Configurations)'
+            $testVariableGroups = '$(Secrets_for_Resource_Provisioner)'
+          }
+          "python" {
+            $internalVariableGroups = '$(Release_Secrets_for_GitHub) $(APIReview_AutoCreate_Configurations)'
+            $testVariableGroups = '$(Secrets_for_Resource_Provisioner)'
+          }
+          "net" {
+            $internalVariableGroups = '$(AzureSDK_Nuget_Release_Pipeline_Secrets) $(Release_Secrets_for_GitHub) $(APIReview_AutoCreate_Configurations)'
+            $testVariableGroups = '$(Secrets_for_Resource_Provisioner)'
+          }
+          "cpp" {
+            $internalVariableGroups = '$(Release_Secrets_for_GitHub) $(APIReview_AutoCreate_Configurations) $(Secrets_for_Resource_Provisioner)'
+          }
+          "ios" {
+            $internalVariableGroups = '$(Release_Secrets_for_GitHub) $(APIReview_AutoCreate_Configurations) $(Secrets_for_Resource_Provisioner) $(AzureSDK_CocoaPods_Release_Pipeline_Secrets)'
+          }
+          "go" {
+            $internalVariableGroups = '$(Release_Secrets_for_GitHub) $(APIReview_AutoCreate_Configurations) $(Secrets_for_Resource_Provisioner)'
+            $generateUnifiedWeekly = 'true'
+          }
+          default {
+            Write-Error "Prefix '$prefix' is not recognized."
+            exit 1
+          }
+        }
+
+        Write-Host "InternalVariableGroups = $internalVariableGroups"
+        Write-Host "TestVariableGroups = $testVariableGroups"
+        Write-Host "GenerateUnifiedWeekly = $generateUnifiedWeekly"
+        Write-Host "##vso[task.setvariable variable=InternalVariableGroups]$internalVariableGroups"
+        Write-Host "##vso[task.setvariable variable=TestVariableGroups]$testVariableGroups"
+        Write-Host "##vso[task.setvariable variable=GenerateUnifiedWeekly]$generateUnifiedWeekly"
+      displayName: Setup pipeline generation variables
+
+    - task: AzureCLI@2
+      inputs:
+        azureSubscription: 'opensource-api-connection'
+        scriptType: pscore
+        scriptLocation: inlineScript
+        inlineScript: >
+          $(Pipeline.Workspace)/pipeline-generator/pipeline-generator generate
+          --organization azure-sdk
+          --project $(ProjectForPRValidation)
+          --prefix $(Prefix)
+          --devopspath "$(DevOpsPath)"
+          --path $(RepositoryPath)
+          --endpoint Azure
+          --repository ${{ parameters.Repository }}
+          --convention ci
+          --agentpool Hosted
+          --branch refs/heads/$(DefaultBranch)
+          --set-managed-variables
+          --debug
+      displayName: 'Generate ci pipelines for: ${{ parameters.Repository }}'
+      continueOnError: true
+    - task: AzureCLI@2
+      inputs:
+        azureSubscription: 'opensource-api-connection'
+        scriptType: pscore
+        scriptLocation: inlineScript
+        inlineScript: >
+          $(Pipeline.Workspace)/pipeline-generator/pipeline-generator generate
+          --organization azure-sdk
+          --project internal
+          --prefix $(Prefix)
+          --devopspath "$(DevOpsPath)"
+          --path $(RepositoryPath)
+          --endpoint Azure
+          --repository ${{ parameters.Repository }}
+          --convention up
+          --agentpool Hosted
+          --branch refs/heads/$(DefaultBranch)
+          --set-managed-variables
+          --debug
+          --variablegroups $(InternalVariableGroups) $(TestVariableGroups)
+      displayName: 'Generate release pipelines for: ${{ parameters.Repository }}'
+      continueOnError: true
+    - task: AzureCLI@2
+      inputs:
+        azureSubscription: 'opensource-api-connection'
+        scriptType: pscore
+        scriptLocation: inlineScript
+        inlineScript: >
+          $(Pipeline.Workspace)/pipeline-generator/pipeline-generator generate
+          --organization azure-sdk
+          --project internal
+          --prefix $(Prefix)
+          --devopspath "$(DevOpsPath)"
+          --path $(RepositoryPath)
+          --endpoint Azure
+          --repository ${{ parameters.Repository }}
+          --convention tests
+          --agentpool Hosted
+          --branch refs/heads/$(DefaultBranch)
+          --set-managed-variables
+          --debug
+          --variablegroups $(TestVariableGroups)
+      displayName: 'Generate test pipelines for: ${{ parameters.Repository }}'
+      condition: and(succeeded(), ne(variables['TestVariableGroups'],''))
+      continueOnError: true
+    - task: AzureCLI@2
+      inputs:
+        azureSubscription: 'opensource-api-connection'
+        scriptType: pscore
+        scriptLocation: inlineScript
+        inlineScript: >
+          $(Pipeline.Workspace)/pipeline-generator/pipeline-generator generate
+          --organization azure-sdk
+          --project internal
+          --prefix $(Prefix)
+          --devopspath "$(DevOpsPath)"
+          --path $(RepositoryPath)
+          --endpoint Azure
+          --repository ${{ parameters.Repository }}
+          --convention testsweekly
+          --agentpool Hosted
+          --branch refs/heads/$(DefaultBranch)
+          --set-managed-variables
+          --debug
+          --variablegroups $(TestVariableGroups)
+      displayName: 'Generate weekly test pipelines (multi-cloud) for: ${{ parameters.Repository }}'
+      condition: and(succeeded(), ne(variables['TestVariableGroups'],''))
+      continueOnError: true
+    - task: AzureCLI@2
+      inputs:
+        azureSubscription: 'opensource-api-connection'
+        scriptType: pscore
+        scriptLocation: inlineScript
+        inlineScript: >
+          $(Pipeline.Workspace)/pipeline-generator/pipeline-generator generate
+          --organization azure-sdk
+          --project internal
+          --prefix $(Prefix)
+          --devopspath "$(DevOpsPath)"
+          --path $(RepositoryPath)
+          --endpoint Azure
+          --repository ${{ parameters.Repository }}
+          --convention upweekly
+          --agentpool Hosted
+          --branch refs/heads/$(DefaultBranch)
+          --set-managed-variables
+          --debug
+          --variablegroups $(InternalVariableGroups) $(TestVariableGroups)
+      displayName: 'Generate weekly unified test pipelines (multi-cloud) for: ${{ parameters.Repository }}'
+      condition: and(succeeded(), ne(variables['GenerateUnifiedWeekly'],''))
+      continueOnError: true