Sync eng/common directory with azure-sdk-tools for PR 9147 (#1709)

* Changes required for sdk-repo based emitter pipelines

* Rename template

* Convert emitter archetype to pipeline template

* Move the archetype template out of the stages folder

---------

Co-authored-by: Patrick Hallisey <pahallis@microsoft.com>

eng/common/pipelines/templates/archetype-typespec-emitter.yml

@@ -0,0 +1,440 @@
+parameters:
+# Pool to use for the pipeline stages
+- name: Pool
+  type: object
+  default:
+    name: $(LINUXPOOL)
+    image: $(LINUXVMIMAGE)
+    os: linux
+
+# Whether to build alpha versions of the packages. This is passed as a flag to the build script.
+- name: BuildPrereleaseVersion
+  type: boolean
+  default: true
+
+# Whether to use the `next` version of TypeSpec. This is passed as a flag to the initialize script.
+- name: UseTypeSpecNext
+  type: boolean
+  default: false
+
+# Custom steps to run after the repository is cloned but before other job steps.
+- name: InitializationSteps
+  type: stepList
+  default: []
+
+# Indicates the build matrix to use for post-build autorest validation
+- name: TestMatrix
+  type: object
+  default: {}
+
+# Whether to run the publish and regeneration stages. If false, only the build stage will run.
+- name: ShouldPublish
+  type: boolean
+  default: false
+
+# Whether to publish to npmjs.org.
+- name: PublishPublic
+  type: boolean
+  default: true
+
+# Indicates if the Publish stage should depend on the Test stage
+- name: PublishDependsOnTest
+  type: boolean
+  default: false
+
+# Whether to regenerate sdk clients using the new emitter.
+- name: ShouldRegenrate
+  type: boolean
+  default: false
+
+# Number of jobs to generate. This is the maximum number of jobs that will be generated. The actual number of jobs will be reduced if it would result in fewer than MinimumPerJob packages per job.
+- name: RegenerationJobCount
+  type: number
+  default: 10
+
+# Minimum number of packages to generate per job.
+- name: MinimumPerJob
+  type: number
+  default: 10
+
+# Indicates if regenration matrix should only contain folders with typespec files
+- name: OnlyGenerateTypespec
+  type: boolean
+  default: false
+
+extends:
+  template: /eng/pipelines/templates/stages/1es-redirect.yml
+  parameters:
+    stages:
+    # Build stage
+    # Responsible for building the autorest generator and typespec emitter packages
+    # Produces the artifact `build_artifacts` which contains the following:
+    #   package-versions.json: Contains a map of package name to version for the packages that were built
+    #   overrides.json: Contains npm package version overrides for the emitter and generator
+    #   packages/: Contains the packages to publish
+    #   lock-files/: Contains package.json and package-lock.json files for use in the test stage to ensure tests are run against the correct dependency versions
+    - stage: Build
+      pool: ${{ parameters.Pool }}
+      jobs:
+      - job: Build
+        steps:
+        - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml
+
+        - ${{ parameters.InitializationSteps }}
+
+        - task: PowerShell@2
+          displayName: 'Run initialize script'
+          inputs:
+            pwsh: true
+            filePath: $(Build.SourcesDirectory)/eng/scripts/typespec/Initialize-WorkingDirectory.ps1
+            arguments: -UseTypeSpecNext:$${{ parameters.UseTypeSpecNext }}
+
+        - task: PowerShell@2
+          displayName: 'Run build script'
+          name: ci_build
+          inputs:
+            pwsh: true
+            filePath: $(Build.SourcesDirectory)/eng/scripts/typespec/Build-Emitter.ps1
+            arguments: >
+              -BuildNumber "$(Build.BuildNumber)"
+              -OutputDirectory "$(Build.ArtifactStagingDirectory)"
+              -TargetNpmJsFeed:$${{ parameters.PublishPublic }}
+              -Prerelease:$${{ parameters.BuildPrereleaseVersion }}
+
+        - pwsh: |
+            $sourceBranch = '$(Build.SourceBranch)'
+            $buildReason = '$(Build.Reason)'
+            $buildNumber = '$(Build.BuildNumber)'
+
+            if ($buildReason -eq 'Schedule') {
+              $branchName = 'validate-typespec-scheduled'
+            } elseif ($sourceBranch -match "^refs/pull/(\d+)/(head|merge)$") {
+              $branchName = "validate-typespec-pr-$($Matches[1])"
+            } else {
+              $branchName = "validate-typespec-$buildNumber"
+            }
+
+            Write-Host "Setting variable 'branchName' to '$branchName'"
+            Write-Host "##vso[task.setvariable variable=branchName;isOutput=true]$branchName"
+          displayName: Set branch name
+          name: set_branch_name
+
+        - template: /eng/common/pipelines/templates/steps/publish-1es-artifact.yml
+          parameters:
+            artifactName: build_artifacts
+            artifactPath: $(Build.ArtifactStagingDirectory)
+
+    # Publish stage
+    # Responsible for publishing the packages in `build_artifacts/packages` and producing `emitter-package-lock.json`
+    # Produces the artifact `publish_artifacts` which contains the following:
+    #   emitter-package.json: Created using the package json from the build step. 
+    #   emitter-package-lock.json: Created by calling `npm install` using `emitter-package.json`
+    - ${{ if parameters.ShouldPublish }}:
+      - stage: Publish
+        dependsOn:
+        - Build
+        - ${{ if and(parameters.PublishDependsOnTest, ne(length(parameters.TestMatrix), 0)) }}:
+          - Test
+        variables:
+          buildArtifactsPath: $(Pipeline.Workspace)/build_artifacts
+        pool: ${{ parameters.Pool }}
+        jobs:
+        - job: Publish
+          steps:
+          - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml
+
+          - download: current
+            artifact: build_artifacts
+            displayName: Download build artifacts
+
+          # Create authenticated .npmrc file for publishing to devops
+          - template: /eng/common/pipelines/templates/steps/create-authenticated-npmrc.yml
+            parameters:
+              npmrcPath: $(buildArtifactsPath)/packages/.npmrc
+              registryUrl: https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-js-test-autorest/npm/registry/
+
+          # publish to devops feed
+          - pwsh: |
+              $packageFiles = Get-ChildItem -Path . -Filter '*.tgz'
+              foreach ($file in $packageFiles.Name) {
+                Write-Host "npm publish $file --verbose --access public"
+                npm publish $file --verbose --access public
+              }
+            displayName: Publish to DevOps feed
+            workingDirectory: $(buildArtifactsPath)/packages
+
+          - ${{ if parameters.PublishPublic }}:
+            # publish to npmjs.org using ESRP
+            - task: EsrpRelease@7
+              inputs:
+                displayName: Publish to npmjs.org
+                ConnectedServiceName: Azure SDK Engineering System
+                ClientId: 5f81938c-2544-4f1f-9251-dd9de5b8a81b
+                KeyVaultName: AzureSDKEngKeyVault
+                AuthCertName: azure-sdk-esrp-release-auth-certificate
+                SignCertName: azure-sdk-esrp-release-sign-certificate
+                Intent: PackageDistribution
+                ContentType: npm
+                FolderLocation: $(buildArtifactsPath)/packages
+                Owners: ${{ coalesce(variables['Build.RequestedForEmail'], 'azuresdk@microsoft.com') }}
+                Approvers: ${{ coalesce(variables['Build.RequestedForEmail'], 'azuresdk@microsoft.com') }}
+                ServiceEndpointUrl: https://api.esrp.microsoft.com
+                MainPublisher: ESRPRELPACMANTEST
+                DomainTenantId: 72f988bf-86f1-41af-91ab-2d7cd011db47
+
+          - task: PowerShell@2
+            displayName: Create emitter-package.json
+            inputs:
+              pwsh: true
+              filePath: ./eng/common/scripts/typespec/New-EmitterPackageJson.ps1
+              arguments: >
+                -PackageJsonPath '$(buildArtifactsPath)/lock-files/package.json'
+                -OverridesPath '$(buildArtifactsPath)/overrides.json'
+                -OutputDirectory '$(Build.ArtifactStagingDirectory)'
+              workingDirectory: $(Build.SourcesDirectory)
+
+          - task: PowerShell@2
+            displayName: Create emitter-package-lock.json
+            inputs:
+              pwsh: true
+              filePath: ./eng/common/scripts/typespec/New-EmitterPackageLock.ps1
+              arguments: >
+                -EmitterPackageJsonPath '$(Build.ArtifactStagingDirectory)/emitter-package.json'
+                -OutputDirectory '$(Build.ArtifactStagingDirectory)'
+              workingDirectory: $(Build.SourcesDirectory)
+
+          - template: /eng/common/pipelines/templates/steps/publish-1es-artifact.yml
+            parameters:
+              artifactName: publish_artifacts
+              artifactPath: $(Build.ArtifactStagingDirectory)
+
+    # Regenerate stage
+    # Responsible for regenerating the SDK code using the emitter package and the generation matrix.
+    - ${{ if and(parameters.ShouldPublish, parameters.ShouldRegenrate) }}:
+      - stage: Regenerate
+        dependsOn:
+        - Build
+        - Publish
+        variables:
+          pullRequestTargetBranch: 'main'
+          publishArtifactsPath: $(Pipeline.Workspace)/publish_artifacts
+          branchName: $[stageDependencies.Build.Build.outputs['set_branch_name.branchName']]
+        pool: ${{ parameters.Pool }}
+        jobs:
+        - job: Initialize
+          steps:
+          - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml
+            parameters:
+              Paths:
+              - "/*"
+              - "!SessionRecords"
+
+          - download: current
+            displayName: Download pipeline artifacts
+
+          - pwsh: |
+              Write-Host "Copying emitter-package.json to $(Build.SourcesDirectory)/eng"
+              Copy-Item $(publishArtifactsPath)/emitter-package.json $(Build.SourcesDirectory)/eng/ -Force
+
+              Write-Host "Copying emitter-package-lock.json to $(Build.SourcesDirectory)/eng"
+              Copy-Item $(publishArtifactsPath)/emitter-package-lock.json $(Build.SourcesDirectory)/eng/ -Force
+            displayName: Copy emitter-package json files
+
+          - ${{ parameters.InitializationSteps }}
+
+          - template: /eng/common/pipelines/templates/steps/git-push-changes.yml
+            parameters:
+              BaseRepoOwner: azure-sdk
+              TargetRepoName: $(Build.Repository.Name)
+              BaseRepoBranch: $(branchName)
+              CommitMsg: Initialize repository for autorest build $(Build.BuildNumber)
+              WorkingDirectory: $(Build.SourcesDirectory)
+              ScriptDirectory: $(Build.SourcesDirectory)/eng/common/scripts
+              # To accomodate scheduled runs and retries, we want to overwrite any existing changes on the branch
+              PushArgs: --force
+
+          - task: PowerShell@2
+            displayName: Get generation job matrix
+            name: generate_matrix
+            inputs:
+              pwsh: true
+              filePath: ./eng/common/scripts/New-RegenerateMatrix.ps1
+              arguments: >
+                -OutputDirectory "$(Build.ArtifactStagingDirectory)"
+                -OutputVariableName matrix
+                -JobCount ${{ parameters.RegenerationJobCount }}
+                -MinimumPerJob ${{ parameters.MinimumPerJob }}
+                -OnlyTypespec ${{ parameters.OnlyGenerateTypespec }}
+              workingDirectory: $(Build.SourcesDirectory)
+
+          - template: /eng/common/pipelines/templates/steps/publish-1es-artifact.yml
+            parameters:
+              artifactName: matrix_artifacts
+              artifactPath: $(Build.ArtifactStagingDirectory)
+
+        - job: Generate
+          dependsOn: Initialize
+          strategy:
+            matrix: $[dependencies.Initialize.outputs['generate_matrix.matrix']]
+          variables:
+            matrixArtifactsPath: $(Pipeline.Workspace)/matrix_artifacts
+          steps:
+          - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml
+            parameters:
+              Paths:
+              - "/*"
+              - "!SessionRecords"
+              
+          - download: current
+            displayName: Download pipeline artifacts
+
+          - ${{ parameters.InitializationSteps }}
+
+          - task: PowerShell@2
+            displayName: Call regeneration script
+            inputs:
+              pwsh: true
+              filePath: ./eng/common/scripts/Update-GeneratedSdks.ps1
+              arguments: >
+                -PackageDirectoriesFile "$(matrixArtifactsPath)/$(DirectoryList)"
+              workingDirectory: $(Build.SourcesDirectory)
+            continueOnError: true
+
+          - template: /eng/common/pipelines/templates/steps/git-push-changes.yml
+            parameters:
+              BaseRepoOwner: azure-sdk
+              TargetRepoName: $(Build.Repository.Name)
+              BaseRepoBranch: $(branchName)
+              CommitMsg: Update SDK code $(JobKey)
+              WorkingDirectory: $(Build.SourcesDirectory)
+              ScriptDirectory: $(Build.SourcesDirectory)/eng/common/scripts
+
+        - job: Create_PR
+          displayName: Create PR
+          dependsOn: 
+          - Generate
+          variables:
+            generateJobResult: $[dependencies.Generate.result]
+            emitterVersion: $[stageDependencies.Build.Build.outputs['ci_build.emitterVersion']]
+          steps:
+          - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml
+
+          - pwsh: |
+              $generateJobResult = '$(generateJobResult)'
+              $emitterVersion = '$(emitterVersion)'
+              $collectionUri = '$(System.CollectionUri)'
+              $project = '$(System.TeamProject)'
+              $definitionName = '$(Build.DefinitionName)'
+              $repoUrl = '$(Build.Repository.Uri)'
+              $repoName = '$(Build.Repository.Name)'
+              $sourceBranch = '$(Build.SourceBranch)'
+              $reason = '$(Build.Reason)'
+              $buildId = '$(Build.BuildId)'
+              $buildNumber = '$(Build.BuildNumber)'
+              $preRelease = '${{ parameters.BuildPrereleaseVersion }}' -eq 'true'
+              
+              $prBody = "Generated by $definitionName build [$buildNumber]($collectionUri/$project/_build/results?buildId=$buildId)<br/>"
+
+              if ($sourceBranch -match "^refs/heads/(.+)$") {
+                $prBody += "Triggered from branch: [$($Matches[1])]($repoUrl/tree/$sourceBranch)"
+              } elseif ($sourceBranch -match "^refs/tags/(.+)$") {
+                $prBody += "Triggered from tag: [$($Matches[1])]($repoUrl/tree/$sourceBranch)"
+              } elseif ($sourceBranch -match "^refs/pull/(\d+)/(head|merge)$") {
+                $prBody += "Triggered from pull request: $repoUrl/pull/$($Matches[1])"
+              } else {
+                $prBody += "Triggered from [$sourceBranch]($repoUrl/tree/$sourceBranch)"
+              }
+
+              if ($reason -eq 'Schedule') {
+                $prTitle = "Scheduled code regeneration test"
+              } else {
+                if ($preRelease) {
+                  $prTitle = "Update typespec emitter version to prerelease $emitterVersion"
+                } else {
+                  $prTitle = "Update typespec emitter version to $emitterVersion"
+                }
+
+                if ($generateJobResult -ne 'Succeeded') {
+                  $prTitle = "Failed: $prTitle"
+                }
+              }
+
+              Write-Host "Setting variable 'PullRequestTitle' to '$prTitle'"
+              Write-Host "##vso[task.setvariable variable=PullRequestTitle]$prTitle"
+
+              Write-Host "Setting variable 'PullRequestBody' to '$prBody'"
+              Write-Host "##vso[task.setvariable variable=PullRequestBody]$prBody"
+
+              $repoNameParts = $repoName.Split('/')
+              if ($repoNameParts.Length -eq 2) {
+                Write-Host "Setting variable 'RepoOwner' to '$($repoNameParts[0])'"
+                Write-Host "##vso[task.setvariable variable=RepoOwner]$($repoNameParts[0])"
+
+                Write-Host "Setting variable 'RepoName' to '$($repoNameParts[1])'"
+                Write-Host "##vso[task.setvariable variable=RepoName]$($repoNameParts[1])"
+              } else {
+                Write-Error "Build.Repository.Name not in the expected {Owner}/{Name} format"
+              }
+            displayName: Get PR title and body
+
+          - task: PowerShell@2
+            displayName: Create pull request
+            inputs:
+              pwsh: true
+              filePath: ./eng/common/scripts/Submit-PullRequest.ps1
+              arguments: >
+                -RepoOwner '$(RepoOwner)'
+                -RepoName '$(RepoName)'
+                -BaseBranch '$(pullRequestTargetBranch)'
+                -PROwner 'azure-sdk'
+                -PRBranch '$(branchName)'
+                -AuthToken '$(azuresdk-github-pat)'
+                -PRTitle '$(PullRequestTitle)'
+                -PRBody '$(PullRequestBody)'
+                -OpenAsDraft $true
+                -PRLabels 'Do Not Merge'
+              workingDirectory: $(Build.SourcesDirectory)
+
+    # Test stage
+    # Responsible for running unit and functional tests using a matrix passed in as the parameter `TestMatrix`.
+    # Will only run if the parameter `TestMatrix` is not empty.
+    # The contents of the artficact `build_artifacts` are available under the path `$(buildArtifactsPath)`.
+    - ${{ if ne(length(parameters.TestMatrix), 0) }}:
+      - stage: Test
+        dependsOn: Build
+        pool: ${{ parameters.Pool }}
+        jobs:
+        - job: Test
+          strategy:
+            matrix: ${{ parameters.TestMatrix }}
+          variables:
+            buildArtifactsPath: $(Pipeline.Workspace)/build_artifacts
+          steps:
+          - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml
+            
+          - download: current
+            artifact: build_artifacts
+            displayName: Download build artifacts
+
+          - ${{ parameters.InitializationSteps }}
+
+          - task: PowerShell@2
+            displayName: 'Run initialize script'
+            inputs:
+              pwsh: true
+              filePath: $(Build.SourcesDirectory)/eng/scripts/typespec/Initialize-WorkingDirectory.ps1
+              arguments: -BuildArtifactsPath '$(buildArtifactsPath)'
+
+          - task: PowerShell@2
+            displayName: 'Run test script'
+            inputs:
+              pwsh: true
+              filePath: $(Build.SourcesDirectory)/eng/scripts/typespec/Test-Emitter.ps1
+              arguments: >
+                $(TestArguments)
+                -OutputDirectory "$(Build.ArtifactStagingDirectory)"
+
+          - template: /eng/common/pipelines/templates/steps/publish-1es-artifact.yml
+            parameters:
+              artifactName: test_artifacts_$(System.JobName)
+              artifactPath: $(Build.ArtifactStagingDirectory)

eng/common/pipelines/templates/steps/create-authenticated-npmrc.yml

@@ -0,0 +1,23 @@
+parameters:
+  - name: npmrcPath
+    type: string
+  - name: registryUrl
+    type: string
+
+steps:
+- pwsh: |
+    Write-Host "Creating .npmrc file ${{ parameters.npmrcPath }} for registry ${{ parameters.registryUrl }}"
+    $parentFolder = Split-Path -Path '${{ parameters.npmrcPath }}' -Parent
+    
+    if (!(Test-Path $parentFolder)) {
+      Write-Host "Creating folder $parentFolder"
+      New-Item -Path $parentFolder -ItemType Directory | Out-Null
+    }
+
+    $content = "registry=${{ parameters.registryUrl }}`n`nalways-auth=true"
+    $content | Out-File '${{ parameters.npmrcPath }}'
+  displayName: 'Create .npmrc'
+- task: npmAuthenticate@0
+  displayName: Authenticate .npmrc
+  inputs:
+    workingFile: ${{ parameters.npmrcPath }}

eng/common/scripts/Helpers/CommandInvocation-Helpers.ps1

@@ -1,5 +1,14 @@
-function Invoke-LoggedCommand($Command, $ExecutePath, [switch]$GroupOutput)
+function Invoke-LoggedCommand
 {
+    [CmdletBinding()]
+    param
+    (
+        [string] $Command,
+        [string] $ExecutePath,
+        [switch] $GroupOutput,
+        [int[]] $AllowedExitCodes = @(0)
+    )
+
     $pipelineBuild = !!$env:TF_BUILD
     $startTime = Get-Date
 
@@ -22,7 +31,7 @@ function Invoke-LoggedCommand($Command, $ExecutePath, [switch]$GroupOutput)
         Write-Host "##[endgroup]"
       }
 
-      if($LastExitCode -ne 0)
+      if($LastExitCode -notin $AllowedExitCodes)
       {
           if($pipelineBuild) {
               Write-Error "##[error]Command failed to execute ($duration): $Command`n"
@@ -40,3 +49,16 @@ function Invoke-LoggedCommand($Command, $ExecutePath, [switch]$GroupOutput)
       }
     }
 }
+
+function Set-ConsoleEncoding
+{
+    [CmdletBinding()]
+    param
+    (
+        [string] $Encoding = 'utf-8'
+    )
+
+    $outputEncoding = [System.Text.Encoding]::GetEncoding($Encoding)
+    [Console]::OutputEncoding = $outputEncoding
+    [Console]::InputEncoding = $outputEncoding
+}

eng/common/scripts/New-RegenerateMatrix.ps1

@@ -14,7 +14,7 @@ param (
   [int]$MinimumPerJob = 10,
 
   [Parameter()]
-  [string]$OnlyTypespec
+  [string]$OnlyTypeSpec
 )
 
 . (Join-Path $PSScriptRoot common.ps1)
@@ -57,14 +57,13 @@ function Split-Items([array]$Items) {
 New-Item -ItemType Directory -Path $OutputDirectory -Force | Out-Null
 
 if (Test-Path "Function:$GetDirectoriesForGenerationFn") {
-  $directoriesForGeneration = &$GetDirectoriesForGenerationFn
+  $directoriesForGeneration = &$GetDirectoriesForGenerationFn -OnlyTypeSpec $OnlyTypespec
 }
 else {
   $directoriesForGeneration = Get-ChildItem "$RepoRoot/sdk" -Directory | Get-ChildItem -Directory
-}
-
-if ($OnlyTypespec) {
-  $directoriesForGeneration = $directoriesForGeneration | Where-Object { Test-Path "$_/tsp-location.yaml" }
+  if ($OnlyTypespec) {
+    $directoriesForGeneration = $directoriesForGeneration | Where-Object { Test-Path "$_/tsp-location.yaml" }
+  }
 }
 
 [array]$packageDirectories = $directoriesForGeneration

eng/common/scripts/common.ps1

@@ -16,6 +16,7 @@ $EngScriptsDir = Join-Path $EngDir "scripts"
 . (Join-Path $EngCommonScriptsDir artifact-metadata-parsing.ps1)
 . (Join-Path $EngCommonScriptsDir "Helpers" git-helpers.ps1)
 . (Join-Path $EngCommonScriptsDir "Helpers" Package-Helpers.ps1)
+. (Join-Path $EngCommonScriptsDir "Helpers" CommandInvocation-Helpers.ps1)
 
 # Setting expected from common languages settings
 $Language = "Unknown"

eng/common/scripts/typespec/New-EmitterPackageJson.ps1

@@ -1,3 +1,4 @@
+#Requires -Version 7.0
 [CmdletBinding()]
 param (
     [parameter(Mandatory = $true)]
@@ -35,9 +36,12 @@ if ($OverridesPath) {
 
 $devDependencies = [ordered]@{}
 
-foreach ($package in $packageJson.peerDependencies.Keys | Sort-Object) {
+$possiblyPinnedPackages = $packageJson['azure-sdk/emitter-package-json-pinning'] ?? $packageJson.peerDependencies.Keys;
+
+foreach ($package in $possiblyPinnedPackages | Sort-Object) {
     $pinnedVersion = $packageJson.devDependencies[$package]
     if ($pinnedVersion -and -not $overrides[$package]) {
+        #We have a dev pinned version that isn't overridden by the overrides.json file
         Write-Host "Pinning $package to $pinnedVersion"
         $devDependencies[$package] = $pinnedVersion
     }