From aebfcb7f67eed2fdb5fdb0a1aa4a854bf6ad8dc3 Mon Sep 17 00:00:00 2001 From: Azure SDK Bot <53356347+azure-sdk@users.noreply.github.com> Date: Wed, 16 Oct 2024 15:06:59 -0700 Subject: [PATCH] Sync eng/common directory with azure-sdk-tools for PR 9147 (#1709) * Changes required for sdk-repo based emitter pipelines * Rename template * Convert emitter archetype to pipeline template * Move the archetype template out of the stages folder --------- Co-authored-by: Patrick Hallisey --- .../templates/archetype-typespec-emitter.yml | 440 ++++++++++++++++++ .../steps/create-authenticated-npmrc.yml | 23 + .../Helpers/CommandInvocation-Helpers.ps1 | 26 +- eng/common/scripts/New-RegenerateMatrix.ps1 | 11 +- eng/common/scripts/common.ps1 | 1 + .../typespec/New-EmitterPackageJson.ps1 | 6 +- 6 files changed, 498 insertions(+), 9 deletions(-) create mode 100644 eng/common/pipelines/templates/archetype-typespec-emitter.yml create mode 100644 eng/common/pipelines/templates/steps/create-authenticated-npmrc.yml diff --git a/eng/common/pipelines/templates/archetype-typespec-emitter.yml b/eng/common/pipelines/templates/archetype-typespec-emitter.yml new file mode 100644 index 000000000..71d16c908 --- /dev/null +++ b/eng/common/pipelines/templates/archetype-typespec-emitter.yml @@ -0,0 +1,440 @@ +parameters: +# Pool to use for the pipeline stages +- name: Pool + type: object + default: + name: $(LINUXPOOL) + image: $(LINUXVMIMAGE) + os: linux + +# Whether to build alpha versions of the packages. This is passed as a flag to the build script. +- name: BuildPrereleaseVersion + type: boolean + default: true + +# Whether to use the `next` version of TypeSpec. This is passed as a flag to the initialize script. +- name: UseTypeSpecNext + type: boolean + default: false + +# Custom steps to run after the repository is cloned but before other job steps. +- name: InitializationSteps + type: stepList + default: [] + +# Indicates the build matrix to use for post-build autorest validation +- name: TestMatrix + type: object + default: {} + +# Whether to run the publish and regeneration stages. If false, only the build stage will run. +- name: ShouldPublish + type: boolean + default: false + +# Whether to publish to npmjs.org. +- name: PublishPublic + type: boolean + default: true + +# Indicates if the Publish stage should depend on the Test stage +- name: PublishDependsOnTest + type: boolean + default: false + +# Whether to regenerate sdk clients using the new emitter. +- name: ShouldRegenrate + type: boolean + default: false + +# Number of jobs to generate. This is the maximum number of jobs that will be generated. The actual number of jobs will be reduced if it would result in fewer than MinimumPerJob packages per job. +- name: RegenerationJobCount + type: number + default: 10 + +# Minimum number of packages to generate per job. +- name: MinimumPerJob + type: number + default: 10 + +# Indicates if regenration matrix should only contain folders with typespec files +- name: OnlyGenerateTypespec + type: boolean + default: false + +extends: + template: /eng/pipelines/templates/stages/1es-redirect.yml + parameters: + stages: + # Build stage + # Responsible for building the autorest generator and typespec emitter packages + # Produces the artifact `build_artifacts` which contains the following: + # package-versions.json: Contains a map of package name to version for the packages that were built + # overrides.json: Contains npm package version overrides for the emitter and generator + # packages/: Contains the packages to publish + # lock-files/: Contains package.json and package-lock.json files for use in the test stage to ensure tests are run against the correct dependency versions + - stage: Build + pool: ${{ parameters.Pool }} + jobs: + - job: Build + steps: + - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml + + - ${{ parameters.InitializationSteps }} + + - task: PowerShell@2 + displayName: 'Run initialize script' + inputs: + pwsh: true + filePath: $(Build.SourcesDirectory)/eng/scripts/typespec/Initialize-WorkingDirectory.ps1 + arguments: -UseTypeSpecNext:$${{ parameters.UseTypeSpecNext }} + + - task: PowerShell@2 + displayName: 'Run build script' + name: ci_build + inputs: + pwsh: true + filePath: $(Build.SourcesDirectory)/eng/scripts/typespec/Build-Emitter.ps1 + arguments: > + -BuildNumber "$(Build.BuildNumber)" + -OutputDirectory "$(Build.ArtifactStagingDirectory)" + -TargetNpmJsFeed:$${{ parameters.PublishPublic }} + -Prerelease:$${{ parameters.BuildPrereleaseVersion }} + + - pwsh: | + $sourceBranch = '$(Build.SourceBranch)' + $buildReason = '$(Build.Reason)' + $buildNumber = '$(Build.BuildNumber)' + + if ($buildReason -eq 'Schedule') { + $branchName = 'validate-typespec-scheduled' + } elseif ($sourceBranch -match "^refs/pull/(\d+)/(head|merge)$") { + $branchName = "validate-typespec-pr-$($Matches[1])" + } else { + $branchName = "validate-typespec-$buildNumber" + } + + Write-Host "Setting variable 'branchName' to '$branchName'" + Write-Host "##vso[task.setvariable variable=branchName;isOutput=true]$branchName" + displayName: Set branch name + name: set_branch_name + + - template: /eng/common/pipelines/templates/steps/publish-1es-artifact.yml + parameters: + artifactName: build_artifacts + artifactPath: $(Build.ArtifactStagingDirectory) + + # Publish stage + # Responsible for publishing the packages in `build_artifacts/packages` and producing `emitter-package-lock.json` + # Produces the artifact `publish_artifacts` which contains the following: + # emitter-package.json: Created using the package json from the build step. + # emitter-package-lock.json: Created by calling `npm install` using `emitter-package.json` + - ${{ if parameters.ShouldPublish }}: + - stage: Publish + dependsOn: + - Build + - ${{ if and(parameters.PublishDependsOnTest, ne(length(parameters.TestMatrix), 0)) }}: + - Test + variables: + buildArtifactsPath: $(Pipeline.Workspace)/build_artifacts + pool: ${{ parameters.Pool }} + jobs: + - job: Publish + steps: + - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml + + - download: current + artifact: build_artifacts + displayName: Download build artifacts + + # Create authenticated .npmrc file for publishing to devops + - template: /eng/common/pipelines/templates/steps/create-authenticated-npmrc.yml + parameters: + npmrcPath: $(buildArtifactsPath)/packages/.npmrc + registryUrl: https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-js-test-autorest/npm/registry/ + + # publish to devops feed + - pwsh: | + $packageFiles = Get-ChildItem -Path . -Filter '*.tgz' + foreach ($file in $packageFiles.Name) { + Write-Host "npm publish $file --verbose --access public" + npm publish $file --verbose --access public + } + displayName: Publish to DevOps feed + workingDirectory: $(buildArtifactsPath)/packages + + - ${{ if parameters.PublishPublic }}: + # publish to npmjs.org using ESRP + - task: EsrpRelease@7 + inputs: + displayName: Publish to npmjs.org + ConnectedServiceName: Azure SDK Engineering System + ClientId: 5f81938c-2544-4f1f-9251-dd9de5b8a81b + KeyVaultName: AzureSDKEngKeyVault + AuthCertName: azure-sdk-esrp-release-auth-certificate + SignCertName: azure-sdk-esrp-release-sign-certificate + Intent: PackageDistribution + ContentType: npm + FolderLocation: $(buildArtifactsPath)/packages + Owners: ${{ coalesce(variables['Build.RequestedForEmail'], 'azuresdk@microsoft.com') }} + Approvers: ${{ coalesce(variables['Build.RequestedForEmail'], 'azuresdk@microsoft.com') }} + ServiceEndpointUrl: https://api.esrp.microsoft.com + MainPublisher: ESRPRELPACMANTEST + DomainTenantId: 72f988bf-86f1-41af-91ab-2d7cd011db47 + + - task: PowerShell@2 + displayName: Create emitter-package.json + inputs: + pwsh: true + filePath: ./eng/common/scripts/typespec/New-EmitterPackageJson.ps1 + arguments: > + -PackageJsonPath '$(buildArtifactsPath)/lock-files/package.json' + -OverridesPath '$(buildArtifactsPath)/overrides.json' + -OutputDirectory '$(Build.ArtifactStagingDirectory)' + workingDirectory: $(Build.SourcesDirectory) + + - task: PowerShell@2 + displayName: Create emitter-package-lock.json + inputs: + pwsh: true + filePath: ./eng/common/scripts/typespec/New-EmitterPackageLock.ps1 + arguments: > + -EmitterPackageJsonPath '$(Build.ArtifactStagingDirectory)/emitter-package.json' + -OutputDirectory '$(Build.ArtifactStagingDirectory)' + workingDirectory: $(Build.SourcesDirectory) + + - template: /eng/common/pipelines/templates/steps/publish-1es-artifact.yml + parameters: + artifactName: publish_artifacts + artifactPath: $(Build.ArtifactStagingDirectory) + + # Regenerate stage + # Responsible for regenerating the SDK code using the emitter package and the generation matrix. + - ${{ if and(parameters.ShouldPublish, parameters.ShouldRegenrate) }}: + - stage: Regenerate + dependsOn: + - Build + - Publish + variables: + pullRequestTargetBranch: 'main' + publishArtifactsPath: $(Pipeline.Workspace)/publish_artifacts + branchName: $[stageDependencies.Build.Build.outputs['set_branch_name.branchName']] + pool: ${{ parameters.Pool }} + jobs: + - job: Initialize + steps: + - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml + parameters: + Paths: + - "/*" + - "!SessionRecords" + + - download: current + displayName: Download pipeline artifacts + + - pwsh: | + Write-Host "Copying emitter-package.json to $(Build.SourcesDirectory)/eng" + Copy-Item $(publishArtifactsPath)/emitter-package.json $(Build.SourcesDirectory)/eng/ -Force + + Write-Host "Copying emitter-package-lock.json to $(Build.SourcesDirectory)/eng" + Copy-Item $(publishArtifactsPath)/emitter-package-lock.json $(Build.SourcesDirectory)/eng/ -Force + displayName: Copy emitter-package json files + + - ${{ parameters.InitializationSteps }} + + - template: /eng/common/pipelines/templates/steps/git-push-changes.yml + parameters: + BaseRepoOwner: azure-sdk + TargetRepoName: $(Build.Repository.Name) + BaseRepoBranch: $(branchName) + CommitMsg: Initialize repository for autorest build $(Build.BuildNumber) + WorkingDirectory: $(Build.SourcesDirectory) + ScriptDirectory: $(Build.SourcesDirectory)/eng/common/scripts + # To accomodate scheduled runs and retries, we want to overwrite any existing changes on the branch + PushArgs: --force + + - task: PowerShell@2 + displayName: Get generation job matrix + name: generate_matrix + inputs: + pwsh: true + filePath: ./eng/common/scripts/New-RegenerateMatrix.ps1 + arguments: > + -OutputDirectory "$(Build.ArtifactStagingDirectory)" + -OutputVariableName matrix + -JobCount ${{ parameters.RegenerationJobCount }} + -MinimumPerJob ${{ parameters.MinimumPerJob }} + -OnlyTypespec ${{ parameters.OnlyGenerateTypespec }} + workingDirectory: $(Build.SourcesDirectory) + + - template: /eng/common/pipelines/templates/steps/publish-1es-artifact.yml + parameters: + artifactName: matrix_artifacts + artifactPath: $(Build.ArtifactStagingDirectory) + + - job: Generate + dependsOn: Initialize + strategy: + matrix: $[dependencies.Initialize.outputs['generate_matrix.matrix']] + variables: + matrixArtifactsPath: $(Pipeline.Workspace)/matrix_artifacts + steps: + - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml + parameters: + Paths: + - "/*" + - "!SessionRecords" + + - download: current + displayName: Download pipeline artifacts + + - ${{ parameters.InitializationSteps }} + + - task: PowerShell@2 + displayName: Call regeneration script + inputs: + pwsh: true + filePath: ./eng/common/scripts/Update-GeneratedSdks.ps1 + arguments: > + -PackageDirectoriesFile "$(matrixArtifactsPath)/$(DirectoryList)" + workingDirectory: $(Build.SourcesDirectory) + continueOnError: true + + - template: /eng/common/pipelines/templates/steps/git-push-changes.yml + parameters: + BaseRepoOwner: azure-sdk + TargetRepoName: $(Build.Repository.Name) + BaseRepoBranch: $(branchName) + CommitMsg: Update SDK code $(JobKey) + WorkingDirectory: $(Build.SourcesDirectory) + ScriptDirectory: $(Build.SourcesDirectory)/eng/common/scripts + + - job: Create_PR + displayName: Create PR + dependsOn: + - Generate + variables: + generateJobResult: $[dependencies.Generate.result] + emitterVersion: $[stageDependencies.Build.Build.outputs['ci_build.emitterVersion']] + steps: + - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml + + - pwsh: | + $generateJobResult = '$(generateJobResult)' + $emitterVersion = '$(emitterVersion)' + $collectionUri = '$(System.CollectionUri)' + $project = '$(System.TeamProject)' + $definitionName = '$(Build.DefinitionName)' + $repoUrl = '$(Build.Repository.Uri)' + $repoName = '$(Build.Repository.Name)' + $sourceBranch = '$(Build.SourceBranch)' + $reason = '$(Build.Reason)' + $buildId = '$(Build.BuildId)' + $buildNumber = '$(Build.BuildNumber)' + $preRelease = '${{ parameters.BuildPrereleaseVersion }}' -eq 'true' + + $prBody = "Generated by $definitionName build [$buildNumber]($collectionUri/$project/_build/results?buildId=$buildId)
" + + if ($sourceBranch -match "^refs/heads/(.+)$") { + $prBody += "Triggered from branch: [$($Matches[1])]($repoUrl/tree/$sourceBranch)" + } elseif ($sourceBranch -match "^refs/tags/(.+)$") { + $prBody += "Triggered from tag: [$($Matches[1])]($repoUrl/tree/$sourceBranch)" + } elseif ($sourceBranch -match "^refs/pull/(\d+)/(head|merge)$") { + $prBody += "Triggered from pull request: $repoUrl/pull/$($Matches[1])" + } else { + $prBody += "Triggered from [$sourceBranch]($repoUrl/tree/$sourceBranch)" + } + + if ($reason -eq 'Schedule') { + $prTitle = "Scheduled code regeneration test" + } else { + if ($preRelease) { + $prTitle = "Update typespec emitter version to prerelease $emitterVersion" + } else { + $prTitle = "Update typespec emitter version to $emitterVersion" + } + + if ($generateJobResult -ne 'Succeeded') { + $prTitle = "Failed: $prTitle" + } + } + + Write-Host "Setting variable 'PullRequestTitle' to '$prTitle'" + Write-Host "##vso[task.setvariable variable=PullRequestTitle]$prTitle" + + Write-Host "Setting variable 'PullRequestBody' to '$prBody'" + Write-Host "##vso[task.setvariable variable=PullRequestBody]$prBody" + + $repoNameParts = $repoName.Split('/') + if ($repoNameParts.Length -eq 2) { + Write-Host "Setting variable 'RepoOwner' to '$($repoNameParts[0])'" + Write-Host "##vso[task.setvariable variable=RepoOwner]$($repoNameParts[0])" + + Write-Host "Setting variable 'RepoName' to '$($repoNameParts[1])'" + Write-Host "##vso[task.setvariable variable=RepoName]$($repoNameParts[1])" + } else { + Write-Error "Build.Repository.Name not in the expected {Owner}/{Name} format" + } + displayName: Get PR title and body + + - task: PowerShell@2 + displayName: Create pull request + inputs: + pwsh: true + filePath: ./eng/common/scripts/Submit-PullRequest.ps1 + arguments: > + -RepoOwner '$(RepoOwner)' + -RepoName '$(RepoName)' + -BaseBranch '$(pullRequestTargetBranch)' + -PROwner 'azure-sdk' + -PRBranch '$(branchName)' + -AuthToken '$(azuresdk-github-pat)' + -PRTitle '$(PullRequestTitle)' + -PRBody '$(PullRequestBody)' + -OpenAsDraft $true + -PRLabels 'Do Not Merge' + workingDirectory: $(Build.SourcesDirectory) + + # Test stage + # Responsible for running unit and functional tests using a matrix passed in as the parameter `TestMatrix`. + # Will only run if the parameter `TestMatrix` is not empty. + # The contents of the artficact `build_artifacts` are available under the path `$(buildArtifactsPath)`. + - ${{ if ne(length(parameters.TestMatrix), 0) }}: + - stage: Test + dependsOn: Build + pool: ${{ parameters.Pool }} + jobs: + - job: Test + strategy: + matrix: ${{ parameters.TestMatrix }} + variables: + buildArtifactsPath: $(Pipeline.Workspace)/build_artifacts + steps: + - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml + + - download: current + artifact: build_artifacts + displayName: Download build artifacts + + - ${{ parameters.InitializationSteps }} + + - task: PowerShell@2 + displayName: 'Run initialize script' + inputs: + pwsh: true + filePath: $(Build.SourcesDirectory)/eng/scripts/typespec/Initialize-WorkingDirectory.ps1 + arguments: -BuildArtifactsPath '$(buildArtifactsPath)' + + - task: PowerShell@2 + displayName: 'Run test script' + inputs: + pwsh: true + filePath: $(Build.SourcesDirectory)/eng/scripts/typespec/Test-Emitter.ps1 + arguments: > + $(TestArguments) + -OutputDirectory "$(Build.ArtifactStagingDirectory)" + + - template: /eng/common/pipelines/templates/steps/publish-1es-artifact.yml + parameters: + artifactName: test_artifacts_$(System.JobName) + artifactPath: $(Build.ArtifactStagingDirectory) diff --git a/eng/common/pipelines/templates/steps/create-authenticated-npmrc.yml b/eng/common/pipelines/templates/steps/create-authenticated-npmrc.yml new file mode 100644 index 000000000..4b6f08359 --- /dev/null +++ b/eng/common/pipelines/templates/steps/create-authenticated-npmrc.yml @@ -0,0 +1,23 @@ +parameters: + - name: npmrcPath + type: string + - name: registryUrl + type: string + +steps: +- pwsh: | + Write-Host "Creating .npmrc file ${{ parameters.npmrcPath }} for registry ${{ parameters.registryUrl }}" + $parentFolder = Split-Path -Path '${{ parameters.npmrcPath }}' -Parent + + if (!(Test-Path $parentFolder)) { + Write-Host "Creating folder $parentFolder" + New-Item -Path $parentFolder -ItemType Directory | Out-Null + } + + $content = "registry=${{ parameters.registryUrl }}`n`nalways-auth=true" + $content | Out-File '${{ parameters.npmrcPath }}' + displayName: 'Create .npmrc' +- task: npmAuthenticate@0 + displayName: Authenticate .npmrc + inputs: + workingFile: ${{ parameters.npmrcPath }} diff --git a/eng/common/scripts/Helpers/CommandInvocation-Helpers.ps1 b/eng/common/scripts/Helpers/CommandInvocation-Helpers.ps1 index 5dc0c8c7d..0b9f810b8 100644 --- a/eng/common/scripts/Helpers/CommandInvocation-Helpers.ps1 +++ b/eng/common/scripts/Helpers/CommandInvocation-Helpers.ps1 @@ -1,5 +1,14 @@ -function Invoke-LoggedCommand($Command, $ExecutePath, [switch]$GroupOutput) +function Invoke-LoggedCommand { + [CmdletBinding()] + param + ( + [string] $Command, + [string] $ExecutePath, + [switch] $GroupOutput, + [int[]] $AllowedExitCodes = @(0) + ) + $pipelineBuild = !!$env:TF_BUILD $startTime = Get-Date @@ -22,7 +31,7 @@ function Invoke-LoggedCommand($Command, $ExecutePath, [switch]$GroupOutput) Write-Host "##[endgroup]" } - if($LastExitCode -ne 0) + if($LastExitCode -notin $AllowedExitCodes) { if($pipelineBuild) { Write-Error "##[error]Command failed to execute ($duration): $Command`n" @@ -40,3 +49,16 @@ function Invoke-LoggedCommand($Command, $ExecutePath, [switch]$GroupOutput) } } } + +function Set-ConsoleEncoding +{ + [CmdletBinding()] + param + ( + [string] $Encoding = 'utf-8' + ) + + $outputEncoding = [System.Text.Encoding]::GetEncoding($Encoding) + [Console]::OutputEncoding = $outputEncoding + [Console]::InputEncoding = $outputEncoding +} diff --git a/eng/common/scripts/New-RegenerateMatrix.ps1 b/eng/common/scripts/New-RegenerateMatrix.ps1 index 1df97420c..9176c23e4 100644 --- a/eng/common/scripts/New-RegenerateMatrix.ps1 +++ b/eng/common/scripts/New-RegenerateMatrix.ps1 @@ -14,7 +14,7 @@ param ( [int]$MinimumPerJob = 10, [Parameter()] - [string]$OnlyTypespec + [string]$OnlyTypeSpec ) . (Join-Path $PSScriptRoot common.ps1) @@ -57,14 +57,13 @@ function Split-Items([array]$Items) { New-Item -ItemType Directory -Path $OutputDirectory -Force | Out-Null if (Test-Path "Function:$GetDirectoriesForGenerationFn") { - $directoriesForGeneration = &$GetDirectoriesForGenerationFn + $directoriesForGeneration = &$GetDirectoriesForGenerationFn -OnlyTypeSpec $OnlyTypespec } else { $directoriesForGeneration = Get-ChildItem "$RepoRoot/sdk" -Directory | Get-ChildItem -Directory -} - -if ($OnlyTypespec) { - $directoriesForGeneration = $directoriesForGeneration | Where-Object { Test-Path "$_/tsp-location.yaml" } + if ($OnlyTypespec) { + $directoriesForGeneration = $directoriesForGeneration | Where-Object { Test-Path "$_/tsp-location.yaml" } + } } [array]$packageDirectories = $directoriesForGeneration diff --git a/eng/common/scripts/common.ps1 b/eng/common/scripts/common.ps1 index 831b4719f..a8c38d0a0 100644 --- a/eng/common/scripts/common.ps1 +++ b/eng/common/scripts/common.ps1 @@ -16,6 +16,7 @@ $EngScriptsDir = Join-Path $EngDir "scripts" . (Join-Path $EngCommonScriptsDir artifact-metadata-parsing.ps1) . (Join-Path $EngCommonScriptsDir "Helpers" git-helpers.ps1) . (Join-Path $EngCommonScriptsDir "Helpers" Package-Helpers.ps1) +. (Join-Path $EngCommonScriptsDir "Helpers" CommandInvocation-Helpers.ps1) # Setting expected from common languages settings $Language = "Unknown" diff --git a/eng/common/scripts/typespec/New-EmitterPackageJson.ps1 b/eng/common/scripts/typespec/New-EmitterPackageJson.ps1 index d9e45038b..24c6c50a9 100644 --- a/eng/common/scripts/typespec/New-EmitterPackageJson.ps1 +++ b/eng/common/scripts/typespec/New-EmitterPackageJson.ps1 @@ -1,3 +1,4 @@ +#Requires -Version 7.0 [CmdletBinding()] param ( [parameter(Mandatory = $true)] @@ -35,9 +36,12 @@ if ($OverridesPath) { $devDependencies = [ordered]@{} -foreach ($package in $packageJson.peerDependencies.Keys | Sort-Object) { +$possiblyPinnedPackages = $packageJson['azure-sdk/emitter-package-json-pinning'] ?? $packageJson.peerDependencies.Keys; + +foreach ($package in $possiblyPinnedPackages | Sort-Object) { $pinnedVersion = $packageJson.devDependencies[$package] if ($pinnedVersion -and -not $overrides[$package]) { + #We have a dev pinned version that isn't overridden by the overrides.json file Write-Host "Pinning $package to $pinnedVersion" $devDependencies[$package] = $pinnedVersion }