Sync eng/common from azure-sdk-tools (#1783)

* In sync as of da75627391
* Adjust for changes in eng/common

eng/common/pipelines/templates/steps/save-package-properties.yml

@@ -0,0 +1,46 @@
+parameters:
+  - name: ServiceDirectory
+    type: string
+    default: ""
+  - name: DiffDirectory
+    type: string
+    default: $(Build.ArtifactStagingDirectory)/diff
+  - name: PackageInfoDirectory
+    type: string
+    default: $(Build.ArtifactStagingDirectory)/PackageInfo
+  - name: TargetPath
+    type: string
+    default: $(Build.SourcesDirectory)
+  - name: ScriptDirectory
+    type: string
+    default: eng/common/scripts
+
+steps:
+  - ${{ if eq(variables['Build.Reason'], 'PullRequest') }}:
+      - task: Powershell@2
+        displayName: Generate PR Diff
+        inputs:
+          filePath: ${{ parameters.ScriptDirectory }}/Generate-PR-Diff.ps1
+          arguments: >
+            -TargetPath '${{ parameters.TargetPath }}'
+            -ArtifactPath '${{ parameters.DiffDirectory }}'
+          pwsh: true
+
+      - task: Powershell@2
+        displayName: Save package properties filtered for PR
+        inputs:
+          filePath: ${{ parameters.ScriptDirectory }}/Save-Package-Properties.ps1
+          arguments: >
+            -PrDiff ${{ parameters.DiffDirectory }}/diff.json
+            -OutDirectory ${{ parameters.PackageInfoDirectory }}
+          pwsh: true
+  - ${{ else }}:
+      - task: Powershell@2
+        displayName: Save package properties
+        inputs:
+          filePath: ${{ parameters.ScriptDirectory }}/Save-Package-Properties.ps1
+          arguments: >
+            -ServiceDirectory ${{parameters.ServiceDirectory}}
+            -OutDirectory ${{ parameters.PackageInfoDirectory }}
+            -AddDevVersion:$${{ eq(variables['SetDevVersion'],'true') }}
+          pwsh: true

eng/common/scripts/Generate-PR-Diff.ps1

@@ -13,27 +13,29 @@ The path under which changes will be detected.
 #>
 [CmdletBinding()]
 Param (
-  [Parameter(Mandatory=$True)]
+  [Parameter(Mandatory = $True)]
   [string] $ArtifactPath,
-  [Parameter(Mandatory=$True)]
+  [Parameter(Mandatory = $True)]
   [string] $TargetPath
 )
 
 . (Join-Path $PSScriptRoot "Helpers" "git-helpers.ps1")
 
-function Get-ChangedServices {
-    Param (
-        [Parameter(Mandatory=$True)]
-        [string[]] $ChangedFiles
-    )
+function Get-ChangedServices
+{
+  Param (
+    [Parameter(Mandatory = $True)]
+    [string[]] $ChangedFiles
+  )
 
-    $changedServices = $ChangedFiles | Foreach-Object { if ($_ -match "sdk/([^/]+)") { $matches[1] } } | Sort-Object -Unique
+  $changedServices = $ChangedFiles | Foreach-Object { if ($_ -match "sdk/([^/]+)") { $matches[1] } } | Sort-Object -Unique
 
-    return $changedServices
+  return $changedServices
 }
 
-if (!(Test-Path $ArtifactPath)) {
-    New-Item -ItemType Directory -Path $ArtifactPath | Out-Null
+if (!(Test-Path $ArtifactPath))
+{
+  New-Item -ItemType Directory -Path $ArtifactPath | Out-Null
 }
 
 $ArtifactPath = Resolve-Path $ArtifactPath
@@ -43,9 +45,13 @@ $changedFiles = Get-ChangedFiles -DiffPath $TargetPath
 $changedServices = Get-ChangedServices -ChangedFiles $changedFiles
 
 $result = [PSCustomObject]@{
-    "ChangedFiles" = $changedFiles
-    "ChangedServices" = $changedServices
-    "PRNumber" = if ($env:SYSTEM_PULLREQUEST_PULLREQUESTNUMBER) { $env:SYSTEM_PULLREQUEST_PULLREQUESTNUMBER } else { "-1" }
+  "ChangedFiles"    = $changedFiles
+  "ChangedServices" = $changedServices
+  "PRNumber"        = if ($env:SYSTEM_PULLREQUEST_PULLREQUESTNUMBER) { $env:SYSTEM_PULLREQUEST_PULLREQUESTNUMBER } else { "-1" }
 }
 
-$result | ConvertTo-Json | Out-File $ArtifactName
+$json = $result | ConvertTo-Json
+$json | Out-File $ArtifactName
+
+Write-Host "`nGenerated diff.json file at $ArtifactName"
+Write-Host "  $($json -replace "`n", "`n  ")"

eng/common/scripts/Package-Properties.ps1

@@ -15,7 +15,7 @@ class PackageProps
     [boolean]$IsNewSdk
     [string]$ArtifactName
     [string]$ReleaseStatus
-    [string[]]$DependentPackages
+    [string[]]$AdditionalValidationPackages
 
     PackageProps([string]$name, [string]$version, [string]$directoryPath, [string]$serviceDirectory)
     {
@@ -113,7 +113,7 @@ function Get-PrPkgProperties([string]$InputDiffJson) {
     $diff = Get-Content $InputDiffJson | ConvertFrom-Json
     $targetedFiles = $diff.ChangedFiles
 
-    $dependentPackagesForInclusion = @()
+    $additionalValidationPackages = @()
     $lookup = @{}
 
     foreach ($pkg in $allPackageProperties)
@@ -129,19 +129,26 @@ function Get-PrPkgProperties([string]$InputDiffJson) {
             if ($shouldInclude) {
                 $packagesWithChanges += $pkg
 
-                if ($pkg.DependentPackages) {
-                    $dependentPackagesForInclusion += $pkg.DependentPackages
+                if ($pkg.AdditionalValidationPackages) {
+                    $additionalValidationPackages += $pkg.AdditionalValidationPackages
                 }
             }
         }
     }
 
-    foreach ($addition in $dependentPackagesForInclusion) {
-        if ($lookup[$addition]) {
-            $packagesWithChanges += $lookup[$addition]
+    foreach ($addition in $additionalValidationPackages) {
+        $key = $addition.Replace($RepoRoot, "").SubString(1)
+
+        if ($lookup[$key]) {
+            $packagesWithChanges += $lookup[$key]
         }
     }
 
+    if ($AdditionalValidationPackagesFromPackageSetFn -and (Test-Path "Function:$AdditionalValidationPackagesFromPackageSetFn"))
+    {
+        $packagesWithChanges += &$AdditionalValidationPackagesFromPackageSetFn $packagesWithChanges $diff
+    }
+
     return $packagesWithChanges
 }
 

eng/common/scripts/Save-Package-Properties.ps1

@@ -7,10 +7,10 @@ Saves package properties in source of a given service directory to JSON files.
 JSON files are named in the form <package name>.json or <artifact name>.json if
 an artifact name property is available in the package properties.
 
-Can optionally add a dev version property which can be used logic for daily 
+Can optionally add a dev version property which can be used logic for daily
 builds.
 
-In cases of collisions where track 2 packages (IsNewSdk = true) have the same 
+In cases of collisions where track 2 packages (IsNewSdk = true) have the same
 filename as track 1 packages (e.g. same artifact name or package name), the
 track 2 package properties will be written.
 
@@ -21,22 +21,22 @@ Service directory in which to search for packages.
 A file path leading to a file generated from Generate-PR-Diff.json. This parameter takes precedence over serviceDirectory, do not provide both.
 
 .PARAMETER outDirectory
-Output location (generally a package artifact directory in DevOps) for JSON 
+Output location (generally a package artifact directory in DevOps) for JSON
 files
 
 .PARAMETER addDevVersion
-Reads the version out of the source and adds a DevVersion property to the 
-package properties JSON file. If the package properties JSON file already 
+Reads the version out of the source and adds a DevVersion property to the
+package properties JSON file. If the package properties JSON file already
 exists, read the Version property from the existing package properties JSON file
 and set that as the Version property for the new output. This has the effect of
-"adding" a DevVersion property to the file which could be different from the 
+"adding" a DevVersion property to the file which could be different from the
 Verison property in that file.
 #>
 
 [CmdletBinding()]
 Param (
   [string] $serviceDirectory,
-  [Parameter(Mandatory=$True)]
+  [Parameter(Mandatory = $True)]
   [string] $outDirectory,
   [string] $prDiff,
   [switch] $addDevVersion
@@ -44,7 +44,8 @@ Param (
 
 . (Join-Path $PSScriptRoot common.ps1)
 
-function SetOutput($outputPath, $incomingPackageSpec) {
+function SetOutput($outputPath, $incomingPackageSpec)
+{
 
   # If there is an exsiting package info json file read that and set that as output object which gets properties updated here.
   if (Test-Path $outputPath)
@@ -56,7 +57,7 @@ function SetOutput($outputPath, $incomingPackageSpec) {
   {
     $outputObject = $incomingPackageSpec
   }
-  
+
 
   if ($addDevVersion)
   {
@@ -75,16 +76,19 @@ function SetOutput($outputPath, $incomingPackageSpec) {
     -Value (ConvertTo-Json -InputObject $outputObject -Depth 100)
 }
 
-function GetRelativePath($path) {
+function GetRelativePath($path)
+{
   # If the path is empty return an empty string
-  if (!$path) {
+  if (!$path)
+  {
     return ''
   }
 
   # If the path is already relative return the path. Calling `GetRelativePath`
   # on a relative path converts the relative path to an absolute path based on
   # the current working directory which can result in unexpected outputs.
-  if (![IO.Path]::IsPathRooted($path)) {
+  if (![IO.Path]::IsPathRooted($path))
+  {
     return $path
   }
 
@@ -98,22 +102,26 @@ $exportedPaths = @{}
 
 $allPackageProperties = @()
 
-if ($prDiff) {
+if ($prDiff)
+{
   Write-Host "Getting package properties for PR diff file: $prDiff"
   $allPackageProperties = Get-PrPkgProperties $prDiff
 
-  if (!$allPackageProperties) {
+  if (!$allPackageProperties)
+  {
     Write-Host "No packages found matching PR diff file $prDiff"
     Write-Host "Setting NoPackagesChanged variable to true"
     Write-Host "##vso[task.setvariable variable=NoPackagesChanged]true"
     exit 0
   }
 }
-else {
+else
+{
   Write-Host "Getting package properties for service directory: $serviceDirectory"
   $allPackageProperties = Get-AllPkgProperties $serviceDirectory
 
-  if (!$allPackageProperties) {
+  if (!$allPackageProperties)
+  {
     Write-Error "Package properties are not available for service directory $serviceDirectory"
     exit 1
   }
@@ -124,21 +132,26 @@ if (-not (Test-Path -Path $outDirectory))
   New-Item -ItemType Directory -Force -Path $outDirectory | Out-Null
 }
 
-foreach($pkg in $allPackageProperties)
+foreach ($pkg in $allPackageProperties)
 {
-  if ($pkg.Name) {
+  if ($pkg.Name)
+  {
+    Write-Host ""
     Write-Host "Package Name: $($pkg.Name)"
     Write-Host "Package Version: $($pkg.Version)"
     Write-Host "Package SDK Type: $($pkg.SdkType)"
     Write-Host "Artifact Name: $($pkg.ArtifactName)"
     Write-Host "Release date: $($pkg.ReleaseStatus)"
     $configFilePrefix = $pkg.Name
+
     if ($pkg.ArtifactName)
     {
       $configFilePrefix = $pkg.ArtifactName
     }
+
     $outputPath = Join-Path -Path $outDirectory "$configFilePrefix.json"
     Write-Host "Output path of json file: $outputPath"
+
     $outDir = Split-Path $outputPath -parent
     if (-not (Test-Path -path $outDir))
     {
@@ -148,14 +161,17 @@ foreach($pkg in $allPackageProperties)
 
     # If package properties for a track 2 (IsNewSdk = true) package has
     # already been written, skip writing to that same path.
-    if ($exportedPaths.ContainsKey($outputPath) -and $exportedPaths[$outputPath].IsNewSdk -eq $true) {
+    if ($exportedPaths.ContainsKey($outputPath) -and $exportedPaths[$outputPath].IsNewSdk -eq $true)
+    {
       Write-Host "Track 2 package info with file name $($outputPath) already exported. Skipping export."
       continue
     }
-    $exportedPaths[$outputPath] = $pkg
 
+    $exportedPaths[$outputPath] = $pkg
     SetOutput $outputPath $pkg
   }
 }
 
-Get-ChildItem -Path $outDirectory
+$fileNames = (Get-ChildItem -Path $outDirectory).Name
+Write-Host "`nFiles written to $outDirectory`:"
+Write-Host "  $($fileNames -join "`n  ")"

eng/common/scripts/Test-SampleMetadata.ps1

@@ -422,6 +422,8 @@ begin {
         "office-word",
         "office-yammer",
         "passport-azure-ad",
+        "playwright",
+        "playwright-testing",
         "power-apps",
         "power-automate",
         "power-bi",

eng/common/scripts/common.ps1

@@ -62,7 +62,8 @@ $GetEmitterAdditionalOptionsFn = "Get-${Language}-EmitterAdditionalOptions"
 $GetEmitterNameFn = "Get-${Language}-EmitterName"
 $GetDirectoriesForGenerationFn = "Get-${Language}-DirectoriesForGeneration"
 $UpdateGeneratedSdksFn = "Update-${Language}-GeneratedSdks"
-$IsApiviewStatusCheckRequiredFn = "Get-${Language}-ApiviewStatusCheckRequirement" 
+$IsApiviewStatusCheckRequiredFn = "Get-${Language}-ApiviewStatusCheckRequirement"
+$AdditionalValidationPackagesFromPackageSetFn = "Get-${Language}-AdditionalValidationPackagesFromPackageSet"
 
 # Expected to be set in eng/scripts/docs/Docs-Onboarding.ps1
 $SetDocsPackageOnboarding = "Set-${Language}-DocsPackageOnboarding"

eng/common/testproxy/target_version.txt

@@ -1 +1 @@
-1.0.0-dev.20240806.1
+1.0.0-dev.20240823.1

eng/pipelines/templates/steps/test-packages.yml

@@ -33,54 +33,9 @@ steps:
     displayName: "Tag scheduled builds"
     condition: and(eq(variables['Build.SourceBranchName'], variables['DefaultBranch']), eq(variables['Build.Reason'],'Schedule'))
 
-  # now we need to call Save-Package-Properties so that we can filter on it
-  - ${{ if eq(variables['Build.Reason'], 'PullRequest') }}:
-      - pwsh: |
-          mkdir -p $(Build.ArtifactStagingDirectory)/diff
-        displayName: Create PR Diff Folder
-
-      - pwsh: |
-          $location = Join-Path "$(Build.ArtifactStagingDirectory)" "diff"
-
-          Write-Host "./eng/common/scripts/Generate-PR-Diff.ps1 -TargetPath `"$(Build.SourcesDirectory)`" -ArtifactPath `"$location`""
-          ./eng/common/scripts/Generate-PR-Diff.ps1 -TargetPath "$(Build.SourcesDirectory)" -ArtifactPath "$location"
-        displayName: Generate PR Diff
-
-      - pwsh: |
-          Write-Host "Freshly generated the PR diff:"
-          Get-ChildItem -R -Force $(Build.ArtifactStagingDirectory)/diff | % { $_.FullName }
-          cat $(Build.ArtifactStagingDirectory)/diff/diff.json
-        displayName: Dump PR Diff
-
-      - task: Powershell@2
-        displayName: Save package properties filtered for PR
-        inputs:
-          filePath: $(Build.SourcesDirectory)/eng/common/scripts/Save-Package-Properties.ps1
-          arguments: >
-            -PrDiff $(Build.ArtifactStagingDirectory)/diff/diff.json
-            -OutDirectory $(Build.ArtifactStagingDirectory)/PackageInfo
-          pwsh: true
-
-  - ${{ else }}:
-      - task: Powershell@2
-        displayName: Save package properties with dev version
-        condition: and(succeeded(), eq(variables['SetDevVersion'],'true'))
-        inputs:
-          filePath: $(Build.SourcesDirectory)/eng/common/scripts/Save-Package-Properties.ps1
-          arguments: >
-            -ServiceDirectory ${{parameters.ServiceDirectory}}
-            -OutDirectory $(Build.ArtifactStagingDirectory)/PackageInfo
-            -AddDevVersion
-          pwsh: true
-      - task: Powershell@2
-        displayName: Save package properties for service
-        condition: and(succeeded(), ne(variables['SetDevVersion'],'true'))
-        inputs:
-          filePath: $(Build.SourcesDirectory)/eng/common/scripts/Save-Package-Properties.ps1
-          arguments: >
-            -ServiceDirectory ${{parameters.ServiceDirectory}}
-            -OutDirectory $(Build.ArtifactStagingDirectory)/PackageInfo
-          pwsh: true
+  - template: /eng/common/pipelines/templates/steps/save-package-properties.yml@self
+    parameters:
+      ServiceDirectory: ${{ parameters.ServiceDirectory }}
 
   - ${{ if parameters.PublishArtifacts}}:
       - template: /eng/common/pipelines/templates/steps/publish-1es-artifact.yml

eng/scripts/Language-Settings.ps1

@@ -84,7 +84,7 @@ function Get-AllPackageInfoFromRepo ([string] $ServiceDirectory) {
       $pkgProp.SdkType = "client"
     }
 
-    $pkgProp.DependentPackages = GetDependentPackages $package | Select-Object -ExpandProperty RelativePath
+    $pkgProp.AdditionalValidationPackages = GetDependentPackages $package | Select-Object -ExpandProperty RelativePath
 
     $allPackageProps += $pkgProp
   }