From 409d7ebd2d12747337cd551ddc253831e157c1d6 Mon Sep 17 00:00:00 2001 From: Scott Beddall <45376673+scbedd@users.noreply.github.com> Date: Wed, 30 Oct 2024 16:55:38 -0700 Subject: [PATCH] `Create-PrJobMatrix` (#9281) * package-properties are now populated with matrix configurations from their ci.yml if present * create new code path for generate-job-matrix.yml which combines Create-JobMatrix and the "distribute-packages-to-matrix" action to generate dynamic matrices for PRs --- .../templates/jobs/generate-job-matrix.yml | 136 +++++++++++------- .../scripts/Helpers/Package-Helpers.ps1 | 80 ++++++++++- eng/common/scripts/Package-Properties.ps1 | 111 +++++++------- .../scripts/job-matrix/Create-PrJobMatrix.ps1 | 126 ++++++++++++++++ 4 files changed, 340 insertions(+), 113 deletions(-) create mode 100644 eng/common/scripts/job-matrix/Create-PrJobMatrix.ps1 diff --git a/eng/common/pipelines/templates/jobs/generate-job-matrix.yml b/eng/common/pipelines/templates/jobs/generate-job-matrix.yml index a7459e6b5..ab67e915d 100644 --- a/eng/common/pipelines/templates/jobs/generate-job-matrix.yml +++ b/eng/common/pipelines/templates/jobs/generate-job-matrix.yml @@ -42,6 +42,12 @@ parameters: - name: PreGenerationSteps type: stepList default: [] +- name: EnablePRGeneration + type: boolean + default: false +- name: PRMatrixSetting + type: string + default: 'ArtifactPackageNames' # Mappings to OS name required at template compile time by 1es pipeline templates - name: Pools type: object @@ -84,57 +90,87 @@ jobs: - ${{ parameters.PreGenerationSteps }} - - ${{ each config in parameters.MatrixConfigs }}: + - ${{ if eq(parameters.EnablePRGeneration, false) }}: + - ${{ each config in parameters.MatrixConfigs }}: + - ${{ each pool in parameters.Pools }}: + - ${{ if eq(config.GenerateVMJobs, 'true') }}: + - task: Powershell@2 + inputs: + pwsh: true + filePath: eng/common/scripts/job-matrix/Create-JobMatrix.ps1 + arguments: > + -ConfigPath ${{ config.Path }} + -Selection ${{ config.Selection }} + -DisplayNameFilter '$(displayNameFilter)' + -Filters '${{ join(''',''', parameters.MatrixFilters) }}', 'container=^$', 'SupportedClouds=^$|${{ parameters.CloudConfig.Cloud }}', 'Pool=${{ pool.filter }}' + -Replace '${{ join(''',''', parameters.MatrixReplace) }}' + -NonSparseParameters '${{ join(''',''', config.NonSparseParameters) }}' + displayName: Create ${{ pool.name }} Matrix ${{ config.Name }} + name: vm_job_matrix_${{ config.Name }}_${{ pool.name }} + - ${{ if eq(config.GenerateContainerJobs, 'true') }}: + - task: Powershell@2 + inputs: + pwsh: true + filePath: eng/common/scripts/job-matrix/Create-JobMatrix.ps1 + arguments: > + -ConfigPath ${{ config.Path }} + -Selection ${{ config.Selection }} + -DisplayNameFilter '$(displayNameFilter)' + -Filters '${{ join(''',''', parameters.MatrixFilters) }}', 'container=^$', 'SupportedClouds=^$|${{ parameters.CloudConfig.Cloud }}', 'Pool=${{ pool.filter }}' + -NonSparseParameters '${{ join(''',''', config.NonSparseParameters) }}' + displayName: Create ${{ pool.name }} Container Matrix ${{ config.Name }} + name: container_job_matrix_${{ config.Name }}_${{ pool.name }} + + # This else being set also currently assumes that the $(Build.ArtifactStagingDirectory)/PackageInfo folder is populated by PreGenerationSteps. + # Not currently not hardcoded, so not doing the needful and populating this folder before we hit this step will result in generation errors. + - ${{ else }}: - ${{ each pool in parameters.Pools }}: - - ${{ if eq(config.GenerateVMJobs, 'true') }}: - - task: Powershell@2 - inputs: - pwsh: true - filePath: eng/common/scripts/job-matrix/Create-JobMatrix.ps1 - arguments: > - -ConfigPath ${{ config.Path }} - -Selection ${{ config.Selection }} - -DisplayNameFilter '$(displayNameFilter)' - -Filters '${{ join(''',''', parameters.MatrixFilters) }}', 'container=^$', 'SupportedClouds=^$|${{ parameters.CloudConfig.Cloud }}', 'Pool=${{ pool.filter }}' - -Replace '${{ join(''',''', parameters.MatrixReplace) }}' - -NonSparseParameters '${{ join(''',''', config.NonSparseParameters) }}' - displayName: Create ${{ pool.name }} Matrix ${{ config.Name }} - name: vm_job_matrix_${{ config.Name }}_${{ pool.name }} + - pwsh: | + # dump the conglomerated CI matrix + '${{ convertToJson(parameters.MatrixConfigs) }}' | Set-Content matrix.json - - ${{ if eq(config.GenerateContainerJobs, 'true') }}: - - task: Powershell@2 - inputs: - pwsh: true - filePath: eng/common/scripts/job-matrix/Create-JobMatrix.ps1 - arguments: > - -ConfigPath ${{ config.Path }} - -Selection ${{ config.Selection }} - -DisplayNameFilter '$(displayNameFilter)' - -Filters '${{ join(''',''', parameters.MatrixFilters) }}', 'container=^$', 'SupportedClouds=^$|${{ parameters.CloudConfig.Cloud }}', 'Pool=${{ pool.filter }}' - -NonSparseParameters '${{ join(''',''', config.NonSparseParameters) }}' - displayName: Create ${{ pool.name }} Container Matrix ${{ config.Name }} - name: container_job_matrix_${{ config.Name }}_${{ pool.name }} + ./eng/common/scripts/job-matrix/Create-PrJobMatrix.ps1 ` + -PackagePropertiesFolder $(Build.ArtifactStagingDirectory)/PackageInfo ` + -PRMatrixFile matrix.json ` + -PRMatrixSetting ${{ parameters.PRMatrixSetting }} ` + -DisplayNameFilter '$(displayNameFilter)' ` + -Filters '${{ join(''',''', parameters.MatrixFilters) }}', 'container=^$', 'SupportedClouds=^$|${{ parameters.CloudConfig.Cloud }}', 'Pool=${{ pool.filter }}' ` + -Replace '${{ join(''',''', parameters.MatrixReplace) }}' + displayName: Create ${{ pool.name }} PR Matrix + name: vm_job_matrix_pr_${{ pool.name }} -- ${{ each config in parameters.MatrixConfigs }}: +- ${{ if eq(parameters.EnablePRGeneration, false) }}: + - ${{ each config in parameters.MatrixConfigs }}: + - ${{ each pool in parameters.Pools }}: + - ${{ if eq(config.GenerateVMJobs, 'true') }}: + - template: ${{ parameters.JobTemplatePath }} + parameters: + UsePlatformContainer: false + OSName: ${{ pool.os }} + Matrix: dependencies.${{ parameters.GenerateJobName }}.outputs['vm_job_matrix_${{ config.Name }}_${{ pool.name }}.matrix'] + DependsOn: ${{ parameters.GenerateJobName }} + CloudConfig: ${{ parameters.CloudConfig }} + ${{ each param in parameters.AdditionalParameters }}: + ${{ param.key }}: ${{ param.value }} + + - ${{ if eq(config.GenerateContainerJobs, 'true') }}: + - template: ${{ parameters.JobTemplatePath }} + parameters: + UsePlatformContainer: true + OSName: ${{ pool.os }} + Matrix: dependencies.${{ parameters.GenerateJobName }}.outputs['vm_job_matrix_${{ config.Name }}_${{ pool.name }}.matrix'] + DependsOn: ${{ parameters.GenerateJobName }} + CloudConfig: ${{ parameters.CloudConfig }} + ${{ each param in parameters.AdditionalParameters }}: + ${{ param.key }}: ${{ param.value }} +- ${{ else }}: - ${{ each pool in parameters.Pools }}: - - ${{ if eq(config.GenerateVMJobs, 'true') }}: - - template: ${{ parameters.JobTemplatePath }} - parameters: - UsePlatformContainer: false - OSName: ${{ pool.os }} - Matrix: dependencies.${{ parameters.GenerateJobName }}.outputs['vm_job_matrix_${{ config.Name }}_${{ pool.name }}.matrix'] - DependsOn: ${{ parameters.GenerateJobName }} - CloudConfig: ${{ parameters.CloudConfig }} - ${{ each param in parameters.AdditionalParameters }}: - ${{ param.key }}: ${{ param.value }} - - - ${{ if eq(config.GenerateContainerJobs, 'true') }}: - - template: ${{ parameters.JobTemplatePath }} - parameters: - UsePlatformContainer: true - OSName: ${{ pool.os }} - Matrix: dependencies.${{ parameters.GenerateJobName }}.outputs['vm_job_matrix_${{ config.Name }}_${{ pool.name }}.matrix'] - DependsOn: ${{ parameters.GenerateJobName }} - CloudConfig: ${{ parameters.CloudConfig }} - ${{ each param in parameters.AdditionalParameters }}: - ${{ param.key }}: ${{ param.value }} + - template: ${{ parameters.JobTemplatePath }} + parameters: + UsePlatformContainer: false + OSName: ${{ pool.os }} + Matrix: dependencies.${{ parameters.GenerateJobName }}.outputs['vm_job_matrix_pr_${{ pool.name }}.matrix'] + DependsOn: ${{ parameters.GenerateJobName }} + CloudConfig: ${{ parameters.CloudConfig }} + ${{ each param in parameters.AdditionalParameters }}: + ${{ param.key }}: ${{ param.value }} diff --git a/eng/common/scripts/Helpers/Package-Helpers.ps1 b/eng/common/scripts/Helpers/Package-Helpers.ps1 index 3c882b311..e83be6643 100644 --- a/eng/common/scripts/Helpers/Package-Helpers.ps1 +++ b/eng/common/scripts/Helpers/Package-Helpers.ps1 @@ -170,10 +170,86 @@ function GetValueSafelyFrom-Yaml { $current = $current[$key] } else { - Write-Host "The '$key' part of the path $($Keys -join "/") doesn't exist or is null." return $null } } return [object]$current -} \ No newline at end of file +} + +function Get-ObjectKey { + param ( + [Parameter(Mandatory = $true)] + [object]$Object + ) + + if (-not $Object) { + return "unset" + } + + if ($Object -is [hashtable] -or $Object -is [System.Collections.Specialized.OrderedDictionary]) { + $sortedEntries = $Object.GetEnumerator() | Sort-Object Name + $hashString = ($sortedEntries | ForEach-Object { "$($_.Key)=$($_.Value)" }) -join ";" + return $hashString.GetHashCode() + } + + elseif ($Object -is [PSCustomObject]) { + $sortedProperties = $Object.PSObject.Properties | Sort-Object Name + $propertyString = ($sortedProperties | ForEach-Object { "$($_.Name)=$($_.Value)" }) -join ";" + return $propertyString.GetHashCode() + } + + elseif ($Object -is [array]) { + $arrayString = ($Object | ForEach-Object { Get-ObjectKey $_ }) -join ";" + return $arrayString.GetHashCode() + } + + else { + return $Object.GetHashCode() + } +} + +function Group-ByObjectKey { + param ( + [Parameter(Mandatory)] + [array]$Items, + + [Parameter(Mandatory)] + [string]$GroupByProperty + ) + + $groupedDictionary = @{} + + foreach ($item in $Items) { + $key = Get-ObjectKey $item."$GroupByProperty" + + if (-not $groupedDictionary.ContainsKey($key)) { + $groupedDictionary[$key] = @() + } + + # Add the current item to the array for this key + $groupedDictionary[$key] += $item + } + + return $groupedDictionary +} + +function Split-ArrayIntoBatches { + param ( + [Parameter(Mandatory = $true)] + [Object[]]$InputArray, + + [Parameter(Mandatory = $true)] + [int]$BatchSize + ) + + $batches = @() + + for ($i = 0; $i -lt $InputArray.Count; $i += $BatchSize) { + $batch = $InputArray[$i..[math]::Min($i + $BatchSize - 1, $InputArray.Count - 1)] + + $batches += , $batch + } + + return , $batches +} diff --git a/eng/common/scripts/Package-Properties.ps1 b/eng/common/scripts/Package-Properties.ps1 index 61054f10f..38d181eaa 100644 --- a/eng/common/scripts/Package-Properties.ps1 +++ b/eng/common/scripts/Package-Properties.ps1 @@ -1,8 +1,7 @@ # Helper functions for retrieving useful information from azure-sdk-for-* repo . "${PSScriptRoot}\logging.ps1" . "${PSScriptRoot}\Helpers\Package-Helpers.ps1" -class PackageProps -{ +class PackageProps { [string]$Name [string]$Version [string]$DevVersion @@ -21,14 +20,13 @@ class PackageProps # additional packages required for validation of this one [string[]]$AdditionalValidationPackages [HashTable]$ArtifactDetails + [HashTable[]]$CIMatrixConfigs - PackageProps([string]$name, [string]$version, [string]$directoryPath, [string]$serviceDirectory) - { + PackageProps([string]$name, [string]$version, [string]$directoryPath, [string]$serviceDirectory) { $this.Initialize($name, $version, $directoryPath, $serviceDirectory) } - PackageProps([string]$name, [string]$version, [string]$directoryPath, [string]$serviceDirectory, [string]$group = "") - { + PackageProps([string]$name, [string]$version, [string]$directoryPath, [string]$serviceDirectory, [string]$group = "") { $this.Initialize($name, $version, $directoryPath, $serviceDirectory, $group) } @@ -37,35 +35,29 @@ class PackageProps [string]$version, [string]$directoryPath, [string]$serviceDirectory - ) - { + ) { $this.Name = $name $this.Version = $version $this.DirectoryPath = $directoryPath $this.ServiceDirectory = $serviceDirectory $this.IncludedForValidation = $false - if (Test-Path (Join-Path $directoryPath "README.md")) - { + if (Test-Path (Join-Path $directoryPath "README.md")) { $this.ReadMePath = Join-Path $directoryPath "README.md" } - else - { + else { $this.ReadMePath = $null } - if (Test-Path (Join-Path $directoryPath "CHANGELOG.md")) - { + if (Test-Path (Join-Path $directoryPath "CHANGELOG.md")) { $this.ChangeLogPath = Join-Path $directoryPath "CHANGELOG.md" # Get release date for current version and set in package property $changeLogEntry = Get-ChangeLogEntry -ChangeLogLocation $this.ChangeLogPath -VersionString $this.Version - if ($changeLogEntry -and $changeLogEntry.ReleaseStatus) - { - $this.ReleaseStatus = $changeLogEntry.ReleaseStatus.Trim().Trim("()") + if ($changeLogEntry -and $changeLogEntry.ReleaseStatus) { + $this.ReleaseStatus = $changeLogEntry.ReleaseStatus.Trim().Trim("()") } } - else - { + else { $this.ChangeLogPath = $null } @@ -78,14 +70,12 @@ class PackageProps [string]$directoryPath, [string]$serviceDirectory, [string]$group - ) - { + ) { $this.Initialize($name, $version, $directoryPath, $serviceDirectory) $this.Group = $group } - hidden [HashTable]ParseYmlForArtifact([string]$ymlPath) { - + hidden [PSCustomObject]ParseYmlForArtifact([string]$ymlPath) { $content = LoadFrom-Yaml $ymlPath if ($content) { $artifacts = GetValueSafelyFrom-Yaml $content @("extends", "parameters", "Artifacts") @@ -95,24 +85,40 @@ class PackageProps $artifactForCurrentPackage = $artifacts | Where-Object { $_["name"] -eq $this.ArtifactName -or $_["name"] -eq $this.Name } } + # if we found an artifact for the current package, we should count this ci file as the source of the matrix for this package if ($artifactForCurrentPackage) { - return [HashTable]$artifactForCurrentPackage + $result = [PSCustomObject]@{ + ArtifactConfig = [HashTable]$artifactForCurrentPackage + MatrixConfigs = @() + } + + # if we know this is the matrix for our file, we should now see if there is a custom matrix config for the package + $matrixConfigList = GetValueSafelyFrom-Yaml $content @("extends", "parameters", "MatrixConfigs") + + if ($matrixConfigList) { + $result.MatrixConfigs = $matrixConfigList + } + + return $result } } return $null } - [void]InitializeCIArtifacts(){ + [void]InitializeCIArtifacts() { $RepoRoot = Resolve-Path (Join-Path $PSScriptRoot ".." ".." "..") $ciFolderPath = Join-Path -Path $RepoRoot -ChildPath (Join-Path "sdk" $this.ServiceDirectory) $ciFiles = Get-ChildItem -Path $ciFolderPath -Filter "ci*.yml" -File if (-not $this.ArtifactDetails) { - foreach($ciFile in $ciFiles) { + foreach ($ciFile in $ciFiles) { $ciArtifactResult = $this.ParseYmlForArtifact($ciFile.FullName) if ($ciArtifactResult) { - $this.ArtifactDetails = [Hashtable]$ciArtifactResult + $this.ArtifactDetails = [Hashtable]$ciArtifactResult.ArtifactConfig + $this.CIMatrixConfigs = $ciArtifactResult.MatrixConfigs + # if this package appeared in this ci file, then we should + # treat this CI file as the source of the Matrix for this package break } } @@ -124,8 +130,7 @@ class PackageProps # Returns important properties of the package relative to the language repo # Returns a PS Object with properties @ { pkgName, pkgVersion, pkgDirectoryPath, pkgReadMePath, pkgChangeLogPath } # Note: python is required for parsing python package properties. -function Get-PkgProperties -{ +function Get-PkgProperties { Param ( [Parameter(Mandatory = $true)] @@ -136,10 +141,8 @@ function Get-PkgProperties $allPkgProps = Get-AllPkgProperties -ServiceDirectory $ServiceDirectory $pkgProps = $allPkgProps.Where({ $_.Name -eq $PackageName -or $_.ArtifactName -eq $PackageName }); - if ($pkgProps.Count -ge 1) - { - if ($pkgProps.Count -gt 1) - { + if ($pkgProps.Count -ge 1) { + if ($pkgProps.Count -gt 1) { Write-Host "Found more than one project with the name [$PackageName], choosing the first one under $($pkgProps[0].DirectoryPath)" } return $pkgProps[0] @@ -159,14 +162,12 @@ function Get-PrPkgProperties([string]$InputDiffJson) { $additionalValidationPackages = @() $lookup = @{} - foreach ($pkg in $allPackageProperties) - { + foreach ($pkg in $allPackageProperties) { $pkgDirectory = Resolve-Path "$($pkg.DirectoryPath)" $lookupKey = ($pkg.DirectoryPath).Replace($RepoRoot, "").TrimStart('\/') $lookup[$lookupKey] = $pkg - foreach ($file in $targetedFiles) - { + foreach ($file in $targetedFiles) { $filePath = Resolve-Path (Join-Path $RepoRoot $file) $shouldInclude = $filePath -like "$pkgDirectory*" if ($shouldInclude) { @@ -191,8 +192,7 @@ function Get-PrPkgProperties([string]$InputDiffJson) { } } - if ($AdditionalValidationPackagesFromPackageSetFn -and (Test-Path "Function:$AdditionalValidationPackagesFromPackageSetFn")) - { + if ($AdditionalValidationPackagesFromPackageSetFn -and (Test-Path "Function:$AdditionalValidationPackagesFromPackageSetFn")) { $packagesWithChanges += &$AdditionalValidationPackagesFromPackageSetFn $packagesWithChanges $diff $allPackageProperties } @@ -202,25 +202,19 @@ function Get-PrPkgProperties([string]$InputDiffJson) { # Takes ServiceName and Repo Root Directory # Returns important properties for each package in the specified service, or entire repo if the serviceName is not specified # Returns a Table of service key to array values of PS Object with properties @ { pkgName, pkgVersion, pkgDirectoryPath, pkgReadMePath, pkgChangeLogPath } -function Get-AllPkgProperties ([string]$ServiceDirectory = $null) -{ +function Get-AllPkgProperties ([string]$ServiceDirectory = $null) { $pkgPropsResult = @() - if (Test-Path "Function:Get-AllPackageInfoFromRepo") - { + if (Test-Path "Function:Get-AllPackageInfoFromRepo") { $pkgPropsResult = Get-AllPackageInfoFromRepo -ServiceDirectory $serviceDirectory } - else - { - if ([string]::IsNullOrEmpty($ServiceDirectory)) - { - foreach ($dir in (Get-ChildItem (Join-Path $RepoRoot "sdk") -Directory)) - { + else { + if ([string]::IsNullOrEmpty($ServiceDirectory)) { + foreach ($dir in (Get-ChildItem (Join-Path $RepoRoot "sdk") -Directory)) { $pkgPropsResult += Get-PkgPropsForEntireService -serviceDirectoryPath $dir.FullName } } - else - { + else { $pkgPropsResult = Get-PkgPropsForEntireService -serviceDirectoryPath (Join-Path $RepoRoot "sdk" $ServiceDirectory) } } @@ -230,29 +224,24 @@ function Get-AllPkgProperties ([string]$ServiceDirectory = $null) # Given the metadata url under https://github.com/Azure/azure-sdk/tree/main/_data/releases/latest, # the function will return the csv metadata back as part of the response. -function Get-CSVMetadata ([string]$MetadataUri=$MetadataUri) -{ +function Get-CSVMetadata ([string]$MetadataUri = $MetadataUri) { $metadataResponse = Invoke-RestMethod -Uri $MetadataUri -method "GET" -MaximumRetryCount 3 -RetryIntervalSec 10 | ConvertFrom-Csv return $metadataResponse } -function Get-PkgPropsForEntireService ($serviceDirectoryPath) -{ +function Get-PkgPropsForEntireService ($serviceDirectoryPath) { $projectProps = @() # Properties from every project in the service $serviceDirectory = $serviceDirectoryPath -replace '^.*[\\/]+sdk[\\/]+([^\\/]+).*$', '$1' - if (!$GetPackageInfoFromRepoFn -or !(Test-Path "Function:$GetPackageInfoFromRepoFn")) - { + if (!$GetPackageInfoFromRepoFn -or !(Test-Path "Function:$GetPackageInfoFromRepoFn")) { LogError "The function for '$GetPackageInfoFromRepoFn' was not found.` Make sure it is present in eng/scripts/Language-Settings.ps1 and referenced in eng/common/scripts/common.ps1.` See https://github.com/Azure/azure-sdk-tools/blob/main/doc/common/common_engsys.md#code-structure" } - foreach ($directory in (Get-ChildItem $serviceDirectoryPath -Directory)) - { + foreach ($directory in (Get-ChildItem $serviceDirectoryPath -Directory)) { $pkgProps = &$GetPackageInfoFromRepoFn $directory.FullName $serviceDirectory - if ($null -ne $pkgProps) - { + if ($null -ne $pkgProps) { $projectProps += $pkgProps } } diff --git a/eng/common/scripts/job-matrix/Create-PrJobMatrix.ps1 b/eng/common/scripts/job-matrix/Create-PrJobMatrix.ps1 new file mode 100644 index 000000000..b7c3b9d3a --- /dev/null +++ b/eng/common/scripts/job-matrix/Create-PrJobMatrix.ps1 @@ -0,0 +1,126 @@ +<# +.SYNOPSIS +Generates a combined PR job matrix from a package properties folder. It is effectively a combination of +Create-JobMatrix and distribute-packages-to-matrix. + +.DESCRIPTION +Create-JobMatrix has a limitation in that it accepts one or multiple matrix files, but it doesn't allow runtime +selection of the matrix file based on what is being built. Due to this, this script exists to provide exactly +that mapping. + +It should be called from a PR build only. + +It generates the matrix by the following algorithm: + - load all package properties files + - group the package properties by their targeted CI Matrix Configs + - for each package group, generate the matrix for each matrix config in the group (remember MatrixConfigs is a list not a single object) + - for each matrix config, generate the matrix + - calculate if batching is necessary for this matrix config + - for each batch + - create combined property name for the batch + - walk each matrix item + - add suffixes for batch and matrix config if nececessary to the job name + - add the combined property name to the parameters of the matrix item + - add the matrix item to the overall result + +.EXAMPLE +./eng/common/scripts/job-matrix/Create-PrJobMatrix.ps1 ` + -PackagePropertiesFolder "path/to/populated/PackageInfo" ` + -PrMatrixSetting "" +#> + +[CmdletBinding()] +param ( + [Parameter(Mandatory = $true)][string] $PackagePropertiesFolder, + [Parameter(Mandatory = $true)][string] $PRMatrixFile, + [Parameter(Mandatory = $true)][string] $PRMatrixSetting, + [Parameter(Mandatory = $False)][string] $DisplayNameFilter, + [Parameter(Mandatory = $False)][array] $Filters, + [Parameter(Mandatory = $False)][array] $Replace, + [Parameter()][switch] $CI = ($null -ne $env:SYSTEM_TEAMPROJECTID) +) + +. $PSScriptRoot/job-matrix-functions.ps1 +. $PSScriptRoot/../Helpers/Package-Helpers.ps1 +$BATCHSIZE = 10 + +if (!(Test-Path $PackagePropertiesFolder)) { + Write-Error "Package Properties folder doesn't exist" + exit 1 +} + +if (!(Test-Path $PRMatrixFile)) { + Write-Error "PR Matrix file doesn't exist" + exit 1 +} + +Write-Host "Generating PR job matrix for $PackagePropertiesFolder" + +$configs = Get-Content -Raw $PRMatrixFile | ConvertFrom-Json + +# calculate general targeting information and create our batches prior to generating any matrix +$packageProperties = Get-ChildItem -Recurse "$PackagePropertiesFolder" *.json ` +| ForEach-Object { Get-Content -Path $_.FullName | ConvertFrom-Json } + +# set default matrix config for each package if there isn't an override +$packageProperties | ForEach-Object { + $_.CIMatrixConfigs = $_.CIMatrixConfigs ?? $configs +} + +# The key here is that after we group the packages by the matrix config objects, we can use the first item's MatrixConfig +# to generate the matrix for the group, no reason to have to parse the key value backwards to get the matrix config. +$matrixBatchesByConfig = Group-ByObjectKey $packageProperties "CIMatrixConfigs" + +$OverallResult = @() +foreach ($matrixBatchKey in $matrixBatchesByConfig.Keys) { + $matrixBatch = $matrixBatchesByConfig[$matrixBatchKey] + $matrixConfigs = $matrixBatch | Select-Object -First 1 -ExpandProperty CIMatrixConfigs + + $matrixResults = @() + foreach ($matrixConfig in $matrixConfigs) { + Write-Host "Generating config for $($matrixConfig.Path)" + $matrixResults = GenerateMatrixForConfig ` + -ConfigPath $matrixConfig.Path ` + -Selection $matrixConfig.Selection ` + -DisplayNameFilter $DisplayNameFilter ` + -Filters $Filters ` + -Replace $Replace + + $packageBatches = Split-ArrayIntoBatches -InputArray $matrixBatch -BatchSize $BATCHSIZE + + # we only need to modify the generated job name if there is more than one matrix config or batch in the matrix + $matrixSuffixNecessary = $matrixConfigs.Count -gt 1 + $batchSuffixNecessary = $packageBatches.Length -gt 1 + + foreach ($batch in $packageBatches) { + # to understand this iteration, one must understand that the matrix is a list of hashtables, each with a couple keys: + # [ + # { "name": "jobname", "parameters": { matrixSetting1: matrixValue1, ...} }, + # ] + foreach ($matrixOutputItem in $matrixResults) { + $namesForBatch = ($batch | ForEach-Object { $_.ArtifactName }) -join "-" + # we just need to iterate across them, grab the parameters hashtable, and add the new key + # if there is more than one batch, we will need to add a suffix including the batch name to the job name + $matrixOutputItem["parameters"]["$PRMatrixSetting"] = $namesForBatch + + if ($matrixSuffixNecessary) { + $matrixOutputItem["name"] = $matrixOutputItem["name"] + $matrixConfig.Name + } + + if ($batchSuffixNecessary) { + $matrixOutputItem["name"] = $matrixOutputItem["name"] + $namesForBatch + } + + $OverallResult += $matrixOutputItem + } + } + } +} + +$serialized = SerializePipelineMatrix $OverallResult + +Write-Output $serialized.pretty + +if ($CI) { + Write-Output "##vso[task.setVariable variable=matrix;isOutput=true]$($serialized.compressed)" +}