From 52ee4555af2c11737399cde55748cdcf620ef945 Mon Sep 17 00:00:00 2001 From: Manuel de la Pena Date: Fri, 6 Sep 2024 07:15:50 -0400 Subject: [PATCH] [CI] Start moving to parse the configuration of the build in the tests pipelines (#21009) At the moment we are recaculating the configuration that was used in the build to decide which tests to run, that is not needed since the configuration was uploaded to the artifacts. This change will allow to do the following: - Load the default variables on the build pipeline this will allow us to set the name of the tests to match those of the build for easy parsing. - Load the default variables to set the property comment in the PR. - Do not recalculate the built platforms on the tests matrix. - Do not calculate the API scan matrix, it is not needed for the tests. --------- Co-authored-by: Rolf Bjarne Kvinge --- tools/devops/automation/scripts/VSTS.psm1 | 35 ++++++ .../templates/common/load_configuration.yml | 106 ++++++++++++++++++ .../automation/templates/tests-stage.yml | 2 +- 3 files changed, 142 insertions(+), 1 deletion(-) create mode 100644 tools/devops/automation/templates/common/load_configuration.yml diff --git a/tools/devops/automation/scripts/VSTS.psm1 b/tools/devops/automation/scripts/VSTS.psm1 index f7c43f276c..655df55a66 100644 --- a/tools/devops/automation/scripts/VSTS.psm1 +++ b/tools/devops/automation/scripts/VSTS.psm1 @@ -448,6 +448,11 @@ class BuildConfiguration { $this.StoreParentBuildVariables($configuration) + # store if dotnet has been enabled + $variableName = "ENABLE_DOTNET" + $variableValue = [Environment]::GetEnvironmentVariable($variableName) + $configuration | Add-Member -NotePropertyName $variableName -NotePropertyValue $variableValue + # For each .NET platform we support, add a INCLUDE_DOTNET_ variable specifying whether that platform is enabled or not. $dotnetPlatforms = $configuration.DOTNET_PLATFORMS.Split(' ', [StringSplitOptions]::RemoveEmptyEntries) foreach ($platform in $dotnetPlatforms) { @@ -479,6 +484,36 @@ class BuildConfiguration { } } + # store all the variables needed when classic xamarin has been enabled + $configuration | Add-Member -NotePropertyName "INCLUDE_XAMARIN_LEGACY" -NotePropertyValue $Env:INCLUDE_XAMARIN_LEGACY + + # if xamarin legacy has been included, check if we need to include the xamarin sdk for each of the platforms, otherewise it will be + # false for all + $xamarinPlatforms = @("ios", "macos", "tvos", "watchos", "maccatalyst") + if ($configuration.INCLUDE_XAMARIN_LEGACY -eq "true") { + foreach ($platform in $xamarinPlatforms) { + $variableName = "INCLUDE_LEGACY_$($platform.ToUpper())" + $variableValue = [Environment]::GetEnvironmentVariable("$variableName") + $configuration | Add-Member -NotePropertyName $variableName -NotePropertyValue $variableValue + } + } else { + foreach ($platform in $xamarinPlatforms) { + $variableName = "INCLUDE_LEGACY_$($platform.ToUpper())" + $configuration | Add-Member -NotePropertyName $variableName -NotePropertyValue "false" + } + } + + # add all the include platforms as well as the nuget os version + foreach ($platform in $xamarinPlatforms) { + $variableName = "INCLUDE_$($platform.ToUpper())" + $variableValue = [Environment]::GetEnvironmentVariable("$variableName") + $configuration | Add-Member -NotePropertyName $variableName -NotePropertyValue $variableValue + + $variableName = "$($platform.ToUpper())__NUGET_OS_VERSION" + $variableValue = [Environment]::GetEnvironmentVariable("$variableName") + $configuration | Add-Member -NotePropertyName $variableName -NotePropertyValue $variableValue + } + # calculate the commit to later share it with the cascade pipelines if ($Env:BUILD_REASON -eq "PullRequest") { $changeId = $configuration.PARENT_BUILD_BUILD_SOURCEBRANCH.Replace("refs/pull/", "").Replace("/merge", "") diff --git a/tools/devops/automation/templates/common/load_configuration.yml b/tools/devops/automation/templates/common/load_configuration.yml new file mode 100644 index 0000000000..5ecac652b1 --- /dev/null +++ b/tools/devops/automation/templates/common/load_configuration.yml @@ -0,0 +1,106 @@ +# yamllint disable rule:line-length +# This job will parse all the labels present in a PR, will set +# the tags for the build AND will set a number of configuration +# variables to be used by the rest of the projects +parameters: + + - name: uploadArtifacts + type: boolean + default: false + + - name: use1ES + type: boolean + default: false + + - name: repositoryAlias + type: string + default: self + + - name: commit + type: string + default: HEAD + + - name: uploadPrefix + type: string + default: '$(MaciosUploadPrefix)' + + - name: testConfigurations + type: object + default: [] + + - name: supportedPlatforms + type: object + default: [] + + - name: testsLabels + type: string + default: '' + + - name: statusContext + type: string + default: '' + +steps: + - template: checkout.yml + parameters: + isPR: true + repositoryAlias: ${{ parameters.repositoryAlias }} + commit: ${{ parameters.commit }} + + - download: macios + displayName: Download Build Config + artifact: build-configuration + + - pwsh: | + Get-ChildItem -Path "$(Pipeline.Workspace)/macios" -Recurse -Force + displayName: 'Display downloads' + timeoutInMinutes: 5 + + - bash: ./xamarin-macios/tools/devops/automation/scripts/bash/configure-platforms.sh + name: configure_platforms + displayName: 'Configure platforms' + + - pwsh: | + Import-Module $Env:SYSTEM_DEFAULTWORKINGDIRECTORY/xamarin-macios/tools/devops/automation/scripts/MaciosCI.psd1 + $jsonPath = Join-Path -Path "$(Build.ArtifactStagingDirectory)" -ChildPath "configuration.json" + Write-Host "##vso[task.setvariable variable=CONFIG_PATH]$jsonPath" + New-BuildConfiguration -ConfigFile $jsonPath + env: + GITHUB_TOKEN: $(GitHub.Token) + ACCESSTOKEN: $(AzDoBuildAccess.Token) + name: labels + displayName: 'Configure build' + + - bash: ./xamarin-macios/tools/devops/automation/scripts/bash/configure-decisions.sh + name: decisions + displayName: 'Make decisions' + + - pwsh: $(System.DefaultWorkingDirectory)/xamarin-macios/tools/devops/automation/scripts/show_env.ps1 + displayName: 'Show Environment' + + - pwsh: | + Import-Module $Env:SYSTEM_DEFAULTWORKINGDIRECTORY/xamarin-macios/tools/devops/automation/scripts/MaciosCI.psd1 + # load the configuration files and set the required variables to be used in the later stages + $configPath = Get-ChildItem -Path "$(Pipeline.Workspace)/macios/build-configuration/configuration.json" -Recurse -Force + $config = Import-BuildConfiguration -ConfigFile $configPath + $testMatrix = $config.TEST_MATRIX + Write-Host "##vso[task.setvariable variable=TEST_MATRIX;isOutput=true]$testMatrix" + name: test_matrix + displayName: 'Create tests strategy matrix' + + # upload config to be consumed later + - ${{ if eq(parameters.uploadArtifacts, true) }}: + - ${{ if eq(parameters.use1ES, true) }}: + - task: 1ES.PublishPipelineArtifact@1 + displayName: 'Publish Artifact: configuration.json' + inputs: + path: '$(Build.ArtifactStagingDirectory)/configuration.json' + artifact: '${{ parameters.uploadPrefix }}build-configuration' + continueOnError: true + - ${{ else }}: + - task: PublishPipelineArtifact@1 + displayName: 'Publish Artifact: configuration.json' + inputs: + targetPath: '$(Build.ArtifactStagingDirectory)/configuration.json' + artifactName: '${{ parameters.uploadPrefix }}build-configuration' + continueOnError: true diff --git a/tools/devops/automation/templates/tests-stage.yml b/tools/devops/automation/templates/tests-stage.yml index b70b751121..10a451545f 100644 --- a/tools/devops/automation/templates/tests-stage.yml +++ b/tools/devops/automation/templates/tests-stage.yml @@ -296,7 +296,7 @@ stages: BRANCH_NAME: $[ replace(variables['Build.SourceBranch'], 'refs/heads/', '') ] steps: - - template: common/configure.yml + - template: common/load_configuration.yml parameters: repositoryAlias: ${{ parameters.repositoryAlias }} commit: ${{ parameters.commit }}