From 66dba36538e7f1ead3a3497e3b21aee0dda6a54e Mon Sep 17 00:00:00 2001 From: Howard Wolosky Date: Wed, 10 Jun 2020 07:09:10 -0700 Subject: [PATCH] Re-enable parallel platform execution in CI build (#231) In a previous commit (b4439f4a6b12f89d755851b313eff0e9ea0b3ab5), the three platforms in the CI pipeline (Windows, Linux, and Mac) were modified to execute serially since they were all operating against the same shared test account, which caused the Pester tests to operate unstabily. I've now created multiple test accounts so that that each platform can operate in isolation of each other. I've updated the UT template to accept parameter input, and modified the CI (and release) pipeline to reference a different AccessToken/OwnerName/OrganizationName based on which platform is being targeted. Additionally: * Had to fix some tests to keep everything passing as well; * Two Contents tests started failing again because GitHub once again changed the HTML output for the standard generated README.md file. This time around, I've attempted to make the tests more robust to future changes. * Two RepositoryForks tests were unreliable in the way that they were written, so they were updated as well to be more robust when being executed in a parallel UT environment (since we can't control when a fork for this repository is being created/removed). * Updates the pipelines to always use the most recent OS images. * Fixes some invalid links on some of the Azure DevOps badges in the README --- README.md | 8 +-- Tests/GitHubContents.tests.ps1 | 18 +++++-- Tests/GitHubRepositoryForks.tests.ps1 | 57 +++++++++++++------- build/pipelines/azure-pipelines.ci.yaml | 23 ++++++-- build/pipelines/azure-pipelines.release.yaml | 7 ++- build/pipelines/templates/run-unitTests.yaml | 19 +++++-- 6 files changed, 93 insertions(+), 39 deletions(-) diff --git a/README.md b/README.md index 2a8bc64..0332ee9 100644 --- a/README.md +++ b/README.md @@ -3,14 +3,14 @@ [![[GitHub version]](https://badge.fury.io/gh/microsoft%2FPowerShellForGitHub.svg)](https://github.com/microsoft/PowerShellForGitHub/releases) [![powershellgallery](https://img.shields.io/powershellgallery/v/PowerShellForGitHub)](https://www.powershellgallery.com/packages/PowerShellForGitHub) [![downloads](https://img.shields.io/powershellgallery/dt/PowerShellForGitHub.svg?label=downloads)](https://www.powershellgallery.com/packages/PowerShellForGitHub) -![GitHub code size in bytes](https://img.shields.io/github/languages/code-size/microsoft/PowerShellForGitHub) +[![GitHub code size in bytes](https://img.shields.io/github/languages/code-size/microsoft/PowerShellForGitHub)](https://github.com/microsoft/PowerShellForGitHub) [![downloads](https://img.shields.io/badge/license-MIT-green)](https://github.com/HowardWolosky/PowerShellForGitHub/blob/master/LICENSE) [![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/3990/badge)](https://bestpractices.coreinfrastructure.org/projects/3990) [![tweet](https://img.shields.io/twitter/url?url=https%3A%2F%2Ftwitter.com%2FQuackFu)](https://twitter.com/intent/tweet?text=%23PowerShellForGitHub%20%40QuackFu%20&original_referer=https://github.com/microsoft/PowerShellForGitHub)
-[![Build status](https://dev.azure.com/ms/PowerShellForGitHub/_apis/build/status/PowerShellForGitHub-CI?branchName=master)](https://dev.azure.com/ms/PowerShellForGitHub/_build/latest?definitionId=109&branchName=master) -[![Azure DevOps tests](https://img.shields.io/azure-devops/tests/ms/PowerShellForGitHub/109/master)](https://dev.azure.com/ms/PowerShellForGitHub/_build/latest?definitionId=109&branchName=master) -[![Azure DevOps coverage](https://img.shields.io/azure-devops/coverage/ms/PowerShellForGitHub/109/master)](https://dev.azure.com/ms/PowerShellForGitHub/_build/latest?definitionId=109&branchName=master) +[![Build status](https://dev.azure.com/ms/PowerShellForGitHub/_apis/build/status/PowerShellForGitHub-CI?branchName=master)](https://dev.azure.com/ms/PowerShellForGitHub/_build?definitionId=109&_a=summary&repositoryFilter=63&branchFilter=2197) +[![Azure DevOps tests](https://img.shields.io/azure-devops/tests/ms/PowerShellForGitHub/109/master)](https://dev.azure.com/ms/PowerShellForGitHub/_build?definitionId=109&_a=summary&repositoryFilter=63&branchFilter=2197) +[![Azure DevOps coverage](https://img.shields.io/azure-devops/coverage/ms/PowerShellForGitHub/109/master)](https://dev.azure.com/ms/PowerShellForGitHub/_build?definitionId=109&_a=summary&repositoryFilter=63&branchFilter=2197)
[![Help Wanted Issues](https://img.shields.io/github/issues/microsoft/PowerShellForGitHub/help%20wanted)](https://github.com/microsoft/PowerShellForGitHub/issues?q=is%3Aissue+is%3Aopen+label%3A%22help+wanted%22) [![GitHub last commit](https://img.shields.io/github/last-commit/microsoft/PowerShellForGitHub)](https://github.com/HowardWolosky/PowerShellForGitHub/commits/master) diff --git a/Tests/GitHubContents.tests.ps1 b/Tests/GitHubContents.tests.ps1 index f0b15b2..7dbf568 100644 --- a/Tests/GitHubContents.tests.ps1 +++ b/Tests/GitHubContents.tests.ps1 @@ -22,7 +22,7 @@ try # Need two separate blocks to set constants because we need to reference a constant from the first block in this block. @{ - htmlOutput = "

$repoGuid

" + htmlOutputStart = '
' rawOutput = "# $repoGuid" }.GetEnumerator() | ForEach-Object { Set-Variable -Force -Scope Script -Option ReadOnly -Visibility Private -Name $_.Key -Value $_.Value @@ -100,9 +100,13 @@ try $readmeFileBytes = Get-GitHubContent -OwnerName $script:ownerName -RepositoryName $repo.name -Path $readmeFileName -MediaType Html $readmeFileString = [System.Text.Encoding]::UTF8.GetString($readmeFileBytes) + # Replace newlines with empty for comparison purposes + $readmeNoBreaks = $readmeFileString.Replace("`n", "").Replace("`r", "") It "Should have the expected content" { - # Replace newlines with empty for comparison - $readmeFileString.Replace("`n", "").Replace("`r", "") | Should be $htmlOutput + # GitHub changes the syntax for this file too frequently, so we'll just do some + # partial matches to make sure we're getting HTML output for the right repo. + $readmeNoBreaks.StartsWith($htmlOutputStart) | Should -BeTrue + $readmeNoBreaks.IndexOf($repoGuid) | Should -BeGreaterOrEqual 0 } } @@ -110,9 +114,13 @@ try $readmeFileString = Get-GitHubContent -OwnerName $script:ownerName -RepositoryName $repo.name -Path $readmeFileName -MediaType Html -ResultAsString + # Replace newlines with empty for comparison purposes + $readmeNoBreaks = $readmeFileString.Replace("`n", "").Replace("`r", "") It "Should have the expected content" { - # Replace newlines with empty for comparison - $readmeFileString.Replace("`n", "").Replace("`r", "") | Should be $htmlOutput + # GitHub changes the syntax for this file too frequently, so we'll just do some + # partial matches to make sure we're getting HTML output for the right repo. + $readmeNoBreaks.StartsWith($htmlOutputStart) | Should -BeTrue + $readmeNoBreaks.IndexOf($repoGuid) | Should -BeGreaterOrEqual 0 } } diff --git a/Tests/GitHubRepositoryForks.tests.ps1 b/Tests/GitHubRepositoryForks.tests.ps1 index 16e1566..c5267a2 100644 --- a/Tests/GitHubRepositoryForks.tests.ps1 +++ b/Tests/GitHubRepositoryForks.tests.ps1 @@ -6,47 +6,64 @@ Tests for GitHubRepositoryForks.ps1 module #> +[CmdletBinding()] +[Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseDeclaredVarsMoreThanAssignments', '', + Justification='Suppress false positives in Pester code blocks')] +param() + # This is common test code setup logic for all Pester test files $moduleRootPath = Split-Path -Path $PSScriptRoot -Parent . (Join-Path -Path $moduleRootPath -ChildPath 'Tests\Common.ps1') try { + # Define Script-scoped, readonly, hidden variables. + @{ + upstreamOwnerName = 'microsoft' + upstreamRepositoryName = 'PowerShellForGitHub' + }.GetEnumerator() | ForEach-Object { + Set-Variable -Force -Scope Script -Option ReadOnly -Visibility Private -Name $_.Key -Value $_.Value + } + Describe 'Creating a new fork for user' { - $originalForks = @(Get-GitHubRepositoryFork -OwnerName Microsoft -RepositoryName PowerShellForGitHub) - Context 'When a new fork is created' { - $repo = New-GitHubRepositoryFork -OwnerName Microsoft -RepositoryName PowerShellForGitHub - $newForks = @(Get-GitHubRepositoryFork -OwnerName Microsoft -RepositoryName PowerShellForGitHub -Sort Newest) - - It 'Should have one more fork than before' { - ($newForks.Count - $originalForks.Count) | Should be 1 + BeforeAll { + $repo = New-GitHubRepositoryFork -OwnerName $script:upstreamOwnerName -RepositoryName $script:upstreamRepositoryName } - It 'Should be the latest fork in the list' { - $newForks[0].full_name | Should be "$($script:ownerName)/PowerShellForGitHub" + AfterAll { + Remove-GitHubRepository -Uri $repo.svn_url -Confirm:$false } - Remove-GitHubRepository -Uri $repo.svn_url -Confirm:$false + $newForks = @(Get-GitHubRepositoryFork -OwnerName $script:upstreamOwnerName -RepositoryName $script:upstreamRepositoryName -Sort Newest) + $ourFork = $newForks | Where-Object { $_.owner.login -eq $script:ownerName } + + It 'Should be in the list' { + # Doing this syntax, because due to odd timing with GitHub, it's possible it may + # think that there's an existing clone out there and so may name this one "...-1" + $ourFork.full_name.StartsWith("$($script:ownerName)/$script:upstreamRepositoryName") | Should -BeTrue + } } } Describe 'Creating a new fork for an org' { - $originalForks = @(Get-GitHubRepositoryFork -OwnerName Microsoft -RepositoryName PowerShellForGitHub) - Context 'When a new fork is created' { - $repo = New-GitHubRepositoryFork -OwnerName Microsoft -RepositoryName PowerShellForGitHub -OrganizationName $script:organizationName - $newForks = @(Get-GitHubRepositoryFork -OwnerName Microsoft -RepositoryName PowerShellForGitHub -Sort Newest) - - It 'Should have one more fork than before' { - ($newForks.Count - $originalForks.Count) | Should be 1 + BeforeAll { + $repo = New-GitHubRepositoryFork -OwnerName $script:upstreamOwnerName -RepositoryName $script:upstreamRepositoryName -OrganizationName $script:organizationName } - It 'Should be the latest fork in the list' { - $newForks[0].full_name | Should be "$($script:organizationName)/PowerShellForGitHub" + AfterAll { + Remove-GitHubRepository -Uri $repo.svn_url -Confirm:$false } - Remove-GitHubRepository -Uri $repo.svn_url -Confirm:$false + $newForks = @(Get-GitHubRepositoryFork -OwnerName $script:upstreamOwnerName -RepositoryName $script:upstreamRepositoryName -Sort Newest) + $ourFork = $newForks | Where-Object { $_.owner.login -eq $script:organizationName } + + It 'Should be in the list' { + # Doing this syntax, because due to odd timing with GitHub, it's possible it may + # think that there's an existing clone out there and so may name this one "...-1" + $ourFork.full_name.StartsWith("$($script:organizationName)/$script:upstreamRepositoryName") | Should -BeTrue + } } } } diff --git a/build/pipelines/azure-pipelines.ci.yaml b/build/pipelines/azure-pipelines.ci.yaml index 9adc9f1..19838c6 100644 --- a/build/pipelines/azure-pipelines.ci.yaml +++ b/build/pipelines/azure-pipelines.ci.yaml @@ -21,26 +21,39 @@ trigger: jobs: - job: Windows pool: - vmImage: 'vs2017-win2016' + vmImage: 'windows-latest' steps: - template: ./templates/verify-testConfigSettingsHash.yaml - template: ./templates/run-staticAnalysis.yaml - template: ./templates/run-unitTests.yaml + parameters: + gitHubAccessToken: $(WindowsCIGitHubAccessToken) + gitHubOwnerName: $(WindowsCIGitHubOwnerName) + gitHubOrganizationName: $(WindowsCIGitHubOrganizationName) + platformName: 'Windows' - job: Linux pool: - vmImage: 'ubuntu-16.04' - dependsOn: Windows # Run in series instead of parallel because the UT's are modifying the same shared state + vmImage: 'ubuntu-latest' steps: - template: ./templates/verify-testConfigSettingsHash.yaml - template: ./templates/run-staticAnalysis.yaml - template: ./templates/run-unitTests.yaml + parameters: + gitHubAccessToken: $(LinuxCIGitHubAccessToken) + gitHubOwnerName: $(LinuxCIGitHubOwnerName) + gitHubOrganizationName: $(LinuxCIGitHubOrganizationName) + platformName: 'Linux' - job: macOS pool: - vmImage: 'macOS-10.14' - dependsOn: Linux # Run in series instead of parallel because the UT's are modifying the same shared state + vmImage: 'macOS-latest' steps: - template: ./templates/verify-testConfigSettingsHash.yaml - template: ./templates/run-staticAnalysis.yaml - template: ./templates/run-unitTests.yaml + parameters: + gitHubAccessToken: $(MacCIGitHubAccessToken) + gitHubOwnerName: $(MacCIGitHubOwnerName) + gitHubOrganizationName: $(MacCIGitHubOrganizationName) + platformName: 'macOS' diff --git a/build/pipelines/azure-pipelines.release.yaml b/build/pipelines/azure-pipelines.release.yaml index a185703..ff0b101 100644 --- a/build/pipelines/azure-pipelines.release.yaml +++ b/build/pipelines/azure-pipelines.release.yaml @@ -27,10 +27,15 @@ variables: jobs: - job: Validate pool: - vmImage: 'vs2017-win2016' + vmImage: 'windows-latest' steps: - template: ./templates/run-staticAnalysis.yaml - template: ./templates/run-unitTests.yaml + parameters: + gitHubAccessToken: $(GitHubAccessToken) + gitHubOwnerName: $(GitHubOwnerName) + gitHubOrganizationName: $(GitHubOrganizationName) + platformName: 'Windows' - job: Release dependsOn: Validate diff --git a/build/pipelines/templates/run-unitTests.yaml b/build/pipelines/templates/run-unitTests.yaml index 925d2a2..dc7d69f 100644 --- a/build/pipelines/templates/run-unitTests.yaml +++ b/build/pipelines/templates/run-unitTests.yaml @@ -13,6 +13,17 @@ # 3. GitHubOrganizationName - The default "organization" that will be used for tests. #-------------------------------------------------------------------------------------------------- +parameters: +- name: 'gitHubAccessToken' + type: string +- name: 'gitHubOwnerName' + type: string +- name: 'gitHubOrganizationName' + type: string +- name: 'platformName' + default: 'Windows' + type: string + steps: - powershell: | Install-Module -Name Pester -Repository PSGallery -Scope CurrentUser -AllowClobber -SkipPublisherCheck -RequiredVersion 4.10.1 -Force -Verbose @@ -24,14 +35,14 @@ steps: workingDirectory: '$(System.DefaultWorkingDirectory)' displayName: 'Run Unit Tests via Pester' env: - ciAccessToken: $(GitHubAccessToken) - ciOwnerName: $(GitHubOwnerName) - ciOrganizationName: $(GitHubOrganizationName) + ciAccessToken: ${{ parameters.gitHubAccessToken }} + ciOwnerName: ${{ parameters.gitHubOwnerName }} + ciOrganizationName: ${{ parameters.gitHubOrganizationName }} - task: PublishTestResults@2 displayName: 'Publish Test Results' inputs: - testRunTitle: 'Windows Test Results for Pester' + testRunTitle: '${{ parameters.platformName }} Test Results for Pester' buildPlatform: 'Windows' testRunner: NUnit testResultsFiles: '../Pester/test-results.xml'