PowerShell-FeatureFlags/test/FeatureFlags.Tests.ps1

650 строки
25 KiB
PowerShell

$ModuleName = "FeatureFlags"
Get-Module $ModuleName | Remove-Module -Force
Import-Module $PSScriptRoot\test-functions.psm1
$VerbosePreference = "Continue"
$Module = Import-Module $PSScriptRoot\..\${ModuleName}.psd1 -Force -PassThru
if ($null -eq $Module) {
Write-Error "Could not import $ModuleName"
exit 1
}
Write-Host "Done."
Describe 'Confirm-FeatureFlagConfig' {
Context 'Validation of invalid configuration' {
It 'Fails on empty or null configuration' {
Confirm-FeatureFlagConfig -EA 0 "{}" | Should -Be $false
Confirm-FeatureFlagConfig -EA 0 $null | Should -Be $false
}
It 'Fails on configuration that is not well-formed JSON' {
Confirm-FeatureFlagConfig '{"stages": "}' -EA 0 | Should -Be $false
}
It 'Fails on well-formed configuration not matching the schema' {
# Missing "stages".
Confirm-FeatureFlagConfig -EA 0 '{"key": "x"}' | Should -Be $False
Confirm-FeatureFlagConfig -EA 0 '{"features": {}}' | Should -Be $False
# Wrong type for "stages".
Confirm-FeatureFlagConfig -EA 0 '{"stages": "x"}' | Should -Be $False
# Extra field "foo".
Confirm-FeatureFlagConfig -EA 0 '{"stages": {}, "foo": ""}' | Should -Be $false
}
It 'Fails if a stage is not an array' {
Confirm-FeatureFlagConfig -EA 0 '{"stages": {"foo": 1}}' | Should -Be $false
Confirm-FeatureFlagConfig -EA 0 '{"stages": {"bar": [1, 2], "foo": 1}}' | Should -Be $false
}
It 'Fails if a stage maps to an empty list of conditions' {
Confirm-FeatureFlagConfig -EA 0 '{"stages": {"foo": []}}' | Should -Be $false
}
}
Context 'Error output of Confirm-FeatureFlagConfig' {
It 'Outputs correct error messages in case of failure' {
# Write-Error will add errors to the $error variable and output them to standard error.
# When run with -EA 0 (ErrorAction SilentlyContinue), the errors will be added to $error
# but not printed to stderr, which is desirable to not litter the unit tests output.
Confirm-FeatureFlagConfig -EA 0 $null | Should -Be $false
$error[0] | Should -BeLike "*null or zero-length*"
Confirm-FeatureFlagConfig -EA 0 "{}" | Should -Be $false
$error[0] | Should -BeLike "*Validation failed*"
Confirm-FeatureFlagConfig -EA 0 '{"stages": {"foo": []}}' | Should -Be $false
$error[0] | Should -BeLike "*Validation failed*"
}
}
Context 'Error output of Get-FeatureFlagConfigFromFile' {
It 'Outputs correct error messages in case of failure' {
# Write-Error will add errors to the $error variable and output them to standard error.
# When run with -EA 0 (ErrorAction SilentlyContinue), the errors will be added to $error
# but not printed to stderr, which is desirable to not litter the unit tests output.
Get-FeatureFlagConfigFromFile -jsonConfigPath "this/file/does/not/exist" -EA 0 | Should -Be $null
}
}
Context 'Validation of configs with typos in sections or properties' {
It 'Fails if "stages" is misspelled' {
$cfg = @"
{
"stags": {
"storage": [
{"allowlist": [".*storage.*"]}
]
}
}
"@
Confirm-FeatureFlagConfig -EA 0 $cfg | Should -Be $false
}
It 'Fails if a deprecated condition is used (whitelist)' {
$cfg = @"
{
"stages": {
"storage": [
{"whitelist": [".*storage.*"]}
]
}
}
"@
Confirm-FeatureFlagConfig -EA 0 $cfg | Should -Be $false
}
It 'Fails if a condition name contains a typo (allwlist)' {
$cfg = @"
{
"stages": {
"storage": [
{"allwlist": [".*storage.*"]}
]
}
}
"@
Confirm-FeatureFlagConfig -EA 0 $cfg | Should -Be $false
}
It 'Fails if "features" is misspelled' {
$cfg = @"
{
"stages": {
"storage": [
{"allowlist": [".*storage.*"]}
]
},
"featurs": {
"foo": "storage"
}
}
"@
Confirm-FeatureFlagConfig -EA 0 $cfg | Should -Be $false
}
It 'Fails if the stage name is an empty string' {
$cfg = @"
{
"stages": {
"": [
{"allowlist": [".*storage.*"]}
]
}
}
"@
Confirm-FeatureFlagConfig -EA 0 $cfg | Should -Be $false
}
It 'Fails if the stage name is a string containing only spaces' {
$cfg = @"
{
"stages": {
" ": [
{"allowlist": [".*storage.*"]}
]
}
}
"@
Confirm-FeatureFlagConfig -EA 0 $cfg | Should -Be $false
}
}
Context 'Successful validation of simple stages' {
It 'Succeeds with a simple stage with two allowlists' {
$cfg = @"
{
"stages": {
"storage": [
{"allowlist": [".*storage.*", ".*compute.*"]}
]
}
}
"@
Confirm-FeatureFlagConfig $cfg | Should -Be $true
}
It 'Succeeds with a simple stage with a allowlist and a denylist' {
$cfg = @"
{
"stages": {
"storage": [
{"allowlist": [".*storage.*"]},
{"denylist": ["ImportantStorage"]}
]
}
}
"@
Confirm-FeatureFlagConfig $cfg | Should -Be $true
}
It 'Succeeds with a simple stage with a probability condition' {
$cfg = @"
{
"stages": {
"1percent": [
{"probability": 0.1}
]
}
}
"@
Confirm-FeatureFlagConfig $cfg | Should -Be $true
}
}
Context 'Successful validation of stages and features' {
It 'Succeeds validating a very simple config file with one feature and one stage' {
$cfg = @"
{
"stages": {
"storage": [
{"allowlist": [".*storage.*"]}
]
},
"features": {
"foo": {
"stages": ["storage"]
}
}
}
"@
Confirm-FeatureFlagConfig -EA 0 $cfg | Should -Be $true
}
It 'Succeeds validating a very simple config file with two features and two stages' {
$cfg = @"
{
"stages": {
"all": [
{"allowlist": [".*"]}
],
"storage": [
{"allowlist": [".*storage.*"]}
]
},
"features": {
"foo": {
"stages": ["storage"]
},
"bar": {
"stages": ["all"]
}
}
}
"@
Confirm-FeatureFlagConfig -EA 0 $cfg | Should -Be $true
}
}
Context 'Validation of configs with features pointing to non-existent stages' {
It 'Fails validation when a feature points to a non-existing stage' {
$cfg = @"
{
"stages": {
"storage": [
{"allowlist": [".*storage.*"]}
]
},
"features": {
"foo": {
"stages": ["all"]
}
}
}
"@
Confirm-FeatureFlagConfig -EA 0 $cfg | Should -Be $false
$error[0] | Should -Be "Stage all is used in the features configuration but is never defined."
}
}
}
Describe 'Get-FeatureFlagConfigFromFile' {
It 'Succeeds to load valid configuration files from a file' {
Get-FeatureFlagConfigFromFile "$PSScriptRoot\multiple-stages.json" | Should -Not -Be $null
Get-FeatureFlagConfigFromFile "$PSScriptRoot\single-stage.json" | Should -Not -Be $null
}
}
Describe 'Test-FeatureFlag' {
Context 'allowlist condition' {
Context 'Simple allowlist configuration' {
BeforeAll {
$serializedConfig = @"
{
"stages": {
"all": [
{"allowlist": [".*"]}
]
},
"features": {
"well-tested": {
"stages": ["all"]
}
}
}
"@
Confirm-FeatureFlagConfig $serializedConfig
$config = ConvertFrom-Json $serializedConfig
}
It 'Rejects non-existing features' {
Test-FeatureFlag "feature1" "Storage/master" $config | Should -Be $false
}
It 'Returns true if the regex matches' {
Test-FeatureFlag "well-tested" "Storage/master" $config | Should -Be $true
}
}
Context 'Chained allowlist configuration' {
BeforeAll {
$serializedConfig = @"
{
"stages": {
"test-repo-and-branch": [
{"allowlist": [
"storage1/.*",
"storage2/dev-branch"
]}
]
},
"features": {
"experimental-feature": {
"stages": ["test-repo-and-branch"]
}
}
}
"@
Confirm-FeatureFlagConfig $serializedConfig
$config = ConvertFrom-Json $serializedConfig
}
It 'Returns true if the regex matches' {
Test-FeatureFlag "experimental-feature" "storage1/master" $config | Should -Be $true
Test-FeatureFlag "experimental-feature" "storage1/dev" $config | Should -Be $true
Test-FeatureFlag "experimental-feature" "storage2/dev-branch" $config | Should -Be $true
}
It 'Returns false if the regex does not match' {
Test-FeatureFlag "experimental-feature" "storage2/master" $config | Should -Be $false
}
}
}
Context 'denylist condition' {
Context 'Reject-all configuration' {
BeforeAll {
$serializedConfig = @"
{
"stages": {
"none": [
{"denylist": [".*"]}
]
},
"features": {
"disabled": {
"stages": ["none"]
}
}
}
"@
Confirm-FeatureFlagConfig $serializedConfig
$config = ConvertFrom-Json $serializedConfig
}
It 'Rejects everything' {
Test-FeatureFlag "disabled" "Storage/master" $config | Should -Be $false
Test-FeatureFlag "disabled" "foo" $config | Should -Be $false
Test-FeatureFlag "disabled" "bar" $config | Should -Be $false
}
}
Context 'Reject single-value configuration' {
BeforeAll {
$serializedConfig = @"
{
"stages": {
"all-except-important": [
{"denylist": ["^important$"]}
]
},
"features": {
"some-feature":
{
"stages": ["all-except-important"]
}
}
}
"@
Confirm-FeatureFlagConfig $serializedConfig
$config = ConvertFrom-Json $serializedConfig
}
# Given that the regex is ^important$, only the exact string "important" will match the denylist.
It 'Allows the flag if the predicate does not match exactly' {
Test-FeatureFlag "some-feature" "Storage/master" $config | Should -Be $true
Test-FeatureFlag "some-feature" "foo" $config | Should -Be $true
Test-FeatureFlag "some-feature" "bar" $config | Should -Be $true
Test-FeatureFlag "some-feature" "Storage/important" $config | Should -Be $true
}
It 'Rejects the flag only if the predicate matches exactly the regex' {
Test-FeatureFlag "some-feature" "important" $config | Should -Be $false
}
}
Context 'Reject multiple-value configuration' {
BeforeAll {
$serializedConfig = @"
{
"stages": {
"all-except-important": [
{"denylist": ["storage-important/master", "storage-important2/master"]}
]
},
"features": {
"some-feature": {
"stages": ["all-except-important"]
}
}
}
"@
Confirm-FeatureFlagConfig $serializedConfig
$config = ConvertFrom-Json $serializedConfig
}
It 'Allows predicates not matching the denylist' {
Test-FeatureFlag "some-feature" "storage1/master" $config | Should -Be $true
Test-FeatureFlag "some-feature" "storage2/master" $config | Should -Be $true
Test-FeatureFlag "some-feature" "storage-important/dev" $config | Should -Be $true
}
It 'Rejects important / important2 master branches' {
Test-FeatureFlag "some-feature" "storage-important/master" $config | Should -Be $false
Test-FeatureFlag "some-feature" "storage-important2/master" $config | Should -Be $false
}
}
}
Context 'Mixed allowlist/denylist configuration' {
BeforeAll {
$serializedConfig = @"
{
"stages": {
"all-storage-important": [
{"allowlist": ["storage.*"]},
{"denylist": ["storage-important/master", "storage-important2/master"]}
]
},
"features": {
"some-feature": {
"stages": ["all-storage-important"]
}
}
}
"@
Confirm-FeatureFlagConfig $serializedConfig
$config = ConvertFrom-Json $serializedConfig
}
It 'Rejects storage important / important2 master branches' {
Test-FeatureFlag "some-feature" "storage-important/master" $config | Should -Be $false
Test-FeatureFlag "some-feature" "storage-important2/master" $config | Should -Be $false
}
It 'Rejects non-storage predicates' {
Test-FeatureFlag "some-feature" "compute/master" $config | Should -Be $false
Test-FeatureFlag "some-feature" "something-else/master" $config | Should -Be $false
}
It 'Allows other storage predicates' {
Test-FeatureFlag "some-feature" "storage-important/dev" $config | Should -Be $true
Test-FeatureFlag "some-feature" "storage-somethingelse/dev" $config | Should -Be $true
Test-FeatureFlag "some-feature" "storage-dev/master" $config | Should -Be $true
}
}
Context 'Probability condition' {
BeforeAll {
$serializedConfig = @"
{
"stages": {
"all": [
{"probability": 1}
],
"none": [
{"probability": 0}
],
"10percent": [
{"probability": 0.1}
]
},
"features": {
"well-tested": {
"stages": ["all"]
},
"not-launched": {
"stages": ["none"]
},
"10pc-feature": {
"stages": ["10percent"]
}
}
}
"@
Confirm-FeatureFlagConfig $serializedConfig
$config = ConvertFrom-Json $serializedConfig
}
It 'Always allows with probability 1' {
Test-FeatureFlag "well-tested" "storage-important/master" $config | Should -Be $true
Test-FeatureFlag "well-tested" "foo" $config | Should -Be $true
Test-FeatureFlag "well-tested" "bar" $config | Should -Be $true
}
It 'Always rejects with probability 0' {
Test-FeatureFlag "not-launched" "storage-important/master" $config | Should -Be $false
Test-FeatureFlag "not-launched" "foo" $config | Should -Be $false
Test-FeatureFlag "not-launched" "bar" $config | Should -Be $false
}
# It's not best practice to test external behavior based on implementation, but this is probably
# the best way to test the probability condition without extracting away its inner logic and mocking
# it.
It 'Allows features if the random value is below the probability threshold' {
# The probability condition generates a random number, computes mod 100 and then scales back the
# results between 0 and 1 to compare it with the given probability, returning true if the generated
# number is less than the probability.
# If Get-Random returns 1, the probability will be 0.01, less than the given 0.1 percent, therefore
# enabling the feature.
Mock Get-Random -ModuleName FeatureFlags {return 1}
Test-FeatureFlag "10pc-feature" "storage-important/master" $config | Should -Be $true
Test-FeatureFlag "10pc-feature" "storage/master" $config | Should -Be $true
Test-FeatureFlag "10pc-feature" "storage/dev" $config | Should -Be $true
Test-FeatureFlag "10pc-feature" "storage-important/dev" $config | Should -Be $true
}
It 'Rejects features if the random value is above the probability threshold' {
# If Get-Random returns 99, the probability will be 0.99, greater than the given 0.1 percent,
# therefore not enabling the feature.
Mock Get-Random -ModuleName FeatureFlags {return 99}
Test-FeatureFlag "10pc-feature" "storage-important/master" $config | Should -Be $false
Test-FeatureFlag "10pc-feature" "storage-important/master" $config | Should -Be $false
Test-FeatureFlag "10pc-feature" "storage-important/master" $config | Should -Be $false
}
}
Context 'Complex allowlist + denylist + probability configuration' {
BeforeAll {
$serializedConfig = @"
{
"stages": {
"all-storage-important-50pc": [
{"allowlist": ["storage.*"]},
{"denylist": ["storage-important/master", "storage-important2/master"]},
{"probability": 0.5}
]
},
"features": {
"some-feature": {
"stages": ["all-storage-important-50pc"]
}
}
}
"@
Confirm-FeatureFlagConfig $serializedConfig
$config = ConvertFrom-Json $serializedConfig
}
It 'Rejects storage important / important2 master branches' {
Test-FeatureFlag "some-feature" "storage-important/master" $config | Should -Be $false
Test-FeatureFlag "some-feature" "storage-important2/master" $config | Should -Be $false
}
It 'Rejects non-storage predicates' {
Test-FeatureFlag "some-feature" "compute/master" $config | Should -Be $false
Test-FeatureFlag "some-feature" "something-else/master" $config | Should -Be $false
}
It 'Rejects storage predicates for high values of Get-Random' {
Mock Get-Random -ModuleName FeatureFlags {return 90}
Test-FeatureFlag "some-feature" "storage-important/dev" $config | Should -Be $false
Test-FeatureFlag "some-feature" "storage-somethingelse/dev" $config | Should -Be $false
Test-FeatureFlag "some-feature" "storage-dev/master" $config | Should -Be $false
}
It 'Accepts storage predicates for low values of Get-Random' {
Mock Get-Random -ModuleName FeatureFlags {return 20}
Test-FeatureFlag "some-feature" "storage-important/dev" $config | Should -Be $true
Test-FeatureFlag "some-feature" "storage-somethingelse/dev" $config | Should -Be $true
Test-FeatureFlag "some-feature" "storage-dev/master" $config | Should -Be $true
}
}
}
Describe 'Get-EvaluatedFeatureFlags' -Tag Features {
Context 'Verify evaluation of all feature flags' {
BeforeAll {
$serializedConfig = Get-Content -Raw "$PSScriptRoot\multiple-stages-features.json"
Confirm-FeatureFlagConfig $serializedConfig
$config = ConvertFrom-Json $serializedConfig;
Mock New-Item -ModuleName FeatureFlags {}
}
It 'Returns expected feature flags' {
$expected = @{ "filetracker"=$true; "newestfeature"=$true; "testfeature"=$false }
$actual = Get-EvaluatedFeatureFlags -predicate "production/some-repo" -config $config
Test-Hashtables $expected $actual
}
}
}
Describe 'Out-EvaluatedFeaturesFiles' -Tag Features {
Context 'Verify output file content' {
BeforeAll {
$global:featuresJsonContent = New-Object 'System.Collections.ArrayList'
$global:featuresIniContent = New-Object 'System.Collections.ArrayList'
$global:featuresEnvConfigContent = New-Object 'System.Collections.ArrayList'
$serializedConfig = Get-Content -Raw "$PSScriptRoot\multiple-stages-features.json"
Confirm-FeatureFlagConfig $serializedConfig
$config = ConvertFrom-Json $serializedConfig
Mock -ModuleName FeatureFlags New-Item {}
Mock -ModuleName $ModuleName Test-Path { Write-Output $true }
Mock -ModuleName $ModuleName Remove-Item {}
Mock -ModuleName $ModuleName Out-File { ${global:featuresJsonContent}.Add($InputObject) } -ParameterFilter { $FilePath.EndsWith("features.json") }
Mock -ModuleName $ModuleName Add-Content { ${global:featuresIniContent}.Add($Value) } -ParameterFilter { $Path.EndsWith("features.ini") }
Mock -ModuleName $ModuleName Add-Content { ${global:featuresEnvConfigContent}.Add($Value) } -ParameterFilter { $Path.EndsWith("features.env.config") }
}
It 'Honors denylist' {
$features = Get-EvaluatedFeatureFlags -predicate "important" -config $config
$expectedFeaturesIniContent = @("filetracker`tfalse", "newestfeature`tfalse", "testfeature`tfalse")
$expectedFeaturesEnvConfigContent = @()
Out-EvaluatedFeaturesFiles -Config $config -EvaluatedFeatures $features -OutputFolder 'outputfolder.mock'
# Validate JSON
$global:featuresJsonContent | Should -Not -Be $null
$global:featuresJsonContent.Count | Should -Be 1
$jsonContent = ConvertFrom-Json -InputObject $global:featuresJsonContent[0]
$jsonContent.filetracker | Should -Be $false
$jsonContent.newestfeature | Should -Be $false
$jsonContent.testfeature | Should -Be $false
# Validate INI
Test-StringArrays ($expectedFeaturesIniContent | Sort-Object) ($global:featuresIniContent | Sort-Object)
# Validate Env
$global:featuresEnvConfigContent.Count | Should -Be 0
$global:featuresEnvConfigContent | Should -Be $expectedFeaturesEnvConfigContent
}
}
}
Remove-Module $ModuleName
Remove-Module test-functions