Updated canary deployment configuration (#2560)

#### Details

Updated canary deployment configuration
Fixed deployment bugs

#### Pull request checklist
<!-- If a checklist item is not applicable to this change, write "n/a"
in the checkbox -->

- [ ] Addresses an existing issue: Fixes #0000
- [ ] Added relevant unit test for your changes. (`yarn test`)
- [ ] Verified code coverage for the changes made. Check coverage report
at: `<rootDir>/test-results/unit/coverage`
- [ ] Ran precheckin (`yarn precheckin`)
- [x] Validated in an Azure resource group
This commit is contained in:
Maxim Laikine 2024-07-11 19:04:48 -07:00 коммит произвёл GitHub
Родитель 9430e0cbe4
Коммит f562dbb56f
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: B5690EEEBB952194
26 изменённых файлов: 133 добавлений и 185 удалений

Просмотреть файл

@ -56,19 +56,19 @@ exports[`ServiceConfiguration verifies custom config 1`] = `
},
"crawlConfig": {
"deepScanDiscoveryLimit": {
"default": 6,
"default": 100,
"doc": "The maximum number of URLs that will be discovered for a deep scan request",
"format": "int",
},
"deepScanUpperLimit": {
"default": 10,
"doc": "The maximum number of URLs that will be processed for a deep scan request",
"default": 5000,
"doc": "The maximum number of URLs that will be accepted for a deep scan request",
"format": "int",
},
},
"featureFlags": {
"sendNotification": {
"default": false,
"default": true,
"doc": "Property to decide if we should notify after scan completed.",
"format": "Boolean",
},
@ -170,7 +170,7 @@ exports[`ServiceConfiguration verifies custom config 1`] = `
},
"queueConfig": {
"maxQueueSize": {
"default": 10,
"default": 100,
"doc": "Maximum message count in scan request queue.",
"format": "int",
},
@ -197,7 +197,7 @@ exports[`ServiceConfiguration verifies custom config 1`] = `
"format": "int",
},
"scanRequestProcessingDelayInSeconds": {
"default": 600,
"default": 300,
"doc": "The scan request processing delay interval in seconds for a new submitted request.",
"format": "int",
},
@ -300,19 +300,19 @@ exports[`ServiceConfiguration verifies dev config 1`] = `
},
"crawlConfig": {
"deepScanDiscoveryLimit": {
"default": 6,
"default": 100,
"doc": "The maximum number of URLs that will be discovered for a deep scan request",
"format": "int",
},
"deepScanUpperLimit": {
"default": 10,
"doc": "The maximum number of URLs that will be processed for a deep scan request",
"default": 5000,
"doc": "The maximum number of URLs that will be accepted for a deep scan request",
"format": "int",
},
},
"featureFlags": {
"sendNotification": {
"default": false,
"default": true,
"doc": "Property to decide if we should notify after scan completed.",
"format": "Boolean",
},
@ -414,7 +414,7 @@ exports[`ServiceConfiguration verifies dev config 1`] = `
},
"queueConfig": {
"maxQueueSize": {
"default": 10,
"default": 100,
"doc": "Maximum message count in scan request queue.",
"format": "int",
},
@ -441,7 +441,7 @@ exports[`ServiceConfiguration verifies dev config 1`] = `
"format": "int",
},
"scanRequestProcessingDelayInSeconds": {
"default": 600,
"default": 300,
"doc": "The scan request processing delay interval in seconds for a new submitted request.",
"format": "int",
},

Просмотреть файл

@ -154,7 +154,7 @@ export class ServiceConfiguration {
featureFlags: {
sendNotification: {
format: 'Boolean',
default: false,
default: true,
doc: 'Property to decide if we should notify after scan completed.',
},
},
@ -168,7 +168,7 @@ export class ServiceConfiguration {
queueConfig: {
maxQueueSize: {
format: 'int',
default: 10,
default: 100,
doc: 'Maximum message count in scan request queue.',
},
messageVisibilityTimeoutInSeconds: {
@ -286,7 +286,7 @@ export class ServiceConfiguration {
},
scanRequestProcessingDelayInSeconds: {
format: 'int',
default: 600,
default: 300,
doc: 'The scan request processing delay interval in seconds for a new submitted request.',
},
minScanPriorityValue: {
@ -357,13 +357,13 @@ export class ServiceConfiguration {
crawlConfig: {
deepScanDiscoveryLimit: {
format: 'int',
default: 6, // Must be at least high enough to allow the largest E2E deep scan test to complete
default: 100, // Must be at least high enough to allow the largest E2E deep scan test to complete
doc: 'The maximum number of URLs that will be discovered for a deep scan request',
},
deepScanUpperLimit: {
format: 'int',
default: 10,
doc: 'The maximum number of URLs that will be processed for a deep scan request',
default: 5000,
doc: 'The maximum number of URLs that will be accepted for a deep scan request',
},
},
privacyScanConfig: {

Просмотреть файл

@ -1,15 +1,5 @@
{
"featureFlags": {
"sendNotification": true
},
"restApiConfig": {
"scanRequestProcessingDelayInSeconds": 300
},
"availabilityTestConfig": {
"environmentDefinition": "canary"
},
"crawlConfig": {
"deepScanDiscoveryLimit": 10,
"deepScanUpperLimit": 5000
}
}

Просмотреть файл

@ -1,23 +1,20 @@
{
"featureFlags": {
"sendNotification": true
},
"availabilityTestConfig": {
"environmentDefinition": "canary"
"environmentDefinition": "dev"
},
"queueConfig": {
"messageVisibilityTimeoutInSeconds": 300,
"maxQueueSize": 50
},
"scanConfig": {
"failedScanRetryIntervalInMinutes": 15,
"failedScanRetryIntervalInMinutes": 5,
"maxFailedScanRetryCount": 1
},
"taskConfig": {
"taskTimeoutInMinutes": 15
},
"crawlConfig": {
"deepScanDiscoveryLimit": 10,
"deepScanUpperLimit": 5000
},
"taskConfig": {
"taskTimeoutInMinutes": 10
"deepScanUpperLimit": 50
}
}

Просмотреть файл

@ -0,0 +1,5 @@
{
"availabilityTestConfig": {
"environmentDefinition": "insider"
}
}

Просмотреть файл

@ -1,18 +0,0 @@
{
"featureFlags": {
"sendNotification": true
},
"queueConfig": {
"maxQueueSize": 100
},
"restApiConfig": {
"scanRequestProcessingDelayInSeconds": 300
},
"availabilityTestConfig": {
"environmentDefinition": "insider"
},
"crawlConfig": {
"deepScanDiscoveryLimit": 100,
"deepScanUpperLimit": 5000
}
}

Просмотреть файл

@ -1,18 +1,5 @@
{
"featureFlags": {
"sendNotification": true
},
"queueConfig": {
"maxQueueSize": 100
},
"restApiConfig": {
"scanRequestProcessingDelayInSeconds": 300
},
"availabilityTestConfig": {
"environmentDefinition": "insider"
},
"crawlConfig": {
"deepScanDiscoveryLimit": 100,
"deepScanUpperLimit": 5000
}
}

Просмотреть файл

@ -0,0 +1,5 @@
{
"availabilityTestConfig": {
"environmentDefinition": "production"
}
}

Просмотреть файл

@ -1,18 +0,0 @@
{
"featureFlags": {
"sendNotification": true
},
"queueConfig": {
"maxQueueSize": 100
},
"restApiConfig": {
"scanRequestProcessingDelayInSeconds": 300
},
"availabilityTestConfig": {
"environmentDefinition": "production"
},
"crawlConfig": {
"deepScanDiscoveryLimit": 100,
"deepScanUpperLimit": 5000
}
}

Просмотреть файл

@ -1,18 +1,5 @@
{
"featureFlags": {
"sendNotification": true
},
"queueConfig": {
"maxQueueSize": 100
},
"restApiConfig": {
"scanRequestProcessingDelayInSeconds": 300
},
"availabilityTestConfig": {
"environmentDefinition": "production"
},
"crawlConfig": {
"deepScanDiscoveryLimit": 100,
"deepScanUpperLimit": 5000
}
}

Просмотреть файл

@ -1,15 +0,0 @@
{
"featureFlags": {
"sendNotification": true
},
"restApiConfig": {
"scanRequestProcessingDelayInSeconds": 300
},
"availabilityTestConfig": {
"environmentDefinition": "canary"
},
"crawlConfig": {
"deepScanDiscoveryLimit": 10,
"deepScanUpperLimit": 5000
}
}

Просмотреть файл

@ -38,14 +38,13 @@ if [[ -z $orgName ]] || [[ -z $publisherEmail ]] || [[ -z $resourceGroupName ]]
exitWithUsageInfo
fi
if [ $environment = "prod" ] || [ $environment = "ppe" ] || [ $environment = "prod-pr" ] || [ $environment = "ppe-pr" ]; then
if [[ ${environment} == prod* ]] || [[ ${environment} == ppe* ]]; then
tier="Standard"
else
tier="Developer"
fi
# Start deployment
echo "[create-api-management] Deploying API management instance. This might take up to 45 mins"
echo " Deploying API management instance. This might take up to 45 minutes"
resources=$(az deployment group create \
--resource-group "$resourceGroupName" \
@ -56,4 +55,4 @@ resources=$(az deployment group create \
. "${0%/*}/get-resource-name-from-resource-paths.sh" -p "Microsoft.ApiManagement/service" -r "$resources"
apiManagementName="$resourceName"
echo "[create-api-management] Successfully deployed API Managment instance - $resourceName"
echo "Successfully deployed API Managment instance $resourceName"

Просмотреть файл

@ -29,17 +29,8 @@ Usage: ${BASH_SOURCE} -r <resource group> -e <environment> -b <Azure Batch objec
. "${0%/*}/process-utilities.sh"
function setParameterFilePath() {
if [ $environment = "prod" ]; then
parameterFilePath="${0%/*}/../templates/batch-account-prod.parameters.json"
elif [ $environment = "prod-pr" ]; then
parameterFilePath="${0%/*}/../templates/batch-account-prod-pr.parameters.json"
elif [ $environment = "ppe" ]; then
parameterFilePath="${0%/*}/../templates/batch-account-ppe.parameters.json"
elif [ $environment = "ppe-pr" ]; then
parameterFilePath="${0%/*}/../templates/batch-account-ppe-pr.parameters.json"
else
parameterFilePath="${0%/*}/../templates/batch-account-dev.parameters.json"
fi
parameterFilePath="${0%/*}/../templates/batch-account-${environment}.parameters.json"
echo "Using configuration file ${parameterFilePath}"
}
function createPublicIp() {

Просмотреть файл

@ -89,9 +89,9 @@ function setupCosmos() {
echo "Creating Cosmos databases in parallel"
runCommandsWithoutSecretsInParallel cosmosSetupProcesses
# Increase autoscale maximum throughput for below collection only in case of prod
# Increase autoscale maximum throughput for below collection only in case of prod or ppe
# Refer to https://docs.microsoft.com/en-us/azure/cosmos-db/time-to-live for item TTL scenarios
if [ $environment = "prod" ] || [ $environment = "ppe" ] || [ $environment = "prod-pr" ] || [ $environment = "ppe-pr" ]; then
if [[ ${environment} == prod* ]] || [[ ${environment} == ppe* ]]; then
cosmosSetupProcesses=(
"createCosmosCollection \"scanRuns\" \"$onDemandScannerDbName\" \"2592000\" \"40000\"" # 30 days
"createCosmosCollection \"scanBatchRequests\" \"$onDemandScannerDbName\" \"604800\" \"4000\"" # 7 days

Просмотреть файл

@ -36,15 +36,16 @@ Usage: ${BASH_SOURCE} \
}
getAllowedApplications() {
if [[ ${environment} == "prod" ]] || [[ ${environment} == "prod-pr" ]]; then
if [[ ${environment} == prod* ]]; then
aclFilePath="${0%/*}/../templates/web-api-aad-acl-prod.txt"
elif [[ ${environment} == "ppe" ]] || [[ ${environment} == "ppe-pr" ]]; then
elif [[ ${environment} == ppe* ]]; then
aclFilePath="${0%/*}/../templates/web-api-aad-acl-ppe.txt"
else
aclFilePath="${0%/*}/../templates/web-api-aad-acl-dev.txt"
aclFilePath="${0%/*}/../templates/web-api-aad-acl-${environment}.txt"
fi
if [[ -f ${aclFilePath} ]]; then
echo "Using Azure Functions ACL configuration ${aclFilePath}"
allowedApplications=$(<"${aclFilePath}")
else
echo "Azure Functions ACL configuration file not found. Expected configuration file ${aclFilePath}"

Просмотреть файл

@ -80,11 +80,14 @@ updateConfigFiles() {
}
updateOpenApiSpec() {
echo "Updating OpenAPI specification"
openApiFilePath="$siteContentFolder/openapi.json"
tempFilePath="${0%/*}/temp-$(date +%s)$RANDOM.json"
gatewayUrl=$(az apim show --name "$apiManagementName" --resource-group "$resourceGroupName" --query "gatewayUrl" -o tsv)
jq "if .servers[0].url then . else .servers[0] += {\"url\": \"$gatewayUrl\"} end" $openApiFilePath >$tempFilePath && mv $tempFilePath $openApiFilePath
apiManagementId=$(az resource list --resource-group "$resourceGroupName" --name "$apiManagementName" --query "[0].id" -o tsv)
if [[ -n ${apiManagementId} ]]; then
echo "Updating OpenAPI specification"
openApiFilePath="$siteContentFolder/openapi.json"
tempFilePath="${0%/*}/temp-$(date +%s)$RANDOM.json"
gatewayUrl=$(az apim show --name "$apiManagementName" --resource-group "$resourceGroupName" --query "gatewayUrl" -o tsv)
jq "if .servers[0].url then . else .servers[0] += {\"url\": \"$gatewayUrl\"} end" $openApiFilePath >$tempFilePath && mv $tempFilePath $openApiFilePath
fi
}
. "${0%/*}/get-resource-names.sh"

Просмотреть файл

@ -43,7 +43,7 @@ Where:
Resource group - The name of the resource group.
Subscription - The Azure subscription name or ID.
Environment - The deployment environment. Supported values dev, prod.
Environment - The deployment environment. Supported values dev, ci, ppe*, prod*.
Organization name - The name of organization.
Publisher email - The notification email.
Azure Batch object ID - The Azure AD object ID for Microsoft Azure Batch enterprise application, application ID ddbf3205-c6bd-46ae-8127-60eb93363864

Просмотреть файл

@ -12,7 +12,7 @@ export MSYS_NO_PATHCONV=1
exitWithUsageInfo() {
echo "
Usage: ${BASH_SOURCE} -r <resource group> [-k <key vault>] [-n <key vault certificate name>] [-s <subscription name or id>] [-e <environment: dev, ci, ppe, prod or selftest>]
Usage: ${BASH_SOURCE} -r <resource group> [-k <key vault>] [-n <key vault certificate name>] [-s <subscription name or id>] [-e <environment>]
"
exit 1
}

Просмотреть файл

@ -44,7 +44,7 @@
},
"resources": [
{
"apiVersion": "2022-08-01",
"apiVersion": "2023-09-01-preview",
"name": "[parameters('name')]",
"location": "[parameters('location')]",
"type": "Microsoft.ApiManagement/service",

Просмотреть файл

@ -0,0 +1,36 @@
{
"$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentParameters.json#",
"contentVersion": "1.0.0.0",
"parameters": {
"environment": {
"value": "canary"
},
"onDemandScanRequestPoolNodes": {
"value": "2"
},
"onDemandScanRequestPoolVmSize": {
"value": "standard_e4ds_v5"
},
"onDemandScanRequestPoolTaskSlotsPerNode": {
"value": "8"
},
"onDemandUrlScanPoolNodes": {
"value": "2"
},
"onDemandUrlScanPoolVmSize": {
"value": "standard_e4ds_v5"
},
"onDemandUrlScanPoolTaskSlotsPerNode": {
"value": "2"
},
"privacyScanPoolNodes": {
"value": "2"
},
"privacyScanPoolVmSize": {
"value": "standard_e4ds_v5"
},
"privacyScanPoolTaskSlotsPerNode": {
"value": "4"
}
}
}

Просмотреть файл

@ -9,7 +9,7 @@
"value": "1"
},
"onDemandScanRequestPoolVmSize": {
"value": "standard_e8ds_v5"
"value": "standard_e4ds_v5"
},
"onDemandScanRequestPoolTaskSlotsPerNode": {
"value": "8"

Просмотреть файл

@ -4,7 +4,7 @@
"parameters": {
"environment": {
"type": "string",
"defaultValue": "dev",
"defaultValue": "prod",
"metadata": {
"description": "The target deployment environment."
}
@ -154,6 +154,7 @@
}
],
"variables": {
"nonVNetEnvironmentName": "dev",
"keyVaultUrl": "[concat('https://', parameters('keyVault'), '.vault.azure.net/')]",
"onDemandUrlScanPoolName": "on-demand-url-scan-pool",
"onDemandScanRequestPoolName": "on-demand-scan-request-pool",
@ -221,7 +222,7 @@
}
}
},
"networkConfiguration": "[if(equals(parameters('environment'), 'dev'), fn.batchPool(), fn.vnetPool(parameters('vnetName'), variables('onDemandUrlScanPoolName')))]",
"networkConfiguration": "[if(equals(parameters('environment'), variables('nonVNetEnvironmentName')), fn.batchPool(), fn.vnetPool(parameters('vnetName'), variables('onDemandUrlScanPoolName')))]",
"scaleSettings": {
"fixedScale": {
"targetDedicatedNodes": "[parameters('onDemandUrlScanPoolNodes')]",
@ -292,7 +293,7 @@
}
}
},
"networkConfiguration": "[if(equals(parameters('environment'), 'dev'), fn.batchPool(), fn.vnetPool(parameters('vnetName'), variables('onDemandScanRequestPoolName')))]",
"networkConfiguration": "[if(equals(parameters('environment'), variables('nonVNetEnvironmentName')), fn.batchPool(), fn.vnetPool(parameters('vnetName'), variables('onDemandScanRequestPoolName')))]",
"scaleSettings": {
"fixedScale": {
"targetDedicatedNodes": "[parameters('onDemandScanRequestPoolNodes')]",
@ -363,7 +364,7 @@
}
}
},
"networkConfiguration": "[if(equals(parameters('environment'), 'dev'), fn.batchPool(), fn.vnetPool(parameters('vnetName'), variables('privacyScanPoolName')))]",
"networkConfiguration": "[if(equals(parameters('environment'), variables('nonVNetEnvironmentName')), fn.batchPool(), fn.vnetPool(parameters('vnetName'), variables('privacyScanPoolName')))]",
"scaleSettings": {
"fixedScale": {
"targetDedicatedNodes": "[parameters('privacyScanPoolNodes')]",

Просмотреть файл

@ -1,38 +0,0 @@
{
"issuerParameters": {
"certificateTransparency": null,
"name": "OneCert"
},
"keyProperties": {
"curve": null,
"exportable": true,
"keySize": 2048,
"keyType": "RSA",
"reuseKey": true
},
"lifetimeActions": [
{
"action": {
"actionType": "AutoRenew"
},
"trigger": {
"daysBeforeExpiry": null,
"lifetimePercentage": 24
}
}
],
"secretProperties": {
"contentType": "application/x-pkcs12"
},
"x509CertificateProperties": {
"ekus": ["1.3.6.1.5.5.7.3.1", "1.3.6.1.5.5.7.3.2"],
"keyUsage": ["cRLSign", "dataEncipherment", "digitalSignature", "keyAgreement", "keyCertSign", "keyEncipherment"],
"subject": "CN=a11yserviceselftest.geneva.keyvault.accessibilityinsights.com",
"subjectAlternativeNames": {
"dnsNames": ["a11yserviceselftest.geneva.keyvault.accessibilityinsights.com"],
"emails": null,
"upns": null
},
"validityInMonths": 12
}
}

Просмотреть файл

@ -0,0 +1,35 @@
diff --git a/node_modules/got-scraping/node_modules/header-generator/.DS_Store b/node_modules/got-scraping/node_modules/header-generator/.DS_Store
new file mode 100644
index 0000000..6406205
Binary files /dev/null and b/node_modules/got-scraping/node_modules/header-generator/.DS_Store differ
diff --git a/node_modules/got-scraping/node_modules/header-generator/header-generator.js b/node_modules/got-scraping/node_modules/header-generator/header-generator.js
index 0c7779f..012cdfc 100644
--- a/node_modules/got-scraping/node_modules/header-generator/header-generator.js
+++ b/node_modules/got-scraping/node_modules/header-generator/header-generator.js
@@ -91,16 +91,22 @@ class HeaderGenerator {
strict,
};
this.uniqueBrowsers = [];
- this.headersOrder = JSON.parse((0, fs_1.readFileSync)(`${__dirname}/data_files/headers-order.json`).toString());
- const uniqueBrowserStrings = JSON.parse((0, fs_1.readFileSync)(`${__dirname}/data_files/browser-helper-file.json`, 'utf8').toString());
+ // Remove unnecessary dependencies to package modules using webpack.
+ // this.headersOrder = JSON.parse((0, fs_1.readFileSync)(`${__dirname}/data_files/headers-order.json`).toString());
+ // const uniqueBrowserStrings = JSON.parse((0, fs_1.readFileSync)(`${__dirname}/data_files/browser-helper-file.json`, 'utf8').toString());
+ this.headersOrder = {};
+ const uniqueBrowserStrings = [];
for (const browserString of uniqueBrowserStrings) {
// There are headers without user agents in the datasets we used to configure the generator. They should be disregarded.
if (browserString !== constants_1.MISSING_VALUE_DATASET_TOKEN) {
this.uniqueBrowsers.push(this.prepareHttpBrowserObject(browserString));
}
}
- this.inputGeneratorNetwork = new generative_bayesian_network_1.BayesianNetwork({ path: `${__dirname}/data_files/input-network-definition.zip` });
- this.headerGeneratorNetwork = new generative_bayesian_network_1.BayesianNetwork({ path: `${__dirname}/data_files/header-network-definition.zip` });
+ // Remove unnecessary dependencies to package modules using webpack.
+ // this.inputGeneratorNetwork = new generative_bayesian_network_1.BayesianNetwork({ path: `${__dirname}/data_files/input-network-definition.zip` });
+ // this.headerGeneratorNetwork = new generative_bayesian_network_1.BayesianNetwork({ path: `${__dirname}/data_files/header-network-definition.zip` });
+ this.inputGeneratorNetwork = {};
+ this.headerGeneratorNetwork = {};
}
/**
* Generates a single set of ordered headers using a combination of the default options specified in the constructor