From fb6c234734f39645d80115e12c7d1b956e58676d Mon Sep 17 00:00:00 2001 From: Sushil Kumar Date: Fri, 17 Jun 2022 17:21:04 -0700 Subject: [PATCH] Adds a flag DEPLOY_PGSQL to manage postgresql database server deployment (#52) --- deploy/infra/groups/data.bicep | 21 +++++++++++---------- deploy/infra/main.bicep | 4 ++++ deploy/install.sh | 2 ++ deploy/package.py | 4 ++-- deploy/package.sh | 32 ++++++++++++++++++++------------ deploy/setup.sh | 7 +++++-- 6 files changed, 44 insertions(+), 26 deletions(-) diff --git a/deploy/infra/groups/data.bicep b/deploy/infra/groups/data.bicep index d4453e6..0c50c30 100644 --- a/deploy/infra/groups/data.bicep +++ b/deploy/infra/groups/data.bicep @@ -46,7 +46,8 @@ param synapseMIStorageAccountRoles array = [ ] param synapseMIPrincipalId string = '' -// Name parameters for Postgres +// Parameters for Postgres +param deployPgSQL bool = true param serverName string = '' param administratorLogin string = '' param postgresAdminLoginPass string = '' @@ -147,7 +148,7 @@ module synapseIdentityForStorageAccess '../modules/storage-role-assignment.bicep ] }] -module postgresqlServer '../modules/postgres.single.svc.bicep' = { +module postgresqlServer '../modules/postgres.single.svc.bicep' = if(deployPgSQL) { name: '${namingPrefix}-postgres' params: { location: location @@ -165,11 +166,11 @@ module postgresqlServer '../modules/postgres.single.svc.bicep' = { } } -resource postgresql_server_resource 'Microsoft.DBforPostgreSQL/servers@2017-12-01' existing = { +resource postgresql_server_resource 'Microsoft.DBforPostgreSQL/servers@2017-12-01' existing = if(deployPgSQL) { name: serverNameVar } -resource azurerm_postgresql_firewall_rule 'Microsoft.DBforPostgreSQL/servers/firewallRules@2017-12-01' = { +resource azurerm_postgresql_firewall_rule 'Microsoft.DBforPostgreSQL/servers/firewallRules@2017-12-01' = if(deployPgSQL) { name: 'AllowAccessToAzureServices' parent: postgresql_server_resource properties: { @@ -188,7 +189,7 @@ module dataUami '../modules/managed.identity.user.bicep' = { uamiName: uamiNameVar } } -module pgAdministratorLoginPassword '../modules/akv.secrets.bicep' = { +module pgAdministratorLoginPassword '../modules/akv.secrets.bicep' = if(deployPgSQL) { name: 'pg-admin-login-pass-${utcValue}' scope: resourceGroup(pipelineResourceGroupName) params: { @@ -202,16 +203,16 @@ module pgAdministratorLoginPassword '../modules/akv.secrets.bicep' = { ] } -module createContainerForTableCreation '../modules/aci.bicep' = { +module createContainerForTableCreation '../modules/aci.bicep' = if(deployPgSQL) { name: '${namingPrefix}-container-for-db-table-creation' params: { name: '${namingPrefix}-container' userManagedIdentityId: dataUami.outputs.uamiId userManagedIdentityPrincipalId: dataUami.outputs.uamiPrincipalId location: location - server: postgresqlServer.outputs.pgServerName - username: postgresqlServer.outputs.pgUserName - dbPassword: postgresAdminLoginPassVar + server: deployPgSQL?postgresqlServer.outputs.pgServerName:'' + username: deployPgSQL?postgresqlServer.outputs.pgUserName:'' + dbPassword: deployPgSQL?postgresAdminLoginPassVar:'' } dependsOn: [ postgresqlServer @@ -220,7 +221,7 @@ module createContainerForTableCreation '../modules/aci.bicep' = { ] } -module deleteContainerForTableCreation '../modules/aci.delete.bicep' = { +module deleteContainerForTableCreation '../modules/aci.delete.bicep' = if(deployPgSQL) { name: 'deleteContainerForTableCreation' params: { location: location diff --git a/deploy/infra/main.bicep b/deploy/infra/main.bicep index aa45c35..c5345df 100644 --- a/deploy/infra/main.bicep +++ b/deploy/infra/main.bicep @@ -32,6 +32,9 @@ param orchestrationModulePrefix string = 'orc' @description('Specify whether or not to deploy batch account') param deployBatchAccount bool = true +@description('Specify whether or not to deploy PostgreSQL') +param deployPgSQL bool = true + @description('Postgres DB administrator login password') @secure() param postgresAdminLoginPass string @@ -138,6 +141,7 @@ module dataModule 'groups/data.bicep' = { synapseMIPrincipalId: pipelineModule.outputs.synapseMIPrincipalId pipelineResourceGroupName: pipelineResourceGroup.name pipelineLinkedSvcKeyVaultName: '${environmentCode}-${pipelineModulePrefix}-kv' + deployPgSQL: deployPgSQL postgresAdminLoginPass: postgresAdminLoginPass } dependsOn: [ diff --git a/deploy/install.sh b/deploy/install.sh index b0f7f1c..82a60d1 100755 --- a/deploy/install.sh +++ b/deploy/install.sh @@ -23,6 +23,7 @@ LOCATION=${2:-${LOCATION}} ENV_TAG=${3:-${ENV_TAG:-"synapse-${ENV_CODE}"}} DEPLOYMENT_NAME=${4:-${DEPLOYMENT_NAME:-"${ENV_TAG}-deploy"}} DEPLOY_BATCH_ACCOUNT=${5:-${DEPLOY_BATCH_ACCOUNT:-"true"}} +DEPLOY_PGSQL=${6:-${DEPLOY_PGSQL:-"true"}} DEPLOYMENT_SCRIPT="az deployment sub create -l $LOCATION -n $DEPLOYMENT_NAME \ -f ./deploy/infra/main.bicep \ @@ -31,6 +32,7 @@ DEPLOYMENT_SCRIPT="az deployment sub create -l $LOCATION -n $DEPLOYMENT_NAME \ environmentCode=$ENV_CODE \ environment=$ENV_TAG \ deployBatchAccount=$DEPLOY_BATCH_ACCOUNT \ + deployPgSQL=$DEPLOY_PGSQL \ postgresAdminLoginPass=$POSTGRES_ADMIN_LOGIN_PASS" $DEPLOYMENT_SCRIPT set +x diff --git a/deploy/package.py b/deploy/package.py index 8ffe335..c22561c 100755 --- a/deploy/package.py +++ b/deploy/package.py @@ -19,8 +19,8 @@ parser.add_argument('--batch_account', type=str, required=True, help="Batch Acco parser.add_argument('--linked_key_vault', type=str, required=True, help="Key Vault to be added as Linked Service") parser.add_argument('--location', type=str, required=True, help="Batch Account Location") parser.add_argument('--pipeline_name', type=str, required=True, help="Name of the pipeline to package") -parser.add_argument('--pg_db_username', type=str, required=True, help="Username to login to postgres db") -parser.add_argument('--pg_db_server_name', type=str, required=True, help="Server name to login to postgres db") +parser.add_argument('--pg_db_username', type=str, required=False, help="Username to login to postgres db", default='') +parser.add_argument('--pg_db_server_name', type=str, required=False, help="Server name to login to postgres db", default='') #Parse Args args = parser.parse_args() diff --git a/deploy/package.sh b/deploy/package.sh index 1d324b4..8dd5c6c 100755 --- a/deploy/package.sh +++ b/deploy/package.sh @@ -21,6 +21,8 @@ SYNAPSE_WORKSPACE_NAME=${10:-${SYNAPSE_WORKSPACE_NAME}} SYNAPSE_STORAGE_ACCOUNT_NAME=${11:-${SYNAPSE_STORAGE_ACCOUNT_NAME}} SYNAPSE_POOL=${12:-${SYNAPSE_POOL}} +DEPLOY_PGSQL=${13:-${DEPLOY_PGSQL:-"true"}} + set -ex @@ -62,16 +64,9 @@ if [[ -z "$SYNAPSE_POOL" ]]; then SYNAPSE_POOL=$(az synapse spark pool list --workspace-name $SYNAPSE_WORKSPACE_NAME --resource-group $SYNAPSE_WORKSPACE_RG --query "[?tags.poolId && tags.poolId == 'default'].name" -o tsv) fi -DB_SERVER_NAME=$(az postgres server list --resource-group $RAW_STORAGE_ACCOUNT_RG --query '[].fullyQualifiedDomainName' -o tsv) -echo $DB_SERVER_NAME -DB_NAME=$(az postgres server list --resource-group $RAW_STORAGE_ACCOUNT_RG --query '[].name' -o tsv) -echo $DB_NAME -DB_USERNAME=$(az postgres server list --resource-group $RAW_STORAGE_ACCOUNT_RG --query '[].administratorLogin' -o tsv)@$DB_NAME -echo $DB_USERNAME - - echo 'Retrieved resource from Azure and ready to package' -PACKAGING_SCRIPT="python3 ${PRJ_ROOT}/deploy/package.py --raw_storage_account_name $RAW_STORAGE_ACCOUNT_NAME \ +PACKAGING_SCRIPT="python3 ${PRJ_ROOT}/deploy/package.py \ + --raw_storage_account_name $RAW_STORAGE_ACCOUNT_NAME \ --synapse_storage_account_name $SYNAPSE_STORAGE_ACCOUNT_NAME \ --batch_storage_account_name $BATCH_STORAGE_ACCOUNT_NAME \ --batch_account $BATCH_ACCOUNT_NAME \ @@ -80,9 +75,22 @@ PACKAGING_SCRIPT="python3 ${PRJ_ROOT}/deploy/package.py --raw_storage_account_na --location $BATCH_ACCOUNT_LOCATION \ --pipeline_name $PIPELINE_NAME \ --synapse_workspace $SYNAPSE_WORKSPACE_NAME \ - --synapse_workspace_id $SYNAPSE_WORKSPACE_ID \ - --pg_db_username $DB_USERNAME \ - --pg_db_server_name $DB_SERVER_NAME" + --synapse_workspace_id $SYNAPSE_WORKSPACE_ID" + +if [[ $DEPLOY_PGSQL == "true" ]]; then + DB_SERVER_NAME=$(az postgres server list --resource-group $RAW_STORAGE_ACCOUNT_RG --query '[].fullyQualifiedDomainName' -o tsv) + echo $DB_SERVER_NAME + DB_NAME=$(az postgres server list --resource-group $RAW_STORAGE_ACCOUNT_RG --query '[].name' -o tsv) + echo $DB_NAME + DB_USERNAME=$(az postgres server list --resource-group $RAW_STORAGE_ACCOUNT_RG --query '[].administratorLogin' -o tsv)@$DB_NAME + echo $DB_USERNAME + + if [[ -n $DB_USERNAME ]] && [[ -n $DB_SERVER_NAME ]]; then + PACKAGING_SCRIPT=$(echo $PACKAGING_SCRIPT \ + --pg_db_username $DB_USERNAME \ + --pg_db_server_name $DB_SERVER_NAME) + fi +fi echo $PACKAGING_SCRIPT echo 'Starting packaging script ...' diff --git a/deploy/setup.sh b/deploy/setup.sh index af0d0a0..aeaa3c1 100755 --- a/deploy/setup.sh +++ b/deploy/setup.sh @@ -8,7 +8,7 @@ LOCATION=${2:-${LOCATION}} PIPELINE_NAME=${3:-${PIPELINE_NAME}} ENV_TAG=${4:-${ENV_TAG}} PRE_PROVISIONED_BATCH_ACCOUNT_NAME=${5:-$PRE_PROVISIONED_BATCH_ACCOUNT_NAME} - +DEPLOY_PGSQL=${6:-${DEPLOY_PGSQL:-"true"}} set -ex @@ -35,9 +35,11 @@ echo "Performing bicep template deployment" if [[ -z "$ENV_TAG" ]] then DEPLOY_BATCH_ACCOUNT=${DEPLOY_BATCH_ACCOUNT} \ + DEPLOY_PGSQL=${DEPLOY_PGSQL} \ ./deploy/install.sh "$ENV_CODE" "$LOCATION" else DEPLOY_BATCH_ACCOUNT=${DEPLOY_BATCH_ACCOUNT} \ + DEPLOY_PGSQL=${DEPLOY_PGSQL} \ ./deploy/install.sh "$ENV_CODE" "$LOCATION" "$ENV_TAG" fi @@ -59,7 +61,8 @@ if [[ -z "$PIPELINE_NAME" ]] echo "Skipping pipeline packaging" else echo "Performing pipeline packaging" - ./deploy/package.sh \ + DEPLOY_PGSQL=${DEPLOY_PGSQL} \ + ./deploy/package.sh \ "$ENV_CODE" \ "$PIPELINE_NAME" \ "$PRE_PROVISIONED_BATCH_ACCOUNT_NAME"