Adds a flag DEPLOY_PGSQL to manage postgresql database server deployment (#52)

deploy/infra/groups/data.bicep

@@ -46,7 +46,8 @@ param synapseMIStorageAccountRoles array = [
 ]
 param synapseMIPrincipalId string = ''
 
-// Name parameters for Postgres
+// Parameters for Postgres
+param deployPgSQL bool = true
 param serverName string = ''
 param administratorLogin string = ''
 param postgresAdminLoginPass string = '' 
@@ -147,7 +148,7 @@ module synapseIdentityForStorageAccess '../modules/storage-role-assignment.bicep
   ]
 }]
 
-module postgresqlServer '../modules/postgres.single.svc.bicep' = {
+module postgresqlServer '../modules/postgres.single.svc.bicep' = if(deployPgSQL) {
   name: '${namingPrefix}-postgres'
   params: {
     location: location
@@ -165,11 +166,11 @@ module postgresqlServer '../modules/postgres.single.svc.bicep' = {
   }
 }
 
-resource postgresql_server_resource 'Microsoft.DBforPostgreSQL/servers@2017-12-01' existing = {
+resource postgresql_server_resource 'Microsoft.DBforPostgreSQL/servers@2017-12-01' existing = if(deployPgSQL) {
   name: serverNameVar
 }
 
-resource azurerm_postgresql_firewall_rule 'Microsoft.DBforPostgreSQL/servers/firewallRules@2017-12-01' = {
+resource azurerm_postgresql_firewall_rule 'Microsoft.DBforPostgreSQL/servers/firewallRules@2017-12-01' = if(deployPgSQL) {
   name: 'AllowAccessToAzureServices'
   parent: postgresql_server_resource
   properties: {
@@ -188,7 +189,7 @@ module dataUami '../modules/managed.identity.user.bicep' = {
     uamiName: uamiNameVar
   }
 }
-module pgAdministratorLoginPassword '../modules/akv.secrets.bicep' = {
+module pgAdministratorLoginPassword '../modules/akv.secrets.bicep' = if(deployPgSQL) {
   name: 'pg-admin-login-pass-${utcValue}'
   scope: resourceGroup(pipelineResourceGroupName)
   params: {
@@ -202,16 +203,16 @@ module pgAdministratorLoginPassword '../modules/akv.secrets.bicep' = {
   ]
 }
 
-module createContainerForTableCreation '../modules/aci.bicep' = {
+module createContainerForTableCreation '../modules/aci.bicep' = if(deployPgSQL) {
   name: '${namingPrefix}-container-for-db-table-creation'
   params: {
     name: '${namingPrefix}-container'
     userManagedIdentityId: dataUami.outputs.uamiId
     userManagedIdentityPrincipalId: dataUami.outputs.uamiPrincipalId
     location: location
-    server: postgresqlServer.outputs.pgServerName
-    username: postgresqlServer.outputs.pgUserName
-    dbPassword: postgresAdminLoginPassVar
+    server: deployPgSQL?postgresqlServer.outputs.pgServerName:''
+    username: deployPgSQL?postgresqlServer.outputs.pgUserName:''
+    dbPassword: deployPgSQL?postgresAdminLoginPassVar:''
   }
   dependsOn: [
     postgresqlServer
@@ -220,7 +221,7 @@ module createContainerForTableCreation '../modules/aci.bicep' = {
   ]
 }
 
-module deleteContainerForTableCreation '../modules/aci.delete.bicep' = {
+module deleteContainerForTableCreation '../modules/aci.delete.bicep' = if(deployPgSQL) {
   name: 'deleteContainerForTableCreation'
   params: {
     location: location

deploy/infra/main.bicep

@@ -32,6 +32,9 @@ param orchestrationModulePrefix string = 'orc'
 @description('Specify whether or not to deploy batch account')
 param deployBatchAccount bool = true
 
+@description('Specify whether or not to deploy PostgreSQL')
+param deployPgSQL bool = true
+
 @description('Postgres DB administrator login password')
 @secure()
 param postgresAdminLoginPass string
@@ -138,6 +141,7 @@ module dataModule 'groups/data.bicep' = {
     synapseMIPrincipalId: pipelineModule.outputs.synapseMIPrincipalId
     pipelineResourceGroupName: pipelineResourceGroup.name
     pipelineLinkedSvcKeyVaultName: '${environmentCode}-${pipelineModulePrefix}-kv'
+    deployPgSQL: deployPgSQL
     postgresAdminLoginPass: postgresAdminLoginPass
   }
   dependsOn: [

deploy/install.sh

@@ -23,6 +23,7 @@ LOCATION=${2:-${LOCATION}}
 ENV_TAG=${3:-${ENV_TAG:-"synapse-${ENV_CODE}"}}
 DEPLOYMENT_NAME=${4:-${DEPLOYMENT_NAME:-"${ENV_TAG}-deploy"}}
 DEPLOY_BATCH_ACCOUNT=${5:-${DEPLOY_BATCH_ACCOUNT:-"true"}}
+DEPLOY_PGSQL=${6:-${DEPLOY_PGSQL:-"true"}}
 
 DEPLOYMENT_SCRIPT="az deployment sub create -l $LOCATION -n $DEPLOYMENT_NAME \
     -f ./deploy/infra/main.bicep \
@@ -31,6 +32,7 @@ DEPLOYMENT_SCRIPT="az deployment sub create -l $LOCATION -n $DEPLOYMENT_NAME \
     environmentCode=$ENV_CODE \
     environment=$ENV_TAG \
     deployBatchAccount=$DEPLOY_BATCH_ACCOUNT \
+    deployPgSQL=$DEPLOY_PGSQL \
     postgresAdminLoginPass=$POSTGRES_ADMIN_LOGIN_PASS"
 $DEPLOYMENT_SCRIPT
 set +x

deploy/package.py

@@ -19,8 +19,8 @@ parser.add_argument('--batch_account', type=str, required=True, help="Batch Acco
 parser.add_argument('--linked_key_vault', type=str, required=True, help="Key Vault to be added as Linked Service")
 parser.add_argument('--location', type=str, required=True, help="Batch Account Location")
 parser.add_argument('--pipeline_name', type=str, required=True, help="Name of the pipeline to package")
-parser.add_argument('--pg_db_username', type=str, required=True, help="Username to login to postgres db")
-parser.add_argument('--pg_db_server_name', type=str, required=True, help="Server name to login to postgres db")
+parser.add_argument('--pg_db_username', type=str, required=False, help="Username to login to postgres db", default='')
+parser.add_argument('--pg_db_server_name', type=str, required=False, help="Server name to login to postgres db", default='')
 
 #Parse Args
 args = parser.parse_args()

deploy/package.sh

@@ -21,6 +21,8 @@ SYNAPSE_WORKSPACE_NAME=${10:-${SYNAPSE_WORKSPACE_NAME}}
 SYNAPSE_STORAGE_ACCOUNT_NAME=${11:-${SYNAPSE_STORAGE_ACCOUNT_NAME}}
 SYNAPSE_POOL=${12:-${SYNAPSE_POOL}}
 
+DEPLOY_PGSQL=${13:-${DEPLOY_PGSQL:-"true"}}
+
 set -ex
 
 
@@ -62,16 +64,9 @@ if [[ -z "$SYNAPSE_POOL" ]]; then
     SYNAPSE_POOL=$(az synapse spark pool list --workspace-name $SYNAPSE_WORKSPACE_NAME --resource-group $SYNAPSE_WORKSPACE_RG --query "[?tags.poolId && tags.poolId == 'default'].name" -o tsv)
 fi
 
-DB_SERVER_NAME=$(az postgres server list --resource-group $RAW_STORAGE_ACCOUNT_RG --query '[].fullyQualifiedDomainName' -o tsv)
-echo $DB_SERVER_NAME
-DB_NAME=$(az postgres server list --resource-group $RAW_STORAGE_ACCOUNT_RG --query '[].name' -o tsv)
-echo $DB_NAME
-DB_USERNAME=$(az postgres server list --resource-group $RAW_STORAGE_ACCOUNT_RG --query '[].administratorLogin' -o tsv)@$DB_NAME
-echo $DB_USERNAME
-
-
 echo 'Retrieved resource from Azure and ready to package'
-PACKAGING_SCRIPT="python3 ${PRJ_ROOT}/deploy/package.py --raw_storage_account_name $RAW_STORAGE_ACCOUNT_NAME \
+PACKAGING_SCRIPT="python3 ${PRJ_ROOT}/deploy/package.py \
+    --raw_storage_account_name $RAW_STORAGE_ACCOUNT_NAME \
     --synapse_storage_account_name $SYNAPSE_STORAGE_ACCOUNT_NAME \
     --batch_storage_account_name $BATCH_STORAGE_ACCOUNT_NAME \
     --batch_account $BATCH_ACCOUNT_NAME \
@@ -80,9 +75,22 @@ PACKAGING_SCRIPT="python3 ${PRJ_ROOT}/deploy/package.py --raw_storage_account_na
     --location $BATCH_ACCOUNT_LOCATION \
     --pipeline_name $PIPELINE_NAME \
     --synapse_workspace $SYNAPSE_WORKSPACE_NAME \
-    --synapse_workspace_id $SYNAPSE_WORKSPACE_ID \
-    --pg_db_username $DB_USERNAME \
-    --pg_db_server_name $DB_SERVER_NAME"
+    --synapse_workspace_id $SYNAPSE_WORKSPACE_ID"
+
+if [[ $DEPLOY_PGSQL == "true" ]]; then
+    DB_SERVER_NAME=$(az postgres server list --resource-group $RAW_STORAGE_ACCOUNT_RG --query '[].fullyQualifiedDomainName' -o tsv)
+    echo $DB_SERVER_NAME
+    DB_NAME=$(az postgres server list --resource-group $RAW_STORAGE_ACCOUNT_RG --query '[].name' -o tsv)
+    echo $DB_NAME
+    DB_USERNAME=$(az postgres server list --resource-group $RAW_STORAGE_ACCOUNT_RG --query '[].administratorLogin' -o tsv)@$DB_NAME
+    echo $DB_USERNAME
+
+    if [[ -n $DB_USERNAME ]] && [[ -n $DB_SERVER_NAME ]]; then
+        PACKAGING_SCRIPT=$(echo $PACKAGING_SCRIPT \
+            --pg_db_username $DB_USERNAME \
+            --pg_db_server_name $DB_SERVER_NAME)
+    fi
+fi
 
 echo $PACKAGING_SCRIPT
 echo 'Starting packaging script ...'

deploy/setup.sh

@@ -8,7 +8,7 @@ LOCATION=${2:-${LOCATION}}
 PIPELINE_NAME=${3:-${PIPELINE_NAME}}
 ENV_TAG=${4:-${ENV_TAG}}
 PRE_PROVISIONED_BATCH_ACCOUNT_NAME=${5:-$PRE_PROVISIONED_BATCH_ACCOUNT_NAME}
-
+DEPLOY_PGSQL=${6:-${DEPLOY_PGSQL:-"true"}}
 
 set -ex
 
@@ -35,9 +35,11 @@ echo "Performing bicep template deployment"
 if [[ -z "$ENV_TAG" ]]
     then
         DEPLOY_BATCH_ACCOUNT=${DEPLOY_BATCH_ACCOUNT} \
+        DEPLOY_PGSQL=${DEPLOY_PGSQL} \
           ./deploy/install.sh "$ENV_CODE" "$LOCATION" 
     else
         DEPLOY_BATCH_ACCOUNT=${DEPLOY_BATCH_ACCOUNT} \
+        DEPLOY_PGSQL=${DEPLOY_PGSQL} \
           ./deploy/install.sh "$ENV_CODE" "$LOCATION" "$ENV_TAG"
 fi
 
@@ -59,7 +61,8 @@ if [[ -z "$PIPELINE_NAME" ]]
     echo "Skipping pipeline packaging"
   else
     echo "Performing pipeline packaging"
-    ./deploy/package.sh \
+    DEPLOY_PGSQL=${DEPLOY_PGSQL} \
+      ./deploy/package.sh \
       "$ENV_CODE" \
       "$PIPELINE_NAME" \
       "$PRE_PROVISIONED_BATCH_ACCOUNT_NAME"