From cf6efb65b494b059883cf0ac2e5792b06152d71b Mon Sep 17 00:00:00 2001 From: Morne Blake Date: Tue, 1 Dec 2020 14:36:51 +0100 Subject: [PATCH] Added Step 5 and updated step 3 --- .../Step 3 - Create External Resources.json | 854 +++++++++++++++++- .../manifest.json | 3 +- .../Step 5 - Delete the backup database.json | 76 ++ .../manifest.json | 24 + 4 files changed, 955 insertions(+), 2 deletions(-) create mode 100644 Step 5 - Delete the backup database/Step 5 - Delete the backup database.json create mode 100644 Step 5 - Delete the backup database/manifest.json diff --git a/Step 3 - Create External Resources/Step 3 - Create External Resources.json b/Step 3 - Create External Resources/Step 3 - Create External Resources.json index b6d0860..49a0b4c 100644 --- a/Step 3 - Create External Resources/Step 3 - Create External Resources.json +++ b/Step 3 - Create External Resources/Step 3 - Create External Resources.json @@ -1 +1,853 @@ -{"$schema":"http://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#","contentVersion":"1.0.0.0","parameters":{"factoryName":{"type":"string","metadata":"Data Factory name"},"conn_BackupServer":{"type":"string"}},"variables":{"factoryId":"[concat('Microsoft.DataFactory/factories/', parameters('factoryName'))]"},"resources":[{"name":"[concat(parameters('factoryName'), '/Step 3 - Create External Resources')]","type":"Microsoft.DataFactory/factories/pipelines","apiVersion":"2018-06-01","properties":{"description":"This pipeline will add the required schema, stored procedures and external resources to the database that will be used to backup the database to BLOB storage","activities":[{"name":"Create backup schema","description":"This creates the backup schema to be used, the schema name is parameterized, if you do change it, make sure that it is a valid value for a schema. \n\nThis schema cannot already exist in the database!","type":"Lookup","dependsOn":[{"activity":"Set user name","dependencyConditions":["Succeeded"]},{"activity":"Set password","dependencyConditions":["Succeeded"]},{"activity":"Set storage location","dependencyConditions":["Succeeded"]}],"policy":{"timeout":"7.00:00:00","retry":0,"retryIntervalInSeconds":30,"secureOutput":true,"secureInput":true},"userProperties":[],"typeProperties":{"source":{"type":"SqlDWSource","sqlReaderQuery":{"value":"EXEC('CREATE SCHEMA [@{pipeline().parameters.sql_BackupSchemaName}]');\nSELECT 1;","type":"Expression"},"queryTimeout":"02:00:00","partitionOption":"None"},"dataset":{"referenceName":"set_blank","type":"DatasetReference","parameters":{"DSBackupServerName":{"value":"@pipeline().parameters.infra_BackupSqlServer","type":"Expression"},"DSBackupDatabaseName":{"value":"@pipeline().parameters.synapse_DatabaseBackupName","type":"Expression"},"DSBackupUsername":{"value":"@variables('security_BackupUserName')","type":"Expression"},"DSBackupUserPassword":{"value":"@variables('security_BackupUserPassword')","type":"Expression"}}}}},{"name":"Create Master Key","description":"This will attempt to create a Master Key on the database, but will fail if one already exists. Failure here would be normal and could just be ignored.","type":"Lookup","dependsOn":[{"activity":"Get password from AKV","dependencyConditions":["Succeeded"]}],"policy":{"timeout":"7.00:00:00","retry":0,"retryIntervalInSeconds":30,"secureOutput":true,"secureInput":true},"userProperties":[],"typeProperties":{"source":{"type":"SqlDWSource","sqlReaderQuery":{"value":"IF NOT EXISTS (SELECT * FROM sys.symmetric_keys WHERE [name] LIKE '%DatabaseMasterKey%')BEGIN\n CREATE MASTER KEY ENCRYPTION BY PASSWORD = '@{activity('Get password from AKV').output.value}';\nEND\nSELECT 1;","type":"Expression"},"queryTimeout":"02:00:00","partitionOption":"None"},"dataset":{"referenceName":"set_blank","type":"DatasetReference","parameters":{"DSBackupServerName":{"value":"@pipeline().parameters.infra_BackupSqlServer","type":"Expression"},"DSBackupDatabaseName":{"value":"@pipeline().parameters.synapse_DatabaseBackupName","type":"Expression"},"DSBackupUsername":{"value":"@variables('security_BackupUserName')","type":"Expression"},"DSBackupUserPassword":{"value":"@variables('security_BackupUserPassword')","type":"Expression"}}}}},{"name":"Create DATABASE SCOPED CREDENTIAL","description":"Created the database scoped credentials that will be used to connect to BLOB storage. It will use the name and secret from AKV, feel free to change how the credentials are specified, but keep the name","type":"Lookup","dependsOn":[{"activity":"Set DSC Secret","dependencyConditions":["Succeeded"]},{"activity":"Set DSC Identity","dependencyConditions":["Succeeded"]}],"policy":{"timeout":"7.00:00:00","retry":0,"retryIntervalInSeconds":30,"secureOutput":true,"secureInput":true},"userProperties":[],"typeProperties":{"source":{"type":"SqlDWSource","sqlReaderQuery":{"value":"CREATE DATABASE SCOPED CREDENTIAL [@{pipeline().parameters.sql_BackupSchemaName}BackupBlobStorageCreds]\nWITH\n IDENTITY = '@{variables('security_DSCIdentity')}'\n, SECRET = '@{variables('security_DSCSecret')}';\nSELECT 1;\n\n","type":"Expression"},"queryTimeout":"02:00:00","partitionOption":"None"},"dataset":{"referenceName":"set_blank","type":"DatasetReference","parameters":{"DSBackupServerName":{"value":"@pipeline().parameters.infra_BackupSqlServer","type":"Expression"},"DSBackupDatabaseName":{"value":"@pipeline().parameters.synapse_DatabaseBackupName","type":"Expression"},"DSBackupUsername":{"value":"@variables('security_BackupUserName')","type":"Expression"},"DSBackupUserPassword":{"value":"@variables('security_BackupUserPassword')","type":"Expression"}}}}},{"name":"CREATE EXTERNAL DATA SOURCE","description":"Creating the external data source, feel free to change it, but leave the name the same","type":"Lookup","dependsOn":[{"activity":"Create DATABASE SCOPED CREDENTIAL","dependencyConditions":["Succeeded"]}],"policy":{"timeout":"7.00:00:00","retry":0,"retryIntervalInSeconds":30,"secureOutput":true,"secureInput":true},"userProperties":[],"typeProperties":{"source":{"type":"SqlDWSource","sqlReaderQuery":{"value":"IF NOT EXISTS (SELECT * FROM sys.external_data_Sources WHERE [name] = '@{pipeline().parameters.sql_BackupSchemaName}BackupBlobStorage')\nBEGIN\nCREATE EXTERNAL DATA SOURCE [@{pipeline().parameters.sql_BackupSchemaName}BackupBlobStorage]\nWITH\n( LOCATION = 'wasbs://@{variables('infra_StorageLocation')}'\n, CREDENTIAL = @{pipeline().parameters.sql_BackupSchemaName}BackupBlobStorageCreds\n, TYPE = HADOOP\n);\nEND\nSELECT 1;","type":"Expression"},"queryTimeout":"02:00:00","partitionOption":"None"},"dataset":{"referenceName":"set_blank","type":"DatasetReference","parameters":{"DSBackupServerName":{"value":"@pipeline().parameters.infra_BackupSqlServer","type":"Expression"},"DSBackupDatabaseName":{"value":"@pipeline().parameters.synapse_DatabaseBackupName","type":"Expression"},"DSBackupUsername":{"value":"@variables('security_BackupUserName')","type":"Expression"},"DSBackupUserPassword":{"value":"@variables('security_BackupUserPassword')","type":"Expression"}}}}},{"name":"CREATE EXTERNAL FILE FORMAT","description":"This is the external file format that will by used to backup the tables, feel free to change the type, but leave the name as is","type":"Lookup","dependsOn":[{"activity":"CREATE EXTERNAL DATA SOURCE","dependencyConditions":["Succeeded"]}],"policy":{"timeout":"7.00:00:00","retry":0,"retryIntervalInSeconds":30,"secureOutput":true,"secureInput":true},"userProperties":[],"typeProperties":{"source":{"type":"SqlDWSource","sqlReaderQuery":"IF NOT EXISTS (SELECT * FROM sys.external_file_formats WHERE [name] = 'Backup_FileFormat')\nBEGIN\n\nCREATE EXTERNAL FILE FORMAT Backup_FileFormat \nWITH ( \n FORMAT_TYPE = PARQUET \n , DATA_COMPRESSION = 'org.apache.hadoop.io.compress.SnappyCodec' \n );\nEND\nSELECT 1;","queryTimeout":"02:00:00","partitionOption":"None"},"dataset":{"referenceName":"set_blank","type":"DatasetReference","parameters":{"DSBackupServerName":{"value":"@pipeline().parameters.infra_BackupSqlServer","type":"Expression"},"DSBackupDatabaseName":{"value":"@pipeline().parameters.synapse_DatabaseBackupName","type":"Expression"},"DSBackupUsername":{"value":"@variables('security_BackupUserName')","type":"Expression"},"DSBackupUserPassword":{"value":"@variables('security_BackupUserPassword')","type":"Expression"}}}}},{"name":"Get password from AKV","type":"WebActivity","dependsOn":[{"activity":"Create createdb schema","dependencyConditions":["Succeeded"]}],"policy":{"timeout":"7.00:00:00","retry":0,"retryIntervalInSeconds":30,"secureOutput":true,"secureInput":true},"userProperties":[],"typeProperties":{"url":{"value":"@pipeline().parameters.security_AKVMasterKeyUri","type":"Expression"},"method":"GET","headers":{},"authentication":{"type":"MSI","resource":"https://vault.azure.net"}}},{"name":"Get identity from AKV for Database Scoped Credentials","type":"WebActivity","dependsOn":[{"activity":"Drop Database Scoped Credentials","dependencyConditions":["Completed"]}],"policy":{"timeout":"7.00:00:00","retry":0,"retryIntervalInSeconds":30,"secureOutput":true,"secureInput":true},"userProperties":[],"typeProperties":{"url":{"value":"@pipeline().parameters.security_AKVDatabaseScopedCredentialsUri","type":"Expression"},"method":"GET","headers":{},"authentication":{"type":"MSI","resource":"https://vault.azure.net"}}},{"name":"Get secret from AKV for Database Scoped Credentials","type":"WebActivity","dependsOn":[{"activity":"Drop Database Scoped Credentials","dependencyConditions":["Completed"]}],"policy":{"timeout":"7.00:00:00","retry":0,"retryIntervalInSeconds":30,"secureOutput":true,"secureInput":true},"userProperties":[],"typeProperties":{"url":{"value":"@pipeline().parameters.security_AKVDatabaseScopedCredentialsSecretUri","type":"Expression"},"method":"GET","headers":{},"authentication":{"type":"MSI","resource":"https://vault.azure.net"}}},{"name":"Set DSC Identity","type":"SetVariable","dependsOn":[{"activity":"Get identity from AKV for Database Scoped Credentials","dependencyConditions":["Succeeded"]}],"userProperties":[],"typeProperties":{"variableName":"security_DSCIdentity","value":{"value":"@activity('Get identity from AKV for Database Scoped Credentials').output.value","type":"Expression"}}},{"name":"Set DSC Secret","type":"SetVariable","dependsOn":[{"activity":"Get secret from AKV for Database Scoped Credentials","dependencyConditions":["Succeeded"]}],"userProperties":[],"typeProperties":{"variableName":"security_DSCSecret","value":{"value":"@activity('Get secret from AKV for Database Scoped Credentials').output.value","type":"Expression"}}},{"name":"Get backup database user name from AKV","type":"WebActivity","dependsOn":[],"policy":{"timeout":"7.00:00:00","retry":0,"retryIntervalInSeconds":30,"secureOutput":true,"secureInput":true},"userProperties":[],"typeProperties":{"url":{"value":"@pipeline().parameters.security_AKVBackupUserNameUri","type":"Expression"},"method":"GET","headers":{},"authentication":{"type":"MSI","resource":"https://vault.azure.net"}}},{"name":"Get backup database user password from AKV","type":"WebActivity","dependsOn":[],"policy":{"timeout":"7.00:00:00","retry":0,"retryIntervalInSeconds":30,"secureOutput":true,"secureInput":true},"userProperties":[],"typeProperties":{"url":{"value":"@pipeline().parameters.security_AKVBackupUserPasswordUri","type":"Expression"},"method":"GET","headers":{},"authentication":{"type":"MSI","resource":"https://vault.azure.net"}}},{"name":"Set user name","type":"SetVariable","dependsOn":[{"activity":"Get backup database user name from AKV","dependencyConditions":["Succeeded"]}],"userProperties":[],"typeProperties":{"variableName":"security_BackupUserName","value":{"value":"@activity('Get backup database user name from AKV').output.value","type":"Expression"}}},{"name":"Set password","type":"SetVariable","dependsOn":[{"activity":"Get backup database user password from AKV","dependencyConditions":["Succeeded"]}],"userProperties":[],"typeProperties":{"variableName":"security_BackupUserPassword","value":{"value":"@activity('Get backup database user password from AKV').output.value","type":"Expression"}}},{"name":"Get storage location","type":"WebActivity","dependsOn":[],"policy":{"timeout":"7.00:00:00","retry":0,"retryIntervalInSeconds":30,"secureOutput":true,"secureInput":true},"userProperties":[],"typeProperties":{"url":{"value":"@pipeline().parameters.security_AKVBackupStorageLocationUri","type":"Expression"},"method":"GET","headers":{},"authentication":{"type":"MSI","resource":"https://vault.azure.net"}}},{"name":"Set storage location","type":"SetVariable","dependsOn":[{"activity":"Get storage location","dependencyConditions":["Succeeded"]}],"userProperties":[],"typeProperties":{"variableName":"infra_StorageLocation","value":{"value":"@activity('Get storage location').output.value","type":"Expression"}}},{"name":"Drop Database Scoped Credentials","type":"Lookup","dependsOn":[{"activity":"Create Master Key","dependencyConditions":["Completed"]}],"policy":{"timeout":"7.00:00:00","retry":0,"retryIntervalInSeconds":30,"secureOutput":true,"secureInput":true},"userProperties":[],"typeProperties":{"source":{"type":"SqlDWSource","sqlReaderQuery":{"value":"DROP DATABASE SCOPED CREDENTIAL [@{pipeline().parameters.sql_BackupSchemaName}BackupBlobStorageCreds];\nSELECT 1;","type":"Expression"},"queryTimeout":"02:00:00","partitionOption":"None"},"dataset":{"referenceName":"set_blank","type":"DatasetReference","parameters":{"DSBackupServerName":{"value":"@pipeline().parameters.infra_BackupSqlServer","type":"Expression"},"DSBackupDatabaseName":{"value":"@pipeline().parameters.synapse_DatabaseBackupName","type":"Expression"},"DSBackupUsername":{"value":"@variables('security_BackupUserName')","type":"Expression"},"DSBackupUserPassword":{"value":"@variables('security_BackupUserPassword')","type":"Expression"}}}}},{"name":"Create createdb schema","description":"This creates the schema that will be used to script the database.\n\nThis schema cannot already exist in the database!","type":"Lookup","dependsOn":[{"activity":"Create backup schema","dependencyConditions":["Succeeded"]}],"policy":{"timeout":"7.00:00:00","retry":0,"retryIntervalInSeconds":30,"secureOutput":true,"secureInput":true},"userProperties":[],"typeProperties":{"source":{"type":"SqlDWSource","sqlReaderQuery":{"value":"EXEC('CREATE SCHEMA [___ibCreateDb___]');\nSELECT 1;","type":"Expression"},"queryTimeout":"02:00:00","partitionOption":"None"},"dataset":{"referenceName":"set_blank","type":"DatasetReference","parameters":{"DSBackupServerName":{"value":"@pipeline().parameters.infra_BackupSqlServer","type":"Expression"},"DSBackupDatabaseName":{"value":"@pipeline().parameters.synapse_DatabaseBackupName","type":"Expression"},"DSBackupUsername":{"value":"@variables('security_BackupUserName')","type":"Expression"},"DSBackupUserPassword":{"value":"@variables('security_BackupUserPassword')","type":"Expression"}}}}}],"parameters":{"infra_BackupSqlServer":{"type":"string","defaultValue":"ibdwtestbackupserver8.database.windows.net"},"synapse_DatabaseBackupName":{"type":"string","defaultValue":"ibTestBackup_Backup"},"sql_BackupSchemaName":{"type":"string","defaultValue":"___ibBackup2___"},"security_AKVMasterKeyUri":{"type":"string","defaultValue":"https://ibdwkeyvault.vault.azure.net/secrets/BackupServerMasterKey/df3ac8be7d684fdebf7bb5bba509468e?api-version=2016-10-01"},"security_AKVDatabaseScopedCredentialsUri":{"type":"string","defaultValue":"https://ibdwkeyvault.vault.azure.net/secrets/BackupDatabaseScopedCredentials/202cc70b4b14488dbbd30d841e10f107?api-version=2016-10-01"},"security_AKVDatabaseScopedCredentialsSecretUri":{"type":"string","defaultValue":"https://ibdwkeyvault.vault.azure.net/secrets/BackupDatabaseScopedCredentialsSecret/0393b28228514735aa1f37ff77c24869?api-version=2016-10-01"},"security_AKVBackupUserNameUri":{"type":"string","defaultValue":"https://ibdwkeyvault.vault.azure.net/secrets/BackupServerUserName/116ace86b67b4e81aa85f846a0dbf06c?api-version=2016-10-01"},"security_AKVBackupUserPasswordUri":{"type":"string","defaultValue":"https://ibdwkeyvault.vault.azure.net/secrets/BackupServerUserPassword/55f0e5b826444293a5ddca55d2154d70?api-version=2016-10-01"},"security_AKVBackupStorageLocationUri":{"type":"string","defaultValue":"https://ibdwkeyvault.vault.azure.net/secrets/BackupBlobStorageLocation/272b9d74238e4e7c9014be2c65ec11c2?api-version=2016-10-01"},"sql_SkipSchemas":{"type":"string"}},"variables":{"SynapseDw_SqlServerName":{"type":"String","defaultValue":"ibdwbacksqlserver.database.windows.net"},"SynapseDw_DatabaseName":{"type":"String","defaultValue":"ibTestBackup"},"BlobStore_BackupContainer":{"type":"String","defaultValue":"backup"},"BlobStore_Path":{"type":"String","defaultValue":"ibdwbackup.blob.core.windows.net"},"SynapseDw_BackupType":{"type":"String","defaultValue":"0"},"security_TemporaryPassword":{"type":"String"},"security_DSCIdentity":{"type":"String"},"security_DSCSecret":{"type":"String"},"security_BackupUserName":{"type":"String"},"security_BackupUserPassword":{"type":"String"},"infra_StorageLocation":{"type":"String"}},"folder":{"name":"Backup DW"},"annotations":[],"lastPublishTime":"2019-09-19T14:34:59Z"},"dependsOn":["[concat(variables('factoryId'), '/datasets/set_blank')]"]},{"name":"[concat(parameters('factoryName'), '/set_blank')]","type":"Microsoft.DataFactory/factories/datasets","apiVersion":"2018-06-01","properties":{"linkedServiceName":{"referenceName":"[parameters('conn_BackupServer')]","type":"LinkedServiceReference","parameters":{"BackupServerName":{"value":"@dataset().DSBackupServerName","type":"Expression"},"BackupDatabaseName":{"value":"@dataset().DSBackupDatabaseName","type":"Expression"},"BackupServerUserName":{"value":"@dataset().DSBackupUsername","type":"Expression"},"BackupUserPassword":{"value":"@dataset().DSBackupUserPassword","type":"Expression"}}},"parameters":{"DSBackupServerName":{"type":"string","defaultValue":"ibdwtestbackupserver.database.windows.net"},"DSBackupDatabaseName":{"type":"string","defaultValue":"ibTestBackup_Backup"},"DSBackupUsername":{"type":"string","defaultValue":"BackupTempLogin"},"DSBackupUserPassword":{"type":"string","defaultValue":"1SQGdWTGNSSFH4M6OYRUCT845N4KPGPWELI6c3aHQCVB2NBMb3P"}},"annotations":[],"type":"AzureSqlDWTable","schema":[],"typeProperties":{"schema":"sys","table":"schemas"}},"dependsOn":[]}]} \ No newline at end of file +{ + "$schema": "http://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "factoryName": { + "type": "string", + "metadata": "Data Factory name" + }, + "conn_BackupServer": { + "type": "string" + } + }, + "variables": { + "factoryId": "[concat('Microsoft.DataFactory/factories/', parameters('factoryName'))]" + }, + "resources": [ + { + "name": "[concat(parameters('factoryName'), '/Step 3 - Create External Resources')]", + "type": "Microsoft.DataFactory/factories/pipelines", + "apiVersion": "2018-06-01", + "properties": { + "description": "This pipeline will add the required schema, stored procedures and external resources to the database that will be used to backup the database to BLOB storage", + "activities": [ + { + "name": "Create backup schema", + "description": "This creates the backup schema to be used, the schema name is parameterized, if you do change it, make sure that it is a valid value for a schema. \n\nThis schema cannot already exist in the database!", + "type": "Lookup", + "dependsOn": [ + { + "activity": "Set user name", + "dependencyConditions": [ + "Succeeded" + ] + }, + { + "activity": "Set password", + "dependencyConditions": [ + "Succeeded" + ] + }, + { + "activity": "Set storage location", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": true, + "secureInput": true + }, + "userProperties": [ + ], + "typeProperties": { + "source": { + "type": "SqlDWSource", + "sqlReaderQuery": { + "value": "EXEC('CREATE SCHEMA [@{pipeline().parameters.sql_BackupSchemaName}]');\nSELECT 1;", + "type": "Expression" + }, + "queryTimeout": "02:00:00", + "partitionOption": "None" + }, + "dataset": { + "referenceName": "set_blank", + "type": "DatasetReference", + "parameters": { + "DSBackupServerName": { + "value": "@pipeline().parameters.infra_BackupSqlServer", + "type": "Expression" + }, + "DSBackupDatabaseName": { + "value": "@pipeline().parameters.synapse_DatabaseBackupName", + "type": "Expression" + }, + "DSBackupUsername": { + "value": "@variables('security_BackupUserName')", + "type": "Expression" + }, + "DSBackupUserPassword": { + "value": "@variables('security_BackupUserPassword')", + "type": "Expression" + } + } + } + } + }, + { + "name": "Create Master Key", + "description": "This will attempt to create a Master Key on the database, but will fail if one already exists. Failure here would be normal and could just be ignored.", + "type": "Lookup", + "dependsOn": [ + { + "activity": "Get password from AKV", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": true, + "secureInput": true + }, + "userProperties": [ + ], + "typeProperties": { + "source": { + "type": "SqlDWSource", + "sqlReaderQuery": { + "value": "IF NOT EXISTS (SELECT * FROM sys.symmetric_keys WHERE [name] LIKE '%DatabaseMasterKey%')BEGIN\n CREATE MASTER KEY ENCRYPTION BY PASSWORD = '@{activity('Get password from AKV').output.value}';\nEND\nSELECT 1;", + "type": "Expression" + }, + "queryTimeout": "02:00:00", + "partitionOption": "None" + }, + "dataset": { + "referenceName": "set_blank", + "type": "DatasetReference", + "parameters": { + "DSBackupServerName": { + "value": "@pipeline().parameters.infra_BackupSqlServer", + "type": "Expression" + }, + "DSBackupDatabaseName": { + "value": "@pipeline().parameters.synapse_DatabaseBackupName", + "type": "Expression" + }, + "DSBackupUsername": { + "value": "@variables('security_BackupUserName')", + "type": "Expression" + }, + "DSBackupUserPassword": { + "value": "@variables('security_BackupUserPassword')", + "type": "Expression" + } + } + } + } + }, + { + "name": "Create DATABASE SCOPED CREDENTIAL", + "description": "Created the database scoped credentials that will be used to connect to BLOB storage. It will use the name and secret from AKV, feel free to change how the credentials are specified, but keep the name", + "type": "Lookup", + "dependsOn": [ + { + "activity": "Set DSC Secret", + "dependencyConditions": [ + "Succeeded" + ] + }, + { + "activity": "Set DSC Identity", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": true, + "secureInput": true + }, + "userProperties": [ + ], + "typeProperties": { + "source": { + "type": "SqlDWSource", + "sqlReaderQuery": { + "value": "CREATE DATABASE SCOPED CREDENTIAL [@{pipeline().parameters.sql_BackupSchemaName}BackupBlobStorageCreds]\nWITH\n IDENTITY = '@{variables('security_DSCIdentity')}'\n, SECRET = '@{variables('security_DSCSecret')}';\nSELECT 1;\n\n", + "type": "Expression" + }, + "queryTimeout": "02:00:00", + "partitionOption": "None" + }, + "dataset": { + "referenceName": "set_blank", + "type": "DatasetReference", + "parameters": { + "DSBackupServerName": { + "value": "@pipeline().parameters.infra_BackupSqlServer", + "type": "Expression" + }, + "DSBackupDatabaseName": { + "value": "@pipeline().parameters.synapse_DatabaseBackupName", + "type": "Expression" + }, + "DSBackupUsername": { + "value": "@variables('security_BackupUserName')", + "type": "Expression" + }, + "DSBackupUserPassword": { + "value": "@variables('security_BackupUserPassword')", + "type": "Expression" + } + } + } + } + }, + { + "name": "CREATE EXTERNAL DATA SOURCE", + "description": "Creating the external data source, feel free to change it, but leave the name the same", + "type": "Lookup", + "dependsOn": [ + { + "activity": "Create DATABASE SCOPED CREDENTIAL", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": true, + "secureInput": true + }, + "userProperties": [ + ], + "typeProperties": { + "source": { + "type": "SqlDWSource", + "sqlReaderQuery": { + "value": "IF NOT EXISTS (SELECT * FROM sys.external_data_Sources WHERE [name] = '@{pipeline().parameters.sql_BackupSchemaName}BackupBlobStorage')\nBEGIN\nCREATE EXTERNAL DATA SOURCE [@{pipeline().parameters.sql_BackupSchemaName}BackupBlobStorage]\nWITH\n( LOCATION = 'wasbs://@{variables('infra_StorageLocation')}'\n, CREDENTIAL = @{pipeline().parameters.sql_BackupSchemaName}BackupBlobStorageCreds\n, TYPE = HADOOP\n);\nEND\nSELECT 1;", + "type": "Expression" + }, + "queryTimeout": "02:00:00", + "partitionOption": "None" + }, + "dataset": { + "referenceName": "set_blank", + "type": "DatasetReference", + "parameters": { + "DSBackupServerName": { + "value": "@pipeline().parameters.infra_BackupSqlServer", + "type": "Expression" + }, + "DSBackupDatabaseName": { + "value": "@pipeline().parameters.synapse_DatabaseBackupName", + "type": "Expression" + }, + "DSBackupUsername": { + "value": "@variables('security_BackupUserName')", + "type": "Expression" + }, + "DSBackupUserPassword": { + "value": "@variables('security_BackupUserPassword')", + "type": "Expression" + } + } + } + } + }, + { + "name": "CREATE EXTERNAL FILE FORMAT", + "description": "This is the external file format that will by used to backup the tables, feel free to change the type, but leave the name as is", + "type": "Lookup", + "dependsOn": [ + { + "activity": "CREATE EXTERNAL DATA SOURCE", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": true, + "secureInput": true + }, + "userProperties": [ + ], + "typeProperties": { + "source": { + "type": "SqlDWSource", + "sqlReaderQuery": "IF NOT EXISTS (SELECT * FROM sys.external_file_formats WHERE [name] = 'Backup_FileFormat')\nBEGIN\n\nCREATE EXTERNAL FILE FORMAT Backup_FileFormat \nWITH ( \n FORMAT_TYPE = PARQUET \n , DATA_COMPRESSION = 'org.apache.hadoop.io.compress.SnappyCodec' \n );\nEND\nSELECT 1;", + "queryTimeout": "02:00:00", + "partitionOption": "None" + }, + "dataset": { + "referenceName": "set_blank", + "type": "DatasetReference", + "parameters": { + "DSBackupServerName": { + "value": "@pipeline().parameters.infra_BackupSqlServer", + "type": "Expression" + }, + "DSBackupDatabaseName": { + "value": "@pipeline().parameters.synapse_DatabaseBackupName", + "type": "Expression" + }, + "DSBackupUsername": { + "value": "@variables('security_BackupUserName')", + "type": "Expression" + }, + "DSBackupUserPassword": { + "value": "@variables('security_BackupUserPassword')", + "type": "Expression" + } + } + } + } + }, + { + "name": "Get password from AKV", + "type": "WebActivity", + "dependsOn": [ + { + "activity": "Create createdb schema", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": true, + "secureInput": true + }, + "userProperties": [ + ], + "typeProperties": { + "url": { + "value": "@pipeline().parameters.security_AKVMasterKeyUri", + "type": "Expression" + }, + "method": "GET", + "headers": { + }, + "authentication": { + "type": "MSI", + "resource": "https://vault.azure.net" + } + } + }, + { + "name": "Get identity from AKV for Database Scoped Credentials", + "type": "WebActivity", + "dependsOn": [ + { + "activity": "Drop Database Scoped Credentials", + "dependencyConditions": [ + "Completed" + ] + } + ], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": true, + "secureInput": true + }, + "userProperties": [ + ], + "typeProperties": { + "url": { + "value": "@pipeline().parameters.security_AKVDatabaseScopedCredentialsUri", + "type": "Expression" + }, + "method": "GET", + "headers": { + }, + "authentication": { + "type": "MSI", + "resource": "https://vault.azure.net" + } + } + }, + { + "name": "Get secret from AKV for Database Scoped Credentials", + "type": "WebActivity", + "dependsOn": [ + { + "activity": "Drop Database Scoped Credentials", + "dependencyConditions": [ + "Completed" + ] + } + ], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": true, + "secureInput": true + }, + "userProperties": [ + ], + "typeProperties": { + "url": { + "value": "@pipeline().parameters.security_AKVDatabaseScopedCredentialsSecretUri", + "type": "Expression" + }, + "method": "GET", + "headers": { + }, + "authentication": { + "type": "MSI", + "resource": "https://vault.azure.net" + } + } + }, + { + "name": "Set DSC Identity", + "type": "SetVariable", + "dependsOn": [ + { + "activity": "Get identity from AKV for Database Scoped Credentials", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "userProperties": [ + ], + "typeProperties": { + "variableName": "security_DSCIdentity", + "value": { + "value": "@activity('Get identity from AKV for Database Scoped Credentials').output.value", + "type": "Expression" + } + } + }, + { + "name": "Set DSC Secret", + "type": "SetVariable", + "dependsOn": [ + { + "activity": "Get secret from AKV for Database Scoped Credentials", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "userProperties": [ + ], + "typeProperties": { + "variableName": "security_DSCSecret", + "value": { + "value": "@activity('Get secret from AKV for Database Scoped Credentials').output.value", + "type": "Expression" + } + } + }, + { + "name": "Get backup database user name from AKV", + "type": "WebActivity", + "dependsOn": [ + ], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": true, + "secureInput": true + }, + "userProperties": [ + ], + "typeProperties": { + "url": { + "value": "@pipeline().parameters.security_AKVBackupUserNameUri", + "type": "Expression" + }, + "method": "GET", + "headers": { + }, + "authentication": { + "type": "MSI", + "resource": "https://vault.azure.net" + } + } + }, + { + "name": "Get backup database user password from AKV", + "type": "WebActivity", + "dependsOn": [ + ], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": true, + "secureInput": true + }, + "userProperties": [ + ], + "typeProperties": { + "url": { + "value": "@pipeline().parameters.security_AKVBackupUserPasswordUri", + "type": "Expression" + }, + "method": "GET", + "headers": { + }, + "authentication": { + "type": "MSI", + "resource": "https://vault.azure.net" + } + } + }, + { + "name": "Set user name", + "type": "SetVariable", + "dependsOn": [ + { + "activity": "Get backup database user name from AKV", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "userProperties": [ + ], + "typeProperties": { + "variableName": "security_BackupUserName", + "value": { + "value": "@activity('Get backup database user name from AKV').output.value", + "type": "Expression" + } + } + }, + { + "name": "Set password", + "type": "SetVariable", + "dependsOn": [ + { + "activity": "Get backup database user password from AKV", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "userProperties": [ + ], + "typeProperties": { + "variableName": "security_BackupUserPassword", + "value": { + "value": "@activity('Get backup database user password from AKV').output.value", + "type": "Expression" + } + } + }, + { + "name": "Get storage location", + "type": "WebActivity", + "dependsOn": [ + ], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": true, + "secureInput": true + }, + "userProperties": [ + ], + "typeProperties": { + "url": { + "value": "@pipeline().parameters.security_AKVBackupStorageLocationUri", + "type": "Expression" + }, + "method": "GET", + "headers": { + }, + "authentication": { + "type": "MSI", + "resource": "https://vault.azure.net" + } + } + }, + { + "name": "Set storage location", + "type": "SetVariable", + "dependsOn": [ + { + "activity": "Get storage location", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "userProperties": [ + ], + "typeProperties": { + "variableName": "infra_StorageLocation", + "value": { + "value": "@activity('Get storage location').output.value", + "type": "Expression" + } + } + }, + { + "name": "Drop Database Scoped Credentials", + "type": "Lookup", + "dependsOn": [ + { + "activity": "Create Master Key", + "dependencyConditions": [ + "Completed" + ] + } + ], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": true, + "secureInput": true + }, + "userProperties": [ + ], + "typeProperties": { + "source": { + "type": "SqlDWSource", + "sqlReaderQuery": { + "value": "DROP DATABASE SCOPED CREDENTIAL [@{pipeline().parameters.sql_BackupSchemaName}BackupBlobStorageCreds];\nSELECT 1;", + "type": "Expression" + }, + "queryTimeout": "02:00:00", + "partitionOption": "None" + }, + "dataset": { + "referenceName": "set_blank", + "type": "DatasetReference", + "parameters": { + "DSBackupServerName": { + "value": "@pipeline().parameters.infra_BackupSqlServer", + "type": "Expression" + }, + "DSBackupDatabaseName": { + "value": "@pipeline().parameters.synapse_DatabaseBackupName", + "type": "Expression" + }, + "DSBackupUsername": { + "value": "@variables('security_BackupUserName')", + "type": "Expression" + }, + "DSBackupUserPassword": { + "value": "@variables('security_BackupUserPassword')", + "type": "Expression" + } + } + } + } + }, + { + "name": "Create createdb schema", + "description": "This creates the schema that will be used to script the database.\n\nThis schema cannot already exist in the database!", + "type": "Lookup", + "dependsOn": [ + { + "activity": "Create backup schema", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": true, + "secureInput": true + }, + "userProperties": [ + ], + "typeProperties": { + "source": { + "type": "SqlDWSource", + "sqlReaderQuery": { + "value": "EXEC('CREATE SCHEMA [___ibCreateDb___]');\nSELECT 1;", + "type": "Expression" + }, + "queryTimeout": "02:00:00", + "partitionOption": "None" + }, + "dataset": { + "referenceName": "set_blank", + "type": "DatasetReference", + "parameters": { + "DSBackupServerName": { + "value": "@pipeline().parameters.infra_BackupSqlServer", + "type": "Expression" + }, + "DSBackupDatabaseName": { + "value": "@pipeline().parameters.synapse_DatabaseBackupName", + "type": "Expression" + }, + "DSBackupUsername": { + "value": "@variables('security_BackupUserName')", + "type": "Expression" + }, + "DSBackupUserPassword": { + "value": "@variables('security_BackupUserPassword')", + "type": "Expression" + } + } + } + } + } + ], + "parameters": { + "infra_BackupSqlServer": { + "type": "securestring" + }, + "synapse_DatabaseBackupName": { + "type": "securestring" + }, + "sql_BackupSchemaName": { + "type": "string", + "defaultValue": "___ibBackup___" + }, + "security_AKVMasterKeyUri": { + "type": "securestring" + }, + "security_AKVDatabaseScopedCredentialsUri": { + "type": "securestring" + }, + "security_AKVDatabaseScopedCredentialsSecretUri": { + "type": "securestring" + }, + "security_AKVBackupUserNameUri": { + "type": "securestring" + }, + "security_AKVBackupUserPasswordUri": { + "type": "securestring" + }, + "security_AKVBackupStorageLocationUri": { + "type": "securestring" + }, + "sql_SkipSchemas": { + "type": "securestring" + } + }, + "variables": { + "SynapseDw_SqlServerName": { + "type": "String" + }, + "SynapseDw_DatabaseName": { + "type": "String" + }, + "BlobStore_BackupContainer": { + "type": "String" + }, + "BlobStore_Path": { + "type": "String" + }, + "SynapseDw_BackupType": { + "type": "String" + }, + "security_TemporaryPassword": { + "type": "String" + }, + "security_DSCIdentity": { + "type": "String" + }, + "security_DSCSecret": { + "type": "String" + }, + "security_BackupUserName": { + "type": "String" + }, + "security_BackupUserPassword": { + "type": "String" + }, + "infra_StorageLocation": { + "type": "String" + } + }, + "folder": { + "name": "Backup DW" + }, + "annotations": [ + ], + "lastPublishTime": "2019-09-19T14:34:59Z" + }, + "dependsOn": [ + "[concat(variables('factoryId'), '/datasets/set_blank')]" + ] + }, + { + "name": "[concat(parameters('factoryName'), '/set_blank')]", + "type": "Microsoft.DataFactory/factories/datasets", + "apiVersion": "2018-06-01", + "properties": { + "linkedServiceName": { + "referenceName": "[parameters('conn_BackupServer')]", + "type": "LinkedServiceReference", + "parameters": { + "BackupServerName": { + "value": "@dataset().DSBackupServerName", + "type": "Expression" + }, + "BackupDatabaseName": { + "value": "@dataset().DSBackupDatabaseName", + "type": "Expression" + }, + "BackupServerUserName": { + "value": "@dataset().DSBackupUsername", + "type": "Expression" + }, + "BackupUserPassword": { + "value": "@dataset().DSBackupUserPassword", + "type": "Expression" + } + } + }, + "parameters": { + "DSBackupServerName": { + "type": "string", + "defaultValue": "ibdwtestbackupserver.database.windows.net" + }, + "DSBackupDatabaseName": { + "type": "string", + "defaultValue": "ibTestBackup_Backup" + }, + "DSBackupUsername": { + "type": "string", + "defaultValue": "BackupTempLogin" + }, + "DSBackupUserPassword": { + "type": "string", + "defaultValue": "1SQGdWTGNSSFH4M6OYRUCT845N4KPGPWELI6c3aHQCVB2NBMb3P" + } + }, + "annotations": [ + ], + "type": "AzureSqlDWTable", + "schema": [ + ], + "typeProperties": { + "schema": "sys", + "table": "schemas" + } + }, + "dependsOn": [ + ] + } + ] +} \ No newline at end of file diff --git a/Step 3 - Create External Resources/manifest.json b/Step 3 - Create External Resources/manifest.json index 081cbfa..6ac933d 100644 --- a/Step 3 - Create External Resources/manifest.json +++ b/Step 3 - Create External Resources/manifest.json @@ -1,7 +1,7 @@ { "name": "Step 3 - Create External Resources", "description": "This pipeline will add the required schema, stored procedures and external resources to the database that will be used to backup the database to BLOB storage", - "image": "LookupCreate backup schemaLookupCreate Master KeyLookupCreate DATABASESCOPED CREDENTIALLookupCREATE EXTERNALDATA SOURCELookupCREATE EXTERNALFILE FORMATWebGet password fromAKVWebGet identity from AKVfor Database Scoped...WebGet secret from AKVfor Database Scoped...Set variableSet DSC IdentitySet variableSet DSC SecretWebGet backup databaseuser name from AKVWebGet backup databaseuser password from...Set variableSet user nameSet variableSet passwordWebGet storage locationSet variableSet storage locationLookupDrop DatabaseScoped CredentialsLookupCreate createdbschema", + "image": "LookupCreate backup schemaLookupCreate Master KeyLookupCreate DATABASESCOPED CREDENTIALLookupCREATE EXTERNALDATA SOURCELookupCREATE EXTERNALFILE FORMATWebGet password fromAKVWebGet identity from AKVfor Database Scoped...WebGet secret from AKVfor Database Scoped...Set variableSet DSC IdentitySet variableSet DSC SecretWebGet backup databaseuser name from AKVWebGet backup databaseuser password from...Set variableSet user nameSet variableSet passwordWebGet storage locationSet variableSet storage locationLookupDrop DatabaseScoped CredentialsLookupCreate createdbschema", "icons": [ "Lookup", "Lookup", @@ -25,6 +25,7 @@ "Azure Blob Storage", "Azure SQL Database", "Azure Synapse Analytics", + "SQL Azure", "Synapse" ], "categories": [ diff --git a/Step 5 - Delete the backup database/Step 5 - Delete the backup database.json b/Step 5 - Delete the backup database/Step 5 - Delete the backup database.json new file mode 100644 index 0000000..decad61 --- /dev/null +++ b/Step 5 - Delete the backup database/Step 5 - Delete the backup database.json @@ -0,0 +1,76 @@ +{ + "$schema": "http://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "factoryName": { + "type": "string", + "metadata": "Data Factory name" + } + }, + "variables": { + "factoryId": "[concat('Microsoft.DataFactory/factories/', parameters('factoryName'))]" + }, + "resources": [ + { + "name": "[concat(parameters('factoryName'), '/Step 5 - Delete the backup database')]", + "type": "Microsoft.DataFactory/factories/pipelines", + "apiVersion": "2018-06-01", + "properties": { + "description": "This pipeline will drop the backup database.", + "activities": [ + { + "name": "Delete the backup database", + "type": "WebActivity", + "dependsOn": [ + ], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": true, + "secureInput": true + }, + "userProperties": [ + ], + "typeProperties": { + "url": { + "value": "https://management.azure.com/subscriptions/@{pipeline().parameters.infra_SubscriptionId}/resourceGroups/@{pipeline().parameters.infra_BackupResourceGroup}/providers/Microsoft.Sql/servers/@{pipeline().parameters.infra_BackupSqlServerName}/databases/@{pipeline().parameters.synapse_DatabaseBackupName}?api-version=2019-06-01-preview", + "type": "Expression" + }, + "method": "DELETE", + "headers": { + "Content-Type": "application/json" + }, + "authentication": { + "type": "MSI", + "resource": "https://management.azure.com/" + } + } + } + ], + "parameters": { + "infra_BackupSqlServerName": { + "type": "securestring" + }, + "synapse_DatabaseBackupName": { + "type": "securestring" + }, + "infra_SubscriptionId": { + "type": "securestring" + }, + "infra_BackupResourceGroup": { + "type": "securestring" + } + }, + "folder": { + "name": "Backup DW" + }, + "annotations": [ + ], + "lastPublishTime": "2019-09-19T14:34:59Z" + }, + "dependsOn": [ + ] + } + ] +} \ No newline at end of file diff --git a/Step 5 - Delete the backup database/manifest.json b/Step 5 - Delete the backup database/manifest.json new file mode 100644 index 0000000..0fdf5ca --- /dev/null +++ b/Step 5 - Delete the backup database/manifest.json @@ -0,0 +1,24 @@ +{ + "name": "Step 5 - Delete the backup database", + "description": "This pipeline will drop the backup database.", + "image": "WebDelete the backupdatabase", + "icons": [ + "WebActivity" + ], + "requires": { + "linkedservices": { + } + }, + "documentation": "https://github.com/microsoft/BackupDwToBlob", + "author": "Mornč Blake", + "annotations": [ + "Delete Synapse Database", + "REST API" + ], + "services": [ + "Azure Synapse Analytics", + "Synapse" + ], + "categories": [ + ] +} \ No newline at end of file