diff --git a/README.md b/README.md index 8e846736..561bac53 100644 --- a/README.md +++ b/README.md @@ -73,12 +73,13 @@ ls ``` >- [ ] You will now see a listing of powershell (*.ps1) files. These are the scripts that drive the deployment. To deploy the solution run the commands below in your terminal window. **Note** it is recommended that you run these one at a time and check output during each command execution for errors. ```powershell -./LocalDevOnly_EnvironmentSetUp.ps1 -./CI_1a_BuildCode.ps1 -./CD_0a_CreateServicePrincipals_AAD_Elevated.ps1 -./CD_1a_DeployServices.ps1 -./CD_2a_CreateMSIs_AAD_Elevated.ps1 -./CD_2b_ConfigureServices.ps1 +az login #Logs you in to your Azure environment +./LocalDevOnly_EnvironmentSetUp.ps1 #Loads your environment settings +./CI_1a_BuildCode.ps1 #Builds the source code binaries +./CD_0a_CreateServicePrincipals_AAD_Elevated.ps1 #Creates your Azure resource group and application registrations +./CD_1a_DeployServices.ps1 #Deploys the core Azure Services +./CD_2a_CreateMSIs_AAD_Elevated.ps1 #Sets up the Azure Managed Service Identities and grants privileges and access +./CD_2b_ConfigureServices.ps1 #Publishes source code to Azure Services. ``` ## Post Deployment Set-up and Instructions diff --git a/solution/DataFactory/IRConfig.json b/solution/DataFactory/IRConfig.json index 44d9321b..ced0808f 100644 --- a/solution/DataFactory/IRConfig.json +++ b/solution/DataFactory/IRConfig.json @@ -53,34 +53,34 @@ ], "pipeline": [ "AZ_SQL_AZ_Storage_Parquet_SH_IR.json", - "AZ-SQL-Create-Table-SH-IR.json", - "AZ-SQL-Full-Load-Chunk-SH-IR.json", - "AZ-SQL-Full-Load-SH-IR.json", - "AZ-SQL-Post-Copy-SH-IR.json", - "AZ-SQL-StoredProcedure-SH-IR.json", - "AZ-SQL-Watermark-Chunk-SH-IR.json", - "AZ-SQL-Watermark-SH-IR.json", - "AZ-Storage-Binary-AZ-Storage-Binary-SH-IR.json", - "AZ-Storage-CSV-AZ-SQL-SH-IR.json", - "AZ-Storage-Excel-AZ-SQL-SH-IR.json", - "AZ-Storage-Excel-AZ-Storage-CSV-SH-IR.json", - "AZ-Storage-JSON-AZ-SQL-SH-IR.json", - "AZ-Storage-Parquet-AZ-SQL-SH-IR.json", - "Create-Task-Master-AZ-SQL-OnP-SH-IR.json", - "Create-Task-Master-AZ-SQL-SH-IR.json", - "GEN-File-Binary-AZ-Storage-Binary-OnP-SH-IR.json", - "GEN-File-Binary-AZ-Storage-Binary-SH-IR.json", - "OnP-SQL-AZ-Storage-Parquet-OnP-SH-IR.json", - "OnP-SQL-Full-Load-Chunk-OnP-SH-IR.json", - "OnP-SQL-Full-Load-OnP-SH-IR.json", - "OnP-SQL-GEN-File-Parquet-OnP-SH-IR.json", - "OnP-SQL-Watermark-Chunk-OnP-SH-IR.json", - "OnP-SQL-Watermark-OnP-SH-IR.json", - "SH-SQL-AZ-Storage-Parquet-SH-IR.json", - "SH-SQL-Full-Load-Chunk-SH-IR.json", - "SH-SQL-Full-Load-SH-IR.json", - "SH-SQL-Watermark-Chunk-SH-IR.json", - "SH-SQL-Watermark-SH-IR.json" + "AZ_SQL_Create_Table_{IR}.json", + "AZ_SQL_Full_Load_Chunk_{IR}.json", + "AZ_SQL_Full_Load_{IR}.json", + "AZ_SQL_Post_Copy_{IR}.json", + "AZ_SQL_StoredProcedure_{IR}.json", + "AZ_SQL_Watermark_Chunk_{IR}.json", + "AZ_SQL_Watermark_{IR}.json", + "AZ_Storage_Binary_AZ_Storage_Binary_{IR}.json", + "AZ_Storage_CSV_AZ_SQL_{IR}.json", + "AZ_Storage_Excel_AZ_SQL_{IR}.json", + "AZ_Storage_Excel_AZ_Storage_CSV_{IR}.json", + "AZ_Storage_JSON_AZ_SQL_{IR}.json", + "AZ_Storage_Parquet_AZ_SQL_{IR}.json", + "Create_Task_Master_AZ_SQL_{IR}.json", + "Create_Task_Master_AZ_SQL_{IR}.json", + "GEN_File_Binary_AZ_Storage_Binary_{IR}.json", + "GEN_File_Binary_AZ_Storage_Binary_{IR}.json", + "OnP_SQL_AZ_Storage_Parquet_{IR}.json", + "OnP_SQL_Full_Load_Chunk_{IR}.json", + "OnP_SQL_Full_Load_{IR}.json", + "OnP_SQL_GEN_File_Parquet_{IR}.json", + "OnP_SQL_Watermark_Chunk_{IR}.json", + "OnP_SQL_Watermark_{IR}.json", + "SH_SQL_AZ_Storage_Parquet_{IR}.json", + "SH_SQL_Full_Load_Chunk_{IR}.json", + "SH_SQL_Full_Load_{IR}.json", + "SH_SQL_Watermark_Chunk_{IR}.json", + "SH_SQL_Watermark_{IR}.json" ] }, { @@ -100,14 +100,14 @@ "GenericAzureDataLakeStorage_OnPrem_Net.json" ], "pipeline": [ - "Create-Task-Master-AZ-SQL-OnPrem_Net.json", - "GEN-File-Binary-AZ-Storage-Binary-OnPrem_Net.json", - "OnP-SQL-AZ-Storage-Parquet-OnPrem_Net.json", - "OnP-SQL-Full-Load-Chunk-OnP-SH-IR.json", - "OnP-SQL-Full-Load-OnP-SH-IR.json", - "OnP-SQL-GEN-File-Parquet-OnP-SH-IR.json", - "OnP-SQL-Watermark-Chunk-OnP-SH-IR.json", - "OnP-SQL-Watermark-OnP-SH-IR.json" + "Create_Task_Master_AZ_SQL_OnPrem_Net.json", + "GEN_File_Binary_AZ_Storage_Binary_OnPrem_Net.json", + "OnP_SQL_AZ_Storage_Parquet_OnPrem_Net.json", + "OnP_SQL_Full_Load_Chunk_{IR}.json", + "OnP_SQL_Full_Load_{IR}.json", + "OnP_SQL_GEN_File_Parquet_{IR}.json", + "OnP_SQL_Watermark_Chunk_{IR}.json", + "OnP_SQL_Watermark_{IR}.json" ] } ] diff --git a/solution/DataFactory/Templates/pipeline/Master_{IR}.json b/solution/DataFactory/Templates/pipeline/Master_{IR}.json index e1dfe675..06e600a5 100644 --- a/solution/DataFactory/Templates/pipeline/Master_{IR}.json +++ b/solution/DataFactory/Templates/pipeline/Master_{IR}.json @@ -1,5 +1,5 @@ { - "name": "Master", + "name": "Master_@GF{IR}", "properties": { "activities": [ { diff --git a/solution/Database/ADSGoFastDbUp/AdsGoFastDbUp/0-1-0-beta/A-Journaled/008-SampleTasks_SourceAndTargetSystems.sql b/solution/Database/ADSGoFastDbUp/AdsGoFastDbUp/0-1-0-beta/A-Journaled/008-SampleTasks_SourceAndTargetSystems.sql index 84a5d127..211fd431 100644 --- a/solution/Database/ADSGoFastDbUp/AdsGoFastDbUp/0-1-0-beta/A-Journaled/008-SampleTasks_SourceAndTargetSystems.sql +++ b/solution/Database/ADSGoFastDbUp/AdsGoFastDbUp/0-1-0-beta/A-Journaled/008-SampleTasks_SourceAndTargetSystems.sql @@ -8,75 +8,75 @@ SET IDENTITY_INSERT [dbo].[SourceAndTargetSystems] ON GO -INSERT [dbo].[SourceAndTargetSystems] ([SystemId], [SystemName], [SystemType], [SystemDescription], [SystemServer], [SystemAuthType], [SystemUserName], [SystemSecretName], [SystemKeyVaultBaseUrl], [SystemJSON], [ActiveYN], [IntegrationRuntime]) VALUES (1, N'Sample - Azure SQL Server ', N'Azure SQL', N'Sample Azure SQL Server Source', N'adsgofastdatakakeaccelsqlsvr.database.windows.net', N'MSI', NULL, NULL, N'https://adsgofastkeyvault.vault.azure.net/', N' +INSERT [dbo].[SourceAndTargetSystems] ([SystemId], [SystemName], [SystemType], [SystemDescription], [SystemServer], [SystemAuthType], [SystemUserName], [SystemSecretName], [SystemKeyVaultBaseUrl], [SystemJSON], [ActiveYN]) VALUES (1, N'Sample - Azure SQL Server ', N'Azure SQL', N'Sample Azure SQL Server Source', N'adsgofastdatakakeaccelsqlsvr.database.windows.net', N'MSI', NULL, NULL, N'https://adsgofastkeyvault.vault.azure.net/', N' { "Database" : "AWSample" } -', 1, NULL) +', 1) GO -INSERT [dbo].[SourceAndTargetSystems] ([SystemId], [SystemName], [SystemType], [SystemDescription], [SystemServer], [SystemAuthType], [SystemUserName], [SystemSecretName], [SystemKeyVaultBaseUrl], [SystemJSON], [ActiveYN], [IntegrationRuntime]) VALUES (2, N'Sample - Exploration SQL', N'Azure SQL', N'Sample - Exploration SQL', N'adsgofastdatakakeaccelsqlsvr.database.windows.net', N'MSI', NULL, NULL, N'https://adsgofastkeyvault.vault.azure.net/', N' { "Database" : "Staging" }', 1, NULL) +INSERT [dbo].[SourceAndTargetSystems] ([SystemId], [SystemName], [SystemType], [SystemDescription], [SystemServer], [SystemAuthType], [SystemUserName], [SystemSecretName], [SystemKeyVaultBaseUrl], [SystemJSON], [ActiveYN]) VALUES (2, N'Sample - Exploration SQL', N'Azure SQL', N'Sample - Exploration SQL', N'adsgofastdatakakeaccelsqlsvr.database.windows.net', N'MSI', NULL, NULL, N'https://adsgofastkeyvault.vault.azure.net/', N' { "Database" : "Staging" }', 1) GO -INSERT [dbo].[SourceAndTargetSystems] ([SystemId], [SystemName], [SystemType], [SystemDescription], [SystemServer], [SystemAuthType], [SystemUserName], [SystemSecretName], [SystemKeyVaultBaseUrl], [SystemJSON], [ActiveYN], [IntegrationRuntime]) VALUES (3, N'Sample - DataLake Raw (Blob)', N'Azure Blob', N'Sample - DataLake Raw', N'https://adsgofastdatalakeaccelst.blob.core.windows.net', N'MSI', NULL, NULL, N'https://adsgofastkeyvault.vault.azure.net/', N' +INSERT [dbo].[SourceAndTargetSystems] ([SystemId], [SystemName], [SystemType], [SystemDescription], [SystemServer], [SystemAuthType], [SystemUserName], [SystemSecretName], [SystemKeyVaultBaseUrl], [SystemJSON], [ActiveYN]) VALUES (3, N'Sample - DataLake Raw (Blob)', N'Azure Blob', N'Sample - DataLake Raw', N'https://adsgofastdatalakeaccelst.blob.core.windows.net', N'MSI', NULL, NULL, N'https://adsgofastkeyvault.vault.azure.net/', N' { "Container" : "datalakeraw" } -', 1, NULL) +', 1) GO -INSERT [dbo].[SourceAndTargetSystems] ([SystemId], [SystemName], [SystemType], [SystemDescription], [SystemServer], [SystemAuthType], [SystemUserName], [SystemSecretName], [SystemKeyVaultBaseUrl], [SystemJSON], [ActiveYN], [IntegrationRuntime]) VALUES (4, N'Sample - DataLake Raw (ADLS)', N'ADLS', N'Sample - DataLake Raw (ADLS)', N'https://adsgofastdatalakeadls.dfs.core.windows.net', N'MSI', NULL, NULL, N'https://adsgofastkeyvault.vault.azure.net/', N' +INSERT [dbo].[SourceAndTargetSystems] ([SystemId], [SystemName], [SystemType], [SystemDescription], [SystemServer], [SystemAuthType], [SystemUserName], [SystemSecretName], [SystemKeyVaultBaseUrl], [SystemJSON], [ActiveYN]) VALUES (4, N'Sample - DataLake Raw (ADLS)', N'ADLS', N'Sample - DataLake Raw (ADLS)', N'https://adsgofastdatalakeadls.dfs.core.windows.net', N'MSI', NULL, NULL, N'https://adsgofastkeyvault.vault.azure.net/', N' { "Container" : "datalakeraw" } -', 1, NULL) +', 1) GO -INSERT [dbo].[SourceAndTargetSystems] ([SystemId], [SystemName], [SystemType], [SystemDescription], [SystemServer], [SystemAuthType], [SystemUserName], [SystemSecretName], [SystemKeyVaultBaseUrl], [SystemJSON], [ActiveYN], [IntegrationRuntime]) VALUES (5, N'Sample - On Premise Fileshare Source', N'File', N'Sample - On Premise Fileshare Source', N'\\\\adsgofastonpremadfir\\D$\\dataingestion\\', N'WindowsAuth', N'AzureUser', N'adsgofast-onpre-file-password', N'https://adsgofastkeyvault.vault.azure.net/', NULL, 1, NULL) +INSERT [dbo].[SourceAndTargetSystems] ([SystemId], [SystemName], [SystemType], [SystemDescription], [SystemServer], [SystemAuthType], [SystemUserName], [SystemSecretName], [SystemKeyVaultBaseUrl], [SystemJSON], [ActiveYN]) VALUES (5, N'Sample - On Premise Fileshare Source', N'File', N'Sample - On Premise Fileshare Source', N'\\\\adsgofastonpremadfir\\D$\\dataingestion\\', N'WindowsAuth', N'AzureUser', N'adsgofast-onpre-file-password', N'https://adsgofastkeyvault.vault.azure.net/', NULL, 1) GO -INSERT [dbo].[SourceAndTargetSystems] ([SystemId], [SystemName], [SystemType], [SystemDescription], [SystemServer], [SystemAuthType], [SystemUserName], [SystemSecretName], [SystemKeyVaultBaseUrl], [SystemJSON], [ActiveYN], [IntegrationRuntime]) VALUES (6, N'Sample - VM Hosted SQL Server', N'SQL Server', N'Sample - VM Hosted SQL Server', N'adsgofast-onpre', N'SQLAuth', N'sqladfir', N'adsgofast-onpre-sqladfir-password', N'https://adsgofastkeyvault.vault.azure.net/', N' +INSERT [dbo].[SourceAndTargetSystems] ([SystemId], [SystemName], [SystemType], [SystemDescription], [SystemServer], [SystemAuthType], [SystemUserName], [SystemSecretName], [SystemKeyVaultBaseUrl], [SystemJSON], [ActiveYN]) VALUES (6, N'Sample - VM Hosted SQL Server', N'SQL Server', N'Sample - VM Hosted SQL Server', N'adsgofast-onpre', N'SQLAuth', N'sqladfir', N'adsgofast-onpre-sqladfir-password', N'https://adsgofastkeyvault.vault.azure.net/', N' { "Database" : "AdventureWorks2017" } -', 1, NULL) +', 1) GO -INSERT [dbo].[SourceAndTargetSystems] ([SystemId], [SystemName], [SystemType], [SystemDescription], [SystemServer], [SystemAuthType], [SystemUserName], [SystemSecretName], [SystemKeyVaultBaseUrl], [SystemJSON], [ActiveYN], [IntegrationRuntime]) VALUES (7, N'Sample - DataLake Landing (Blob)', N'Azure Blob', N'Sample - DataLake Landing (Blob)', N'https://adsgofastdatalakeaccelst.blob.core.windows.net', N'MSI', NULL, NULL, N'https://adsgofastkeyvault.vault.azure.net/', N' +INSERT [dbo].[SourceAndTargetSystems] ([SystemId], [SystemName], [SystemType], [SystemDescription], [SystemServer], [SystemAuthType], [SystemUserName], [SystemSecretName], [SystemKeyVaultBaseUrl], [SystemJSON], [ActiveYN]) VALUES (7, N'Sample - DataLake Landing (Blob)', N'Azure Blob', N'Sample - DataLake Landing (Blob)', N'https://adsgofastdatalakeaccelst.blob.core.windows.net', N'MSI', NULL, NULL, N'https://adsgofastkeyvault.vault.azure.net/', N' { "Container" : "datalakelanding" } -', 1, NULL) +', 1) GO -INSERT [dbo].[SourceAndTargetSystems] ([SystemId], [SystemName], [SystemType], [SystemDescription], [SystemServer], [SystemAuthType], [SystemUserName], [SystemSecretName], [SystemKeyVaultBaseUrl], [SystemJSON], [ActiveYN], [IntegrationRuntime]) VALUES (8, N'Sample - DataLake Landing (ADLS)', N'ADLS', N'Sample - DataLake Landing (ADLS)', N'https://adsgofastdatalakeadls.dfs.core.windows.net', N'MSI', NULL, NULL, N'https://adsgofastkeyvault.vault.azure.net/', N' +INSERT [dbo].[SourceAndTargetSystems] ([SystemId], [SystemName], [SystemType], [SystemDescription], [SystemServer], [SystemAuthType], [SystemUserName], [SystemSecretName], [SystemKeyVaultBaseUrl], [SystemJSON], [ActiveYN]) VALUES (8, N'Sample - DataLake Landing (ADLS)', N'ADLS', N'Sample - DataLake Landing (ADLS)', N'https://adsgofastdatalakeadls.dfs.core.windows.net', N'MSI', NULL, NULL, N'https://adsgofastkeyvault.vault.azure.net/', N' { "Container" : "datalakelanding" } -', 1, NULL) +', 1) GO -INSERT [dbo].[SourceAndTargetSystems] ([SystemId], [SystemName], [SystemType], [SystemDescription], [SystemServer], [SystemAuthType], [SystemUserName], [SystemSecretName], [SystemKeyVaultBaseUrl], [SystemJSON], [ActiveYN], [IntegrationRuntime]) VALUES (9, N'Sample - Transient-In (Blob)', N'Azure Blob', N'Sample - Transient-In (Blob)', N'https://adsgofasttransientstg.blob.core.windows.net', N'MSI', NULL, NULL, N'https://adsgofastkeyvault.vault.azure.net/', N' +INSERT [dbo].[SourceAndTargetSystems] ([SystemId], [SystemName], [SystemType], [SystemDescription], [SystemServer], [SystemAuthType], [SystemUserName], [SystemSecretName], [SystemKeyVaultBaseUrl], [SystemJSON], [ActiveYN]) VALUES (9, N'Sample - Transient-In (Blob)', N'Azure Blob', N'Sample - Transient-In (Blob)', N'https://adsgofasttransientstg.blob.core.windows.net', N'MSI', NULL, NULL, N'https://adsgofastkeyvault.vault.azure.net/', N' { "Container" : "transientin" } -', 1, NULL) +', 1) GO -INSERT [dbo].[SourceAndTargetSystems] ([SystemId], [SystemName], [SystemType], [SystemDescription], [SystemServer], [SystemAuthType], [SystemUserName], [SystemSecretName], [SystemKeyVaultBaseUrl], [SystemJSON], [ActiveYN], [IntegrationRuntime]) VALUES (10, N'Sample - Sendgrid Email Account', N'SendGrid', N'Sample - Sendgrid Email Account', N'ADSGoFastSendGrid', N'Key', NULL, NULL, N'https://adsgofastkeyvault.vault.azure.net/', N' { +INSERT [dbo].[SourceAndTargetSystems] ([SystemId], [SystemName], [SystemType], [SystemDescription], [SystemServer], [SystemAuthType], [SystemUserName], [SystemSecretName], [SystemKeyVaultBaseUrl], [SystemJSON], [ActiveYN]) VALUES (10, N'Sample - Sendgrid Email Account', N'SendGrid', N'Sample - Sendgrid Email Account', N'ADSGoFastSendGrid', N'Key', NULL, NULL, N'https://adsgofastkeyvault.vault.azure.net/', N' { "SenderEmail" : "noreply@cleverchiro.com", "SenderDescription" : "ADS Go Fast (No Reply)", "Subject" : "ADS GO Fast SAS Uri for File Upload", "PlainTextContent" : "Hello, Email!", "HtmlContent" : "Hi ,

Use the link below to upload files:


" - }', 1, NULL) + }', 1) GO -INSERT [dbo].[SourceAndTargetSystems] ([SystemId], [SystemName], [SystemType], [SystemDescription], [SystemServer], [SystemAuthType], [SystemUserName], [SystemSecretName], [SystemKeyVaultBaseUrl], [SystemJSON], [ActiveYN], [IntegrationRuntime]) VALUES (11, N'Sample - AdsGoFast Database', N'Azure SQL', N'Sample - AdsGoFast Database', N'adsgofastdatakakeaccelsqlsvr.database.windows.net', N'MSI', NULL, NULL, N'https://adsgofastkeyvault.vault.azure.net/', N' +INSERT [dbo].[SourceAndTargetSystems] ([SystemId], [SystemName], [SystemType], [SystemDescription], [SystemServer], [SystemAuthType], [SystemUserName], [SystemSecretName], [SystemKeyVaultBaseUrl], [SystemJSON], [ActiveYN]) VALUES (11, N'Sample - AdsGoFast Database', N'Azure SQL', N'Sample - AdsGoFast Database', N'adsgofastdatakakeaccelsqlsvr.database.windows.net', N'MSI', NULL, NULL, N'https://adsgofastkeyvault.vault.azure.net/', N' { "Database" : "AdsGoFast" } -', 1, NULL) +', 1) GO -INSERT [dbo].[SourceAndTargetSystems] ([SystemId], [SystemName], [SystemType], [SystemDescription], [SystemServer], [SystemAuthType], [SystemUserName], [SystemSecretName], [SystemKeyVaultBaseUrl], [SystemJSON], [ActiveYN], [IntegrationRuntime]) VALUES (12, N'Sample - Sendgrid Email Account2', N'SendGrid', N'Sample - Sendgrid Email Account2', N'ADSGoFastSendGrid', N'Key', NULL, NULL, N'https://adsgofastkeyvault.vault.azure.net/', N' { +INSERT [dbo].[SourceAndTargetSystems] ([SystemId], [SystemName], [SystemType], [SystemDescription], [SystemServer], [SystemAuthType], [SystemUserName], [SystemSecretName], [SystemKeyVaultBaseUrl], [SystemJSON], [ActiveYN]) VALUES (12, N'Sample - Sendgrid Email Account2', N'SendGrid', N'Sample - Sendgrid Email Account2', N'ADSGoFastSendGrid', N'Key', NULL, NULL, N'https://adsgofastkeyvault.vault.azure.net/', N' { "SenderEmail" : "noreply@cleverchiro.com", "SenderDescription" : "ADS Go Fast (No Reply)", "Subject" : "ADS GO Fast SAS Uri for File Upload", "PlainTextContent" : "Hello, Email!", "HtmlContent" : "Hi ,

Use the link below to upload files:


" - }', 1, NULL) + }', 1) GO -INSERT [dbo].[SourceAndTargetSystems] ([SystemId], [SystemName], [SystemType], [SystemDescription], [SystemServer], [SystemAuthType], [SystemUserName], [SystemSecretName], [SystemKeyVaultBaseUrl], [SystemJSON], [ActiveYN], [IntegrationRuntime]) VALUES (13, N'Sample - VM Target', N'AzureVM', N'Sample - VM Target', N'adsgofastadfir', N'MSI', NULL, NULL, N'https://adsgofastkeyvault.vault.azure.net/', N' { "SubscriptionUid" : "035a1364-f00d-48e2-b582-4fe125905ee3", "VMname" : "adsgofastadfir", "ResourceGroup":"ADSGOFASTDATALAKEACCEL" }', 1, NULL) +INSERT [dbo].[SourceAndTargetSystems] ([SystemId], [SystemName], [SystemType], [SystemDescription], [SystemServer], [SystemAuthType], [SystemUserName], [SystemSecretName], [SystemKeyVaultBaseUrl], [SystemJSON], [ActiveYN]) VALUES (13, N'Sample - VM Target', N'AzureVM', N'Sample - VM Target', N'adsgofastadfir', N'MSI', NULL, NULL, N'https://adsgofastkeyvault.vault.azure.net/', N' { "SubscriptionUid" : "035a1364-f00d-48e2-b582-4fe125905ee3", "VMname" : "adsgofastadfir", "ResourceGroup":"ADSGOFASTDATALAKEACCEL" }', 1) GO SET IDENTITY_INSERT [dbo].[SourceAndTargetSystems] OFF GO diff --git a/solution/Database/ADSGoFastDbUp/AdsGoFastDbUp/0-1-0-beta/A-Journaled/011-SampleTasks_SourceAndTargetSystems_Updated.sql b/solution/Database/ADSGoFastDbUp/AdsGoFastDbUp/0-1-0-beta/A-Journaled/011-SampleTasks_SourceAndTargetSystems_Updated.sql new file mode 100644 index 00000000..0c9276d2 --- /dev/null +++ b/solution/Database/ADSGoFastDbUp/AdsGoFastDbUp/0-1-0-beta/A-Journaled/011-SampleTasks_SourceAndTargetSystems_Updated.sql @@ -0,0 +1,88 @@ +/*----------------------------------------------------------------------- + + Copyright (c) Microsoft Corporation. + Licensed under the MIT license. + +-----------------------------------------------------------------------*/ + +--Alteration needed as ADF Activity Errors now includes varchar in ErrorCode. +Alter Table [dbo].[ADFActivityErrors] alter column ErrorCode varchar(50) + +Alter table [dbo].[SourceAndTargetSystems] add DataFactoryIR varchar(20) + +/* +SET IDENTITY_INSERT [dbo].[SourceAndTargetSystems] ON +GO +INSERT [dbo].[SourceAndTargetSystems] ([SystemId], [SystemName], [SystemType], [SystemDescription], [SystemServer], [SystemAuthType], [SystemUserName], [SystemSecretName], [SystemKeyVaultBaseUrl], [SystemJSON], [ActiveYN], [DataFactoryIR]) VALUES (1, N'Sample - Azure SQL Server ', N'Azure SQL', N'Sample Azure SQL Server Source', N'adsgofastdatakakeaccelsqlsvr.database.windows.net', N'MSI', NULL, NULL, N'https://adsgofastkeyvault.vault.azure.net/', N' + { + "Database" : "AWSample" + } +', 1, 'IRA') +GO +INSERT [dbo].[SourceAndTargetSystems] ([SystemId], [SystemName], [SystemType], [SystemDescription], [SystemServer], [SystemAuthType], [SystemUserName], [SystemSecretName], [SystemKeyVaultBaseUrl], [SystemJSON], [ActiveYN], [DataFactoryIR]) VALUES (2, N'Sample - Exploration SQL', N'Azure SQL', N'Sample - Exploration SQL', N'adsgofastdatakakeaccelsqlsvr.database.windows.net', N'MSI', NULL, NULL, N'https://adsgofastkeyvault.vault.azure.net/', N' { "Database" : "Staging" }', 1, NULL) +GO +INSERT [dbo].[SourceAndTargetSystems] ([SystemId], [SystemName], [SystemType], [SystemDescription], [SystemServer], [SystemAuthType], [SystemUserName], [SystemSecretName], [SystemKeyVaultBaseUrl], [SystemJSON], [ActiveYN], [DataFactoryIR]) VALUES (3, N'Sample - DataLake Raw (Blob)', N'Azure Blob', N'Sample - DataLake Raw', N'https://adsgofastdatalakeaccelst.blob.core.windows.net', N'MSI', NULL, NULL, N'https://adsgofastkeyvault.vault.azure.net/', N' + { + "Container" : "datalakeraw" + } +', 1, 'IRA') +GO +INSERT [dbo].[SourceAndTargetSystems] ([SystemId], [SystemName], [SystemType], [SystemDescription], [SystemServer], [SystemAuthType], [SystemUserName], [SystemSecretName], [SystemKeyVaultBaseUrl], [SystemJSON], [ActiveYN], [DataFactoryIR]) VALUES (4, N'Sample - DataLake Raw (ADLS)', N'ADLS', N'Sample - DataLake Raw (ADLS)', N'https://adsgofastdatalakeadls.dfs.core.windows.net', N'MSI', NULL, NULL, N'https://adsgofastkeyvault.vault.azure.net/', N' + { + "Container" : "datalakeraw" + } +', 1, 'IRA') +GO +INSERT [dbo].[SourceAndTargetSystems] ([SystemId], [SystemName], [SystemType], [SystemDescription], [SystemServer], [SystemAuthType], [SystemUserName], [SystemSecretName], [SystemKeyVaultBaseUrl], [SystemJSON], [ActiveYN], [DataFactoryIR]) VALUES (5, N'Sample - On Premise Fileshare Source', N'File', N'Sample - On Premise Fileshare Source', N'\\\\adsgofastonpremadfir\\D$\\dataingestion\\', N'WindowsAuth', N'AzureUser', N'adsgofast-onpre-file-password', N'https://adsgofastkeyvault.vault.azure.net/', NULL, 1, NULL) +GO +INSERT [dbo].[SourceAndTargetSystems] ([SystemId], [SystemName], [SystemType], [SystemDescription], [SystemServer], [SystemAuthType], [SystemUserName], [SystemSecretName], [SystemKeyVaultBaseUrl], [SystemJSON], [ActiveYN], [DataFactoryIR]) VALUES (6, N'Sample - VM Hosted SQL Server', N'SQL Server', N'Sample - VM Hosted SQL Server', N'adsgofast-onpre', N'SQLAuth', N'sqladfir', N'adsgofast-onpre-sqladfir-password', N'https://adsgofastkeyvault.vault.azure.net/', N' + { + "Database" : "AdventureWorks2017" + } +', 1, 'IRA') +GO +INSERT [dbo].[SourceAndTargetSystems] ([SystemId], [SystemName], [SystemType], [SystemDescription], [SystemServer], [SystemAuthType], [SystemUserName], [SystemSecretName], [SystemKeyVaultBaseUrl], [SystemJSON], [ActiveYN], [DataFactoryIR]) VALUES (7, N'Sample - DataLake Landing (Blob)', N'Azure Blob', N'Sample - DataLake Landing (Blob)', N'https://adsgofastdatalakeaccelst.blob.core.windows.net', N'MSI', NULL, NULL, N'https://adsgofastkeyvault.vault.azure.net/', N' + { + "Container" : "datalakelanding" + } +', 1, 'IRA') +GO +INSERT [dbo].[SourceAndTargetSystems] ([SystemId], [SystemName], [SystemType], [SystemDescription], [SystemServer], [SystemAuthType], [SystemUserName], [SystemSecretName], [SystemKeyVaultBaseUrl], [SystemJSON], [ActiveYN], [DataFactoryIR]) VALUES (8, N'Sample - DataLake Landing (ADLS)', N'ADLS', N'Sample - DataLake Landing (ADLS)', N'https://adsgofastdatalakeadls.dfs.core.windows.net', N'MSI', NULL, NULL, N'https://adsgofastkeyvault.vault.azure.net/', N' + { + "Container" : "datalakelanding" + } +', 1, 'IRA') +GO +INSERT [dbo].[SourceAndTargetSystems] ([SystemId], [SystemName], [SystemType], [SystemDescription], [SystemServer], [SystemAuthType], [SystemUserName], [SystemSecretName], [SystemKeyVaultBaseUrl], [SystemJSON], [ActiveYN], [DataFactoryIR]) VALUES (9, N'Sample - Transient-In (Blob)', N'Azure Blob', N'Sample - Transient-In (Blob)', N'https://adsgofasttransientstg.blob.core.windows.net', N'MSI', NULL, NULL, N'https://adsgofastkeyvault.vault.azure.net/', N' + { + "Container" : "transientin" + } +', 1, 'IRA') +GO +INSERT [dbo].[SourceAndTargetSystems] ([SystemId], [SystemName], [SystemType], [SystemDescription], [SystemServer], [SystemAuthType], [SystemUserName], [SystemSecretName], [SystemKeyVaultBaseUrl], [SystemJSON], [ActiveYN], [DataFactoryIR]) VALUES (10, N'Sample - Sendgrid Email Account', N'SendGrid', N'Sample - Sendgrid Email Account', N'ADSGoFastSendGrid', N'Key', NULL, NULL, N'https://adsgofastkeyvault.vault.azure.net/', N' { + "SenderEmail" : "noreply@cleverchiro.com", + "SenderDescription" : "ADS Go Fast (No Reply)", + "Subject" : "ADS GO Fast SAS Uri for File Upload", + "PlainTextContent" : "Hello, Email!", + "HtmlContent" : "Hi ,

Use the link below to upload files:


" + }', 1, 'IRA') +GO +INSERT [dbo].[SourceAndTargetSystems] ([SystemId], [SystemName], [SystemType], [SystemDescription], [SystemServer], [SystemAuthType], [SystemUserName], [SystemSecretName], [SystemKeyVaultBaseUrl], [SystemJSON], [ActiveYN], [DataFactoryIR]) VALUES (11, N'Sample - AdsGoFast Database', N'Azure SQL', N'Sample - AdsGoFast Database', N'adsgofastdatakakeaccelsqlsvr.database.windows.net', N'MSI', NULL, NULL, N'https://adsgofastkeyvault.vault.azure.net/', N' + { + "Database" : "AdsGoFast" + } +', 1, 'IRA') +GO +INSERT [dbo].[SourceAndTargetSystems] ([SystemId], [SystemName], [SystemType], [SystemDescription], [SystemServer], [SystemAuthType], [SystemUserName], [SystemSecretName], [SystemKeyVaultBaseUrl], [SystemJSON], [ActiveYN], [DataFactoryIR]) VALUES (12, N'Sample - Sendgrid Email Account2', N'SendGrid', N'Sample - Sendgrid Email Account2', N'ADSGoFastSendGrid', N'Key', NULL, NULL, N'https://adsgofastkeyvault.vault.azure.net/', N' { + "SenderEmail" : "noreply@cleverchiro.com", + "SenderDescription" : "ADS Go Fast (No Reply)", + "Subject" : "ADS GO Fast SAS Uri for File Upload", + "PlainTextContent" : "Hello, Email!", + "HtmlContent" : "Hi ,

Use the link below to upload files:


" + }', 1, 'IRA') +GO +INSERT [dbo].[SourceAndTargetSystems] ([SystemId], [SystemName], [SystemType], [SystemDescription], [SystemServer], [SystemAuthType], [SystemUserName], [SystemSecretName], [SystemKeyVaultBaseUrl], [SystemJSON], [ActiveYN], [DataFactoryIR]) VALUES (13, N'Sample - VM Target', N'AzureVM', N'Sample - VM Target', N'adsgofastadfir', N'MSI', NULL, NULL, N'https://adsgofastkeyvault.vault.azure.net/', N' { "SubscriptionUid" : "035a1364-f00d-48e2-b582-4fe125905ee3", "VMname" : "adsgofastadfir", "ResourceGroup":"ADSGOFASTDATALAKEACCEL" }', 1, 'IRA') +GO +SET IDENTITY_INSERT [dbo].[SourceAndTargetSystems] OFF +GO +*/ \ No newline at end of file diff --git a/solution/Database/ADSGoFastDbUp/AdsGoFastDbUp/AdsGoFastDbUp.csproj b/solution/Database/ADSGoFastDbUp/AdsGoFastDbUp/AdsGoFastDbUp.csproj index 427a1add..677068e3 100644 --- a/solution/Database/ADSGoFastDbUp/AdsGoFastDbUp/AdsGoFastDbUp.csproj +++ b/solution/Database/ADSGoFastDbUp/AdsGoFastDbUp/AdsGoFastDbUp.csproj @@ -8,6 +8,8 @@ + + @@ -29,6 +31,8 @@ Never + + diff --git a/solution/Deployment/environments/development.json b/solution/Deployment/environments/development.json index ebd23061..f7774738 100644 --- a/solution/Deployment/environments/development.json +++ b/solution/Deployment/environments/development.json @@ -31,13 +31,13 @@ "Enable": true, "Name": "AdsGFWebAuthSP", "ApplyNamePostFix": true, - "ClientId": "6112fad2-f8f2-4313-987c-690703898480" + "ClientId": "15ebf945-aec3-43b8-9fc4-2f2309570fb5" }, "FunctionAppAuthenticationSP": { "Enable": true, "Name": "AdsGFFuncAppAuthSP", "ApplyNamePostFix": true, - "ClientId": "e7bf45f2-019c-4f6e-b908-58b71278bbe1" + "ClientId": "59f4da37-a8bb-4e24-b810-6b7dabd6ad20" } }, "FolderPaths": { @@ -194,4 +194,4 @@ } } } -} \ No newline at end of file +} diff --git a/solution/Deployment/environments/environment.schema.json b/solution/Deployment/environments/environment.schema.json index 6b68c9dd..fd8b9c76 100644 --- a/solution/Deployment/environments/environment.schema.json +++ b/solution/Deployment/environments/environment.schema.json @@ -1726,9 +1726,25 @@ "title": "Name", "description": "", "default": "", + "options": { + "inputAttributes": { + "placeholder": "eg. AdsGoFastVnet" + } + }, "examples": [ "AdsGoFastVnet" ] + }, + "ApplyNamePostFix": { + "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/Vnet/properties/Logging/properties/ApplyNamePostFix", + "type": "boolean", + "format": "checkbox", + "title": "Apply Postfix to Resource Name", + "description": "", + "default": false, + "examples": [ + true + ] } }, "additionalProperties": true diff --git a/solution/Deployment/workflows/CD_0a_CreateServicePrincipals_AAD_Elevated.ps1 b/solution/Deployment/workflows/CD_0a_CreateServicePrincipals_AAD_Elevated.ps1 index f50280a8..4f5145be 100644 --- a/solution/Deployment/workflows/CD_0a_CreateServicePrincipals_AAD_Elevated.ps1 +++ b/solution/Deployment/workflows/CD_0a_CreateServicePrincipals_AAD_Elevated.ps1 @@ -40,6 +40,7 @@ if($env:AdsOpts_CD_ServicePrincipals_WebAppAuthenticationSP_Enable -eq "True") $subid = ((az account show -s $env:AdsOpts_CD_ResourceGroup_Subscription) | ConvertFrom-Json).id $appid = ((az ad app create --display-name $env:AdsOpts_CD_ServicePrincipals_WebAppAuthenticationSP_Name --homepage "api://$env:AdsOpts_CD_ServicePrincipals_WebAppAuthenticationSP_Name" --identifier-uris "api://$env:AdsOpts_CD_ServicePrincipals_WebAppAuthenticationSP_Name" --app-roles $roles --reply-urls $replyurls) | ConvertFrom-Json).appId + $appid = ((az ad app show --id "api://$env:AdsOpts_CD_ServicePrincipals_WebAppAuthenticationSP_Name") | ConvertFrom-Json).appId $spid = ((az ad sp create --id $appid) | ConvertFrom-Json).ObjectId } @@ -54,6 +55,7 @@ if($env:AdsOpts_CD_ServicePrincipals_FunctionAppAuthenticationSP_Enable -eq "Tru $subid = ((az account show -s $env:AdsOpts_CD_ResourceGroup_Subscription) | ConvertFrom-Json).id $appid = ((az ad app create --display-name $env:AdsOpts_CD_ServicePrincipals_FunctionAppAuthenticationSP_Name --homepage "api://$env:AdsOpts_CD_ServicePrincipals_FunctionAppAuthenticationSP_Name" --identifier-uris "api://$env:AdsOpts_CD_ServicePrincipals_FunctionAppAuthenticationSP_Name" --app-roles $roles) | ConvertFrom-Json).appId + $appid = ((az ad app show --id "api://$env:AdsOpts_CD_ServicePrincipals_FunctionAppAuthenticationSP_Name") | ConvertFrom-Json).appId $spid = ((az ad sp create --id $appid) | ConvertFrom-Json).ObjectId #Will need to do below during service creation to add the Azure Function MSI to role diff --git a/solution/Deployment/workflows/CD_2b_ConfigureServices.ps1 b/solution/Deployment/workflows/CD_2b_ConfigureServices.ps1 index 95ceb354..ea6a694a 100644 --- a/solution/Deployment/workflows/CD_2b_ConfigureServices.ps1 +++ b/solution/Deployment/workflows/CD_2b_ConfigureServices.ps1 @@ -27,6 +27,8 @@ if (([Environment]::GetEnvironmentVariable("AdsOpts_CD_EnableConfigure")) -eq "T Invoke-Expression -Command ".\Steps\CD_ConfigureAzureSqlServer_UpdateTaskTypeMappingJson.ps1" + Invoke-Expression -Command ".\Steps\CD_ConfigureSampleData.ps1" + Write-Host "Finishing CD.." } diff --git a/solution/Deployment/workflows/Cleanup_RemoveAll.ps1 b/solution/Deployment/workflows/Cleanup_RemoveAll.ps1 index 8e53ce52..aa56fd98 100644 --- a/solution/Deployment/workflows/Cleanup_RemoveAll.ps1 +++ b/solution/Deployment/workflows/Cleanup_RemoveAll.ps1 @@ -1,4 +1,4 @@ -[Environment]::SetEnvironmentVariable("ENVIRONMENT_NAME", "development_test") +[Environment]::SetEnvironmentVariable("ENVIRONMENT_NAME", "development") . .\Steps\PushEnvFileIntoVariables.ps1 ParseEnvFile("$env:ENVIRONMENT_NAME") Invoke-Expression -Command ".\Steps\CD_SetResourceGroupHash.ps1" diff --git a/solution/Deployment/workflows/FileForUpload.json b/solution/Deployment/workflows/FileForUpload.json new file mode 100644 index 00000000..32112d7f --- /dev/null +++ b/solution/Deployment/workflows/FileForUpload.json @@ -0,0 +1,71 @@ +{ + "name": "SqlServerTable_IRA", + "properties": { + "linkedServiceName": { + "referenceName": "GenericMsSqlServer_IRA", + "type": "LinkedServiceReference", + "parameters": { + "KeyVaultBaseUrl": { + "value": "@dataset().KeyVaultBaseUrl", + "type": "Expression" + }, + "PasswordSecret": { + "value": "@dataset().PasswordSecret", + "type": "Expression" + }, + "Server": { + "value": "@dataset().Server", + "type": "Expression" + }, + "Database": { + "value": "@dataset().Database", + "type": "Expression" + }, + "UserName": { + "value": "@dataset().UserName", + "type": "Expression" + } + } + }, + "parameters": { + "TableSchema": { + "type": "string" + }, + "TableName": { + "type": "string" + }, + "KeyVaultBaseUrl": { + "type": "string" + }, + "PasswordSecret": { + "type": "string" + }, + "Server": { + "type": "string" + }, + "Database": { + "type": "string" + }, + "UserName": { + "type": "string" + } + }, + "folder": { + "name": "ADS Go Fast/Generic/IRA" + }, + "annotations": [], + "type": "SqlServerTable", + "schema": [], + "typeProperties": { + "schema": { + "value": "@dataset().TableSchema", + "type": "Expression" + }, + "table": { + "value": "@dataset().TableName", + "type": "Expression" + } + } + }, + "type": "Microsoft.DataFactory/factories/datasets" +} diff --git a/solution/Deployment/workflows/Steps/CD_ConfigureADF.ps1 b/solution/Deployment/workflows/Steps/CD_ConfigureADF.ps1 index 99fc944d..226a5a5f 100644 --- a/solution/Deployment/workflows/Steps/CD_ConfigureADF.ps1 +++ b/solution/Deployment/workflows/Steps/CD_ConfigureADF.ps1 @@ -2,7 +2,7 @@ $logsSetting = "[{'category':'ActivityRuns','enabled':true,'retentionPolicy':{'days': 30,'enabled': true}},{'category':'PipelineRuns','enabled':true,'retentionPolicy':{'days': 30,'enabled': true}},{'category':'TriggerRuns','enabled':true,'retentionPolicy':{'days': 30,'enabled': true}}]".Replace("'",'\"') $metricsSetting = "[{'category':'AllMetrics','enabled':true,'retentionPolicy':{'days': 30,'enabled': true}}]".Replace("'",'\"') -az monitor diagnostic-settings create --name ADF-Diagnostics --export-to-resource-specific true --resource "$env:AdsOpts_CD_ResourceGroup_Id/providers/Microsoft.DataFactory/factories/$env:AdsOpts_CD_Services_DataFactory_Name" --logs $logsSetting --metrics $metricsSetting --storage-account "$env:AdsOpts_CD_ResourceGroup_Id/providers/Microsoft.Storage/storageAccounts/$env:AdsOpts_CD_Services_Storage_Logging_Name" --workspace "$env:AdsOpts_CD_ResourceGroup_Id/providers/microsoft.operationalinsights/workspaces/$env:AdsOpts_CD_Services_LogAnalytics_Name" +$result = az monitor diagnostic-settings create --name ADF-Diagnostics --export-to-resource-specific true --resource "$env:AdsOpts_CD_ResourceGroup_Id/providers/Microsoft.DataFactory/factories/$env:AdsOpts_CD_Services_DataFactory_Name" --logs $logsSetting --metrics $metricsSetting --storage-account "$env:AdsOpts_CD_ResourceGroup_Id/providers/Microsoft.Storage/storageAccounts/$env:AdsOpts_CD_Services_Storage_Logging_Name" --workspace "$env:AdsOpts_CD_ResourceGroup_Id/providers/microsoft.operationalinsights/workspaces/$env:AdsOpts_CD_Services_LogAnalytics_Name" #Create IRs diff --git a/solution/Deployment/workflows/Steps/CD_ConfigureAzureSQLServer.ps1 b/solution/Deployment/workflows/Steps/CD_ConfigureAzureSQLServer.ps1 index 4a213e29..c6fc2556 100644 --- a/solution/Deployment/workflows/Steps/CD_ConfigureAzureSQLServer.ps1 +++ b/solution/Deployment/workflows/Steps/CD_ConfigureAzureSQLServer.ps1 @@ -23,8 +23,15 @@ function GeneratePassword { Write-Debug "Configuring Azure SQL Server" + #Install Sql Server Module -Install-Module -Name SqlServer -Force +if (Get-Module -ListAvailable -Name SqlServer) { + Write-Host "SqlServer Module exists" +} +else { + Write-Host "Module does not exist.. installing.." + Install-Module -Name SqlServer -Force +} #Get Access Token for SQL --Note that the deployment principal or user running locally will need rights on the database $token=$(az account get-access-token --resource=https://database.windows.net/ --query accessToken --output tsv) @@ -36,9 +43,9 @@ if($env:AdsOpts_CD_Services_AzureSQLServer_AdsGoFastDB_Enable -eq "True") #Add Ip to SQL Firewall $myIp = (Invoke-WebRequest ifconfig.me/ip).Content - az sql server firewall-rule create -g $env:AdsOpts_CD_ResourceGroup_Name -s $env:AdsOpts_CD_Services_AzureSQLServer_Name -n "MySetupIP" --start-ip-address $myIp --end-ip-address $myIp + $result = az sql server firewall-rule create -g $env:AdsOpts_CD_ResourceGroup_Name -s $env:AdsOpts_CD_Services_AzureSQLServer_Name -n "MySetupIP" --start-ip-address $myIp --end-ip-address $myIp #Allow Azure services and resources to access this server - az sql server firewall-rule create -g $env:AdsOpts_CD_ResourceGroup_Name -s $env:AdsOpts_CD_Services_AzureSQLServer_Name -n "Azure" --start-ip-address 0.0.0.0 --end-ip-address 0.0.0.0 + $result = az sql server firewall-rule create -g $env:AdsOpts_CD_ResourceGroup_Name -s $env:AdsOpts_CD_Services_AzureSQLServer_Name -n "Azure" --start-ip-address 0.0.0.0 --end-ip-address 0.0.0.0 $CurrentPath = (Get-Location).Path Set-Location "..\bin\publish\unzipped\database\" diff --git a/solution/Deployment/workflows/Steps/CD_ConfigureKeyVault.ps1 b/solution/Deployment/workflows/Steps/CD_ConfigureKeyVault.ps1 index 1857c68c..9bcbe3ec 100644 --- a/solution/Deployment/workflows/Steps/CD_ConfigureKeyVault.ps1 +++ b/solution/Deployment/workflows/Steps/CD_ConfigureKeyVault.ps1 @@ -1,4 +1,4 @@ -Write-Debug"Configuring Key Vault" +Write-Debug " Configuring Key Vault" if($env:AdsOpts_CD_Services_KeyVault_Enable -eq "True") { @@ -12,14 +12,14 @@ if($env:AdsOpts_CD_Services_KeyVault_Enable -eq "True") $functionkey = (az functionapp keys list -g $env:AdsOpts_CD_ResourceGroup_Name -n $env:AdsOpts_CD_Services_CoreFunctionApp_Name | ConvertFrom-Json).functionKeys.default - Write-Debug"Enabling Access to KeyVault and Adding Secrets" + Write-Debug " Enabling Access to KeyVault and Adding Secrets" #Set KeyVault Policy to allow logged in user to add key - az keyvault set-policy --name $env:AdsOpts_CD_Services_KeyVault_Name --certificate-permissions backup create delete deleteissuers get getissuers import list listissuers managecontacts manageissuers purge recover restore setissuers update --key-permissions backup create decrypt delete encrypt get import list purge recover restore sign unwrapKey update verify wrapKey --object-id $AADUserId --resource-group $env:AdsOpts_CD_ResourceGroup_Name --secret-permissions backup delete get list purge recover restore set --storage-permissions backup delete deletesas get getsas list listsas purge recover regeneratekey restore set setsas update --subscription $env:AdsOpts_CD_ResourceGroup_Subscription + $result = az keyvault set-policy --name $env:AdsOpts_CD_Services_KeyVault_Name --certificate-permissions backup create delete deleteissuers get getissuers import list listissuers managecontacts manageissuers purge recover restore setissuers update --key-permissions backup create decrypt delete encrypt get import list purge recover restore sign unwrapKey update verify wrapKey --object-id $AADUserId --resource-group $env:AdsOpts_CD_ResourceGroup_Name --secret-permissions backup delete get list purge recover restore set --storage-permissions backup delete deletesas get getsas list listsas purge recover regeneratekey restore set setsas update --subscription $env:AdsOpts_CD_ResourceGroup_Subscription #Set KeyVault Policy to allow MSI for ADF to Retrieve Key Vault Key #az keyvault set-policy --name $env:AdsOpts_CD_Services_KeyVault_Name --certificate-permissions backup create delete deleteissuers get getissuers import list listissuers managecontacts manageissuers purge recover restore setissuers update --key-permissions backup create decrypt delete encrypt get import list purge recover restore sign unwrapKey update verify wrapKey --object-id $AADUserId --resource-group $env:AdsOpts_CD_ResourceGroup_Name --secret-permissions backup delete get list purge recover restore set --storage-permissions backup delete deletesas get getsas list listsas purge recover regeneratekey restore set setsas update --subscription $env:AdsOpts_CD_ResourceGroup_Subscription #Save Function Key to KeyVault - az keyvault secret set --name "AdsGfCoreFunctionAppKey" --vault-name $env:AdsOpts_CD_Services_KeyVault_Name --disabled false --subscription $env:AdsOpts_CD_ResourceGroup_Subscription --value $functionkey --output none + $result = az keyvault secret set --name "AdsGfCoreFunctionAppKey" --vault-name $env:AdsOpts_CD_Services_KeyVault_Name --disabled false --subscription $env:AdsOpts_CD_ResourceGroup_Subscription --value $functionkey --output none } else diff --git a/solution/Deployment/workflows/Steps/CD_ConfigureSampleData.ps1 b/solution/Deployment/workflows/Steps/CD_ConfigureSampleData.ps1 index bdff6184..23ce8e88 100644 --- a/solution/Deployment/workflows/Steps/CD_ConfigureSampleData.ps1 +++ b/solution/Deployment/workflows/Steps/CD_ConfigureSampleData.ps1 @@ -4,7 +4,7 @@ $files = @("yellow_tripdata_2017-03.xlsx","yellow_tripdata_2017-03.csv") $files | ForEach-Object -Parallel { - az storage blob upload --file $using:pathbase/$_ --container-name "datalakeraw" --name samples/$_ --account-name $env:AdsOpts_CD_Services_Storage_ADLS_Name - az storage blob upload --file $using:pathbase/$_ --container-name "datalakeraw" --name samples/$_ --account-name $env:AdsOpts_CD_Services_Storage_Blob_Name + $result = az storage blob upload --file $using:pathbase/$_ --container-name "datalakeraw" --name samples/$_ --account-name $env:AdsOpts_CD_Services_Storage_ADLS_Name + $result = az storage blob upload --file $using:pathbase/$_ --container-name "datalakeraw" --name samples/$_ --account-name $env:AdsOpts_CD_Services_Storage_Blob_Name } diff --git a/solution/Deployment/workflows/Steps/CD_ConfigureVnet.ps1 b/solution/Deployment/workflows/Steps/CD_ConfigureVnet.ps1 index 5d6e6fbd..74f48029 100644 --- a/solution/Deployment/workflows/Steps/CD_ConfigureVnet.ps1 +++ b/solution/Deployment/workflows/Steps/CD_ConfigureVnet.ps1 @@ -3,16 +3,16 @@ if($env:AdsOpts_CD_Services_Vnet_Enable -eq "True") ############################################################################################################## #Firewall and Virtual Network Rules for Services ############################################################################################################## - Write-Debug"Configuring VNet rules for provisioned services" + Write-Debug " Configuring VNet rules for provisioned services" #Enable Service Endpoints on the subnet. Required to be done before adding network rules. - az network vnet subnet update --resource-group $env:AdsOpts_CD_ResourceGroup_Name --vnet-name $env:AdsOpts_CD_Services_Vnet_Name --name $env:AdsOpts_CD_Services_Vnet_DataSubnetName --service-endpoints Microsoft.Sql Microsoft.Storage Microsoft.KeyVault Microsoft.Web - Write-Debug"Configured Microsoft.Storage Service Endpoint to subnet $env:AdsOpts_CD_Services_Vnet_DataSubnetName" + $result = az network vnet subnet update --resource-group $env:AdsOpts_CD_ResourceGroup_Name --vnet-name $env:AdsOpts_CD_Services_Vnet_Name --name $env:AdsOpts_CD_Services_Vnet_DataSubnetName --service-endpoints Microsoft.Sql Microsoft.Storage Microsoft.KeyVault Microsoft.Web + Write-Debug " Configured Microsoft.Storage Service Endpoint to subnet $env:AdsOpts_CD_Services_Vnet_DataSubnetName" #adls - az storage account network-rule add --resource-group $env:AdsOpts_CD_ResourceGroup_Name --account-name $env:AdsOpts_CD_Services_Storage_ADLS_Name --vnet-name $env:AdsOpts_CD_Services_Vnet_Name --subnet $env:AdsOpts_CD_Services_Vnet_DataSubnetName - az storage account update --resource-group $env:AdsOpts_CD_ResourceGroup_Name --name $env:AdsOpts_CD_Services_Storage_ADLS_Name --default-action Deny #Default action to apply when no rule matches i.e. allow access from selected network/PE only. - Write-Debug"Completed network rule configuration for storage $env:AdsOpts_CD_Services_Storage_ADLS_Name" + $result = az storage account network-rule add --resource-group $env:AdsOpts_CD_ResourceGroup_Name --account-name $env:AdsOpts_CD_Services_Storage_ADLS_Name --vnet-name $env:AdsOpts_CD_Services_Vnet_Name --subnet $env:AdsOpts_CD_Services_Vnet_DataSubnetName + $result = az storage account update --resource-group $env:AdsOpts_CD_ResourceGroup_Name --name $env:AdsOpts_CD_Services_Storage_ADLS_Name --default-action Deny #Default action to apply when no rule matches i.e. allow access from selected network/PE only. + Write-Debug " Completed network rule configuration for storage $env:AdsOpts_CD_Services_Storage_ADLS_Name" #adlstran #Note: Skipping as it will be outside for importing data into it. @@ -21,40 +21,40 @@ if($env:AdsOpts_CD_Services_Vnet_Enable -eq "True") #Note: Commented below to allow App Insights access to Storage account. # az storage account network-rule add --resource-group $env:AdsOpts_CD_ResourceGroup_Name --account-name $env:AdsOpts_CD_Services_Storage_Logging_Name --vnet-name $env:AdsOpts_CD_Services_Vnet_Name --subnet $env:AdsOpts_CD_Services_Vnet_DataSubnetName # az storage account update --resource-group $env:AdsOpts_CD_ResourceGroup_Name --name $env:AdsOpts_CD_Services_Storage_Logging_Name --default-action Deny #Default action to apply when no rule matches i.e. allow access from selected network/PE only. - # Write-Debug"Completed network rule configuration for storage $env:AdsOpts_CD_Services_Storage_Logging_Name" + # Write-Debug " Completed network rule configuration for storage $env:AdsOpts_CD_Services_Storage_Logging_Name" #Blob - az storage account network-rule add --resource-group $env:AdsOpts_CD_ResourceGroup_Name --account-name $env:AdsOpts_CD_Services_Storage_Blob_Name --vnet-name $env:AdsOpts_CD_Services_Vnet_Name --subnet $env:AdsOpts_CD_Services_Vnet_DataSubnetName - az storage account update --resource-group $env:AdsOpts_CD_ResourceGroup_Name --name $env:AdsOpts_CD_Services_Storage_Blob_Name --default-action Deny #Default action to apply when no rule matches i.e. allow access from selected network/PE only. - Write-Debug"Completed network rule configuration for storage $env:AdsOpts_CD_Services_Storage_Blob_Name" + $result = az storage account network-rule add --resource-group $env:AdsOpts_CD_ResourceGroup_Name --account-name $env:AdsOpts_CD_Services_Storage_Blob_Name --vnet-name $env:AdsOpts_CD_Services_Vnet_Name --subnet $env:AdsOpts_CD_Services_Vnet_DataSubnetName + $result = az storage account update --resource-group $env:AdsOpts_CD_ResourceGroup_Name --name $env:AdsOpts_CD_Services_Storage_Blob_Name --default-action Deny #Default action to apply when no rule matches i.e. allow access from selected network/PE only. + Write-Debug " Completed network rule configuration for storage $env:AdsOpts_CD_Services_Storage_Blob_Name" #Azure Sql #Note: Commenting below as will use the PE for SQL, so vNet rule is not required. # az sql server vnet-rule create --server $env:AdsOpts_CD_Services_AzureSQLServer_Name --name $env:AdsOpts_CD_Services_Vnet_Name --resource-group $env:AdsOpts_CD_ResourceGroup_Name --vnet-name $env:AdsOpts_CD_Services_Vnet_Name --subnet $env:AdsOpts_CD_Services_Vnet_DataSubnetName - # Write-Debug"Completed network rule configuration for Azure SQL Server $env:AdsOpts_CD_Services_AzureSQLServer_Name" + # Write-Debug " Completed network rule configuration for Azure SQL Server $env:AdsOpts_CD_Services_AzureSQLServer_Name" #Key Vault #Note: Commenting below as will use the PE for AKV, so vNet rule is not required. # az keyvault network-rule add --name $env:AdsOpts_CD_Services_KeyVault_Name --resource-group $env:AdsOpts_CD_ResourceGroup_Name --vnet-name $env:AdsOpts_CD_Services_Vnet_Name --subnet $env:AdsOpts_CD_Services_Vnet_DataSubnetName # az keyvault update --name $env:AdsOpts_CD_Services_KeyVault_Name --resource-group $env:AdsOpts_CD_ResourceGroup_Name --default-action Deny #Default action to apply when no rule matches i.e. allow access from selected network/PE only. - # Write-Debug"Completed network rule configuration for Azure Key Vault $env:AdsOpts_CD_Services_KeyVault_Name" + # Write-Debug " Completed network rule configuration for Azure Key Vault $env:AdsOpts_CD_Services_KeyVault_Name" #Azure Function App - az appservice plan update --name $env:AdsOpts_CD_Services_AppPlans_FunctionApp_Name --resource-group $env:AdsOpts_CD_ResourceGroup_Name --sku P1V2 #Upgrade SKU to 'P1V2' to support PE and vNet Integration. - az functionapp vnet-integration add --resource-group $env:AdsOpts_CD_ResourceGroup_Name --name $env:AdsOpts_CD_Services_CoreFunctionApp_Name --vnet $env:AdsOpts_CD_Services_Vnet_Name --subnet $env:AdsOpts_CD_Services_Vnet_FuncAppSubnetName - az functionapp config access-restriction add --priority 100 --rule-name "Allow FuncApp Subnet" --resource-group $env:AdsOpts_CD_ResourceGroup_Name --name $env:AdsOpts_CD_Services_CoreFunctionApp_Name --vnet-name $env:AdsOpts_CD_Services_Vnet_Name --subnet $env:AdsOpts_CD_Services_Vnet_FuncAppSubnetName --description "Allow vNet" --action Allow #Configure Access Restrictions. - az functionapp config access-restriction add --priority 200 --rule-name "Allow Data Subnet" --resource-group $env:AdsOpts_CD_ResourceGroup_Name --name $env:AdsOpts_CD_Services_CoreFunctionApp_Name --vnet-name $env:AdsOpts_CD_Services_Vnet_Name --subnet $env:AdsOpts_CD_Services_Vnet_DataSubnetName --description "Allow vNet" --action Allow #Configure Access Restrictions. - az functionapp config access-restriction add --priority 300 --rule-name "Allow WebApp Subnet" --resource-group $env:AdsOpts_CD_ResourceGroup_Name --name $env:AdsOpts_CD_Services_CoreFunctionApp_Name --vnet-name $env:AdsOpts_CD_Services_Vnet_Name --subnet $env:AdsOpts_CD_Services_Vnet_WebAppSubnetName --description "Allow vNet" --action Allow #Configure Access Restrictions. - az functionapp config access-restriction add --priority 400 --rule-name "Allow Azure Portal" --resource-group $env:AdsOpts_CD_ResourceGroup_Name --name $env:AdsOpts_CD_Services_CoreFunctionApp_Name --service-tag AzurePortal --description "Allow vNet" --action Allow #Configure Access Restrictions. - az functionapp config access-restriction add --priority 500 --rule-name "Allow ADF" --resource-group $env:AdsOpts_CD_ResourceGroup_Name --name $env:AdsOpts_CD_Services_CoreFunctionApp_Name --service-tag DataFactory --description "Allow vNet" --action Allow #Configure Access Restrictions. - Write-Debug"Completed network rule configuration for Azure Function App $env:AdsOpts_CD_Services_CoreFunctionApp_Name" + $result = az appservice plan update --name $env:AdsOpts_CD_Services_AppPlans_FunctionApp_Name --resource-group $env:AdsOpts_CD_ResourceGroup_Name --sku P1V2 #Upgrade SKU to 'P1V2' to support PE and vNet Integration. + $result = az functionapp vnet-integration add --resource-group $env:AdsOpts_CD_ResourceGroup_Name --name $env:AdsOpts_CD_Services_CoreFunctionApp_Name --vnet $env:AdsOpts_CD_Services_Vnet_Name --subnet $env:AdsOpts_CD_Services_Vnet_FuncAppSubnetName + $result = az functionapp config access-restriction add --priority 100 --rule-name "Allow FuncApp Subnet" --resource-group $env:AdsOpts_CD_ResourceGroup_Name --name $env:AdsOpts_CD_Services_CoreFunctionApp_Name --vnet-name $env:AdsOpts_CD_Services_Vnet_Name --subnet $env:AdsOpts_CD_Services_Vnet_FuncAppSubnetName --description "Allow vNet" --action Allow #Configure Access Restrictions. + $result = az functionapp config access-restriction add --priority 200 --rule-name "Allow Data Subnet" --resource-group $env:AdsOpts_CD_ResourceGroup_Name --name $env:AdsOpts_CD_Services_CoreFunctionApp_Name --vnet-name $env:AdsOpts_CD_Services_Vnet_Name --subnet $env:AdsOpts_CD_Services_Vnet_DataSubnetName --description "Allow vNet" --action Allow #Configure Access Restrictions. + $result = az functionapp config access-restriction add --priority 300 --rule-name "Allow WebApp Subnet" --resource-group $env:AdsOpts_CD_ResourceGroup_Name --name $env:AdsOpts_CD_Services_CoreFunctionApp_Name --vnet-name $env:AdsOpts_CD_Services_Vnet_Name --subnet $env:AdsOpts_CD_Services_Vnet_WebAppSubnetName --description "Allow vNet" --action Allow #Configure Access Restrictions. + $result = az functionapp config access-restriction add --priority 400 --rule-name "Allow Azure Portal" --resource-group $env:AdsOpts_CD_ResourceGroup_Name --name $env:AdsOpts_CD_Services_CoreFunctionApp_Name --service-tag AzurePortal --description "Allow vNet" --action Allow #Configure Access Restrictions. + $result = az functionapp config access-restriction add --priority 500 --rule-name "Allow ADF" --resource-group $env:AdsOpts_CD_ResourceGroup_Name --name $env:AdsOpts_CD_Services_CoreFunctionApp_Name --service-tag DataFactory --description "Allow vNet" --action Allow #Configure Access Restrictions. + Write-Debug " Completed network rule configuration for Azure Function App $env:AdsOpts_CD_Services_CoreFunctionApp_Name" #Azure Web App - az appservice plan update --name $env:AdsOpts_CD_Services_AppPlans_WebApp_Name --resource-group $env:AdsOpts_CD_ResourceGroup_Name --sku P1V2 #Upgrade SKU to 'P1V2' to support PE. - az webapp vnet-integration add --resource-group $env:AdsOpts_CD_ResourceGroup_Name --name $env:AdsOpts_CD_Services_Website_Name --vnet $env:AdsOpts_CD_Services_Vnet_Name --subnet $env:AdsOpts_CD_Services_Vnet_WebAppSubnetName + $result = az appservice plan update --name $env:AdsOpts_CD_Services_AppPlans_WebApp_Name --resource-group $env:AdsOpts_CD_ResourceGroup_Name --sku P1V2 #Upgrade SKU to 'P1V2' to support PE. + $result = az webapp vnet-integration add --resource-group $env:AdsOpts_CD_ResourceGroup_Name --name $env:AdsOpts_CD_Services_Website_Name --vnet $env:AdsOpts_CD_Services_Vnet_Name --subnet $env:AdsOpts_CD_Services_Vnet_WebAppSubnetName #Note: Commented below to keep Web App accessible over internet. For customers who have VPN/Express Route/use Bastion -> Uncomment below. #az functionapp config access-restriction add --priority 100 --resource-group $env:AdsOpts_CD_ResourceGroup_Name --name $env:AdsOpts_CD_Services_Website_Name --vnet-name $env:AdsOpts_CD_Services_Vnet_Name --subnet $env:AdsOpts_CD_Services_Vnet_WebAppSubnetName --description "Allow vNet" --action Allow #Configure Access Restrictions. - Write-Debug"Completed network rule configuration for Azure Web App $env:AdsOpts_CD_Services_Website_Name" + Write-Debug " Completed network rule configuration for Azure Web App $env:AdsOpts_CD_Services_Website_Name" } else { @@ -66,7 +66,7 @@ if($env:AdsOpts_CD_Services_Vnet_Enable -eq "True") ############################################################################################################## #Private Endpoints for Services (ADF Managed and vNet Managed) ############################################################################################################## - Write-Debug"Configuring Private Endpoints for provisioned services" + Write-Debug " Configuring Private Endpoints for provisioned services" ########## ADF Managed PE @@ -78,48 +78,48 @@ if($env:AdsOpts_CD_Services_Vnet_Enable -eq "True") $managedPrivateEndpointName = "ADF-Managed-PE-"+$env:AdsOpts_CD_Services_Storage_ADLS_Name $privateEndpointResourceId = "$env:AdsOpts_CD_ResourceGroup_Id/providers/Microsoft.DataFactory/factories/$env:AdsOpts_CD_Services_DataFactory_Name/managedVirtualNetworks/default/managedprivateendpoints/${managedPrivateEndpointName}" $privateLinkResourceId = az storage account show --name $env:AdsOpts_CD_Services_Storage_ADLS_Name --resource-group $env:AdsOpts_CD_ResourceGroup_Name --query '[id][0]' --output tsv - Write-Debug"Creating $managedPrivateEndpointName" + Write-Debug " Creating $managedPrivateEndpointName" New-AzResource -Force -ApiVersion "${apiVersion}" -ResourceId "${privateEndpointResourceId}" -Properties @{ privateLinkResourceId = "${privateLinkResourceId}" groupId = "dfs" } - Write-Debug"Created $managedPrivateEndpointName" + Write-Debug " Created $managedPrivateEndpointName" #Blob (ADF Managed PE) $managedPrivateEndpointName = "ADF-Managed-PE-"+$env:AdsOpts_CD_Services_Storage_Blob_Name $privateEndpointResourceId = "$env:AdsOpts_CD_ResourceGroup_Id/providers/Microsoft.DataFactory/factories/$env:AdsOpts_CD_Services_DataFactory_Name/managedVirtualNetworks/default/managedprivateendpoints/${managedPrivateEndpointName}" $privateLinkResourceId = az storage account show --name $env:AdsOpts_CD_Services_Storage_Blob_Name --resource-group $env:AdsOpts_CD_ResourceGroup_Name --query '[id][0]' --output tsv - Write-Debug"Creating $managedPrivateEndpointName" + Write-Debug " Creating $managedPrivateEndpointName" New-AzResource -Force -ApiVersion "${apiVersion}" -ResourceId "${privateEndpointResourceId}" -Properties @{ privateLinkResourceId = "${privateLinkResourceId}" groupId = "blob" } - Write-Debug"Created $managedPrivateEndpointName" + Write-Debug " Created $managedPrivateEndpointName" #Note: Commented below to allow On-Prem SHIR to communicate with AKV (Non-VPN,Non-ExpressRoute, Non-Peering scenario) # #Key Vault (ADF Managed PE) # $managedPrivateEndpointName = "ADF-Managed-PE-"+$env:AdsOpts_CD_Services_KeyVault_Name # $privateEndpointResourceId = "$env:AdsOpts_CD_ResourceGroup_Id/providers/Microsoft.DataFactory/factories/$env:AdsOpts_CD_Services_DataFactory_Name/managedVirtualNetworks/default/managedprivateendpoints/${managedPrivateEndpointName}" # $privateLinkResourceId = az keyvault show --name $env:AdsOpts_CD_Services_KeyVault_Name --resource-group $env:AdsOpts_CD_ResourceGroup_Name --query '[id][0]' --output tsv - # Write-Debug"Creating $managedPrivateEndpointName" + # Write-Debug " Creating $managedPrivateEndpointName" # New-AzResource -Force -ApiVersion "${apiVersion}" -ResourceId "${privateEndpointResourceId}" -Properties @{ # privateLinkResourceId = "${privateLinkResourceId}" # groupId = "vault" # } - # Write-Debug"Created $managedPrivateEndpointName" + # Write-Debug " Created $managedPrivateEndpointName" ########## PEs #Note: Not using to allow SHIR to communicate with AKV (Non-VPN,Non-ExpressRoute, Peering scenario) #Key Vault (vNet Managed PE) - Write-Debug"Creating PE for Azure Key Vault" + Write-Debug " Creating PE for Azure Key Vault" $PE_Name = "PE-AKV" $Private_DNS_Zone_Name = "privatelink.vaultcore.azure.net" $id= az keyvault list --resource-group $env:AdsOpts_CD_ResourceGroup_Name --query '[].[id]' --output tsv #PE - az network private-endpoint create ` + $result = az network private-endpoint create ` --name $PE_Name ` --resource-group $env:AdsOpts_CD_ResourceGroup_Name ` --vnet-name $env:AdsOpts_CD_Services_Vnet_Name --subnet $env:AdsOpts_CD_Services_Vnet_DataSubnetName ` @@ -128,35 +128,35 @@ if($env:AdsOpts_CD_Services_Vnet_Enable -eq "True") --connection-name "PE-AKV-Connection" #Private DNS Zone - az network private-dns zone create ` + $result = az network private-dns zone create ` --resource-group $env:AdsOpts_CD_ResourceGroup_Name ` --name $Private_DNS_Zone_Name - az network private-dns link vnet create ` + $result = az network private-dns link vnet create ` --resource-group $env:AdsOpts_CD_ResourceGroup_Name ` --zone-name $Private_DNS_Zone_Name ` --name "DnsLink-AzKeyVault" ` --virtual-network $env:AdsOpts_CD_Services_Vnet_Name ` --registration-enabled false - az network private-endpoint dns-zone-group create ` + $result = az network private-endpoint dns-zone-group create ` --resource-group $env:AdsOpts_CD_ResourceGroup_Name ` --endpoint-name $PE_Name ` --name "AkvZoneGroup" ` --private-dns-zone $Private_DNS_Zone_Name ` --zone-name "akv" - Write-Debug"Completed PE creation for Azure Key Vault" + Write-Debug " Completed PE creation for Azure Key Vault" #Azure Sql (vNet Managed) - Write-Debug"Creating PE for Azure SQL Server" + Write-Debug " Creating PE for Azure SQL Server" $PE_Name = "PE-SQL" $Private_DNS_Zone_Name = "privatelink.database.windows.net" $id= az sql server list --resource-group $env:AdsOpts_CD_ResourceGroup_Name --query '[].[id]' --output tsv #PE - az network private-endpoint create ` + $result = az network private-endpoint create ` --name $PE_Name ` --resource-group $env:AdsOpts_CD_ResourceGroup_Name ` --vnet-name $env:AdsOpts_CD_Services_Vnet_Name --subnet $env:AdsOpts_CD_Services_Vnet_DataSubnetName ` @@ -165,35 +165,35 @@ if($env:AdsOpts_CD_Services_Vnet_Enable -eq "True") --connection-name "PE-SQL-Connection" #Private DNS Zone - az network private-dns zone create ` + $result = az network private-dns zone create ` --resource-group $env:AdsOpts_CD_ResourceGroup_Name ` --name $Private_DNS_Zone_Name - az network private-dns link vnet create ` + $result = az network private-dns link vnet create ` --resource-group $env:AdsOpts_CD_ResourceGroup_Name ` --zone-name $Private_DNS_Zone_Name ` --name "DnsLink-AzSQL" ` --virtual-network $env:AdsOpts_CD_Services_Vnet_Name ` --registration-enabled false - az network private-endpoint dns-zone-group create ` + $result = az network private-endpoint dns-zone-group create ` --resource-group $env:AdsOpts_CD_ResourceGroup_Name ` --endpoint-name $PE_Name ` --name "SqlZoneGroup" ` --private-dns-zone $Private_DNS_Zone_Name ` --zone-name "sql" - Write-Debug"Completed PE creation for Azure SQL Server" + Write-Debug " Completed PE creation for Azure SQL Server" ############################## #Post PE Addition Task - az sql server update --name $env:AdsOpts_CD_Services_AzureSQLServer_Name --resource-group $env:AdsOpts_CD_ResourceGroup_Name --enable-public-network false #Disable Public Access (Requires PE to be enabled first). - Write-Debug"Disabled Public Access to Azure SQL" + $result = az sql server update --name $env:AdsOpts_CD_Services_AzureSQLServer_Name --resource-group $env:AdsOpts_CD_ResourceGroup_Name --enable-public-network false #Disable Public Access (Requires PE to be enabled first). + Write-Debug " Disabled Public Access to Azure SQL" #Enable soft-delete on AKV - az keyvault update --name $env:AdsOpts_CD_Services_KeyVault_Name --enable-soft-delete true - Write-Debug"Soft-delete enabled on AKV" + $result = az keyvault update --name $env:AdsOpts_CD_Services_KeyVault_Name --enable-soft-delete true + Write-Debug " Soft-delete enabled on AKV" ############################## } diff --git a/solution/Deployment/workflows/Steps/CD_ConfigureWebApp.ps1 b/solution/Deployment/workflows/Steps/CD_ConfigureWebApp.ps1 index 1cf34829..80a2725f 100644 --- a/solution/Deployment/workflows/Steps/CD_ConfigureWebApp.ps1 +++ b/solution/Deployment/workflows/Steps/CD_ConfigureWebApp.ps1 @@ -1,4 +1,4 @@ -Write-Debug"Configuring Web App" +Write-Debug " Configuring Web App" $SourceFile = $env:AdsOpts_CD_FolderPaths_PublishZip + "/webapplication/Publish.zip" if($env:AdsOpts_CD_Services_WebSite_Enable -eq "True") @@ -16,7 +16,7 @@ if($env:AdsOpts_CD_Services_WebSite_Enable -eq "True") $AppInsightsWPId = (az monitor app-insights component show --app $env:AdsOpts_CD_Services_AppInsights_Name -g $env:AdsOpts_CD_ResourceGroup_Name | ConvertFrom-Json).appId $appSettings.ApplicationOptions.AppInsightsWorkspaceId = $AppInsightsWPId - $LogAnalyticsWorkspaceId = (az monitor log-analytics workspace show --workspace-name $env:AdsOpts_CD_Services_LogAnalytics_Name -g $env:AdsOpts_CD_ResourceGroup_Name | ConvertFrom-Json).customerId + $LogAnalyticsWorkspaceId = (az monitor log-analytics workspace show --workspace-name $env:AdsOpts_CD_Services_LogAnalytics_Name -g $env:AdsOpts_CD_ResourceGroup_Name | ConvertFrom-Json -AsHashtable).customerId $appSettings.ApplicationOptions.LogAnalyticsWorkspaceId = $LogAnalyticsWorkspaceId $appSettings.AzureAdAuth.Domain=$env:AdsOpts_CD_ResourceGroup_Domain @@ -35,7 +35,7 @@ if($env:AdsOpts_CD_Services_WebSite_Enable -eq "True") Set-Location $CurrentPath # Deploy Web App - az webapp deployment source config-zip --resource-group $env:AdsOpts_CD_ResourceGroup_Name --name $env:AdsOpts_CD_Services_WebSite_Name --src $SourceFile + $result = az webapp deployment source config-zip --resource-group $env:AdsOpts_CD_ResourceGroup_Name --name $env:AdsOpts_CD_Services_WebSite_Name --src $SourceFile #Enable App Insights #az resource create --resource-group $env:AdsOpts_CD_ResourceGroup_Name --resource-type "Microsoft.Insights/components" --name $env:AdsOpts_CD_Services_WebSite_Name --location $env:AdsOpts_CD_ResourceGroup_Location --properties '{\"Application_Type\":\"web\"}' diff --git a/solution/Deployment/workflows/Steps/CD_DeployADF.ps1 b/solution/Deployment/workflows/Steps/CD_DeployADF.ps1 index 525cb189..1099ae63 100644 --- a/solution/Deployment/workflows/Steps/CD_DeployADF.ps1 +++ b/solution/Deployment/workflows/Steps/CD_DeployADF.ps1 @@ -1,8 +1,8 @@ if ($env:AdsOpts_CD_Services_DataFactory_Enable -eq "True") { - Write-Debug"Creating Data Factory" - az deployment group create -g $env:AdsOpts_CD_ResourceGroup_Name --template-file ./../arm/DataFactory.json --parameters location=$env:AdsOpts_CD_ResourceGroup_Location adf-name=$env:AdsOpts_CD_Services_DataFactory_Name + Write-Debug " Creating Data Factory" + $result = az deployment group create -g $env:AdsOpts_CD_ResourceGroup_Name --template-file ./../arm/DataFactory.json --parameters location=$env:AdsOpts_CD_ResourceGroup_Location adf-name=$env:AdsOpts_CD_Services_DataFactory_Name } else { diff --git a/solution/Deployment/workflows/Steps/CD_DeployADFOnPremSHIR.ps1 b/solution/Deployment/workflows/Steps/CD_DeployADFOnPremSHIR.ps1 index 37cc000c..89b68b70 100644 --- a/solution/Deployment/workflows/Steps/CD_DeployADFOnPremSHIR.ps1 +++ b/solution/Deployment/workflows/Steps/CD_DeployADFOnPremSHIR.ps1 @@ -12,16 +12,16 @@ $ADFJDKDownloadURL = $env:AdsOpts_CD_Services_DataFactory_OnPremVnetIr_IrInstall $ADFJDKLocalFileName = $ADFJDKDownloadURL.Split("/")[$ADFJDKDownloadURL.Split("/").Length-1] #Get the .msi filename. $ADFJDKInstallerLocalFileLocation = $ADFLocalDrive + '\' + $ADFLocalVMFolder + '\' + $ADFJDKLocalFileName #Local Path of downloaded installer. -Write-Debug"Creating directory to download the SHIR installable." +Write-Debug " Creating directory to download the SHIR installable." New-Item -Path $ADFLocalDrive -Name $ADFLocalVMFolder -ItemType Directory -Force #'-Force' Ok if directory already exists. -Write-Debug"Downloading the SHIR installable at $ADFIRInstallerLocalFileLocation." +Write-Debug " Downloading the SHIR installable at $ADFIRInstallerLocalFileLocation." Invoke-WebRequest -Uri $ADFIRDownloadURL -OutFile $ADFIRInstallerLocalFileLocation #Download SHIR installable. -Write-Debug"Downloading the OpenJDK for SHIR at $ADFJDKInstallerLocalFileLocation." +Write-Debug " Downloading the OpenJDK for SHIR at $ADFJDKInstallerLocalFileLocation." Invoke-WebRequest -Uri $ADFJDKDownloadURL -OutFile $ADFJDKInstallerLocalFileLocation #Download OpenJDK. -Write-Debug"Installing OpenJDK." +Write-Debug " Installing OpenJDK." #msiexec /i $ADFJDKInstallerLocalFileLocation ADDLOCAL=FeatureMain,FeatureEnvironment,FeatureJarFileRunWith,FeatureJavaHome /quiet #Ensure command prompt is run as administrator @@ -34,17 +34,17 @@ $MSIInstallArguments = @( "/qb!" "/norestart" ) -Write-Debug$MSIInstallArguments +Write-Debug $MSIInstallArguments Start-Process "msiexec.exe" -ArgumentList $MSIInstallArguments -Wait -NoNewWindow -Write-Debug"Installing SHIR." +Write-Debug " Installing SHIR." # Data Factory - VM AZ IR - Install IR # $irKey1 = az datafactory integration-runtime list-auth-key --factory-name $DataFactoryName --name "SelfHostedIntegrationRuntime-Azure-VNET" --resource-group $ResourceGroupName --query authKey1 --out tsv # az vm run-command invoke --command-id RunPowerShellScript --name $VMAzIR -g $ResourceGroupName --scripts "$ADFIRInstallerLocalFileLocation -path $ADFIRLocalFileLocation -authKey '$irKey1'" # $irKey1 = az datafactory integration-runtime list-auth-key --factory-name $env:AdsOpts_CD_Services_DataFactory_Name --name $env:AdsOpts_CD_Services_DataFactory_OnPremVnetIr_Name --resource-group $env:AdsOpts_CD_ResourceGroup_Name --query authKey1 --out tsv -Write-Debug"irKey1 retrieved." +Write-Debug " irKey1 retrieved." # #Ensure command prompt is run as administrator # $MSIInstallArguments = @( @@ -54,7 +54,7 @@ Write-Debug"irKey1 retrieved." # "/qb!" # "/norestart" # ) -# Write-Debug$MSIInstallArguments +# Write-Debug $MSIInstallArguments # Start-Process "msiexec.exe" -ArgumentList $MSIInstallArguments -Wait -NoNewWindow . .\Steps\InstallGatewayFunctions.ps1 -path "$ADFIRInstallerLocalFileLocation" -authKey "$irKey1" \ No newline at end of file diff --git a/solution/Deployment/workflows/Steps/CD_DeployAppInsights.ps1 b/solution/Deployment/workflows/Steps/CD_DeployAppInsights.ps1 index 884bfa9b..1403813f 100644 --- a/solution/Deployment/workflows/Steps/CD_DeployAppInsights.ps1 +++ b/solution/Deployment/workflows/Steps/CD_DeployAppInsights.ps1 @@ -1,10 +1,10 @@ if ($env:AdsOpts_CD_Services_AppInsights_Enable -eq "True") { - Write-Debug"Creating App Insights" + Write-Debug " Creating App Insights" $storageaccountkey = (az storage account keys list -g $env:AdsOpts_CD_ResourceGroup_Name -n $env:AdsOpts_CD_Services_Storage_Logging_Name | ConvertFrom-Json)[0].value - az deployment group create -g $env:AdsOpts_CD_ResourceGroup_Name --template-file ./../arm/ApplicationInsights.json --parameters location=$env:AdsOpts_CD_ResourceGroup_Location appinsights_name=$env:AdsOpts_CD_Services_AppInsights_Name + $result = az deployment group create -g $env:AdsOpts_CD_ResourceGroup_Name --template-file ./../arm/ApplicationInsights.json --parameters location=$env:AdsOpts_CD_ResourceGroup_Location appinsights_name=$env:AdsOpts_CD_Services_AppInsights_Name } else { diff --git a/solution/Deployment/workflows/Steps/CD_DeployAppService.ps1 b/solution/Deployment/workflows/Steps/CD_DeployAppService.ps1 index 146f0c99..94d8a1bf 100644 --- a/solution/Deployment/workflows/Steps/CD_DeployAppService.ps1 +++ b/solution/Deployment/workflows/Steps/CD_DeployAppService.ps1 @@ -1,11 +1,11 @@ if ($env:AdsOpts_CD_Services_AppPlans_WebApp_Enable -eq "True") { - Write-Debug"Creating App Service for Web App" + Write-Debug " Creating App Service for Web App" #App Service (Includes both functions and web) $storageaccountkey = (az storage account keys list -g $env:AdsOpts_CD_ResourceGroup_Name -n $env:AdsOpts_CD_Services_Storage_Logging_Name | ConvertFrom-Json)[0].value - az deployment group create -g $env:AdsOpts_CD_ResourceGroup_Name --template-file ./../arm/AppService_Web.json --parameters location=$env:AdsOpts_CD_ResourceGroup_Location asp_name=$env:AdsOpts_CD_Services_AppPlans_WebApp_Name + $result = az deployment group create -g $env:AdsOpts_CD_ResourceGroup_Name --template-file ./../arm/AppService_Web.json --parameters location=$env:AdsOpts_CD_ResourceGroup_Location asp_name=$env:AdsOpts_CD_Services_AppPlans_WebApp_Name } else { @@ -15,11 +15,11 @@ else if ($env:AdsOpts_CD_Services_AppPlans_FunctionApp_Enable -eq "True") { - Write-Debug"Creating App Service for Function App" + Write-Debug " Creating App Service for Function App" #App Service (Includes both functions and web) $storageaccountkey = (az storage account keys list -g $env:AdsOpts_CD_ResourceGroup_Name -n $env:AdsOpts_CD_Services_Storage_Logging_Name | ConvertFrom-Json)[0].value - az deployment group create -g $env:AdsOpts_CD_ResourceGroup_Name --template-file ./../arm/AppService_Func.json --parameters location=$env:AdsOpts_CD_ResourceGroup_Location asp_name=$env:AdsOpts_CD_Services_AppPlans_FunctionApp_Name + $result = az deployment group create -g $env:AdsOpts_CD_ResourceGroup_Name --template-file ./../arm/AppService_Func.json --parameters location=$env:AdsOpts_CD_ResourceGroup_Location asp_name=$env:AdsOpts_CD_Services_AppPlans_FunctionApp_Name } else { diff --git a/solution/Deployment/workflows/Steps/CD_DeployAzureSqlServer.ps1 b/solution/Deployment/workflows/Steps/CD_DeployAzureSqlServer.ps1 index df1421f9..9ad0e882 100644 --- a/solution/Deployment/workflows/Steps/CD_DeployAzureSqlServer.ps1 +++ b/solution/Deployment/workflows/Steps/CD_DeployAzureSqlServer.ps1 @@ -1,17 +1,17 @@ -Write-Debug"Creating Azure SQL Server" -Write-Debug$env:AdsOpts_CD_Services_AzureSQLServer_Name -Write-Debug$env:AdsOpts_CD_Services_AzureSQLServer_SampleDB_Name -Write-Debug$env:AdsOpts_CD_Services_AzureSQLServer_AdsGoFastDB_Name -Write-Debug$env:AdsOpts_CD_Services_AzureSQLServer_AdminUser -Write-Debug$env:AdsOpts_CD_Services_AzureSQLServer_StagingDB_Name +Write-Debug " Creating Azure SQL Server" +Write-Debug $env:AdsOpts_CD_Services_AzureSQLServer_Name +Write-Debug $env:AdsOpts_CD_Services_AzureSQLServer_SampleDB_Name +Write-Debug $env:AdsOpts_CD_Services_AzureSQLServer_AdsGoFastDB_Name +Write-Debug $env:AdsOpts_CD_Services_AzureSQLServer_AdminUser +Write-Debug $env:AdsOpts_CD_Services_AzureSQLServer_StagingDB_Name if($env:AdsOpts_CD_Services_AzureSQLServer_Enable -eq "True") { - Write-Debug"Creating Azure SQL Server" + Write-Debug " Creating Azure SQL Server" #StorageAccount For Logging - az deployment group create -g $env:AdsOpts_CD_ResourceGroup_Name --template-file ./../arm/AzureSQLServer.json --parameters location=$env:AdsOpts_CD_ResourceGroup_Location sql_server_name=$env:AdsOpts_CD_Services_AzureSQLServer_Name sql_admin_login=$env:AdsOpts_CD_Services_AzureSQLServer_AdminUser sql_admin_password=$env:AdsOpts_CD_Services_AzureSQLServer_AdminPassword sample_db_name=$env:AdsOpts_CD_Services_AzureSQLServer_SampleDB_Name ads_go_fast_db_name=$env:AdsOpts_CD_Services_AzureSQLServer_AdsGoFastDB_Name staging_db_name=$env:AdsOpts_CD_Services_AzureSQLServer_StagingDB_Name vnet_name=$env:AdsOpts_CD_Services_Vnet_Name + $result = az deployment group create -g $env:AdsOpts_CD_ResourceGroup_Name --template-file ./../arm/AzureSQLServer.json --parameters location=$env:AdsOpts_CD_ResourceGroup_Location sql_server_name=$env:AdsOpts_CD_Services_AzureSQLServer_Name sql_admin_login=$env:AdsOpts_CD_Services_AzureSQLServer_AdminUser sql_admin_password=$env:AdsOpts_CD_Services_AzureSQLServer_AdminPassword sample_db_name=$env:AdsOpts_CD_Services_AzureSQLServer_SampleDB_Name ads_go_fast_db_name=$env:AdsOpts_CD_Services_AzureSQLServer_AdsGoFastDB_Name staging_db_name=$env:AdsOpts_CD_Services_AzureSQLServer_StagingDB_Name vnet_name=$env:AdsOpts_CD_Services_Vnet_Name #Make sure password is correct - az sql server update -n $env:AdsOpts_CD_Services_AzureSQLServer_Name -g $env:AdsOpts_CD_ResourceGroup_Name -p ($env:AdsOpts_CD_Services_AzureSQLServer_AdminPassword | Out-String) + $result = az sql server update -n $env:AdsOpts_CD_Services_AzureSQLServer_Name -g $env:AdsOpts_CD_ResourceGroup_Name -p ($env:AdsOpts_CD_Services_AzureSQLServer_AdminPassword | Out-String) } else { diff --git a/solution/Deployment/workflows/Steps/CD_DeployFunctionApp.ps1 b/solution/Deployment/workflows/Steps/CD_DeployFunctionApp.ps1 index 509d142e..1ff86481 100644 --- a/solution/Deployment/workflows/Steps/CD_DeployFunctionApp.ps1 +++ b/solution/Deployment/workflows/Steps/CD_DeployFunctionApp.ps1 @@ -1,5 +1,5 @@ -Write-Debug"Creating Function App" +Write-Debug " Creating Function App" if ($env:AdsOpts_CD_Services_CoreFunctionApp_Enable -eq "True") { if ($env:AdsOpts_CD_Services_AppPlans_FunctionApp_ResourceGroup -eq $null) @@ -18,7 +18,7 @@ if ($env:AdsOpts_CD_Services_CoreFunctionApp_Enable -eq "True") $key = az storage account keys list -g $env:AdsOpts_CD_ResourceGroup_Name -n $env:AdsOpts_CD_Services_Storage_Logging_Name --query [0].value -o tsv - az deployment group create -g $rg --template-file ./../arm/FunctionApp.json --parameters azure-function-site-name=$env:AdsOpts_CD_Services_CoreFunctionApp_Name app-insights-name=$env:AdsOpts_CD_Services_AppInsights_Name storage-log-account-name=$env:AdsOpts_CD_Services_Storage_Logging_Name storage-log-account-key=$key appservice-name=$sn + $result = az deployment group create -g $rg --template-file ./../arm/FunctionApp.json --parameters azure-function-site-name=$env:AdsOpts_CD_Services_CoreFunctionApp_Name app-insights-name=$env:AdsOpts_CD_Services_AppInsights_Name storage-log-account-name=$env:AdsOpts_CD_Services_Storage_Logging_Name storage-log-account-key=$key appservice-name=$sn } else { diff --git a/solution/Deployment/workflows/Steps/CD_DeployKeyVault.ps1 b/solution/Deployment/workflows/Steps/CD_DeployKeyVault.ps1 index 9f429ded..db02cff0 100644 --- a/solution/Deployment/workflows/Steps/CD_DeployKeyVault.ps1 +++ b/solution/Deployment/workflows/Steps/CD_DeployKeyVault.ps1 @@ -1,8 +1,8 @@ if ($env:AdsOpts_CD_Services_KeyVault_Enable -eq "True") { - Write-Debug"Creating Key Vault" - az deployment group create -g $env:AdsOpts_CD_ResourceGroup_Name --template-file ./../arm/KeyVault.json --parameters location=$env:AdsOpts_CD_ResourceGroup_Location keyvault-name=$env:AdsOpts_CD_Services_KeyVault_Name tenant-id=$env:AdsOpts_CD_ResourceGroup_TenantId + Write-Debug "Creating Key Vault" + $result = az deployment group create -g $env:AdsOpts_CD_ResourceGroup_Name --template-file ./../arm/KeyVault.json --parameters location=$env:AdsOpts_CD_ResourceGroup_Location keyvault-name=$env:AdsOpts_CD_Services_KeyVault_Name tenant-id=$env:AdsOpts_CD_ResourceGroup_TenantId } else { diff --git a/solution/Deployment/workflows/Steps/CD_DeployLogAnalytics.ps1 b/solution/Deployment/workflows/Steps/CD_DeployLogAnalytics.ps1 index 394b1052..25eef11d 100644 --- a/solution/Deployment/workflows/Steps/CD_DeployLogAnalytics.ps1 +++ b/solution/Deployment/workflows/Steps/CD_DeployLogAnalytics.ps1 @@ -1,8 +1,8 @@ if ($env:AdsOpts_CD_Services_DataFactory_Enable -eq "True") { - Write-Debug"Creating Log Analyticss" - az deployment group create -g $env:AdsOpts_CD_ResourceGroup_Name --template-file ./../arm/LogAnalytics.json --parameters location=$env:AdsOpts_CD_ResourceGroup_Location workspaces_adsgofastloganalytics_name=$env:AdsOpts_CD_Services_LogAnalytics_Name + Write-Debug " Creating Log Analyticss" + $results = az deployment group create -g $env:AdsOpts_CD_ResourceGroup_Name --template-file ./../arm/LogAnalytics.json --parameters location=$env:AdsOpts_CD_ResourceGroup_Location workspaces_adsgofastloganalytics_name=$env:AdsOpts_CD_Services_LogAnalytics_Name } else { diff --git a/solution/Deployment/workflows/Steps/CD_DeployStorageADLS.ps1 b/solution/Deployment/workflows/Steps/CD_DeployStorageADLS.ps1 index 7319fdbb..a1d1dd46 100644 --- a/solution/Deployment/workflows/Steps/CD_DeployStorageADLS.ps1 +++ b/solution/Deployment/workflows/Steps/CD_DeployStorageADLS.ps1 @@ -1,9 +1,9 @@ -Write-Debug"Creating Storage Account (ADLS) For Data Lake" +Write-Debug " Creating Storage Account (ADLS) For Data Lake" if($env:AdsOpts_CD_Services_Storage_ADLS_Enable -eq "True") { #StorageAccount For Logging - az deployment group create -g $env:AdsOpts_CD_ResourceGroup_Name --template-file ./../arm/Storage_ADLS.json --parameters location=$env:AdsOpts_CD_ResourceGroup_Location storage-account-name=$env:AdsOpts_CD_Services_Storage_ADLS_Name - Write-Debug"Creating Storage Account (ADLS) For Data Lake" + $result = az deployment group create -g $env:AdsOpts_CD_ResourceGroup_Name --template-file ./../arm/Storage_ADLS.json --parameters location=$env:AdsOpts_CD_ResourceGroup_Location storage-account-name=$env:AdsOpts_CD_Services_Storage_ADLS_Name + Write-Debug " Creating Storage Account (ADLS) For Data Lake" } else { @@ -13,9 +13,9 @@ else #Transient Storage Account if($env:AdsOpts_CD_Services_Storage_ADLSTransient_Enable -eq "True") { - Write-Debug"Creating Transient Storage Account (ADLS)" + Write-Debug " Creating Transient Storage Account (ADLS)" - az deployment group create -g $env:AdsOpts_CD_ResourceGroup_Name --template-file ./../arm/Storage_ADLS.json --parameters location=$env:AdsOpts_CD_ResourceGroup_Location storage-account-name=$env:AdsOpts_CD_Services_Storage_ADLSTransient_Name storage-raw-container-name=transient + $result = az deployment group create -g $env:AdsOpts_CD_ResourceGroup_Name --template-file ./../arm/Storage_ADLS.json --parameters location=$env:AdsOpts_CD_ResourceGroup_Location storage-account-name=$env:AdsOpts_CD_Services_Storage_ADLSTransient_Name storage-raw-container-name=transient } else { diff --git a/solution/Deployment/workflows/Steps/CD_DeployStorageBlob.ps1 b/solution/Deployment/workflows/Steps/CD_DeployStorageBlob.ps1 index 0aae0caa..f125d152 100644 --- a/solution/Deployment/workflows/Steps/CD_DeployStorageBlob.ps1 +++ b/solution/Deployment/workflows/Steps/CD_DeployStorageBlob.ps1 @@ -1,9 +1,9 @@ -Write-Debug"Creating Storage Account (Blob) For Data Lake" +Write-Debug " Creating Storage Account (Blob) For Data Lake" if($env:AdsOpts_CD_Services_Storage_Blob_Enable -eq "True") { #StorageAccount For Logging - az deployment group create -g $env:AdsOpts_CD_ResourceGroup_Name --template-file ./../arm/Storage_Blob.json --parameters location=$env:AdsOpts_CD_ResourceGroup_Location storage-account-name=$env:AdsOpts_CD_Services_Storage_Blob_Name - Write-Debug"Creating Storage Account (Blob) For Data Lake" + $result = az deployment group create -g $env:AdsOpts_CD_ResourceGroup_Name --template-file ./../arm/Storage_Blob.json --parameters location=$env:AdsOpts_CD_ResourceGroup_Location storage-account-name=$env:AdsOpts_CD_Services_Storage_Blob_Name + Write-Debug " Creating Storage Account (Blob) For Data Lake" } else { diff --git a/solution/Deployment/workflows/Steps/CD_DeployStorageForLogging.ps1 b/solution/Deployment/workflows/Steps/CD_DeployStorageForLogging.ps1 index 600ed29e..e724fc67 100644 --- a/solution/Deployment/workflows/Steps/CD_DeployStorageForLogging.ps1 +++ b/solution/Deployment/workflows/Steps/CD_DeployStorageForLogging.ps1 @@ -1,10 +1,10 @@ -Write-Debug"Creating Storage Account For Logging" -Write-Debug$env:AdsOpts_CD_Services_Storage_Logging_Name +Write-Debug " Creating Storage Account For Logging" +Write-Debug $env:AdsOpts_CD_Services_Storage_Logging_Name if($env:AdsOpts_CD_Services_Storage_Logging_Enable -eq "True") { #StorageAccount For Logging - az deployment group create -g $env:AdsOpts_CD_ResourceGroup_Name --template-file ./../arm/Storage_Logging.json --parameters location=$env:AdsOpts_CD_ResourceGroup_Location storage-log-account-name=$env:AdsOpts_CD_Services_Storage_Logging_Name - Write-Debug"Creating Storage Account For Logging" + $result = az deployment group create -g $env:AdsOpts_CD_ResourceGroup_Name --template-file ./../arm/Storage_Logging.json --parameters location=$env:AdsOpts_CD_ResourceGroup_Location storage-log-account-name=$env:AdsOpts_CD_Services_Storage_Logging_Name + Write-Debug " Creating Storage Account For Logging" } else { diff --git a/solution/Deployment/workflows/Steps/CD_DeployVnet.ps1 b/solution/Deployment/workflows/Steps/CD_DeployVnet.ps1 index 330c1450..f6553d6e 100644 --- a/solution/Deployment/workflows/Steps/CD_DeployVnet.ps1 +++ b/solution/Deployment/workflows/Steps/CD_DeployVnet.ps1 @@ -2,11 +2,11 @@ if($env:AdsOpts_CD_Services_Vnet_Enable -eq "True") { - Write-Debug"Creating Vnet + Subnets (Bastion, Data, WebApp)" - Write-Debug$env:AdsOpts_CD_Services_Vnet_Name + Write-Debug " Creating Vnet + Subnets (Bastion, Data, WebApp)" + Write-Debug $env:AdsOpts_CD_Services_Vnet_Name #vNet - az deployment group create -g $env:AdsOpts_CD_ResourceGroup_Name --template-file ./../arm/Networking.json --parameters location=$env:AdsOpts_CD_ResourceGroup_Location vnet-name=$env:AdsOpts_CD_Services_Vnet_Name vnet-address-prefix=$env:AdsOpts_CD_Services_Vnet_vNetAddressRange ` + $result = az deployment group create -g $env:AdsOpts_CD_ResourceGroup_Name --template-file ./../arm/Networking.json --parameters location=$env:AdsOpts_CD_ResourceGroup_Location vnet-name=$env:AdsOpts_CD_Services_Vnet_Name vnet-address-prefix=$env:AdsOpts_CD_Services_Vnet_vNetAddressRange ` bastion-subnet-ip-prefix=$env:AdsOpts_CD_Services_Vnet_BastionSubnetAddressRange ` data-subnet-ip-prefix=$env:AdsOpts_CD_Services_Vnet_DataSubnetAddressRange ` webapp-subnet-ip-prefix=$env:AdsOpts_CD_Services_Vnet_WebAppSubnetAddressRange ` @@ -17,7 +17,7 @@ if($env:AdsOpts_CD_Services_Vnet_Enable -eq "True") webapp-subnet-name=$env:AdsOpts_CD_Services_Vnet_WebAppSubnetName ` funcapp-subnet-name=$env:AdsOpts_CD_Services_Vnet_FuncAppSubnetName - Write-Debug"Creating Vnet" + Write-Debug " Creating Vnet" } else { diff --git a/solution/Deployment/workflows/Steps/CD_DeployWebSite.ps1 b/solution/Deployment/workflows/Steps/CD_DeployWebSite.ps1 index eec85573..7f92a8c4 100644 --- a/solution/Deployment/workflows/Steps/CD_DeployWebSite.ps1 +++ b/solution/Deployment/workflows/Steps/CD_DeployWebSite.ps1 @@ -13,8 +13,8 @@ if ($env:AdsOpts_CD_Services_WebSite_Enable -eq "True") $sn = $env:AdsOpts_CD_Services_AppPlans_WebApp_Name - Write-Debug"Deploying Wesite to $sn in resource group $rg" - az deployment group create -g $rg --template-file ./../arm/WebApp.json --parameters resource_group_name=$rg sites_AdsGoFastWebApp_name=$env:AdsOpts_CD_Services_WebSite_Name appservice_name=$sn} + Write-Debug " Deploying Wesite to $sn in resource group $rg" + $result = az deployment group create -g $rg --template-file ./../arm/WebApp.json --parameters resource_group_name=$rg sites_AdsGoFastWebApp_name=$env:AdsOpts_CD_Services_WebSite_Name appservice_name=$sn} else { Write-Warning "Skipped Creation of Web Site" diff --git a/solution/Deployment/workflows/Steps/CD_GrantRBAC.ps1 b/solution/Deployment/workflows/Steps/CD_GrantRBAC.ps1 index e17e8564..e27ba05a 100644 --- a/solution/Deployment/workflows/Steps/CD_GrantRBAC.ps1 +++ b/solution/Deployment/workflows/Steps/CD_GrantRBAC.ps1 @@ -38,7 +38,7 @@ #Transient Storage Account if($env:AdsOpts_CD_Services_Storage_ADLSTransient_Enable -eq "True") { - Write-Debug"Granting RBAC on Transient Storage Account (ADLS)" + Write-Debug " Granting RBAC on Transient Storage Account (ADLS)" # MSI Access from AF to ADLS Gen2 $result = az role assignment create --assignee $AzureFunctionId --role "Storage Blob Data Contributor" --scope "$basescope/Microsoft.Storage/storageAccounts/$env:AdsOpts_CD_Services_Storage_ADLSTransient_Name" diff --git a/solution/Deployment/workflows/Steps/CD_GrantWebAppAccess.ps1 b/solution/Deployment/workflows/Steps/CD_GrantWebAppAccess.ps1 index 50458483..b1ea53ae 100644 --- a/solution/Deployment/workflows/Steps/CD_GrantWebAppAccess.ps1 +++ b/solution/Deployment/workflows/Steps/CD_GrantWebAppAccess.ps1 @@ -18,4 +18,4 @@ $body.appRoleId = ($authapp.appRoles | Where-Object {$_.value -eq "Administrato $body.principalId = $callinguser $body = ($body | ConvertTo-Json -compress | Out-String).Replace('"','\"') -az rest --method post --uri "https://graph.microsoft.com/v1.0/servicePrincipals/$authappobjectid/appRoleAssignedTo" --headers '{\"Content-Type\":\"application/json\"}' --body $body +$result = az rest --method post --uri "https://graph.microsoft.com/v1.0/servicePrincipals/$authappobjectid/appRoleAssignedTo" --headers '{\"Content-Type\":\"application/json\"}' --body $body diff --git a/solution/Deployment/workflows/Steps/InstallGatewayFunctions.ps1 b/solution/Deployment/workflows/Steps/InstallGatewayFunctions.ps1 index 1892fe72..135b11e5 100644 --- a/solution/Deployment/workflows/Steps/InstallGatewayFunctions.ps1 +++ b/solution/Deployment/workflows/Steps/InstallGatewayFunctions.ps1 @@ -8,20 +8,20 @@ function Install-Gateway([string] $gwPath) # uninstall any existing gateway UnInstall-Gateway - Write-Debug"Start Gateway installation" + Write-Debug " Start Gateway installation" Start-Process "msiexec.exe" "/i $path /quiet /passive" -Wait Start-Sleep -Seconds 30 - Write-Debug"Succeed to install gateway" + Write-Debug " Succeed to install gateway" } function Register-Gateway([string] $key) { - Write-Debug"Start to register gateway with key: $key" + Write-Debug " Start to register gateway with key: $key" $cmd = Get-CmdFilePath Start-Process $cmd "-k $key" -Wait - Write-Debug"Succeed to register gateway" + Write-Debug " Succeed to register gateway" } @@ -58,11 +58,11 @@ function UnInstall-Gateway() if ($installed -eq $false) { - Write-Debug"Microsoft Integration Runtime Preview is not installed." + Write-Debug " Microsoft Integration Runtime Preview is not installed." return } - Write-Debug"Microsoft Integration Runtime has been uninstalled from this machine." + Write-Debug " Microsoft Integration Runtime has been uninstalled from this machine." } function Get-CmdFilePath() diff --git a/solution/TaskTypeJson/AZ_SQL_AZ_Storage_Parquet_IRA.json b/solution/TaskTypeJson/AZ_SQL_AZ_Storage_Parquet_IRA.json index b4b5feb5..eca92786 100644 --- a/solution/TaskTypeJson/AZ_SQL_AZ_Storage_Parquet_IRA.json +++ b/solution/TaskTypeJson/AZ_SQL_AZ_Storage_Parquet_IRA.json @@ -71,11 +71,8 @@ "options": { "inputAttributes": { "placeholder": 0 - }, - "dependencies": { - "Type": "ChunkField" - }, - "infoText": "Number of rows to use for each 'chunk' of data" + }, + "infoText": "Number of rows to use for each 'chunk' of data." }, "default": 0 } diff --git a/solution/WebApplication/WebApplication.Tests/WebApplication.Tests.csproj b/solution/WebApplication/WebApplication.Tests/WebApplication.Tests.csproj index 48748405..295d6f47 100644 --- a/solution/WebApplication/WebApplication.Tests/WebApplication.Tests.csproj +++ b/solution/WebApplication/WebApplication.Tests/WebApplication.Tests.csproj @@ -4,6 +4,7 @@ netcoreapp3.1 false + false