From 66ad02966edc68347bf748c92f6103a66d3edf19 Mon Sep 17 00:00:00 2001 From: John Rampono Date: Sun, 21 Nov 2021 19:55:34 +0800 Subject: [PATCH] Major changes in this version including: - Complete refactor of ADF Code Components to improve naming conventions and consistency - Addition of Node Web App to facilitate configuration editing On branch development/0.1 Changes to be committed: modified: README.md new file: documentation/images/DeploymentForm.png renamed: solution/DataFactory/dataset/ADLSBinary_AZ_IR.json -> solution/DataFactory/ADF/dataset/ADLSBinary_IRA.json renamed: solution/DataFactory/dataset/ADLSDelimitedText_SH_IR.json -> solution/DataFactory/ADF/dataset/ADLSDelimitedText_IRA.json renamed: solution/DataFactory/dataset/ADLSExcel_SH_IR.json -> solution/DataFactory/ADF/dataset/ADLSExcel_IRA.json renamed: solution/DataFactory/dataset/ADLSJson_SH_IR.json -> solution/DataFactory/ADF/dataset/ADLSJson_IRA.json renamed: solution/DataFactory/dataset/ADLSParquet_AZ_IR.json -> solution/DataFactory/ADF/dataset/ADLSParquet_IRA.json renamed: solution/DataFactory/dataset/AzureSqlTable_AZ_IR.json -> solution/DataFactory/ADF/dataset/AzureSqlTable_IRA.json renamed: solution/DataFactory/dataset/BlobBinary_AZ_IR.json -> solution/DataFactory/ADF/dataset/BlobBinary_IRA.json renamed: solution/DataFactory/dataset/BlobDelimitedText_SH_IR.json -> solution/DataFactory/ADF/dataset/BlobDelimitedText_IRA.json renamed: solution/DataFactory/dataset/BlobExcel_SH_IR.json -> solution/DataFactory/ADF/dataset/BlobExcel_IRA.json renamed: solution/DataFactory/dataset/BlobJson_SH_IR.json -> solution/DataFactory/ADF/dataset/BlobJson_IRA.json renamed: solution/DataFactory/dataset/BlobParquet_AZ_IR.json -> solution/DataFactory/ADF/dataset/BlobParquet_IRA.json renamed: solution/DataFactory/dataset/FileBinary_SH_IR.json -> solution/DataFactory/ADF/dataset/FileBinary_IRA.json renamed: solution/DataFactory/dataset/FileParquet_SH_IR.json -> solution/DataFactory/ADF/dataset/FileParquet_IRA.json renamed: solution/DataFactory/dataset/SqlServerTable_SH_IR.json -> solution/DataFactory/ADF/dataset/SqlServerTable_IRA.json renamed: solution/DataFactory/factory/adsgofastdatakakeacceladf.json -> solution/DataFactory/ADF/factory/adsgofastdatakakeacceladf.json new file: solution/DataFactory/ADF/integrationRuntime/IRA.json renamed: solution/DataFactory/linkedService/AdsGoFastKeyVault.json -> solution/DataFactory/ADF/linkedService/AdsGoFastKeyVault.json renamed: solution/DataFactory/linkedService/AzureFunctionAdsGoFastDataLakeAccelFunApp.json -> solution/DataFactory/ADF/linkedService/AzureFunctionAdsGoFastDataLakeAccelFunApp.json renamed: solution/DataFactory/linkedService/GenericAzureDataLakeStorage_SH_IR.json -> solution/DataFactory/ADF/linkedService/GenericAzureDataLakeStorage_IRA.json renamed: solution/DataFactory/linkedService/GenericAzureKeyVault.json -> solution/DataFactory/ADF/linkedService/GenericAzureKeyVault_IRA.json renamed: solution/DataFactory/linkedService/GenericAzureSQL_SH_IR.json -> solution/DataFactory/ADF/linkedService/GenericAzureSQL_IRA.json renamed: solution/DataFactory/linkedService/GenericBlobStorage_SH_IR.json -> solution/DataFactory/ADF/linkedService/GenericBlobStorage_IRA.json renamed: solution/DataFactory/linkedService/GenericFileServer_SH_IR.json -> solution/DataFactory/ADF/linkedService/GenericFileServer_IRA.json renamed: solution/DataFactory/linkedService/GenericMsSqlServer_SH_IR.json -> solution/DataFactory/ADF/linkedService/GenericMsSqlServer_IRA.json new file: solution/DataFactory/ADF/pipeline/AZ_Function_Generic.json renamed: solution/DataFactory/pipeline/AZ-SQL-AZ-Storage-Parquet-SH-IR.json -> solution/DataFactory/ADF/pipeline/AZ_SQL_AZ_Storage_Parquet_IRA.json renamed: solution/DataFactory/pipeline/AZ-SQL-Create-Table-SH-IR.json -> solution/DataFactory/ADF/pipeline/AZ_SQL_Create_Table_IRA.json renamed: solution/DataFactory/pipeline/AZ-SQL-Full-Load-Chunk-SH-IR.json -> solution/DataFactory/ADF/pipeline/AZ_SQL_Full_Load_Chunk_IRA.json renamed: solution/DataFactory/pipeline/AZ-SQL-Full-Load-SH-IR.json -> solution/DataFactory/ADF/pipeline/AZ_SQL_Full_Load_IRA.json renamed: solution/DataFactory/pipeline/AZ-SQL-Post-Copy-SH-IR.json -> solution/DataFactory/ADF/pipeline/AZ_SQL_Post_Copy_IRA.json renamed: solution/DataFactory/pipeline/AZ-SQL-StoredProcedure-SH-IR.json -> solution/DataFactory/ADF/pipeline/AZ_SQL_StoredProcedure_IRA.json renamed: solution/DataFactory/pipeline/AZ-SQL-Watermark-Chunk-SH-IR.json -> solution/DataFactory/ADF/pipeline/AZ_SQL_Watermark_Chunk_IRA.json renamed: solution/DataFactory/pipeline/AZ-SQL-Watermark-SH-IR.json -> solution/DataFactory/ADF/pipeline/AZ_SQL_Watermark_IRA.json renamed: solution/DataFactory/pipeline/AZ-Storage-Binary-AZ-Storage-Binary-SH-IR.json -> solution/DataFactory/ADF/pipeline/AZ_Storage_Binary_AZ_Storage_Binary_IRA.json renamed: solution/DataFactory/pipeline/AZ-Storage-CSV-AZ-SQL-SH-IR.json -> solution/DataFactory/ADF/pipeline/AZ_Storage_CSV_AZ_SQL_IRA.json renamed: solution/DataFactory/pipeline/AZ-Storage-Excel-AZ-SQL-SH-IR.json -> solution/DataFactory/ADF/pipeline/AZ_Storage_Excel_AZ_SQL_IRA.json renamed: solution/DataFactory/pipeline/AZ-Storage-Excel-AZ-Storage-CSV-SH-IR.json -> solution/DataFactory/ADF/pipeline/AZ_Storage_Excel_AZ_Storage_CSV_IRA.json renamed: solution/DataFactory/pipeline/AZ-Storage-JSON-AZ-SQL-SH-IR.json -> solution/DataFactory/ADF/pipeline/AZ_Storage_JSON_AZ_SQL_IRA.json renamed: solution/DataFactory/pipeline/AZ-Storage-Parquet-AZ-SQL-SH-IR.json -> solution/DataFactory/ADF/pipeline/AZ_Storage_Parquet_AZ_SQL_IRA.json renamed: solution/DataFactory/pipeline/Create-Task-Master-AZ-SQL-SH-IR.json -> solution/DataFactory/ADF/pipeline/Create_Task_Master_AZ_SQL_IRA.json renamed: solution/DataFactory/pipeline/GEN-File-Binary-AZ-Storage-Binary-SH-IR.json -> solution/DataFactory/ADF/pipeline/GEN_File_Binary_AZ_Storage_Binary_IRA.json renamed: solution/DataFactory/pipeline/Master.json -> solution/DataFactory/ADF/pipeline/Master_IRA.json renamed: solution/DataFactory/pipeline/SH-SQL-AZ-Storage-Parquet-SH-IR.json -> solution/DataFactory/ADF/pipeline/SH_SQL_AZ_Storage_Parquet_IRA.json renamed: solution/DataFactory/pipeline/SH-SQL-Full-Load-Chunk-SH-IR.json -> solution/DataFactory/ADF/pipeline/SH_SQL_Full_Load_Chunk_IRA.json renamed: solution/DataFactory/pipeline/SH-SQL-Full-Load-SH-IR.json -> solution/DataFactory/ADF/pipeline/SH_SQL_Full_Load_IRA.json renamed: solution/DataFactory/pipeline/OnP-SQL-GEN-File-Parquet-OnP-SH-IR.json -> solution/DataFactory/ADF/pipeline/SH_SQL_GEN_File_Parquet_IRA.json renamed: solution/DataFactory/pipeline/SH-SQL-Watermark-Chunk-SH-IR.json -> solution/DataFactory/ADF/pipeline/SH_SQL_Watermark_Chunk_IRA.json renamed: solution/DataFactory/pipeline/SH-SQL-Watermark-SH-IR.json -> solution/DataFactory/ADF/pipeline/SH_SQL_Watermark_IRA.json renamed: solution/DataFactory/trigger/Trigger on File End.json -> solution/DataFactory/ADF/trigger/Trigger on File End.json new file: solution/DataFactory/GenerateADFArtefacts.ps1 new file: solution/DataFactory/IRConfig.json renamed: solution/DataFactory/dataset/ADLSBinary_SH_IR.json -> solution/DataFactory/Templates/dataset/ADLSBinary_{IR}.json new file: solution/DataFactory/Templates/dataset/ADLSDelimitedText_{IR}.json new file: solution/DataFactory/Templates/dataset/ADLSExcel_{IR}.json new file: solution/DataFactory/Templates/dataset/ADLSJson_{IR}.json renamed: solution/DataFactory/dataset/ADLSParquet_SH_IR.json -> solution/DataFactory/Templates/dataset/ADLSParquet_{IR}.json renamed: solution/DataFactory/dataset/AzureSqlTable_SH_IR.json -> solution/DataFactory/Templates/dataset/AzureSqlTable_{IR}.json renamed: solution/DataFactory/dataset/BlobBinary_SH_IR.json -> solution/DataFactory/Templates/dataset/BlobBinary_{IR}.json new file: solution/DataFactory/Templates/dataset/BlobDelimitedText_{IR}.json new file: solution/DataFactory/Templates/dataset/BlobExcel_{IR}.json new file: solution/DataFactory/Templates/dataset/BlobJson_{IR}.json renamed: solution/DataFactory/dataset/BlobParquet_SH_IR.json -> solution/DataFactory/Templates/dataset/BlobParquet_{IR}.json renamed: solution/DataFactory/dataset/FileBinary_OnPrem_SH_IR.json -> solution/DataFactory/Templates/dataset/FileBinary_{IR}.json renamed: solution/DataFactory/dataset/FileParquet_OnPrem_SH_IR.json -> solution/DataFactory/Templates/dataset/FileParquet_{IR}.json renamed: solution/DataFactory/dataset/SqlServerTable_OnPrem_SH_IR.json -> solution/DataFactory/Templates/dataset/SqlServerTable_{IR}.json new file: solution/DataFactory/Templates/factory/adsgofastdatakakeacceladf.json new file: solution/DataFactory/Templates/integrationRuntime/IRA.json new file: solution/DataFactory/Templates/integrationRuntime/IRB.json renamed: solution/DataFactory/linkedService/GenericAzureDataLakeStorage_OnPrem_Net.json -> solution/DataFactory/Templates/linkedService/GenericAzureDataLakeStorage_{IR}.json new file: solution/DataFactory/Templates/linkedService/GenericAzureKeyVault_{IR}.json renamed: solution/DataFactory/linkedService/GenericAzureSQL.json -> solution/DataFactory/Templates/linkedService/GenericAzureSQL_{IR}.json renamed: solution/DataFactory/linkedService/GenericBlobStorage_OnPrem_Net.json -> solution/DataFactory/Templates/linkedService/GenericBlobStorage_{IR}.json renamed: solution/DataFactory/linkedService/GenericFileServer_OnPrem_Net.json -> solution/DataFactory/Templates/linkedService/GenericFileServer_{IR}.json renamed: solution/DataFactory/linkedService/GenericMsSqlServer_OnPrem_Net.json -> solution/DataFactory/Templates/linkedService/GenericMsSqlServer_{IR}.json new file: solution/DataFactory/Templates/linkedService/NamingConventions.md renamed: solution/DataFactory/pipeline/AZ-Function-Generic.json -> solution/DataFactory/Templates/pipeline/AZ_Function_Generic.json new file: solution/DataFactory/Templates/pipeline/AZ_SQL_AZ_Storage_Parquet_{IR}.json new file: solution/DataFactory/Templates/pipeline/AZ_SQL_Create_Table_{IR}.json renamed: solution/DataFactory/pipeline/OnP-SQL-Full-Load-Chunk-OnP-SH-IR.json -> solution/DataFactory/Templates/pipeline/AZ_SQL_Full_Load_Chunk_{IR}.json new file: solution/DataFactory/Templates/pipeline/AZ_SQL_Full_Load_{IR}.json new file: solution/DataFactory/Templates/pipeline/AZ_SQL_Post_Copy_{IR}.json new file: solution/DataFactory/Templates/pipeline/AZ_SQL_StoredProcedure_{IR}.json renamed: solution/DataFactory/pipeline/OnP-SQL-Watermark-Chunk-OnP-SH-IR.json -> solution/DataFactory/Templates/pipeline/AZ_SQL_Watermark_Chunk_{IR}.json new file: solution/DataFactory/Templates/pipeline/AZ_SQL_Watermark_{IR}.json renamed: solution/DataFactory/pipeline/AZ-Storage-Binary-AZ-Storage-Binary-AZ-IR.json -> solution/DataFactory/Templates/pipeline/AZ_Storage_Binary_AZ_Storage_Binary_{IR}.json new file: solution/DataFactory/Templates/pipeline/AZ_Storage_CSV_AZ_SQL_{IR}.json new file: solution/DataFactory/Templates/pipeline/AZ_Storage_Excel_AZ_SQL_{IR}.json new file: solution/DataFactory/Templates/pipeline/AZ_Storage_Excel_AZ_Storage_CSV_{IR}.json new file: solution/DataFactory/Templates/pipeline/AZ_Storage_JSON_AZ_SQL_{IR}.json new file: solution/DataFactory/Templates/pipeline/AZ_Storage_Parquet_AZ_SQL_{IR}.json renamed: solution/DataFactory/pipeline/Create-Task-Master-AZ-SQL-OnP-SH-IR.json -> solution/DataFactory/Templates/pipeline/Create_Task_Master_AZ_SQL_{IR}.json renamed: solution/DataFactory/pipeline/GEN-File-Binary-AZ-Storage-Binary-OnP-SH-IR.json -> solution/DataFactory/Templates/pipeline/GEN_File_Binary_AZ_Storage_Binary_{IR}.json new file: solution/DataFactory/Templates/pipeline/Master_{IR}.json renamed: solution/DataFactory/pipeline/OnP-SQL-AZ-Storage-Parquet-OnP-SH-IR.json -> solution/DataFactory/Templates/pipeline/SH_SQL_AZ_Storage_Parquet_{IR}.json new file: solution/DataFactory/Templates/pipeline/SH_SQL_Full_Load_Chunk_{IR}.json renamed: solution/DataFactory/pipeline/OnP-SQL-Full-Load-OnP-SH-IR.json -> solution/DataFactory/Templates/pipeline/SH_SQL_Full_Load_{IR}.json new file: solution/DataFactory/Templates/pipeline/SH_SQL_GEN_File_Parquet_{IR}.json new file: solution/DataFactory/Templates/pipeline/SH_SQL_Watermark_Chunk_{IR}.json renamed: solution/DataFactory/pipeline/OnP-SQL-Watermark-OnP-SH-IR.json -> solution/DataFactory/Templates/pipeline/SH_SQL_Watermark_{IR}.json new file: solution/DataFactory/Templates/trigger/Trigger on File End.json deleted: solution/DataFactory/dataset/ADLSBinary_OnPrem_SH_IR.json deleted: solution/DataFactory/dataset/ADLSParquet_OnPrem_SH_IR.json deleted: solution/DataFactory/dataset/BlobBinary_OnPrem_SH_IR.json deleted: solution/DataFactory/dataset/BlobParquet_OnPrem_SH_IR.json deleted: solution/DataFactory/integrationRuntime/SelfHostedIntegrationRuntime-Azure-VNET.json deleted: solution/DataFactory/integrationRuntime/SelfHostedIntegrationRuntime-OnPem-Net.json deleted: solution/DataFactory/linkedService/GenericAzureDataLakeStorage.json deleted: solution/DataFactory/linkedService/GenericBlobStorage.json deleted: solution/DataFactory/pipeline/AZ-SQL-AZ-Storage-Parquet-AZ-IR.json deleted: solution/DataFactory/pipeline/AZ-Storage-Parquet-AZ-SQL-AZ-IR.json modified: solution/Database/ADSGoFastDbUp/AdsGoFastDbUp/0-1-0-beta/A-Journaled/006-SampleTasks_TaskMaster.sql modified: solution/Database/ADSGoFastDbUp/AdsGoFastDbUp/0-1-0-beta/A-Journaled/008-SampleTasks_SourceAndTargetSystems.sql new file: solution/Database/ADSGoFastDbUp/AdsGoFastDbUp/0-1-0-beta/A-Journaled/010-TaskTypeMapping_Update.sql modified: solution/Deployment/.devcontainer/devcontainer.json new file: solution/Deployment/arm/AppService_Func.bicep new file: solution/Deployment/arm/AppService_Web.bicep new file: solution/Deployment/arm/ApplicationInsights.bicep modified: solution/Deployment/arm/AzureSQLServer.json new file: solution/Deployment/arm/AzureSqlServer.bicep new file: solution/Deployment/arm/DataFactory.bicep new file: solution/Deployment/arm/FunctionApp.bicep new file: solution/Deployment/arm/KeyVault.bicep new file: solution/Deployment/arm/LogAnalytics.bicep new file: solution/Deployment/arm/Networking.bicep new file: solution/Deployment/arm/Storage_ADLS.bicep new file: solution/Deployment/arm/Storage_Blob.bicep new file: solution/Deployment/arm/Storage_Logging.bicep new file: solution/Deployment/arm/VirtualMachine.bicep new file: solution/Deployment/arm/WebApp.bicep modified: solution/Deployment/arm/WebApp.json new file: solution/Deployment/environments/EditSettings.html new file: solution/Deployment/environments/Node/package-lock.json new file: solution/Deployment/environments/Node/package.json new file: solution/Deployment/environments/Node/server.js modified: solution/Deployment/environments/development.json new file: solution/Deployment/environments/development_bak.json new file: solution/Deployment/environments/development_test.json modified: solution/Deployment/environments/environment.schema.json modified: solution/Deployment/workflows/CD_0a_CreateServicePrincipals_AAD_Elevated.ps1 modified: solution/Deployment/workflows/CD_1a_DeployServices.ps1 modified: solution/Deployment/workflows/CD_2a_CreateMSIs_AAD_Elevated.ps1 modified: solution/Deployment/workflows/CD_2b_ConfigureServices.ps1 modified: solution/Deployment/workflows/LocalDevOnly_EnvironmentSetUp.ps1 modified: solution/Deployment/workflows/Steps/CD_ConfigureADF.ps1 modified: solution/Deployment/workflows/Steps/CD_ConfigureAzureSQLServer.ps1 modified: solution/Deployment/workflows/Steps/CD_ConfigureAzureSqlServer_UpdateTaskTypeMappingJson.ps1 modified: solution/Deployment/workflows/Steps/CD_ConfigureFunctionApp.ps1 modified: solution/Deployment/workflows/Steps/CD_ConfigureKeyVault.ps1 new file: solution/Deployment/workflows/Steps/CD_ConfigureSampleData.ps1 modified: solution/Deployment/workflows/Steps/CD_ConfigureVnet.ps1 modified: solution/Deployment/workflows/Steps/CD_ConfigureWebApp.ps1 modified: solution/Deployment/workflows/Steps/CD_DeployADF.ps1 modified: solution/Deployment/workflows/Steps/CD_DeployADFOnPremSHIR.ps1 modified: solution/Deployment/workflows/Steps/CD_DeployAppInsights.ps1 modified: solution/Deployment/workflows/Steps/CD_DeployAppService.ps1 modified: solution/Deployment/workflows/Steps/CD_DeployAzureSqlServer.ps1 modified: solution/Deployment/workflows/Steps/CD_DeployFunctionApp.ps1 modified: solution/Deployment/workflows/Steps/CD_DeployKeyVault.ps1 modified: solution/Deployment/workflows/Steps/CD_DeployLogAnalytics.ps1 modified: solution/Deployment/workflows/Steps/CD_DeployResourceGroup.ps1 modified: solution/Deployment/workflows/Steps/CD_DeployStorageADLS.ps1 modified: solution/Deployment/workflows/Steps/CD_DeployStorageBlob.ps1 modified: solution/Deployment/workflows/Steps/CD_DeployStorageForLogging.ps1 modified: solution/Deployment/workflows/Steps/CD_DeployVnet.ps1 modified: solution/Deployment/workflows/Steps/CD_DeployWebSite.ps1 modified: solution/Deployment/workflows/Steps/CD_GrantRBAC.ps1 modified: solution/Deployment/workflows/Steps/CD_SetResourceGroupHash.ps1 modified: solution/Deployment/workflows/Steps/CI_BuildDataFactory.ps1 modified: solution/Deployment/workflows/Steps/InstallGatewayFunctions.ps1 modified: solution/Deployment/workflows/Steps/PushEnvFileIntoVariables.ps1 modified: solution/FunctionApp/FunctionApp.csproj modified: solution/FunctionApp/Startup.cs modified: solution/FunctionApp/TaskMetaData/TaskInstance.cs new file: solution/SampleFiles/yellow_tripdata_2017-03.csv new file: solution/SampleFiles/yellow_tripdata_2017-03.xlsx renamed: solution/TaskTypeJson/AZ-SQL-AZ-Storage-Parquet-SH-IR.json -> solution/TaskTypeJson/AZ_SQL_AZ_Storage_Parquet_IRA.json renamed: solution/TaskTypeJson/AZ-SQL-StoredProcedure-SH-IR.json -> solution/TaskTypeJson/AZ_SQL_StoredProcedure_IRA.json renamed: solution/TaskTypeJson/AZ-Storage-Binary-AZ-Storage-Binary-SH-IR.json -> solution/TaskTypeJson/AZ_Storage_Binary_AZ_Storage_Binary_IRA.json renamed: solution/TaskTypeJson/AZ-Storage-CSV-AZ-SQL-SH-IR.json -> solution/TaskTypeJson/AZ_Storage_CSV_AZ_SQL_IRA.json renamed: solution/TaskTypeJson/AZ-Storage-Excel-AZ-SQL-SH-IR.json -> solution/TaskTypeJson/AZ_Storage_Excel_AZ_SQL_IRA.json renamed: solution/TaskTypeJson/AZ-Storage-Excel-AZ-Storage-CSV-SH-IR.json -> solution/TaskTypeJson/AZ_Storage_Excel_AZ_Storage_CSV_IRA.json renamed: solution/TaskTypeJson/AZ-Storage-JSON-AZ-SQL-SH-IR.json -> solution/TaskTypeJson/AZ_Storage_JSON_AZ_SQL_IRA.json renamed: solution/TaskTypeJson/AZ-Storage-Parquet-AZ-SQL-SH-IR.json -> solution/TaskTypeJson/AZ_Storage_Parquet_AZ_SQL_IRA.json renamed: solution/TaskTypeJson/GEN-File-Binary-AZ-Storage-Binary-OnP-SH-IR.json -> solution/TaskTypeJson/GEN_File_Binary_AZ_Storage_Binary_IRA.json renamed: solution/TaskTypeJson/GEN-File-Binary-AZ-Storage-Binary-SH-IR.json -> solution/TaskTypeJson/GEN_File_Binary_AZ_Storage_Binary_IRB.json renamed: solution/TaskTypeJson/OnP-SQL-GEN-File-Parquet-OnP-SH-IR.json -> solution/TaskTypeJson/OnP_SQL_AZ_Storage_Parquet_IRB.json renamed: solution/TaskTypeJson/OnP-SQL-AZ-Storage-Parquet-OnP-SH-IR.json -> solution/TaskTypeJson/OnP_SQL_GEN_File_Parquet_IRB.json modified: solution/WebApplication/WebApplication/Controllers/TaskMasterController.cs modified: solution/WebApplication/WebApplication/Startup.cs modified: solution/WebApplication/WebApplication/WebApplication.csproj --- README.md | 12 +- documentation/images/DeploymentForm.png | Bin 0 -> 106314 bytes .../dataset/ADLSBinary_IRA.json} | 8 +- .../dataset/ADLSDelimitedText_IRA.json} | 8 +- .../dataset/ADLSExcel_IRA.json} | 8 +- .../dataset/ADLSJson_IRA.json} | 8 +- .../dataset/ADLSParquet_IRA.json} | 8 +- .../dataset/AzureSqlTable_IRA.json} | 8 +- .../dataset/BlobBinary_IRA.json} | 8 +- .../dataset/BlobDelimitedText_IRA.json} | 8 +- .../dataset/BlobExcel_IRA.json} | 8 +- .../dataset/BlobJson_IRA.json} | 8 +- .../dataset/BlobParquet_IRA.json} | 8 +- .../dataset/FileBinary_IRA.json} | 8 +- .../dataset/FileParquet_IRA.json} | 8 +- .../dataset/SqlServerTable_IRA.json} | 8 +- .../factory/adsgofastdatakakeacceladf.json | 0 .../ADF/integrationRuntime/IRA.json | 22 + .../linkedService/AdsGoFastKeyVault.json | 0 ...eFunctionAdsGoFastDataLakeAccelFunApp.json | 0 .../GenericAzureDataLakeStorage_IRA.json} | 6 +- .../GenericAzureKeyVault_IRA.json} | 6 +- .../linkedService/GenericAzureSQL_IRA.json} | 6 +- .../GenericBlobStorage_IRA.json} | 6 +- .../linkedService/GenericFileServer_IRA.json} | 6 +- .../GenericMsSqlServer_IRA.json} | 6 +- .../ADF/pipeline/AZ_Function_Generic.json | 104 + .../AZ_SQL_AZ_Storage_Parquet_IRA.json} | 38 +- .../pipeline/AZ_SQL_Create_Table_IRA.json} | 14 +- .../pipeline/AZ_SQL_Full_Load_Chunk_IRA.json} | 12 +- .../pipeline/AZ_SQL_Full_Load_IRA.json} | 32 +- .../pipeline/AZ_SQL_Post_Copy_IRA.json} | 26 +- .../pipeline/AZ_SQL_StoredProcedure_IRA.json} | 10 +- .../pipeline/AZ_SQL_Watermark_Chunk_IRA.json} | 14 +- .../pipeline/AZ_SQL_Watermark_IRA.json} | 32 +- ...Storage_Binary_AZ_Storage_Binary_IRA.json} | 48 +- .../pipeline/AZ_Storage_CSV_AZ_SQL_IRA.json} | 30 +- .../AZ_Storage_Excel_AZ_SQL_IRA.json} | 30 +- .../AZ_Storage_Excel_AZ_Storage_CSV_IRA.json} | 26 +- .../pipeline/AZ_Storage_JSON_AZ_SQL_IRA.json} | 30 +- .../AZ_Storage_Parquet_AZ_SQL_IRA.json} | 32 +- .../Create_Task_Master_AZ_SQL_IRA.json} | 14 +- ...EN_File_Binary_AZ_Storage_Binary_IRA.json} | 28 +- .../pipeline/Master_IRA.json} | 172 +- .../SH_SQL_AZ_Storage_Parquet_IRA.json} | 38 +- .../pipeline/SH_SQL_Full_Load_Chunk_IRA.json} | 10 +- .../pipeline/SH_SQL_Full_Load_IRA.json} | 30 +- .../SH_SQL_GEN_File_Parquet_IRA.json} | 22 +- .../pipeline/SH_SQL_Watermark_Chunk_IRA.json} | 12 +- .../pipeline/SH_SQL_Watermark_IRA.json} | 30 +- .../trigger/Trigger on File End.json | 0 solution/DataFactory/GenerateADFArtefacts.ps1 | 29 + solution/DataFactory/IRConfig.json | 114 + .../dataset/ADLSBinary_{IR}.json} | 6 +- .../dataset/ADLSDelimitedText_{IR}.json | 63 + .../Templates/dataset/ADLSExcel_{IR}.json | 67 + .../Templates/dataset/ADLSJson_{IR}.json | 53 + .../dataset/ADLSParquet_{IR}.json} | 6 +- .../dataset/AzureSqlTable_{IR}.json} | 6 +- .../dataset/BlobBinary_{IR}.json} | 6 +- .../dataset/BlobDelimitedText_{IR}.json | 63 + .../Templates/dataset/BlobExcel_{IR}.json | 67 + .../Templates/dataset/BlobJson_{IR}.json | 53 + .../dataset/BlobParquet_{IR}.json} | 6 +- .../dataset/FileBinary_{IR}.json} | 6 +- .../dataset/FileParquet_{IR}.json} | 6 +- .../dataset/SqlServerTable_{IR}.json} | 6 +- .../factory/adsgofastdatakakeacceladf.json | 4 + .../Templates/integrationRuntime/IRA.json | 22 + .../Templates/integrationRuntime/IRB.json | 22 + .../GenericAzureDataLakeStorage_{IR}.json} | 4 +- .../GenericAzureKeyVault_{IR}.json | 21 + .../linkedService/GenericAzureSQL_{IR}.json} | 6 +- .../GenericBlobStorage_{IR}.json} | 4 +- .../GenericFileServer_{IR}.json} | 8 +- .../GenericMsSqlServer_{IR}.json} | 4 +- .../linkedService/NamingConventions.md | 1 + .../pipeline/AZ_Function_Generic.json} | 2 +- .../AZ_SQL_AZ_Storage_Parquet_{IR}.json | 526 +++ .../pipeline/AZ_SQL_Create_Table_{IR}.json | 260 ++ .../AZ_SQL_Full_Load_Chunk_{IR}.json} | 12 +- .../pipeline/AZ_SQL_Full_Load_{IR}.json | 633 +++ .../pipeline/AZ_SQL_Post_Copy_{IR}.json | 556 +++ .../pipeline/AZ_SQL_StoredProcedure_{IR}.json | 92 + .../AZ_SQL_Watermark_Chunk_{IR}.json} | 14 +- .../pipeline/AZ_SQL_Watermark_{IR}.json | 636 +++ ...torage_Binary_AZ_Storage_Binary_{IR}.json} | 368 +- .../pipeline/AZ_Storage_CSV_AZ_SQL_{IR}.json | 460 ++ .../AZ_Storage_Excel_AZ_SQL_{IR}.json | 420 ++ .../AZ_Storage_Excel_AZ_Storage_CSV_{IR}.json | 406 ++ .../pipeline/AZ_Storage_JSON_AZ_SQL_{IR}.json | 408 ++ .../AZ_Storage_Parquet_AZ_SQL_{IR}.json | 494 ++ .../Create_Task_Master_AZ_SQL_{IR}.json} | 28 +- ...N_File_Binary_AZ_Storage_Binary_{IR}.json} | 196 +- .../Templates/pipeline/Master_{IR}.json | 470 ++ .../SH_SQL_AZ_Storage_Parquet_{IR}.json} | 36 +- .../pipeline/SH_SQL_Full_Load_Chunk_{IR}.json | 118 + .../pipeline/SH_SQL_Full_Load_{IR}.json} | 41 +- .../SH_SQL_GEN_File_Parquet_{IR}.json | 364 ++ .../pipeline/SH_SQL_Watermark_Chunk_{IR}.json | 153 + .../pipeline/SH_SQL_Watermark_{IR}.json} | 41 +- .../trigger/Trigger on File End.json | 18 + .../dataset/ADLSBinary_OnPrem_SH_IR.json | 52 - .../dataset/ADLSParquet_OnPrem_SH_IR.json | 54 - .../dataset/BlobBinary_OnPrem_SH_IR.json | 52 - .../dataset/BlobParquet_OnPrem_SH_IR.json | 54 - ...lfHostedIntegrationRuntime-Azure-VNET.json | 6 - ...elfHostedIntegrationRuntime-OnPem-Net.json | 6 - .../GenericAzureDataLakeStorage.json | 17 - .../linkedService/GenericBlobStorage.json | 18 - .../AZ-SQL-AZ-Storage-Parquet-AZ-IR.json | 745 --- .../AZ-Storage-Parquet-AZ-SQL-AZ-IR.json | 376 -- .../006-SampleTasks_TaskMaster.sql | 9 +- ...008-SampleTasks_SourceAndTargetSystems.sql | 116 +- .../010-TaskTypeMapping_Update.sql | 20 + .../.devcontainer/devcontainer.json | 8 +- solution/Deployment/arm/AppService_Func.bicep | 28 + solution/Deployment/arm/AppService_Web.bicep | 27 + .../Deployment/arm/ApplicationInsights.bicep | 288 ++ solution/Deployment/arm/AzureSQLServer.json | 334 +- solution/Deployment/arm/AzureSqlServer.bicep | 120 + solution/Deployment/arm/DataFactory.bicep | 14 + solution/Deployment/arm/FunctionApp.bicep | 48 + solution/Deployment/arm/KeyVault.bicep | 28 + solution/Deployment/arm/LogAnalytics.bicep | 36 + solution/Deployment/arm/Networking.bicep | 107 + solution/Deployment/arm/Storage_ADLS.bicep | 53 + solution/Deployment/arm/Storage_Blob.bicep | 53 + solution/Deployment/arm/Storage_Logging.bicep | 21 + solution/Deployment/arm/VirtualMachine.bicep | 254 ++ solution/Deployment/arm/WebApp.bicep | 42 + solution/Deployment/arm/WebApp.json | 136 +- .../Deployment/environments/EditSettings.html | 67 + .../environments/Node/package-lock.json | 872 ++++ .../Deployment/environments/Node/package.json | 16 + .../Deployment/environments/Node/server.js | 37 + .../Deployment/environments/development.json | 81 +- .../environments/development_bak.json | 196 + .../environments/development_test.json | 175 + .../environments/environment.schema.json | 4042 +++++++++-------- ...a_CreateServicePrincipals_AAD_Elevated.ps1 | 14 +- .../workflows/CD_1a_DeployServices.ps1 | 6 +- .../CD_2a_CreateMSIs_AAD_Elevated.ps1 | 31 +- .../workflows/CD_2b_ConfigureServices.ps1 | 2 + .../LocalDevOnly_EnvironmentSetUp.ps1 | 11 +- .../workflows/Steps/CD_ConfigureADF.ps1 | 97 +- .../Steps/CD_ConfigureAzureSQLServer.ps1 | 12 +- ...ureSqlServer_UpdateTaskTypeMappingJson.ps1 | 6 + .../Steps/CD_ConfigureFunctionApp.ps1 | 6 +- .../workflows/Steps/CD_ConfigureKeyVault.ps1 | 6 +- .../Steps/CD_ConfigureSampleData.ps1 | 10 + .../workflows/Steps/CD_ConfigureVnet.ps1 | 48 +- .../workflows/Steps/CD_ConfigureWebApp.ps1 | 4 +- .../workflows/Steps/CD_DeployADF.ps1 | 4 +- .../Steps/CD_DeployADFOnPremSHIR.ps1 | 16 +- .../workflows/Steps/CD_DeployAppInsights.ps1 | 4 +- .../workflows/Steps/CD_DeployAppService.ps1 | 8 +- .../Steps/CD_DeployAzureSqlServer.ps1 | 18 +- .../workflows/Steps/CD_DeployFunctionApp.ps1 | 4 +- .../workflows/Steps/CD_DeployKeyVault.ps1 | 4 +- .../workflows/Steps/CD_DeployLogAnalytics.ps1 | 4 +- .../Steps/CD_DeployResourceGroup.ps1 | 4 +- .../workflows/Steps/CD_DeployStorageADLS.ps1 | 10 +- .../workflows/Steps/CD_DeployStorageBlob.ps1 | 6 +- .../Steps/CD_DeployStorageForLogging.ps1 | 8 +- .../workflows/Steps/CD_DeployVnet.ps1 | 8 +- .../workflows/Steps/CD_DeployWebSite.ps1 | 6 +- .../workflows/Steps/CD_GrantRBAC.ps1 | 34 +- .../Steps/CD_SetResourceGroupHash.ps1 | 8 +- .../workflows/Steps/CI_BuildDataFactory.ps1 | 16 +- .../Steps/InstallGatewayFunctions.ps1 | 12 +- .../Steps/PushEnvFileIntoVariables.ps1 | 23 +- solution/FunctionApp/FunctionApp.csproj | 7 +- solution/FunctionApp/Startup.cs | 2 +- .../FunctionApp/TaskMetaData/TaskInstance.cs | 6 +- .../SampleFiles/yellow_tripdata_2017-03.csv | 61 + .../SampleFiles/yellow_tripdata_2017-03.xlsx | Bin 0 -> 16320 bytes ...son => AZ_SQL_AZ_Storage_Parquet_IRA.json} | 12 +- ...R.json => AZ_SQL_StoredProcedure_IRA.json} | 0 ...Storage_Binary_AZ_Storage_Binary_IRA.json} | 6 +- ...IR.json => AZ_Storage_CSV_AZ_SQL_IRA.json} | 4 +- ....json => AZ_Storage_Excel_AZ_SQL_IRA.json} | 10 +- ... AZ_Storage_Excel_AZ_Storage_CSV_IRA.json} | 2 +- ...R.json => AZ_Storage_JSON_AZ_SQL_IRA.json} | 4 +- ...son => AZ_Storage_Parquet_AZ_SQL_IRA.json} | 6 +- ...EN_File_Binary_AZ_Storage_Binary_IRA.json} | 4 +- ...EN_File_Binary_AZ_Storage_Binary_IRB.json} | 4 +- ...on => OnP_SQL_AZ_Storage_Parquet_IRB.json} | 2 +- ...json => OnP_SQL_GEN_File_Parquet_IRB.json} | 2 +- .../Controllers/TaskMasterController.cs | 1 + .../WebApplication/WebApplication/Startup.cs | 3 + .../WebApplication/WebApplication.csproj | 1 + 192 files changed, 12780 insertions(+), 4836 deletions(-) create mode 100644 documentation/images/DeploymentForm.png rename solution/DataFactory/{dataset/ADLSBinary_AZ_IR.json => ADF/dataset/ADLSBinary_IRA.json} (87%) rename solution/DataFactory/{dataset/ADLSDelimitedText_SH_IR.json => ADF/dataset/ADLSDelimitedText_IRA.json} (89%) rename solution/DataFactory/{dataset/ADLSExcel_SH_IR.json => ADF/dataset/ADLSExcel_IRA.json} (90%) rename solution/DataFactory/{dataset/ADLSJson_SH_IR.json => ADF/dataset/ADLSJson_IRA.json} (88%) rename solution/DataFactory/{dataset/ADLSParquet_AZ_IR.json => ADF/dataset/ADLSParquet_IRA.json} (88%) rename solution/DataFactory/{dataset/AzureSqlTable_AZ_IR.json => ADF/dataset/AzureSqlTable_IRA.json} (87%) rename solution/DataFactory/{dataset/BlobBinary_AZ_IR.json => ADF/dataset/BlobBinary_IRA.json} (88%) rename solution/DataFactory/{dataset/BlobDelimitedText_SH_IR.json => ADF/dataset/BlobDelimitedText_IRA.json} (90%) rename solution/DataFactory/{dataset/BlobExcel_SH_IR.json => ADF/dataset/BlobExcel_IRA.json} (91%) rename solution/DataFactory/{dataset/BlobJson_SH_IR.json => ADF/dataset/BlobJson_IRA.json} (88%) rename solution/DataFactory/{dataset/BlobParquet_AZ_IR.json => ADF/dataset/BlobParquet_IRA.json} (89%) rename solution/DataFactory/{dataset/FileBinary_SH_IR.json => ADF/dataset/FileBinary_IRA.json} (90%) rename solution/DataFactory/{dataset/FileParquet_SH_IR.json => ADF/dataset/FileParquet_IRA.json} (90%) rename solution/DataFactory/{dataset/SqlServerTable_SH_IR.json => ADF/dataset/SqlServerTable_IRA.json} (90%) rename solution/DataFactory/{ => ADF}/factory/adsgofastdatakakeacceladf.json (100%) create mode 100644 solution/DataFactory/ADF/integrationRuntime/IRA.json rename solution/DataFactory/{ => ADF}/linkedService/AdsGoFastKeyVault.json (100%) rename solution/DataFactory/{ => ADF}/linkedService/AzureFunctionAdsGoFastDataLakeAccelFunApp.json (100%) rename solution/DataFactory/{linkedService/GenericAzureDataLakeStorage_SH_IR.json => ADF/linkedService/GenericAzureDataLakeStorage_IRA.json} (77%) rename solution/DataFactory/{linkedService/GenericAzureKeyVault.json => ADF/linkedService/GenericAzureKeyVault_IRA.json} (80%) rename solution/DataFactory/{linkedService/GenericAzureSQL_SH_IR.json => ADF/linkedService/GenericAzureSQL_IRA.json} (83%) rename solution/DataFactory/{linkedService/GenericBlobStorage_SH_IR.json => ADF/linkedService/GenericBlobStorage_IRA.json} (79%) rename solution/DataFactory/{linkedService/GenericFileServer_SH_IR.json => ADF/linkedService/GenericFileServer_IRA.json} (90%) rename solution/DataFactory/{linkedService/GenericMsSqlServer_SH_IR.json => ADF/linkedService/GenericMsSqlServer_IRA.json} (91%) create mode 100644 solution/DataFactory/ADF/pipeline/AZ_Function_Generic.json rename solution/DataFactory/{pipeline/AZ-SQL-AZ-Storage-Parquet-SH-IR.json => ADF/pipeline/AZ_SQL_AZ_Storage_Parquet_IRA.json} (94%) rename solution/DataFactory/{pipeline/AZ-SQL-Create-Table-SH-IR.json => ADF/pipeline/AZ_SQL_Create_Table_IRA.json} (96%) rename solution/DataFactory/{pipeline/AZ-SQL-Full-Load-Chunk-SH-IR.json => ADF/pipeline/AZ_SQL_Full_Load_Chunk_IRA.json} (92%) rename solution/DataFactory/{pipeline/AZ-SQL-Full-Load-SH-IR.json => ADF/pipeline/AZ_SQL_Full_Load_IRA.json} (96%) rename solution/DataFactory/{pipeline/AZ-SQL-Post-Copy-SH-IR.json => ADF/pipeline/AZ_SQL_Post_Copy_IRA.json} (96%) rename solution/DataFactory/{pipeline/AZ-SQL-StoredProcedure-SH-IR.json => ADF/pipeline/AZ_SQL_StoredProcedure_IRA.json} (93%) rename solution/DataFactory/{pipeline/AZ-SQL-Watermark-Chunk-SH-IR.json => ADF/pipeline/AZ_SQL_Watermark_Chunk_IRA.json} (93%) rename solution/DataFactory/{pipeline/AZ-SQL-Watermark-SH-IR.json => ADF/pipeline/AZ_SQL_Watermark_IRA.json} (96%) rename solution/DataFactory/{pipeline/AZ-Storage-Binary-AZ-Storage-Binary-SH-IR.json => ADF/pipeline/AZ_Storage_Binary_AZ_Storage_Binary_IRA.json} (91%) rename solution/DataFactory/{pipeline/AZ-Storage-CSV-AZ-SQL-SH-IR.json => ADF/pipeline/AZ_Storage_CSV_AZ_SQL_IRA.json} (95%) rename solution/DataFactory/{pipeline/AZ-Storage-Excel-AZ-SQL-SH-IR.json => ADF/pipeline/AZ_Storage_Excel_AZ_SQL_IRA.json} (95%) rename solution/DataFactory/{pipeline/AZ-Storage-Excel-AZ-Storage-CSV-SH-IR.json => ADF/pipeline/AZ_Storage_Excel_AZ_Storage_CSV_IRA.json} (95%) rename solution/DataFactory/{pipeline/AZ-Storage-JSON-AZ-SQL-SH-IR.json => ADF/pipeline/AZ_Storage_JSON_AZ_SQL_IRA.json} (95%) rename solution/DataFactory/{pipeline/AZ-Storage-Parquet-AZ-SQL-SH-IR.json => ADF/pipeline/AZ_Storage_Parquet_AZ_SQL_IRA.json} (95%) rename solution/DataFactory/{pipeline/Create-Task-Master-AZ-SQL-SH-IR.json => ADF/pipeline/Create_Task_Master_AZ_SQL_IRA.json} (96%) rename solution/DataFactory/{pipeline/GEN-File-Binary-AZ-Storage-Binary-SH-IR.json => ADF/pipeline/GEN_File_Binary_AZ_Storage_Binary_IRA.json} (87%) rename solution/DataFactory/{pipeline/Master.json => ADF/pipeline/Master_IRA.json} (75%) rename solution/DataFactory/{pipeline/SH-SQL-AZ-Storage-Parquet-SH-IR.json => ADF/pipeline/SH_SQL_AZ_Storage_Parquet_IRA.json} (87%) rename solution/DataFactory/{pipeline/SH-SQL-Full-Load-Chunk-SH-IR.json => ADF/pipeline/SH_SQL_Full_Load_Chunk_IRA.json} (92%) rename solution/DataFactory/{pipeline/SH-SQL-Full-Load-SH-IR.json => ADF/pipeline/SH_SQL_Full_Load_IRA.json} (96%) rename solution/DataFactory/{pipeline/OnP-SQL-GEN-File-Parquet-OnP-SH-IR.json => ADF/pipeline/SH_SQL_GEN_File_Parquet_IRA.json} (96%) rename solution/DataFactory/{pipeline/SH-SQL-Watermark-Chunk-SH-IR.json => ADF/pipeline/SH_SQL_Watermark_Chunk_IRA.json} (93%) rename solution/DataFactory/{pipeline/SH-SQL-Watermark-SH-IR.json => ADF/pipeline/SH_SQL_Watermark_IRA.json} (96%) rename solution/DataFactory/{ => ADF}/trigger/Trigger on File End.json (100%) create mode 100644 solution/DataFactory/GenerateADFArtefacts.ps1 create mode 100644 solution/DataFactory/IRConfig.json rename solution/DataFactory/{dataset/ADLSBinary_SH_IR.json => Templates/dataset/ADLSBinary_{IR}.json} (87%) create mode 100644 solution/DataFactory/Templates/dataset/ADLSDelimitedText_{IR}.json create mode 100644 solution/DataFactory/Templates/dataset/ADLSExcel_{IR}.json create mode 100644 solution/DataFactory/Templates/dataset/ADLSJson_{IR}.json rename solution/DataFactory/{dataset/ADLSParquet_SH_IR.json => Templates/dataset/ADLSParquet_{IR}.json} (88%) rename solution/DataFactory/{dataset/AzureSqlTable_SH_IR.json => Templates/dataset/AzureSqlTable_{IR}.json} (86%) rename solution/DataFactory/{dataset/BlobBinary_SH_IR.json => Templates/dataset/BlobBinary_{IR}.json} (88%) create mode 100644 solution/DataFactory/Templates/dataset/BlobDelimitedText_{IR}.json create mode 100644 solution/DataFactory/Templates/dataset/BlobExcel_{IR}.json create mode 100644 solution/DataFactory/Templates/dataset/BlobJson_{IR}.json rename solution/DataFactory/{dataset/BlobParquet_SH_IR.json => Templates/dataset/BlobParquet_{IR}.json} (88%) rename solution/DataFactory/{dataset/FileBinary_OnPrem_SH_IR.json => Templates/dataset/FileBinary_{IR}.json} (89%) rename solution/DataFactory/{dataset/FileParquet_OnPrem_SH_IR.json => Templates/dataset/FileParquet_{IR}.json} (89%) rename solution/DataFactory/{dataset/SqlServerTable_OnPrem_SH_IR.json => Templates/dataset/SqlServerTable_{IR}.json} (89%) create mode 100644 solution/DataFactory/Templates/factory/adsgofastdatakakeacceladf.json create mode 100644 solution/DataFactory/Templates/integrationRuntime/IRA.json create mode 100644 solution/DataFactory/Templates/integrationRuntime/IRB.json rename solution/DataFactory/{linkedService/GenericAzureDataLakeStorage_OnPrem_Net.json => Templates/linkedService/GenericAzureDataLakeStorage_{IR}.json} (76%) create mode 100644 solution/DataFactory/Templates/linkedService/GenericAzureKeyVault_{IR}.json rename solution/DataFactory/{linkedService/GenericAzureSQL.json => Templates/linkedService/GenericAzureSQL_{IR}.json} (77%) rename solution/DataFactory/{linkedService/GenericBlobStorage_OnPrem_Net.json => Templates/linkedService/GenericBlobStorage_{IR}.json} (78%) rename solution/DataFactory/{linkedService/GenericFileServer_OnPrem_Net.json => Templates/linkedService/GenericFileServer_{IR}.json} (84%) rename solution/DataFactory/{linkedService/GenericMsSqlServer_OnPrem_Net.json => Templates/linkedService/GenericMsSqlServer_{IR}.json} (90%) create mode 100644 solution/DataFactory/Templates/linkedService/NamingConventions.md rename solution/DataFactory/{pipeline/AZ-Function-Generic.json => Templates/pipeline/AZ_Function_Generic.json} (98%) create mode 100644 solution/DataFactory/Templates/pipeline/AZ_SQL_AZ_Storage_Parquet_{IR}.json create mode 100644 solution/DataFactory/Templates/pipeline/AZ_SQL_Create_Table_{IR}.json rename solution/DataFactory/{pipeline/OnP-SQL-Full-Load-Chunk-OnP-SH-IR.json => Templates/pipeline/AZ_SQL_Full_Load_Chunk_{IR}.json} (90%) create mode 100644 solution/DataFactory/Templates/pipeline/AZ_SQL_Full_Load_{IR}.json create mode 100644 solution/DataFactory/Templates/pipeline/AZ_SQL_Post_Copy_{IR}.json create mode 100644 solution/DataFactory/Templates/pipeline/AZ_SQL_StoredProcedure_{IR}.json rename solution/DataFactory/{pipeline/OnP-SQL-Watermark-Chunk-OnP-SH-IR.json => Templates/pipeline/AZ_SQL_Watermark_Chunk_{IR}.json} (92%) create mode 100644 solution/DataFactory/Templates/pipeline/AZ_SQL_Watermark_{IR}.json rename solution/DataFactory/{pipeline/AZ-Storage-Binary-AZ-Storage-Binary-AZ-IR.json => Templates/pipeline/AZ_Storage_Binary_AZ_Storage_Binary_{IR}.json} (60%) create mode 100644 solution/DataFactory/Templates/pipeline/AZ_Storage_CSV_AZ_SQL_{IR}.json create mode 100644 solution/DataFactory/Templates/pipeline/AZ_Storage_Excel_AZ_SQL_{IR}.json create mode 100644 solution/DataFactory/Templates/pipeline/AZ_Storage_Excel_AZ_Storage_CSV_{IR}.json create mode 100644 solution/DataFactory/Templates/pipeline/AZ_Storage_JSON_AZ_SQL_{IR}.json create mode 100644 solution/DataFactory/Templates/pipeline/AZ_Storage_Parquet_AZ_SQL_{IR}.json rename solution/DataFactory/{pipeline/Create-Task-Master-AZ-SQL-OnP-SH-IR.json => Templates/pipeline/Create_Task_Master_AZ_SQL_{IR}.json} (90%) rename solution/DataFactory/{pipeline/GEN-File-Binary-AZ-Storage-Binary-OnP-SH-IR.json => Templates/pipeline/GEN_File_Binary_AZ_Storage_Binary_{IR}.json} (73%) create mode 100644 solution/DataFactory/Templates/pipeline/Master_{IR}.json rename solution/DataFactory/{pipeline/OnP-SQL-AZ-Storage-Parquet-OnP-SH-IR.json => Templates/pipeline/SH_SQL_AZ_Storage_Parquet_{IR}.json} (87%) create mode 100644 solution/DataFactory/Templates/pipeline/SH_SQL_Full_Load_Chunk_{IR}.json rename solution/DataFactory/{pipeline/OnP-SQL-Full-Load-OnP-SH-IR.json => Templates/pipeline/SH_SQL_Full_Load_{IR}.json} (95%) create mode 100644 solution/DataFactory/Templates/pipeline/SH_SQL_GEN_File_Parquet_{IR}.json create mode 100644 solution/DataFactory/Templates/pipeline/SH_SQL_Watermark_Chunk_{IR}.json rename solution/DataFactory/{pipeline/OnP-SQL-Watermark-OnP-SH-IR.json => Templates/pipeline/SH_SQL_Watermark_{IR}.json} (95%) create mode 100644 solution/DataFactory/Templates/trigger/Trigger on File End.json delete mode 100644 solution/DataFactory/dataset/ADLSBinary_OnPrem_SH_IR.json delete mode 100644 solution/DataFactory/dataset/ADLSParquet_OnPrem_SH_IR.json delete mode 100644 solution/DataFactory/dataset/BlobBinary_OnPrem_SH_IR.json delete mode 100644 solution/DataFactory/dataset/BlobParquet_OnPrem_SH_IR.json delete mode 100644 solution/DataFactory/integrationRuntime/SelfHostedIntegrationRuntime-Azure-VNET.json delete mode 100644 solution/DataFactory/integrationRuntime/SelfHostedIntegrationRuntime-OnPem-Net.json delete mode 100644 solution/DataFactory/linkedService/GenericAzureDataLakeStorage.json delete mode 100644 solution/DataFactory/linkedService/GenericBlobStorage.json delete mode 100644 solution/DataFactory/pipeline/AZ-SQL-AZ-Storage-Parquet-AZ-IR.json delete mode 100644 solution/DataFactory/pipeline/AZ-Storage-Parquet-AZ-SQL-AZ-IR.json create mode 100644 solution/Database/ADSGoFastDbUp/AdsGoFastDbUp/0-1-0-beta/A-Journaled/010-TaskTypeMapping_Update.sql create mode 100644 solution/Deployment/arm/AppService_Func.bicep create mode 100644 solution/Deployment/arm/AppService_Web.bicep create mode 100644 solution/Deployment/arm/ApplicationInsights.bicep create mode 100644 solution/Deployment/arm/AzureSqlServer.bicep create mode 100644 solution/Deployment/arm/DataFactory.bicep create mode 100644 solution/Deployment/arm/FunctionApp.bicep create mode 100644 solution/Deployment/arm/KeyVault.bicep create mode 100644 solution/Deployment/arm/LogAnalytics.bicep create mode 100644 solution/Deployment/arm/Networking.bicep create mode 100644 solution/Deployment/arm/Storage_ADLS.bicep create mode 100644 solution/Deployment/arm/Storage_Blob.bicep create mode 100644 solution/Deployment/arm/Storage_Logging.bicep create mode 100644 solution/Deployment/arm/VirtualMachine.bicep create mode 100644 solution/Deployment/arm/WebApp.bicep create mode 100644 solution/Deployment/environments/EditSettings.html create mode 100644 solution/Deployment/environments/Node/package-lock.json create mode 100644 solution/Deployment/environments/Node/package.json create mode 100644 solution/Deployment/environments/Node/server.js create mode 100644 solution/Deployment/environments/development_bak.json create mode 100644 solution/Deployment/environments/development_test.json create mode 100644 solution/Deployment/workflows/Steps/CD_ConfigureSampleData.ps1 create mode 100644 solution/SampleFiles/yellow_tripdata_2017-03.csv create mode 100644 solution/SampleFiles/yellow_tripdata_2017-03.xlsx rename solution/TaskTypeJson/{AZ-SQL-AZ-Storage-Parquet-SH-IR.json => AZ_SQL_AZ_Storage_Parquet_IRA.json} (92%) rename solution/TaskTypeJson/{AZ-SQL-StoredProcedure-SH-IR.json => AZ_SQL_StoredProcedure_IRA.json} (100%) rename solution/TaskTypeJson/{AZ-Storage-Binary-AZ-Storage-Binary-SH-IR.json => AZ_Storage_Binary_AZ_Storage_Binary_IRA.json} (97%) rename solution/TaskTypeJson/{AZ-Storage-CSV-AZ-SQL-SH-IR.json => AZ_Storage_CSV_AZ_SQL_IRA.json} (99%) rename solution/TaskTypeJson/{AZ-Storage-Excel-AZ-SQL-SH-IR.json => AZ_Storage_Excel_AZ_SQL_IRA.json} (94%) rename solution/TaskTypeJson/{AZ-Storage-Excel-AZ-Storage-CSV-SH-IR.json => AZ_Storage_Excel_AZ_Storage_CSV_IRA.json} (98%) rename solution/TaskTypeJson/{AZ-Storage-JSON-AZ-SQL-SH-IR.json => AZ_Storage_JSON_AZ_SQL_IRA.json} (99%) rename solution/TaskTypeJson/{AZ-Storage-Parquet-AZ-SQL-SH-IR.json => AZ_Storage_Parquet_AZ_SQL_IRA.json} (97%) rename solution/TaskTypeJson/{GEN-File-Binary-AZ-Storage-Binary-OnP-SH-IR.json => GEN_File_Binary_AZ_Storage_Binary_IRA.json} (98%) rename solution/TaskTypeJson/{GEN-File-Binary-AZ-Storage-Binary-SH-IR.json => GEN_File_Binary_AZ_Storage_Binary_IRB.json} (98%) rename solution/TaskTypeJson/{OnP-SQL-GEN-File-Parquet-OnP-SH-IR.json => OnP_SQL_AZ_Storage_Parquet_IRB.json} (99%) rename solution/TaskTypeJson/{OnP-SQL-AZ-Storage-Parquet-OnP-SH-IR.json => OnP_SQL_GEN_File_Parquet_IRB.json} (99%) diff --git a/README.md b/README.md index b316a8f4..59215ac8 100644 --- a/README.md +++ b/README.md @@ -43,7 +43,17 @@ Once you have your Prerequisite items, please move on to the Deployment Configur >- [Docker Desktop](https://www.docker.com/products/docker-desktop) >- [Windows Store Ubuntu 18.04 LTS](https://www.microsoft.com/store/productId/9N9TNGVNDL3Q) -The deployment uses a concept of **Developing inside a Container** to containerize all the necessary pre-requisite component without requiring them to be installed on the local machine. Follow our [Configuring your System for Development Containers](docs%2Fdeveloping_inside_a_container.md) guide. +The deployment uses a concept of **Developing inside a Container** to containerize all the necessary pre-requisite component without requiring them to be installed on the local machine. Follow our [Configuring your System for Development Containers](https://code.visualstudio.com/docs/remote/containers) guide. + +Once you have set up these pre-requisites you will then need to + +>- [ ] [Clone](https://docs.github.com/en/enterprise-server@3.1/repositories/creating-and-managing-repositories/cloning-a-repository) this repository to your local machine +>- [ ] Navigate to the root solution folder and then navigate to "solution/Deployment/environments". Within this folder you will find a file called "environment.schema.json". +>- [ ] Open a web browser and navigate to the website [https://pmk65.github.io/jedemov2/dist/demo.html](https://pmk65.github.io/jedemov2/dist/demo.html) + + +![Form](./documentation/images/DeploymentForm.png) + --- diff --git a/documentation/images/DeploymentForm.png b/documentation/images/DeploymentForm.png new file mode 100644 index 0000000000000000000000000000000000000000..82e9f0cfd3b8a027c600c2d6f195555b51119051 GIT binary patch literal 106314 zcmdSBWmp_b)HRxfBm@W$JUBr@aCZwHoZwEd0S0%6Kp;2-cL;8S4(_glySux~FmQ)+ z&imf``#$%_ck6i=db+E-t9I?)UA5NQ6RapNiHb~s{Nlw6RH<)bN-ti#;eYW0;qu)p z_?~yk#uV^>FCCO5MP8H+6Yasjyfzh<6MpfcA`<1n;0^pclI=H5hZip}9iRVRE(1ti zU%YtAl@b$Han(D7p*s@nq%l247IQ9|m6>@lmAWukc~VXZL>~%#*ijGi@LwmPZqS`$?e z4gz(8jml1YEUp{kw2tJps9Nzq{`mTHRETilWM*xa4dML=|Hb=twvkqj)`!!co^V1W zczd5qA4`V8BjUf?KZ$<*CH{AlO!AZ2e|!7Bzng#g?_PUsKj;6c8t{L5)yZUiZ&#y@ zo^Z%~^n=5xK4X2LVCp-_&=5;XJOdNc$zX0bAJxKB)u}?F+`pZA5t+0#U8RiCc*n31 z9^6K%#%P$Y=#hObb(AzO1V+*jnbB$XDjjQk4@P@o>jMes(eY|#eClEc|6?AtDSM0g zr+gSBMDma&V`y+eMeAWj1)Aveju5&=(oN`lYH0VXS-&w3OYiO9+i-0aPJpJ<_Aj-U z;P>U2+dT{(u}hGd$bDI#Vltto_s=aV2Gk2fYxTX-LS|=}E$9v>Y!@G4GiYoFGUCf5 zq(t6O;#wPfR%YkGFw0Hc1RR)M@Hg^!RVi9delfeUR_%gH2rHfcWx>psHCDQEOPADe zOc*mbI*gI+F5p$g>0ViI@gOpf8{}HY?=LJ<IB1w+Nh@mW1GYdQ3 zTC58V-iY#jlBKbY9e{*SLD76MSl4Zq*L&pm=TdGfT?W?BjJfE)<=1^4DSrt+Xa(7?w~GiL`Id;YGjnM=w`&T#5&zb_c@8UtBdH#}QihF( zn8@J@->InP;u8RCL778Rmcy_5b<{{qb?e;EGE#UGeooacZlC!~EcerwB)1;y|A?>T z!Xp}`U10niaPTHKC4k&cDhKF(|GU@2$R1OLDjTlEPFj;e0Wn|ko=Xb z%u6Hk+Su|B@+v8x^Bg`P#ora0&m3b9Bqd&gA&@-nQh7+qeUh;MrGlKX{29BOWIP!l zcYtxgbK+6Tc61(cd#k;fDDKsVZ#?~iwQEc5VNZ>vW6c#5gRQ^i$mWZ-24 z_hd>x4;x%%ISyEPdy-6kUW#yeK`_O?d>QKIdG~x8Ss*{qYd-&8 zWlDGKY~eh&mytkYDl?Rm`1Jil^TD{sOqqVDqtW*1Ku88|-_3MxN(hZ=@0iC10`*E| zy+nu63|_1?7Tvy{QV^EKCRi>h3?DcWCU&G)=aNeFgsXe>Yyd?Qx(%V-Wjx81F8BRHt_z}r0#HS4_ zf(PeK2c(flhEQ$Bz_$*+$tspVwU<&ym?yTM;b$#OtW}`+{Uw^?0udR}WIQceX1w z*`CXrx@VVg6dThYUfn(Ap$Alwq0{KaY#4WG8gbi`zU41vz*QYUZ>iLet>=L>x1Dyk zfK8bs2Y9@=Ju@vI190QBb~v3PJiV5v)1qFY6htdimGGR()swA>&bg&1sbw3W!*)+n09ZSQywH(V=+_)= z*j@q0a%Ic+qSPQGBZ-busl1g@?J!E6e#7QDPPZXk22E8V1~PR?>fkkJUeR_gohgGsQXDXM~k6DCXGcvd1nD8$%~bBDEkL{*)P&FCX^NEniM~;Bi8}P@F|(Dqd)$3rMWR(; z*I~>w2>a*LI~%FMrZJZw+F2m4_Bz-@0=Y&?i1}&$pZCf)el1~-z9)Hp0MkNqNus6> z<$1ExinWtVPyKpc8PQU$;x3=pl*)=RsSHQ)#HF}yj8&YrvH~<5m0-cOs9+<#v1E+5 zX1w)KopuE`n23YUME5wZ73WX5+yf90%qoZ>+h@fqdc99Xqwq4E2yC&xa+>dve4bLA zm#1brKrRNkZzR+r4jq)TKMX7)270qY&On5w!p`9KcYA;$wT(jh+)}{nxD9S=sZv1Tqy;#grOw|W50)9R~Q8TFlT`h&t$&6Fgpy~&@up5_>8ix z=aQ5@Qk0C}mak1JjGXXJs3&@amCaM14`Lr(_bnhE@xGyx!m>#v2!ko@SnR@-(H;1C z$$>)P0gai_qRXioOzTTo6YFrZ9*Lf@4RHHFd14w z?XJTIH~LRx52sLmJP$;4z&z{P`Jw7_t5j#MVGR-cp(35&DZ zj#BOmt6Nchzo0O75c^w12TW+Bng3aQH#q$o-c%LO{13bD(Ps4Z)QjLIO}+G51y0OJ58S zG)(3~kOPSLiq#MQuqW#nq4cDNGOB3iCQ);UsSyVTKxE#DIMxZ>Cu5wni;oD@@1BtE zkM53>bl<-^(_yxtn)E}eZ@u4ovn1Cn1IN^uS8%D>c`=WyEYY9vW|kRF_brn)Dj_b{B4B~LQJq~B zVygUjf=wzvAC&9jpx;S=Ou?2FQOpt)`O+unT2qxn0a35isx;?+9FmTP8f_WP_7R|G zP^b_EEkSK{lUEY`ra1L%zb0oH37N!GauvwWUR7&%lJb^)MMMAIpM5?s-_)Ea+5E%H zPBo_b#+F|vm4GN5(J5BjE6s*EfnQ)VN1P$VX=fWdgnb&T$SfHqNqol1jgc6#aa@29Yv z7l%QV&neuIz_Nc3j;rJxWb}C!t9Z^_wzZ_vSX)dkr=>Zu6x2X_ioFN&oybKN8l+DO z?KA-MIW_hbT+>|OVM0)+SG1dnRL@$Ueiwfc<94)HN74uBSJ2q znzQw0eVK)-%=@?mF{CwpL*E?C(NirYy*TO3v9GaYrElh%t9x~>cVPrAYJ9DIc49u6 zAd-!00<`h^vu-9^wwHT1nn@#_s^kq0 zt;$V*R*TQ}0|K>CAmoI5I@wn8R}YP94RGSkvO}@$@8@U6zK4SI!;-xml>Zlmy#4!h z4q5tb~l@5aq4s?~!s2(nSXA!c0j3!$~glgHE&^D@b zuiyr%W1_$Bf%sSuAxGbEgrUnV{)nHQ#m_1*h(|Cim#EW|0Y|bn+Pnekbb1 z@Dd<-2n&}7d=CROiCP1qLT-VS<`O6JyDjO8cv_#Wf&ygwcOs@Zy%VZ5b7&5^-v!--nvsZyO&Ts;D51dx_%; z#2q&VFnLg2GkitwR)@)M2`&}-THM^>lE6*ZR-^$p9jT4%<2E|voWJ#H2E9zjBt9V$ z+8c1D-`sN6PbFq${2-8O_97m+ks55XPXVAA!wNkKY>8&g^T{nSZedW;vpcf0z-lBu z7I^w8l|M^^YXR!4pczu@zubh5xyF$Vq(9nU)?m37IzF>s6XOiuebH|tBnDd zD`~0I^a@!`D!qIUb30W#>fdhfY|Py{CGUlK#^3q0C~c((Ag#P&%f^eSp~=lkBN$X9 zGe8!F_xWew&_3eP)AbWrV{t~Q`v~6Y;-`y~;cn6NEjmV1j%ueCFtP)^q#gI!%MI65 z@dsWnNQr!lUBD2`I`F${b$YWTE4#7vB5Wt{{#CI$TAt)G$q(F4EZiJm&JY7*@oZC@ z(b78Ny%s};Nm@KpxjbHgc0Jd8LOTCW{qSoMcn8H1H?t7!88$DX&)KFn1gmW|(X5l2 z2CZ!(ZWhtz?S)uT3zvzm4IEp=m=VM$_Ow(fNeHcmM?#vb?D7o`Ce^$yyh?RW5Eez*kxCI z_js<3M_R_;+?cRO`uu4SE29gCY`=UY%@lA$ejBk%2;7z(y4DW~JH)ji)Gzpue}Z38 zR0EWU434`+!isr<+XniXEP$%ro$@ng4`N`Dbmo8?(#5Zw4u%-ukoC!EJywa*%L5Cy z`z&EQc21{f9J5Vy+Z$CZ;bUze#Wncp)F!GLM@6wZh9m#aQ^A>Mk#OP!-8>~A{+;0| z@V>CB@4$!xLViWj$bQtPjUGeKyAXJs_ns_1!^Xl5<%%r5-Xlw2p5K%s3Bb6sKXJ7zf`-#wL%H6r$=0*6qfS1Dg;`8^c}Ef?>KYYk zn&j_pp!o7*;b#Vbs8vfEDMl<4!#5MENry0@%l6)do(t8=OWiGXk}_t&uS-+?I4Rdc zc;$%nKQxm>;$mi9-UpWHD&XlVEPdlr@OZVa{kN#y7GTyjMnY=fdw4+Fx6Svwq9D6j zEc=^=8NT52G)60a$bK9FlRK*b^{t_8ecB;kcTWGysz%J^G>)I*%HLnLFgHk#nJUm> z53tK!+Hngz*Eq`Zn!)O4g&d5kpTho zrk;t#ajaXG-**Kd$nAyW=CelIcvcua^Jwog93vPQa#EdirRTbh7;X0~AF_*U0m`j% zX;~V@2YhVXegZk0d zM9M83mC270eMuaCr?T|NxQuYyd@Zeu6y^&ES$OJo(T*rp>yjU&v)EMvZwDpgAN~oM zRI^F0=<3&gTxJw)db?e?bFhA)azi}Mt30{7qGWi+k6 z!SO&iS?P+mD;wfG)iumCH{-r*8S)M=P-xNGM~~GF6fepmUKz34Io%p_Nvt`GZqPN) zH`OP8qk5ifcHifVmPWDk-qeL(!XsH~Uj3(yVFg?95_6w(VR-4;q`05-(T&lK+`IKR z-Ve*u1K70EYUbCB=3hprXmRP*yJwe{Sx!*=?;po%uJan_FwVIp#@Pje@C?wXRZno> zhW)wd;`8eo>eo4=p{J1O0=y5Y_s&-&`%bdY>XJf%uAHJpgLUynN!3*U&(%WJ*Xg22 zIX7{%Jrn7)Ar3(&8$h!ABGUT>#}I*g4LN)BP774Jh47n5O$MVfVk<6a2PrX;;DP4V zb~zHotJhWpsva3a_N&V3(D}IY>&*A9YY9mvDro>wG}*L4qT-9?notj5J&%GrB=oYAKCvnijs%T zYjCAleJa`L(-+BUt(Zy*FJy~cEiVur0uLyGuIe&X;}QvZ2P{_{>(m8Y4JFi5&RxFH zI+M>!eT@joPvkY&yPyH^>6;K4pjy-jDM0m1Qola`~82>@|&R40ONM z32X`ktv#m>n{irX$CreXPtqd?tj0L|1=9*=a2SkA1&z3?o6kD*7Ms)|4R)2OB5|t8 zZUG>kn|s-`;j<53u!x%mwUbgwu}oQlcidU@c+w32WxW1lmjg%g`a6HFVSmkyC>GEuy9R;|?_5TS3<5I920xc^dbBmLt z$d2t(a6sOBnHZv8$}4`=i;KFy-Nuqv>cnx<2(;jAz%;gzzAi^x@gce2 zvHFI!ZiDmHYyFvaF2^1)gSXkIe2GO23%YdamfOBJVEREj#A9wks(aUs%{SO8wNK>- zOapa=odEkuP*H7|X-L0$Fi*ey*WA1$izDaUC1LYlOJf5HVMeU+ZS|w4H?32B$jB34 z3cQ=2G?W%z;+}^H=NubRg(_QIIa1KB~pvzcaYr1H8qqnkq2pqa>|rJ^4~Lw zN7`g}EZkOvGGB&QF?6Q28xB>#9!19lX^^O`*9PO2uNn@zP%aeMpT|RV6$4}*xvE;j z%czlpmH#9a3~sSqihqgE{Vbcvui#rSC777y?R1BXqb1jkkX&r;O%d?%m=X5V zT5|j4NPL{lL2fVoek|fJjq(lANP{2`M!u}~q0Tf0lpOmV(9v{}q8B(0r#>RS8o4+Q zu%*kP*+A6?TH8R7&V^#r$b#OULd7HxJ*kjcyy)Iy-bJB5JW>_`^AkdYA1ai&Ma&sI z(C-G8il-%wUlQi@rcQ5L;ZqB043^=5Y$sskZWa!C_UTAvVi4`!S34j)t_;Va&{-N8 zre`#Pg14Nj^T-nsUi{f+kmznr}$tPRBN*eUyVzMla1 ztI!@ibMk#N1huqr(e#h%ejF7GcV!FjP!~0Nm#`2af!%SQ>Bd5Auk+n9%I}xOdNL;Jnul*uuX-x?36`R~ zZ+11koSJGM&PQW>QA~>B84q|yJSa@(EncInpsdCYTty7-$XCs%*f53$08_7I&m45l zT_I39Rxzc!?&_Wl)I@FLg%Aq2ugwX(Rakh7XeEiYv6vs_qkEk`dw!3VSg2BN#5qz^ z4%{ZLsYr}b5AqqG`~f+(^{jDS zHM$o}5HL8cy$8wqJW-W?v%c0jL%k;CP2N?);CnOnN+M1>;U>CXDv@zWhCyWx4)h4{ zx=w#J0k9jW$V(H5(Rt2LR5x|(4?9d2TK*9OQ$0)xdp*5hP+A!?0ws{v|2P7sC47ob zMl)*Nk@lH^k=Q=N)IX>MDu*^+l1QV}QjZ`R;TCof;SZF8Hbi`^_V2&{jAOqREnG(h zybw<^-~6^O@dbX$i^ycjhCJdIY6yD{sbOe>BMIuC39-XHE69W2Gl;lK1Hgb^Z8a(oPVwSy)LiiW-nE4h zsvuGpochPv_L@}P=8eB6QCQyP%c)RcvG8C0>1a-&{5-ql5BpGs(Dq8=$i26E^s;%i zU_zxVY^Zi5s=9@;k2!>-jrBhG?aB7hs%fP&Hp&+`1Yf`5+K^?^_w$Sg|hU$5G8Beuxq=} z5Wn1qyY*7651D~&8Xh)k2Y7R}KA|d<5k7k;wnA#{evB65G2E>$vSNa*) zW&i8Csi{dJsXI>cAD;Ks3-kdW13)nKG;XMK?I^g*!hdgj0Z{`v{c<)g3hukw6os=y z3^uWFiZq3)y5ycM!BekTD>g-$?AZjMyshDhN1-EAYuVS2%v`;ovVxKAEh}1vN1Gxq z2rs+_ytHi?*K-`f>1QES$r(EB6_N$&Y99dBe+P{K@Pn~FXpe(bH=}`AzV&g%HY0d-1b-JQqd^6jujob?&1!&nYEh@$H0nc8pIyNo%J`nGsv( z?K^<8%@JgRpYxNb?d8#^QGfa>Ar~K`Z#D_hu>Ep3fx2Z)n+c35>HXR8fh#`XnvasZ ztw1hL>%~401Bu&$kbBJCox_X~5R3;b;lPP`De znlw~honp{L&XpUh%WV~XO;`YFj9B&&*K11S(diTg$3fPJ1Wo2V66LM?rs{Mh9UBz- z61?i4pNmD8!4mTw%hk{tB#KB=t`dr%j{NLma$ZB9*`dW|aZGKQ5PN#Vd{55{JOuUc z=Jl^PyGC@gTMg7mQb(a)$9s~^4d}U-7}v~ci;A|bFJGK0 zrlh4C#y{JHMDT{iEQ7?wv9L?T`z~Hu6FfCYK=H-xvJy`5=M}H$O1XsLNnh{jBzfD& zMWQEaVXyhFN9#kD(XTRrX+ymvNndzH^h7%3!zHpWZHp+6C|oj z)r2DX47Aw3dW{)w_k}x-+j6QPSbw?88-dD>I_i6&bzl=NYFgNCXnwD555R=mz;o;v zSBCx!E{4#)T}ML~p1m1E3XQnn@^LKQ+WN}-X{C@ePVDPZ%k4(zIO{xcvcU+ELIPM)Yo=N(qdG;7A_Xq>Xu zzAZUL_aSYJdMqR(g{K%ypNn!x>F3VrEhcid+@%ydL?vM|5Z+ZB?3CEQtvmMA&fgw= zN}x*Uq${pq{g5rH@+nwXEzj4jn_D=~=>7Hd*`SM7Z=&j3w{Ego@@k!I+$4n@8Ts3W zztP0~owM=w6-5(Cf6QZT)5YXrP1q6&%DW*~inl2KybyWyKW>~${b`ZIueYT$*=TD5 zXCIAbSe=50m}c=?1jv>2a}LK8_8>IsbA?nXwCO5&s{#7$-){2#VGPNa)NkPqv)WRp zFAObXB1CVxe@vM4nXwWWYx!lHa*zW3ye{N02YBl=ejk%*icWG=p{1~!qqAZ>UvOFj zwhOF0S`7PUl%=<5G|4Z3Wvs`4aH1xC5UEh=amEabt*;I_{fPdJFl|gf9HSqJui@l} z{w9~E@3$o*4tC4WK|@6plO!PE7+!}$`Oq8Uq04-1FJ*=Nnkzb&p@F=Z7zjA@YYX)E zu~O3V#)W>Zf1~cQUCq&y2hX^Ti!!NIfiB;;*Q)kX6iaO@8NXRI$@e%6e=!+q$oz!} zQnNsRr2|??u*a)GX&(*&qUdnfJPb;Y&_G@5th}2XUSentxH$4#weGSoZ-_Ryr z^BVUcTkL5_@UR(E=o;B8#pP81sfpX&5?}mLmtzfXfWBplJ|{^_z4(4=v`qsxba^Pm zVlXt^G;3Dbs3Z2~4Qa9~s+g8Ot&yc_N(x=?~;^ymaB*P)BQt*DwxIAR~ zs6`&7kXz6mhS2Iv*KDxwDRZTlW~6o-67^VDY7_q&>cpCB&MvnocrBVMd$MsnmGN#z z!EN`xtCdY*NHik(I%vv9zBJCo-ywTLe0V7`vHVNph+N9lT)dLjeVXF7AHuwlx>?%9 zLWk*Mk&q(hVXkzPISayQq&ALyvrEZ?kAGQS9z$lu?l+?T%KEINnb#R;K1W2#39=;W z&zRS7IK(wfF0X=VMlk`$30L~_k(E=46cweluMvIhlvKeiHnL z%>&}ySqOjLy&mndu`8Dw^k@Qgt2-rUrI{6L!F5PYxyihj&>O(97T)dv<%*k0MH5TU zA_b2HucST}6T&5hJ+GlD^}{<-n~pWb$gfY{g2io=sWPE0E9iF!7G}Ff#Qn5o!F7q~0()_O#cTD){v3aiBl;s+;0Z;GTw@2akQyZ)%}hBC~z0W$^9TKB-1NPeQfJ zIq_rAaQvD*+h+N0<6TRAYlyA>X*p0j8nZ@KM~sTz$iXmvln+m!t27~5!-MSnt{NHu zIX6_XVK9m(pOyjAX?A|zN)XbO`Vceb{2E0ojVRIf1{DiVw4fw8Nw=KV6L(Dt=_o{% z+Y-?3>tk2QD~_pCk#}4@mL;l6AIaMtZn*gkAo~@}-WWcl#-9}<8D3^2O~0I@`Z`-4 zNh*|M*zQ0WRQQx~B$ey`I~4&OT`GU~@bX>1ew}44N($O%!MG72+pn@Zj;LS{533!q0z>guGsEE{nI zE2L4kyD&nabxo|A9XQ1DO5sX94VBy@LB+wV&_mY5+gIfJAlJe_w&8jTV@+Q|jW*x8 zDDylik7!&TWqz9Z;VqYz^HVebFixerH2KPJXZN!RN64(;#X$P9uMbooc%r;`1!tGV-IU?352bmr@7m76yhaRQWL$$d%qk5Qlv~xQf@T}5q3uULMK3$)4u*k+7#qLC)Hp@^GmK~if`vywIo7}S*{PO@320C1Fa z(FF)DtI*KUVqk(dlxffk747Q8FD*P^>U1}I@~DX8H;eMh?i7`ei(3FEjo)dq3c50J zeN7$XC9iqXj}UR1Z8aJhjQJF4iYDfMznY*S-w7J^w`r$Kmd6~+M{*I26ubqx`}_6Z z?%~D7QF@_xp0z4T;(T)&sN|JspP3S`EkS@-g{ocdQw9E+Dkw zqT(sF`4lC+o!zcY4qOKxYEFx<1Ze}X&V*So%j9U^>MP)^iL;~ayi=9D#`U;TayNUC z??ZdnsG|{;Dg{z*9j%r89qL%Y3Zv=2KY3ojx9!)%r&5ONvst?CfxS}`mA`E9uaB_l zAo#pxg6((o;LE?*{AaJhRrIMj$f<=F4CH6&qSUa7y8ynMT43XQIhn#%{WmGV4uOXl z5~fOuw>A4}7fpED=ph;gcZLkCFJ&$zQ^m38aH?YY1d+H?Zf@R(bBA$=*MhBLLU zUY>4iXO5>_oDUL!D3-lNkCcT&0g+}NgI^ntx8}B_%&m->{kjl~!R*vmZuF~y{qzj; z0CR+lL)gli0mdtmz}E7;gNHsg8LBa@ZDNsuD86|5vU07JlV;;7@9?@GimIV;OX_ng zTmAa(?-{wAU*X@IJ`*RVr?j?|3x#9C{K;!A13pcCrz z!VK%<8yn_gq?R^1@}M>JR}dg1tsiMW@P{GJuD$I!(#fsyxJ}*415_74vkhdH zlihx3*0nAj;PqTH3zSf%!wIR08!8JPF?Ot&di4Qu8) zQGvXnF2xjor6y8Y*HxF2JM^}3#;fXD%4NYNMj|Eh-A?~2b#_>CumbYyek#Q;^VqF8 zTP9Ntfl)>?{9V7JJFvUi2aR$9vnfXEC8Z1xv%JDMUN2$#xAOl=?Qjb$57)06=PKqV z^shbwjBI%yP20)DxUqYq&}5n*hc^jwieP*3q)Po4Vdy3@qJ!ukN&X2j>5C}yg&B@# z%fH^>@KKWcHFC0T0zklRyWN5B@r1MO-uI0!oIr}DPgC97Jd%u z_mOb}nM_@Ic3dIIh-^_C(PX5k-77Veuha5m)ZMBnk>?Lg2&gf@mI1kquuV{O#5n&z=-{0^f|NIP1r+$R<>y<`e;TlUHv5RBT>8=1JF-aK`7vk7tFQS54iv1GTZ^m3A>*UUQ z$}1bl7Zk^lIxU}2VLI74lr!toqn*=-&m>g}863$4(6Wo1D|OoWlio%?5oMebI6oC;rPWO5d58zKhVaIerCt8{PqWo8y z#Vz|Y*Y~>nyGU2umcJ!REcS#K*D?QKmN{Ytesn*miy4T~pk5Q}=8)v=R_0WVo8SDH zC1f&dZhe4EokAf?>vSo#l3R&UMAB@`!s20IP9;2Ff(v{aZV|pdyth=K$~SJyQ$8H? z2ymAXUGEEc-)h$w!n2{y-<+1G=eI&8btj#rV!k`?y^t3@nUdNW1vZ*?l8_(H65^^3 zklyPEc_gQaVoXt^IXDgGU9k-LY}_tmd5PK(=#$f?s2+1h|EXXl>U{dwY6^<}7!Iwd zD9gb>7Hj^pKUp+e%dr11N(w$}%Ma7&>(uQH++`>qNFd5x`k_CEwv8LFCT48ek)$blS3{s!A&TG*0YljuLF!-2PIj+u(1T7l5^|X8G~5>S1#@d_ zdyEhOQK%L6rzQ{3iCq+gD2%ndcP^@7rC5l_)IK2elYs|81+R{&GC86wxB6)rg*uQH zl3n{(s=97&vBT;_G)0zA3+{Ur6ZiA$QlV#~PF0~K7o+M4fB6^qC19O!YvRHdWc z;@i7m6g4X<{9#gB)V$>}YUgRBgs~>~D5P)Vwxgv>f*ZSvGcK-q6WAr>^>>xk0qJvk zqCNvI@(QI;X}8UMUe$dfMh9ikWD@KEEfEo_j4F_Zq;R$pIc-gFG+jVA^iux5bl6c^ z{=isPdNNIjN}LzEb?o>HfG5}(*qeF=Xx>8Y3x{Hmsn2Dh7qaA2WqjWZdd+8$uK(Vk=o*TX-!%|kU)%cL4bSZXvx}*9 z^NbzDmDQ*B9w}7t`3q5Yz{A^Uss*`QkO}s5u^Eea*;MOHz0uokv&?bkKbii<4q@zY ziO9Z_Z`+apW_x98Pa>v#18?Wf&7Z;(7&VoXzAhxUezc$AOPOc9@xVytM$34@QG(N1 zA^AJQrQuE?-cL#Va%mjZdC=w#tqmSYItvotfEEE(ot)6PYlVKp|H60le;H#)Mz1?r z<4Vki(n7q|Cs=u_cKnO;>{b5`TB(6d zIpHtj7hienc|R#Z~Gn?2c@U1srS-F@0PFQ7%}SsCs`FTo44=CO8uoF3$PSt}5DY;}82g1(AUWw#YIVe;EBwqJZT8|75%Uzst<4{=YeEl~&l&i8pj%#alKy!~32F z{686j@uqM5u03v!VQc8EJN80P^&?)#Y#G8+0?+n>&oT!ztixanE3gQ7mRbpvJg`&v zpQJ4Kp>n@?;;N_XTAkyaq-Qn6bJ5@#M4cD<+z-Eq1LQf?^0|Ogx85E--&CvnKQaUK zwrkS*13QxBr?#zt=pA><9#1} ztK2=2!TP^ck1l#k9&E~Yq$=YHGUcO32G31o`l~0=oam17KV!8zcJ(t`zZYuZeZHJL zYLhf&smPSKEYcjM`Cl%ltC+y*Ul_y4I{EdxeTGTGq&P(C|6x&R0*K@u>|-)CwY`iu zbH6UK+6HRs(ze>6w2t8$PAfG4X;-rB=x=@1~;*X zJluW?vCHd4gf!eQ#Z_!znlBaOs66@VG>74HbPvniev17P%aG=k9&En-ru^N6uYpXc zECerzPJD8sxC4$z4{f6iv5frVJp9%qwxcaSgsvTN&wAj~2{jA!gUm00^nUZ&WkIj! zR1Fa{u)K-|#U7UK=t>ZKkuCQ7GKL=(d(7(!Z@UGh;8!19 z!8%txEnB8h_}_q4W?bU3#wBhOVUhaj}MW-^qpajkF(i*hvhatkdqKb1GB z{i3}3Q#!tgt$hp<9QBf`1e41eRqdHT8o>hQ?q;x`6p5}ymm;3O+-pQ=@ zLS~#x1ivM<=~NV!kW2XEZ_VJwBYu^c-YFj99++x3|1jx8Jm6rLlv`YTORL!9hkk)v z{e$;&vLjq2nGr~4CZyLWakJxCr&mu?xyJJKo{0L_l+y?v%R%8LTw<2&p-=M}`o6{j zQWMy<31#$~Z4v!50uu6@j-PT~ND1vANyD&5!AcV#OvEx!1(%Jru?L2yp?PoM~F0^PpN2=0?@i=_L8lnb;_M`C^UQ z^erk&{d={=9c5A}lXY`#KZB3qIkA~zINzRtdKejGrFt0}Kna#W9zt8k!d<*X#v3@R za&VuQxP!0kRdmk4fN)dLtwQl?IoMRb-gP9C^xo*D0q2gHftyJ+B2^b$t%eFfpqfa} zHi8Ct=Fh8O3R9|np6Q~$@B{`+elO$O3Qw0h5AmIsTIkO4MfY>mt8Uo+Wg-{_r>Cx^ z{2h^IQVDW1f~?j;_myJrT#*Z9eqRNjdBS@jFITlGc8%|yENB;fjp0}RJ^!~n8(W+I zQ+`ilGQf@9Fn{xmwkGcEJdvZ}ptoJEd$iD3`@iSfh`iR%nO4jPS~f#eX0-0?-1Yc? zE_n5@KhbxNOm3431XW`;qLG5NA`6hqZuX2<&?8~Ec2ksU+9&Zx`5xUV+UbomGkP=0 zlM*f+b6O{q=@tU>MjS4OPJvbL*L zjk`Z}un&gQ+qKpl;{E{;F&tL@1KIn(dBxX@4tTk+-WTx=*EG8`N!}k>H^g`z%|{(0 ztnAB6UxoF~8WTY6hgQCqu@uSt!w5t;!a_`85?kptpF}dV8=YPFg*HZyzdtbFwPhjW zloTno3TLvTQby+X_K|1%<)aFb-L&k%+I*Uw0zvX3w`rX(x9iK>$|pt&OA6(u2>s?; zWwVZ3+CwSpB&l~iu^+haRd9E{&@GyFsfwM#cKiDQBNIYg90a{|!1vSzuiX9`kLM|y zrAcu`8}=0gx{q(ar71lZgrCIOeaC(~Zs3*Yx38VwZA*{u#0G7#)XXCh1Ng>xO` z@?N6^>FaZnxL(1S3Jaqhvb+%21RGAs9E5Ve=Hv}%cLMxor!YzW`r`jEyxf5uJSyp6 zppVgb8;x_{-F58fo$8+(B>wa4yEbjAjWRBa1@5jhjoCp%#1ARl?ds$s-PoyQO|}IH zXYVL1Vq%Z~&`)SYPTBV9?by-}^wXW#Hh5@lsV?|u9dYPNYSRJ)Oa%_h+pX&}8!W9< zE%UF*)Xa_|IvJ&F#$O+6X3{)=>H{&utz*XlnNsGi$$DO(M2fn>|Y=x%=!cqrkLJ zjxfP-5nQa+S|5MvNTM(&m_#H{3XUpDf6PXWO0}%yy<89Gl1I&R=Li5-SKl~>u0}Ll z)rx^CKBNE2P?f0fw8X9>rsiyIjXvP((9c^Ro+qq{NbEg%@JzX(roi}3opjH|v21(i=GHLw(oCzyLOZOjZ~X>$e{3~Ph!WMfOa<*xPVF|L zUm&%5kv;I6B4o^eL#M3}{%Xp{)+S~KOr^hD_d_7CrR5B5=2Bive+JNAxcYCw2OHK- zUHiz?{I&atPOUec-Tgt>KqUL3nYJ6AaANHT#X-Umk?qZk*pac|s_<)(dxe3sL+y`h zg6b3X0XKf&dN8Ps?-BdTcauq{W+Aw2gbw@Y17_)%6d+B+{VBl-s2+Yu1MvF9HesHX z=;stb)Zw$Y1B&eevd%oS2`oz9cN1~1S@30NmBW>ANAUMK47+6lavU!2AGGwxUD|O$ z1wP-q_OtaaRfB_Uo1f?aPJ)a+J|m++odxt)DTFREaFQCNE$oa>JW{<9TdCY#_d#o+eBDN6A{k5tMNQ!k zI8ePXbS}l7vd=+dJ^<-4|3;g{g2II%U~f8;r?sS^Hu~D*e92ih{SqxBx!;6L{mwIL z2AA;Q!heavgWft)!&mK0D!cchP?56~s4ODD;X;fvaQ|7*P>p8u=*(Bp!9_h(q%byE zl@s6)R?be-uzfyFg?Js7)Tpw_wp^d0dej=AUvh&X_Je$S^D!Oj$kYyOZu&dAl%itW zld=L!qfjB9(>|P%d4m}~wwECdtn^`8b^1pD948$hChfu|Pf1H>V2ZsJ`V==%+&1|@ z0d+D9L5|j6=@cx5!}Lrz8z!ZALvWz7W<1lwV)go{A@j-bRFeS%9gEFjfN-`(%LlM}0X8#)L|bn%bh{1;#>2Qw*Yi}!X2G{j z6e1qKG+j$8Gu3jnQUceq;NXL}Q_);HL@F>8D|ybOgPK}6x$Mg35sKvsBC34|Y!5>c2DPL>H(X>=L)bSa%>!D`A79 z(Nj!9A}NbF&^(x{{(x#oetyqkLG#5(*!*=HYyylSeI%v20=&ruxS~Hk9eFyH$ ziB5thq%fL?uo)1+Ma&i+fzIwgt>;^4Pd&0M6K(q>4VbQOxO`WA69tP^hT4uhW!+aJ za&$h8pbDTSHU66PC4aeTz?d!9#eIJ5nMhFKM35S&<(W|>@M2K z`eAT8Gcy4);b<7c(9GvTa~4)%^BC*x_Fj5N!Sa5;++$EV{``C_7q7QC*YS z{`6pkjh&rn^gXYz^Xi$;?;%Gn1@8kQ54-QxAdv9AAOe)Ir`uv^1nczU>FxAenez>_ zT8+|Ba@~|NXWS2bEU#XF0l!5$7J7#yyw!Ys_St&Ky(*0B^@8lMilg_kg#`QSU73dD z1)JYYv-X=7hNFKV5t&^Qp$AaNA@lQ-^Vr|1g%`RNdwV11u}0vAmeqPRn!CEf;@q*& zN7?B#sG*0XB8iyF%^pj$kT7k{?e!178zPzNf6hi#%`k}skqF~Br>RCVH@Z$CR^Z}5 zPiA-=tI?m}_}y-{eR%YCRl`armXuQ49`&u$;B9=z$MPj~{O~XoE4(AZ?~k%S{uDy2 ze1;API$b<=2RPYOlAEvm9(LMICdih$N&nEmRl94@W{y4q-3Cw^bqCF~jIn6`VIKt% z`M+8x%q8f$A0v6#d(LYhjWSO=i}lYuk-__y;+J)t`^gDb%B}%Ag8G3n@7OdW#y4T9 zx=_1fQ!i3EcUD9k{dMTAo@Ls(_O)5J6cfc~TVgSsI}`1B9%lA25Z z6xybFywsRd0pyZK5)v1+J!nv!YMg!7^D}T+`~0W(B@e|99ras+MJLPC1qyB&s5RjQ zqi#>{??AWK$r&Bnkt3|qJjdkJVx5#AWLtO7?BOefF;@qb@yJf@jPJRk>Rt7U`3GoA zl?4Uk#9h*?MEsI!aXANWUp*2&aVCL1oOZoMZq*wNYz@!wMwDU>`1dMuJ+t%?ZaXm8 zOOQ=7@Yl<$9HFdCqvYh_4MUE%D~U)8y_4ax}>$>v-| zGgpukT;##)$-78#X;uf}Fg6Pc=92lP>7q^H?4U`_$d}zCaSWKNuZ&ZlWO^Ts2}^9) zYF5u!898Pfm3SP7sSPGugoJI!PNeh+;3woK6}~!O{5SNAKxPXf9RC~L$0W;Wla#xs zjtP;z*hcl5g9CrJDjISch4>5M?3gLy8k6Ep_My+ptq6?znT>-fBX%YstrQFGUwjwY z53_7q*gqtdYkA+rOK|I_U2)>Pjj)Buo(rtCT_0D-m7$6Nt>Cg5#!^AFd7dAq{M!H& zKoh{Tac^kA<~pvH$3jm5UuR$`SFkq|b~c0XS`tPu_oCADenHjV9dg)uByaeh=`1k< z6Mz5ElKSI~Btxa)y|HW|*ADv^DK!O+uPmBXRq_$Vk_ezk{+y<()aY?{rXS62dk15f z@JUS(Rm66~A4Qvs#}08oR3!%$uD*o`cNGY0@VykNL|mQSu(($-O=iu7LBXgR%5&!C zi_8bx!{Y^Nq0l~dNs|=Nlwl&t$VDO%YOY#dHA@qlY98bJa|qw?SL}5XYURq9b(mL` zQud9L;~cl`gY4RgylvMKnEkl*-@P8Y7QqTESo<2XX_a-bstMnXzvl$^W~F^6NPv6Y ziy0`HaE}^sH{!N+zFt=nEnIFRihsZ?6c{_HWg9AK3EnBMZ=AgMYIj+oylk-8TEPc} zi>KrSCaaF!YiJjpBzwnDi2{H;E`V6Ctc$uk`58Rcz1sY9lN-GPV-^mTn9#X2INn#!+xY+_NNMSZz zn2IbMM;;e>Tggp{Tyf9yuSHjPfQF}`%$m^kZo8xQ8LpCawqIu8aki8oW$v5h%k#gd zNxeB|njz`G&D7~9P;ip>&fQwV{Mpq8_!^N9Z!l9RHgP)}q*a!O&^k$s`ei$_Ia~T3 z3Ws04j!Qa>eR55E(W1NZa;icu@3?-?N2=;!Cl}s9E?47QW3bCfCT1CG){r&5vicP# zfrGNM9bxDOBClhr^(2NB7?$7}o^{&ix(0ySv?5grjs^dg99{b-xpjJHtxLs!RH4!Rm4M{AP)LRG0p;uE48!Bf~f0&vdf6qjYZgKP9sjTDz z^GAX6{yzX&0OVUXoz=v>7Kh#p{kY$PA#i}Z_-j{xJP2IvfL-QudQPI2m zSe1b{7dp>?pZ&kC>ZkDdE5zBQQ%Y)`u(W4{(AkPlQsM?3!)mh$Qm^Ba4N|(f3X-aJT1vA8i8tZfV+eLn6truV zCk)PfHshaH*~{8jzzYTjT9ym(XR%PkEmL(>MiQhu6PK*NxZDZyky!d*6K|iPiY?ll zo+(=Cit2cpk#6mnUD(Ei{%Ajl%y<#z652b-u*Vk4MgY;MR}u-vF^?t>QBoqXXx=z9 zo75kh{~SaK(|JB2XHIU~=|H4C<{{=?qw0P>+8DV&zJOd%TcEH|p>*B8-@6#`4nfl@ zBBl=CC=1vmPb9rf0leF$mfFMAsge5cf>j$UB3Nz_xk~m^)=du%WttyvVITH9mb9wf z7KYT?UC+I%6Q1X73;)5&#}SFE3#PG!)bt1aNlbn79?0Bpg2_xfsxxtFY}*;(8}bTD`eGVWa;=diDdM#FAlYV*{@akt zI>YjwC}A2grJ*hsPdY36bgVU%<}%3%+`N6(y-Q#{C=p_oq&$*loHJM=)2pjdLBJ=6 zkJ6qKiN3Ja+fAOS2w+?3b=<{+JG6%-pjU1#Dh2j%dmmsy4!*<=l`PPF zy?fY0Dor>ejLp8nW^?&rSWkGk%cH8M*kYzewSKOv5yFYNZ9axN%Tk}^(;qv8p*Z0m z#Eq@85fH(~F$^+xZGh4t3=`?&8WqX%qqDoq930J* z^hS0+Jl^5z0)@#w(uXh=u0w1>VAxupvpkWLS)(%eDOV|n4qiqZo ze1;pjLjNn=sD}LnH!7RyUwHPeeQ1AwLkH=VO#ha_sb?e6PcP-;192}AXj#v)j}DdG zwpA@INBy0v&evSlKz8QjLiSakjA|@hP;4aQ)P#Dya%Z2G{R~q>tlu7+4rK8TX^! z68e3bAvp+zSi@`1uY7sS5yQ+rK-Q($u*Gc*52s$&c!=Inr1nU%Gc)KvfJ&-=u=FKh+bRGh+0gSlMYdqj}Uw z>Z#X`Cbj`ysuSPeW)yz8>rC-(6`Z;u);DJMtLkpp{SOX3h|ks(eT5D$oSD6?TK?H! zjP3$;Ncp-lcqanVn6><~f&eR0W}`~th=xmxReQ2$@;2C>-EYWw`)8DzE-NnxSf%}o zOU-*xPU^%Ew|K4S*LSQxCp=li{0+NPkQw~>tLp8+WD4f@R;jO_OKxBT z$`|G^u;>WRR=VzP%THPgjyy#r^mhU-aeqU%o4nm4icyJZq>YWhIk66v!>*-G@dQk^ zyuu9kWP(fMawTjdw=wkw`eQw!n=^u+m)!Xs!Qw|JHzjK8b~F7}2YPWbeM2vMw}+rp z_t~t9c&&GxSv{1eK0}Sm&Ffh;9=d10W<#0#7$eAq08Kc)@;Gz^odl&Fvz7p#Phgund`*DF7S=R zM;n{6ppD3c4KH_YhD>&|(Uy7vd;JZY3wRE_MDZf!-l;@EQ%oOGfpTeFMn)6*OD=n1(ttf;RrUTVktO_< z1%A}eLbw#$c-HxX679fSI?&JKIcgu;JP-6@WGKR4jh{$_L=kEJC>7?()qzaAGISt3m&Bd? zpYZYj4w<^iPV8x*7ckKXAEUp!@OENnw&&DTtq(vyH%IZ$b0S+2wo+-+Y=?Eqjs6EP zU3qx}A9vuex_1UA$npEPX|%#Ck^hF5dPHFsX09qcF@9xH3gXmgz(soZt3x!Nb&;Qm zz2CX!%k)maRBlo}k^QoR)5zU?!n0Ga1+O^XObW=)qywACk&%n9JUa?=^yKIqWna%- zRe*@}gtj8o&wCP{NZXM)uL9c4EyMBU0fwFi;uO*8q&dOcur6$Pb}+wKp8XF*#aIJp zBk*Y;d(qBm{`ODeG6Xi^{&hj z)Ouf8)|O6PP_#-wzzQgPI2{2`WD zaO%fm-Fiz3a&aKe(fmOEr9;^3&3OVM{h@xSfc?Te!EI&ZLqW{> z>mv%zT6PLC2uDT7rou$5vw8PaT(oS*rs6fC7@np>n$_<5TxLZ4AdC8;=_fX*|tMhAXSHQsJn<7jYK&XC8=)9LF)y<8OJf0$HQeXijrnhO`!(({w`GPQfC-}@hr*I0Ob^YmDN>b#-5y%MoVA@DqtwXOm=7#U zk&nW`5a)OX4Xu=v`abr2n&Q{jMxPc=HEM+9Ezz@__%*aEo0eRA^6#;%|D@$(P}FBa z(xgC^$N^V4>^rH0H1oC13Um5KiAu0NqPQz)R;hZik@D4vTexu-X+{qNziWWr>CHVaC{MV3s%+^c_@nBYAmOTBk6a=w~Z-3Od7vpXO ze;von1N9){^fX z`()x`Nl-1o*O_^kXE;VG>EMnIPSYgW!M2r!p9AX^`s&Cy0j>F}~L}$<7&JvvN!gG=r`0>1@EueEG zCV88|uIMR=BrMGS5SE&Zjlcg%Qy@Y_f5oespf?M_P3QPpt?bWVa^0uLaUoGsPF*BX zSaEloOGor#bV}ZG{j$oepAz&O5A@3U*jkV$jiwUnqdJSlzJ=nkuvHxm)ny_)&F1wy zSeyz^3vg=lItz8HFfaKXwL3YUN0ichW4})D*1q87k)_S``AC`D&Y-+*tgX+giO&4x zRpH1EY6N%ghC{29Zi4s|o;sFk?k*LTp8|rTjej)$H;eYQH4Q7B7+P#;-MiPEhVK13 z?`Mtzk-CN7v4&kC1N7e<*g&yXbnfb)k9#*@7XFH=kJ&LcFITFP#c>87s?MBvK1sdU z+jg2Gns`ljW4WKUz|yh17G89tlDLpgMNq=p3IfVbQJd{$8$BcVsaTSv%rFWl?`HNc zHJn|M1qni6;58;?nXbbqU%I->>D@NUmvi0ele;<~`EBpvYkBrotM~Ob0pB4WZQiHH>TQECV-4DlA642UUWpWYckOEpL+38Wd&t# z6!WChHR?#?#CawRd!}-k3YGlyotk0LfMBy}OU9zLM_r&ZwNU@XeGK0ci}=xuzKj?a z)SnzgtleK4|2G#^J)4=$x0URZ&1@Z`xj6bs|MCnsfpG&}EEtJk$MGU@05sq#ps7%W zR5X_I5E08Yiy^k%AXFIO5&jQbH81s;^TF1w)pYz52yYj>!rhoLpvhPgVCh3^%cWy z{MIj7s8TOA#He(8CO3rw3?6XAYCam2{#su7?N1+^r7oS{vGSsgESnZvs!58&?(}sc zUQoW>d_x$XX^UQ_ZE$b#arg)EP<_|yzBc507Iy{i3o`PYa1M!!Q;)&L-?IO(qXi|C zyR_veBx8q;9wR>onHr`CtJdO7f95fWlKQj`cK}}b=ATSKx}ew(BFrrw<%s?lwpwJU z0sk}|kV=5*(ERlw`YJVHV_(KHeX+Zcayzk6qNv!AXV8#Eok2K{tEfbM?`SfAFUcO$ zBbQw_FJTI;>RQXu)AmodVPz^Q_CrGf6f((9YguI~s@nqVX{+>#$(u z&C)-lDdTzw5*Fdkl>ICy6S7eWau^D){FPc|Qk>Rssxqn@>^5wzxzl&q$8PPkeOr&%#`u0+pkh z>b3YjRc!1z)5eB2dAUOtmV745J94?9&BAiVAbzzU8XJG}l@rcQ2ae9hNq*e-uxPUK zb-Y{@_?&gEx=MO0771R~H~yP?ET3q9bjNIdK~+*dVO{$u5*2^;YfJ=_R-(Sg!+}xx zdV_CMK+pNQuR_D6Re$2U|FMcG9U`(TRnDvZgwLdaUrZi+gVQLV=JKuBkAXS%9lncE zl^-1N2IIXOFSyFv=I#fGny*7;36}l*KCZ(TB!L{8UB+!CH9@rHpPz0kXW``CX&W{+ zi-q1K#?j_LQlB%&2Eh8eTW@4apQbd0Pbau+cH11A(o3b?3k#$Rl3$TQNJ;V3d|{IS z*a$I=>~Me4JVgks*d{oUt6ekP-LNzAiw8q?O}MtY!&RzIw|4&!9t{jH;Gd=}Q_~(i z*%iLv)Wfrq;P8QKuGhHM+e`JR(_*m)G>3h8m^?1Po9b_77hoSER3k!Z`<7iR;ZtKI zYxqj052Wg+7}tlgr%g7e4W63>M@ySorO9lrIu!wxb-S&6l@3=UzUjI`6rVnc#t#0q zVsF`#${TRw9Df34Utm18AaS+1-@a+A|GpIH?;dbBxh>+f^6O3gxB@wJd=g_#^UIUH z0Kfr$@8$4wp8g~@=OoY1vGj3zQGek4QW?5dZWwU#(w2g2G)dLdt>W8=akJT+=C97R zr?VWF@}2=998c~47@gPD<7Oo!1e=B4G|2I6b6}$ZbUO6MvvwWt){20Zg{8|(^_{b` zvnDmwBH7GCu8(J{*>n96;9lf$E+;yW>##KI*V7?iw>^0gn)r=dfI)Kjf6RJuZPV$L zG)ww;=myMygK@-R-9ovyjw?R^=@dYNcLvIjfPYA$)lZ50`f{E6Y;LcaXu_wUiLY=) z>2d{;6A40w2?Ky_Ow|rxaJ#U3N$s8~JG=d!UhR49mI*u0`7l?~)P#lX+MoOSWTv6v zsM#T2WvDeTpn0g1l4PZjhGKyr+&oTE>_!mBU@hZjz>A%tr|<*)Sa*avOg~(|W7iDn z2oeKD4}qaTnOKvp5&+K3yhVy~eR(u|oc(2LThEG;w#}@6O9AZ+?os@1U$IO?S+Q5` zwByo03Gw=k-~7?t(a6=SwJ)2O_Ac-LyOinSDO^D?pV=EYhZvHh;ILX~FmC>?Avs`)zLS06^zYD!Cc6lRQLsW_HCYIukrh`aC*kfrr30Mi zeF%f59H{t5JptULxz>W?~u#vC$=mKJ4TgYw0&bqnFuq$uA zHMRsFPssl=watLFVvfDnOlQqsqs+8=41uJ^ZQ@@8V86u8nb#~ose54nvIJRVO zPb;RP#S&H`{i#9+55gBNcT_ugSub=%w} zzFX>sS1MC-YRxlqU-2{M01WH}4Zh8r_zC~Yj*RaoV?HM)(>vIBM-wLj$CZ;>KMYDD$ft6R*z&EW3k_#wxg4A*l+l?mXP6z1+uCH8k1#g5}1i-n9} zGlzm9w~;=z!c(UywXAUmV%(5{XEikp){t=d5sgfDM#a;)6<@7Lt1}N^Z z2P=MHVQ@M%;?x_BKd3*47dSvdQOeuN3lS!fd$ht;pR-kbPO`u&i|a-3-U?+lqgc;+*T>edn*WCX zRjLY%&fzU|@ES9;)hM4zjEnS*<=zAFPOm)V8H)7wNpnA$40IGr5bWJ9n(Ig5{D{LF zldO^zEG|hW7Orc+)m6_~<{t%7w;T}*0=Oll>C3Mn0?0h-p}kKwGe3p#y)KDJmAiJC zxGrfMV;5uqKK)75n{gqZz{`D3O?{*`YN@*BlRb?V^UPqvX_ zeVmqNuGP}cmnZRiBv4Mtsgm#2xRUOY(xCetg*lUC%(##nl`=Tpw|lEdUbnf<^M_J~ z*-C|i@~JLtSS^Vw$$bCeKoO6gP1rKml9Ud}QSk8);}arNV>ch9o2+^Bu6x-?b}{fu z;M9S9V5<~T4VVE{5x-?i%%GXhu%6eh64FyzSJr{;K#PYNe2_h*3IU%@E>3^m!j>lq zKFt=|Co4|du{O*NY!Z5nqHq*)OpI|_m5PTe0nUODxu9LJL)J{f#nlyOobd*xzXI+r zBbd=Fg4Drz(NYUyO_Cf?PSnDakL9N@c&6?4OYO3-8p()GF7>K9+47&$ZPvzVU0*zw zDz60F8|A)t6d}OgfmPSh%_W>n2k9(fx*`+|=UmeVh{hPNo2ODKUgmktn%Vr2aGT;B zoBB~`S|uwKW4;Nc9mAwq8@zsTp$k1v7^LaBl`iT_7BsBGditMZ;b4 z)0ELH=Kh=#2Styvz0*;*cbdWH^g+M1s-3cl-QX7Vd~(RPBI-6wlmZJBT>4-%^~@N0 z?X=v6bC#;*3Tf#RdV~NEunN#^hg37sm(1c?SBcwS&SkXd@59(G;L7YYWfvFW1x~9} z3(Xvh9+odZ1^wtYEDFPwr{oJtxj*X1M|V59iCw&I>2WH{W3SPS9-^edcb<^1S6Xt7 zX%gz@cp!7wXX82?e!>kzp0e^0`VNN`5+VB z!hIQ-3G-aZgw}m2kUr$s;^g-T*{(VRL*l%0&>=M3oSM;09`0)|^KRdz>Su?U-+cj% zXI)L}(C0LrD`a3g#@F`c9~6?40f586``w8`#D<`&hwNUP7;ia!WouIEJkijq_?7XLB;Qjg3oj>Gl)A@3@j+K*Ul@RQo8YkQ&xbd46DTsEK?fprxyQU(YcWp#ymMvPrJ&edvB{jGe(s(x?rJ<|1S#Vms5=Qm z+giIZ0=KO^^$71-yGh=9FhER-^)>jwY{;&F)rs0ZRbr1E=nYb>;ryZgo}K;t1mCmcG+&}$C-P6!rHmJ$Q90smE6)SxJ+84v0wSB3;ttd`u|0bazQ)u! zdydI)Kk@q9pqi`gq1#A<$t!nTWjL^E(r!kg@+7sY*zu8$ENwNKS%r&Vu_#%yJ>=nV zXbsKWk}DmRq2M(g`>GTspZMjpt6_j_?IImo=*^NXXd3UH4LW{iUmq{HT4n2`q0UiY z&Y7#IQJbat2VZI_DS(sjWwT@Vrya5u96b+}u`9;E6j}M*bFONru}08Yl=sz7=`9!+ zH!WfvLBmRPrg}$V>isf)cSr4qf0;c6I#B5g#wTBNWMHXNxpJQUJcGt68!69QjUEh7 z`3E|$fOnj-=hjvGhFq|8qHbW#L6>C{;2v?c^~2icA*`3h6_ai^kt!6JzJ2Ii&MAdG zyY`<_LQ*4%Hq?X|990~>9tvp{hTb~Ftz`+@NO@pXA7rI=Z?UK(eCwMlDR@g_->R(x z!cZ96cqmTX{vt;rvp9QJ#pI!CF~{MmWLVxZ_bXs#^-XAiUFNbSwg{HNj0M zat6w9lB|DB;hTWmi@H_LPh@i|!q;=YZH1jpa-@YbpQbRfl5js$k&nIILC&o-cBKnj zY`e2;?#nET0w~bs4ajP_p@ThjZ{T50CmE=1&G~$ZY92J)1Uv+D)3qr~*-Hcc#bgBOJshK!SS@$D_meTQq2y> z$q^s1n_GrXV$dJfN?p;D1j)Iw-l+7xF;+)_hy>Rxg{)8h*uOK--zPvPXZWsplcLCK z30K_k*9gUuY3Rib606@>k_cwLT14|N}lKmI0uY-NTJrd)1E}h5AZ8`EgwH-g5vy= zi}t~AK$yo2-8SDTTmi?Bb77*B?l0QYP`_e(rBp1?QoBJYQ_hR#Y(jtv6Gb+=Xh|tR7W09uZXkTmm>`czuxY zH8|={T)(on<#1IDMr5}4d_>qo!q2xkc{yvFrsM4J?Q!oTPvKIgtTU4-4%9y*(urU3 zXrQo-jM@?x68%ZDK!RhN#j&1KafpYBHF0<#euBAayXDM_(N!6)siMuzqDHf1Iy!k~ zj9Y3yw;_Xkqjnhx;wbN9!1xqy<}TU*X{ltXB$lfCwba(>=ZRIokHNAZYQHZPJ-p=^ zfvkEdi&@)J%7D=Xko$i)^<13-bR%hv#b0wq$*4G#5ILnCE{TAy% z<{;Tq*3+>VprUfRFt;_tlX`r>V$TAErfl9*l*v%BR?1rEqlj5mi7FRep`w`wRx{Yx zZM&3YADlf6@Z)dmZo?1b8jUCQJR;ERqa}|N=aqYf&n5?s6$XJlY~R78ZDT5=H#&jD z37U5$MExiiYiiQ;&9~vzbnXnp@oYA?l3M41mT5!t&owD}I@h|KbMg(2R2>Yt<+tdt z8O`}6Y>kB<{DHnL)K;qdU?O^#;>bfFTh$C|SOF3NGOEegVbxYMRgZKvct4=ZMSOe8 zznCQU!m}v4P3U#6lY2EKd+`Yo$k>2KWKn|KT;LBN3lXk;H}<+XZCud8<@DaPpP-Ca z13gG)OwFW8|3IkMzf?fG^0FP@*NGMMF86YeEYbWo_j#9U&{N7rwK;XMe`@Rrc;64N zU&9yi&WDJ=Ab?$B44*+PD549MDS=CA{>dYFUn=3kZjvb2vTHEBZ0hPONMh5vWw62P zbHksB)Sb`X106o#CeE2956V9Q*RL&34ad-v08^jqTFI8V^x!Mz zPuNR~cgz_Os^!_>0Og6_xbbk@!#tu7x)SAfGDU&b5Fiq z<6>*3H~HcS=mo#GV+MY8y#!B~t1AXAY&0^b91M*~5G9Psm#O0wwPQCghuj|;R)%c# z6_nQ$g!}G_>)DYuThcBvI~kOUIp?v2XRy1K6=`JBIfG9JQiv~aK%W||Dnj4_98DjH+>EtkLJyS!MzL2iCN2v4~fp21YTN|?qqQIRb zG^7i}Xk~c&>90poV{;k?m7zm!321kscNn7?N8EaZwqRvsXHMRb!}scYDIRY1l!M4~ zuPkVrYQzb1*8oE{k);bSexS}p)CcUHCzK$)*v?(Ld(wUFG0?hOp|Q|hpvPWVrLKFN zT_Zg{gh1NJ)-IPM*ax^CkDBRB3)1=Y{T1$gggq|RuWE-rUIvXS<#k45P=Byl@&rgK z0w|RyUOee3j^@@Kn}8X#vDyCBdjV`r_B?uH{)$)^yY8;`_dt*kpe)TbORQ&gPR=mj zP4RF?m*WvaY%cES3}&mY31;u_53?_ut!nryBFFQ(FkRa<>xkR?kfCr!u1J0K$0Uv< zr$hf%7Ce5eE>qzA>M4~s&AU}^;{LiN^)lY2K+L+z$`^yEC#-ROJ#K{5)gzUMlhq-G z?^Z^EF{P5~_C}8L0sd^Yt3&Qom;W$D(U{L0M#rFHsMM0>Vxb(1W#*|1$}gnPjiu15 zA1!Q_J&P^cC0EP?r=Y-vYxG;*4$(hu1%A_mOCEg^1R>`?Y`79}2P}LN73H&@*&J3HQ7kG?X*LBPI zY%{7aOR{TlsX-Fd-(FJ<%73c)3Lh#iX@H|bYsauS=($igUAt_xb;YS-eZ>c4oW$_V zqt}MDMO8Vp3j;56PcFd5=9lXK4JT9~piw`7&pEu{oPY@i+PL-qMZvfU? zRlW;v>~C0O5(&!^J5-dCzPKn61Xt57`q>je>26SffH8%0sT*YB`mY#W5$^Y|B7`9O zGJ9?*Bj=6?#GPw@v?p?$qAU*UUP9*q7o)RMYre9tYMRvjuJ`xLJ>}dEj+E7VUH-wsrr!qYak7gk z@$f&`vGLCuTudD@85XUtsE6f~+_Ambq5Y2nDh;lG%`C;GOZ0ltSO4%Ar88MGAuING zhxxEp6$Gx=5Pz;(I!p#t`o`(5Nr!`lrJf>MYVh$g(m;G22PKd8Bm|fGaKvGeT$pxt z&9ydB^0op?(hM?7f`FW42w7z@O_9>t#mrynG6U-M>_;h)@?IF(3D}tK4g|WdWJOJoC!u`THcvEG9GV=T=h*89%IG{ zV-ri$Wi70T1^eg8+0NHBoN7pSDvm2xftJ6r8TV15|(Md-HsS# z-v9s`PtuLSBME9#O2D=E!pT5T8-d|k}?`O^R64Va!>fs zt@~!&>HBVX(+(8QjBIde*bLk&85AAsc_oIZQe$=;6SLqB$c_%8S^hHUbdl(sHh-J3 zU&|A;=(ntR7$O1BvuCnrm4gKzj)6r#Lk~oIb&){)XWdb+ED64ailES}$Za+=4c3eC zX)rN0r(EUGuxl!z5+5fm3NAROg;@?(P~!Lov;{DtoO=B`^k%KA3xtDh+}D!~Ow;-Lk9cDj+y zj~g5{a>3TQ0wW#yo+jESfs25jnPNnnO1+5)3kD2l zv`;xEu|X5DPRF%q=;ZDV@~`(Vh6;8>8xDUX-Dp^i8|N=&2ZAsL9n{Z4fB%80;j_Lo z=rwoE{Jtc%@ol{kh?x9X*($^PlSL|v)x-)j~5b?5!jM1jdb*~dsi(;}7rY~Ik? z(8(jTK!<9(+2l4QH;vmBK=E__$(f5=`GxRpB;{g`FX8MI3bt7!&S+mWJ9bQ#@#e;Q z<8u6-yqbEpvTid6O^RW4G09UJiFMua{HG)PGD2-UQ^`u}wSw_D@tTiFY<^0$k%@%S zEFGvHOTk|+Fs78R$8s%0kd&WFWda_z+HXp*_3Gt8ThHvMw8AzF{=K@`YG>`c8^4Kx=lCl>|7Pa` z%1a)0p9=;QK|h9B2aw!EvB7}dEZ+MLiLLHPJ<$;*PmehW*ivz66Hc zxr2~6ZzZ{hiT{i>d1#TrqN!DIR5BKbnY&_o6it5bU_fn+Ea!~3Q5fd|Ekb_6;*4m5 zpN44(qjP3<*J>BVk2xM7tEo^bZfZL^zdBQ+pwwt8y^&CF(0NP=OKBHF;noANNWTT_ zqMog2uxs#XMpmhF(fSLD4_~`LAbCWA#kmcfq>XVrivX71!8F-1$_?q%#iDfjAEZvc zgU5rAwXy~Z^HPTc)n2XNV=o;`M3?dur^b*Z>cKQC!Dt08=K0|Hhw*Tab9eE|2)PVD!;oAp{dl-E)`F|~X) zVwmj8s0ntl828Xtq%cKe%nkCLZ1q65s+%+Nhc~X^J!;uj!cPPWYC24p9lMr&a*|X} zLDp70pN%}tqQP}YL2K4(IJp@;F5()kg+<{RB|WHoiQjJ|9g>b?I>GXe9c`Q691WLV zqmo#?#;VVh!^5YhhWRJ2gz|HP`sq>R8O`{_jj-kcVk3J9J-SNJ@=V6*DQ8`1GK z=u8IRu2tWBkWu?(u1<+|nkJ!$bv(|WKP+Kch`I3j93Wiq(6n1a-_gw<9E_)dkH8CE^OjRJ#p5O840hb9UeU>9^Y10C;H&w~|C~LtCg?di22rq#Vy?i( ziQ~jOdk9p<_$R{SN=H<}uWZRSyP0C6U#gD=r|M%Thh7{d2?AA{MJeI(8LScc3mIJF>ZI^g>a(PDNHs-)qiMJkc%? zBf&B#Qd3z_f6DdnC8YQl`!4 z6E2R`?aEh+IE`j{?^}jh;W7K8oc(IE+;zN=h9OR*-4zk6r--vMGz6?dCk*peXUWk8 zc|c9KSm%cGTe6xE884-IMBN&kPBAQdJ3bXTR{laL!(!djhD_Q1e-zR0EoV6rvM0_R z)p7e#=6M)i6~&li6815XERF$rc=Ukz0Ei8@RwhJ+c*`$dTw&jYaTDw0%Ey$kw>;JC*4bYllwx&{ zG*^ew_W`0HwM5fEVi%R9?SO zM#yw%TvJUgdNNP%!`}9-Nmid0jV7HW2z8(4bB=99a{ouVEls~N_ zFmO>p^rSL-n3%C=I>Ebc5aS60L)aHuNtgAMCo|VOqO;f>`};e4V%UX2*f&pBKA&MZ zm(#dtXpZCztQDj=a=E=!enA6M`fgu);J<2dKNg0sbQ+?OvrDn9C~iHE1wQejU?0$Q z0(6Ckd6PF2#0sd1H_>aPA@E~7d1SuB?vR)Zo?`H^yYfWC%gW#MJy>4 zuTY!N=H6-?T_U0pdbS8d%7o>Vg7UyVcHG69Bk!nk9FPJ=y}XqfJvl1`qv z2*6qK@Q`Y9KybO>G#5GJ6GNrCCeZAcO0~bGd@qiVU0Rq9o_-$C!V!hSp6)Ktx@F$Z zK6NwDZ2rGlr1=0LIM3~~@E4>1_}2e19=TSlo1qJqp3u24(bF+09)0G&Ps{mcvQXvh zwclAPndiB1KXEK9e#KI~;}MeQHCG&<-~)WnQg7ayJpAkMWxCTIlGMA0tLDc6OON~7 zA+9puy<=bssP!T}q4i5L$j~}_&2=kQ|LMO!V9nprDlc%?K~wyAf5elcqX!)9jX3KG zRinN-`^RAo@Z>s6ef$fZkby$xNro5yl?T|2(^35E%3g#~kh4F1lIY2~>%aK0NspY| zjm#j^K1~Cl{a=+E^S{xS?UkEr@Y+3-rSs&SoYlY)k9JkodF{{#PsQsUu~jK_DbL=2 zNIKmJXS(ac>HjNdw$Dmj=Vb-JS3XH+x2yY2nDa2@o;iE}^lX+Nw~=Qbq{g4i%LV>I z$Y6uF_Stc5UVBm&j*V$UO+2$7t0nO4ZyDwRve2OTQYQX74&-?tET zcS_Y0=110=egt2Ns!1ldXV%p9nNN(Kg@Mj4U!n*NKJzBcjE?<2eLi>zh=4;K#L}!# zQz>EKDCYJcu^{?gd}*Km)yt4vQ0Ml`GJFTtSj+YMGsdSe@Xf+?`(L8Ts0(a>b5ZGc z!AeT5mEtS{c&d>Bo@!j0tUJtrOen=OxwU!j&>>qOr%YoAbDt}ZM|;u~?DjTfnq8jc zzF*8n+Nbvex9MiFqJ2@GRi~k4E`d~xyPkuy4jJ(6APOZoBeSyIc2*<0mu7C67bp(W z^pfyVt0z&kub?XdY8h&|jz`%E)jR*Wr#tkyiwlDndNRvY$CKN3(JK*w5LVXEy5$fM zFmn0or>vA&xqCe;3BC`Zls*1Ri#8Tgw1ST8Cjd$o)KJ*U-BRQAOlCG)lZXMWw!`SZ7L4pwQ3oOhdP`_RaBN>J(0 z_m`k5>yPNjHjF2I5*|7(^CQ`Cj8<$g%sUpx%~sdwc>u4zH&{1q>D+(A6*1~=48Emf#Sy%^uELKS<^?U)5vPK zA)n+D=D#ztpi_&5B)2w06umzNIyErb@f?`p5*YVTBkVS0&T4yq&4QgKNpzn+ZXEK~ zQ1brpJNYND&}$KU2Dr{Y3jHQXP?cANLA zQjC^(Yr+nJk=DT}97Z}>rF66*(!3zyNLKQ9%1i))O|GYf31g_z_(w_Qzw<=kiWsoM z`x*WtOJ+f$aqZ@lpF`NHUl5og`Qj?&)Y|0#A?_{X+SvMc@9y0eN+|`36)0}S-K|B7 zJHb=j-MzS#;w~xf?q1wof>YdsCTQTK?cVqOf6jT%yXW(H=4~>WnKf%BYpvgPeJ`r) zPp52nP8>3aQOKaVln?0sWA6#;c zvNj*qJ$#U=Op^X;UUeKf=(;wcOE>N5^>B#G%iL-_Q!82O;-0T!f?V7q3HF2mZ!;*8_%rd(e39~ z)hKYSvDE(q?jp9a1Jxh~$q0X7(z!{5)W>HGcndio_pf*|&39;M%FRz}{^5(^Gl3j|PtG7C_}>TX2b>46V`bjqBy&mHQF59!tl?@}?YXg;uOipSqbFH;;}^LodhJ7f(ZM zD)y_>2-|C*LI58#vnQ*Nb8_{-&(;b_wt`Fg6s9>(0KI>T8lee#W9a_BDHXpmYDK3HpgsO|758E1mPo;@kcZ-!}|=q^34cmC_%wnJlq% z#q#8KxCyzQS*Om0J++l!!b;g0E!XYoQ4pAXW~5f!X8M%h7kh!G4*9T(;Pt}cyL_Oc zqBIfsUn^v3D2dQ|rTMyO*<7 zG?B!Q1}HUxvW*=smICkpp@LO?Hf+nSXCwah=4M*~@TSKonuub?M`eikb>#1O55Cow zvC^8DcGUXM;_!$g+zxSILg)+dU5A0X3IRv);p~hk*c`+#+tQk!s%(PpC*w_=EkV14 zp_yl&i3z~!ysDTsXV8%B6@HKrHdSL_mqP2X2g}Hn8Wv!;eJ>wH*LoP==rgKV)#_uI z6bX>WYm)I5lMm7#q+a&!7L!}CnNvY90$ngv9x>!?qVX);wnWhZG2}ecDjO&t81u`^5A@=i<&OfXOiF#Uw7|i zONehgZt+Q5_<=j7FE#y_k4V1tzh=a>a=PUvfUHidz`GXfb>Ah2v^7%W46&@%;$%%q z9^-L!rN_a;+^QON(s^o18wDhaB&ozakBV?-avuMRPlefRaBY*{L;IhYu)^oZsy)OJ zOB9JKbVBBr*BgP0Eu&Q8Bg*qy#5hw5l{x30e^p$lHYfoqROu+EV3;P3@s-(>WJ2UG zR{1RO!u$j`!CAEaZ#}&g1a@+X`Js9TShgnNWQ0fSgXOFIu43|hPK>25SNDr7pZ!V- zk^YqwLiq8O!c+{IWFcR5!+c57uL9(>FpjOVTu0SL%K29K$Ung&bZ6!$GFT2QJgl)6 z5sJS3F!E@2{qN&uZ+!XD_8~LQk-C)L9XM zFEc|UmmML1YI=lU=M5ymnCF>|# zq*5RKezxw_yrcssxBK1ea#RY`S2e=+y_`z)xTTBkzaMmsc1aeq0!7TbM6l30m+$!# z3KxU1wWaNy;?i!sZ!TsWbxy9!)$V@Tw&X=aiGR{GnpniuE)b+M!)TSBVwfrMf;Mq;K7_;MfFgabBJ>dL6ac`#TrJRj`TN zk%`AX;r*NV9?`~<&Op9X{y3}=Wj?3a*7mb_wOj?S_UOisb`4`P*2)(l?;$R{D>{#E zw|;Q8>%XJ;G`mTjCC?_GlBw3v`LuNgs=RIHJz*3hb@vx+_D{mNHxFlef*+H!1NJoS zl{^2WeOW3`Vo$;4MrvrVEQxVH6=?GK_1|2R!zy9UeG0DAoLx?VcNuG(UJ=PI{GJP< z6O5hbGR_u3;Q^f8wyoQvEO_141!s}_`FL0!OuuFvPqtDKlcUFF<*3htwj*POYH;sEqPUFM!yA)ko}Wn z!Y&}r zFS^i7Vz>_Xy$#b5c_h-#pqyG|Cer1^xG;*PH5cL&8kyg0Hx&?ry45Magh0UV3d*lm zl#V}OSyMHzbeA`l8D~;3jN{}&HT3&LRMCd+&PNFSIBSVk#Plny9Fa?ZhL|n-syFsD zUzV=D^#T!lWI#`mYb+?_xhJSvB{C-WvsO8m3jnn@5wU6gNnt{4YTTO0dxD-GQ0d;o zhpAvEKQX^jH6Robp2XyA1e(!)$0eFGSC`yAnNC;N9)4Pzwinh(ry3k=@FzP2hqu{J zNJTq?lTF0umj#d&;Tx1Z{s*mAY*z`m#0xGzZm_14Y8&L0?U*#4rZXlIQrbD__F4#dSuJohv?)tV35( z$ttz^5!+Is7oo%ANU#LT$`!wTbTj30eU?^Rh%aZfUuz@Hmd$CI=~P`xiPkr|dcB)B(`26xcPBBl%<%4Ql*6s~(H@YDQ=ls-<$WhsA#w7;&i7 zs#okCq|ofNLO~fb4$g!KXEU_=C`+DHYMMHR(9KMppQ8Suu1TPI&xCxtlLpoU=fYBI zw{hs2bxjvRG{LfwPW7!2(jcM&-J|H(+wX=55?pT_?!jIvG^sJQ!Fg(?id@@mT(Hc* zWPJ2?m9W)xm;xtP^(~AdBu2WtF7iR~KZG+@D~82961%0A`1|OF@lP43k*f~&@z}-9 zM-U}6_5fWA)t$>0`FHQr@2iB;mv*k)53tI87%fM zt^ajMa{P8k4)%PNHXb1sqOd@ed~Drf8)|w5?3z00g^)Zo`lkEmbA{}+x``&IQ;JR~ z=(Rm`c6!Lpu&@V5lN1h(o;j}a`$~s9!G>2HVDFIQvvoOV{eCQ|^SrbQGHq82{vGN5#9=KJ=;LO% z*%vvoxUtgee(_hk;KsiW!O~sd?Ehg9d{RQaH51AoKJGO^_nB*1flM+k*EF^6@fgpOA(z|F-`v)qIK){=k$n~qC_U-l?34Xy`rGyhlk}AiWK4* z)WResW|LKx?jOVSjbixPSN#DY9w zc!kKZHad2tIJU^JhTRwnmSuaH{6VypW=j4yszdTe4ll+(>$E>8Q2u`jzb-_q50h}7 z4L4L0ZwNx0RaP-Bjtq_XSgfXJPZC~a-+VXY;w|5?kXW|o`W0MkIEc!d8atjNQg-Na zE9yA%&sr=oca*GLiw=&%QL5Ga>TA5TL)6*aWa zt*!;8_Q8AKGMqh`Q!K%EEr)s#G#2LHZ=EM@UIN+Pi`=2E>TkA?ouH&+F0*Pr%Kz>pnS@r(WoUjpNCH=Xy#2#Zedv-?)yZr(-up`%T=*SI5Z zSj$zOc3H$Xv@5P@lm4pOQ30;!7YjXwJt;?RhhnN<+CN`)iqZ1_UWj?})1FnGKRYh2 zB~T*;zgkKJGHcvx<-0KMofwC@W((D3T^M4_+wKjE(FVA;Ov@DsCI%6WsTF@6WoUF+ z(8tzf|1wKOVGt zrGZU^{WiwRZ~;Rwm{j_4Ex5O^-(MaAF! zLCvTB^pJ>o-DZ!dXFz)t^2JydG7xVoewA(#k%Upom9DC zl1PeHTW|6A-kY0|nLFk`8@wWqVl7|uzp|24@0AoXUBsvSy`4vWL`ko zs+G*1VbHTBm%geERQr8OH?Brw1Ac^jCgzm?!r1% z)lFAZ`(DK~9k~Tt$Ok);5Y3exl`j5BCa$FZqWYl7viHUlE!Vlb>UY8W$rmLP#95XI zGh^s5V{BzC;Za8U(jPTews3Ol;>vAsRoOBFUNL2{2IeMA#eAQ=I0^Ke>unVRZzdeG zCiP5}Keq*|VQg#~4O6$sR;S5-nj8Hn9`jl~ZeF3bc`6oH&N3ewx0K0qODW?VnP zcQS*Z3qG&3qUy+&ezzy&O?MY&Y-A!|mTRRL(_wPXM+bVXPHpw(cMJYxWu05Bb+k%4 zy7?TZrli_vQFpXYb~n$X?(VkwEi?b5lIhlAuE?8Ag&y5j3&JYQkQJ_JqG3OC>s-+H z^`uVP6WF9?+MlK{AnrFM`m!tKNCerX{;EW;3?zPTq)0GRQ7& z%_C}`4vxZ3FEaJgqm`t!zBVvAh`mL0uT(nW&3<$BgC6Y|CmZ=zi&tb>8*;ohRj`m> zXL|8oR0pwY5h8SXS<+t0)byO;49XyKJ%@gq?dA^YT&Z#ZMf$adz#9L&I6oT^M1165 zKLxJgW`o-AY{O(nt&l-@1<{QvZHXZIjzp{2we07C@pCKLLXOLLL3VTx(>V9^0~hkd z3e&*q0bNWHLQUQ`?mN)f@`Pn9AWz1a4Vih*PTjD4Er<;p;UhFU`Ao4-e2wMC4 z`Xn10MdWtY+1;7weI84n^ZR4y681|4(FnmyJv>gc@!vUSstor=vdYzAwy>yFVu1^*b(lU_rQ4RTE9bB)zpbjJx>>L2P7V{TB6}!`#xSiAMd9 zmCB3shh~MxrbYRXh1f8v0v0Ybm0I?#g5>9)DSMkX6z zAWQR2fg18ld=sfiUi^0N+A6T16NC0z?6k59*(WzBhP&UYC$(BuYNE{a<7HeK8GfArKJe_Xn1#R*^deRn^X9^z9( zwUE)(S5@h&1OU|4VK49i14}384kHKSsP$(Hm|s0N=4Xdp@sLTHYF9$SU*?IUOg;*E zaW&=oVR*A=MtJ_9PZQZuSjSLyb$|3Ep~}aZ9YpCL(TC8d)t4Fe)o9U8%O5>n4-a=~ z@z#+d{=NDlJB3sdmduA`Hga5uzmAtAI8wXY&f`^m+`TTH-4Gl^4oGHNJJ{fMKdDw5 zCk`4^mL;z-!P1Uu^0N)DSf{_$Iuva1E*y@096xKUT_*RSHQ-sXpJO>UBAXTI-5Nlsda7zg&*j}~&8mDS$Ni*#&A^C4i1!8Iq#K#F&{)-kTQMr>AC%f7( zWUvr@k&qybIwgfv=g(<=kqmt)Q}i?brP9JB3qJH?OR1WTmC+CCf6ot-jweqMdUxCgtuA$eByX-Jhd2><^+F&TGiKJmadaIymiY8F!LxOx z12#9eHS`AMtKg27tL#8Ip*I%p*L6`W0z1U#jEGB~0NHbk^>T4>Sy`M84Gj^mZ(!@6 zjVthv8DUWmtf8u1L(&+oo&s;s!_VSWI#}ockn?lv%sn zKu99s{X>UT3e_7;ezHxp(saBE-^m5lkc?0CLWIN_GWnj*J>Gso83)-;J@PFPI*et= zH!AD-Chf^rZrvcYZdjKIWN;+pdnH_!Ri-W$mWS2t+Nzi40!mma9aoeHTJAKct1fhp zEWAP*uldrdu{fd=jlArI)HEg+^YaY(oLznl%d9ll&lPzRi)=3}ZwEPBULzfGph!=hJ+hOq#}U7I)J7%nx2xBO2>;O446LAS(#ez&iKuurB$ja|zstw|F-Y zD4FxE7j^08QUWeW^J&W9@yut4G=eq4fxSpW(9HUKNR2HBv!#^OW;}*+U03hJKscw2 zsUwwU|C(^Fx|n91!~2a-u^PicnKVY?UurT-mHOnT?Ve#T=JggK!z75u2(b{fay=7(H9i(@HXKVkI{raOasFNCA4sXl6`< z{-q4v`;8j%4r^psTN^(Acz`)h`qywifOI;%FaddtwRBJb?o+CGY*o*R03%95coMZM zTvRpD-QcgL%4TYK1;4G6Ld3QaH&x71|K6{GV)IS7mYlVF`!J%x99>dm4}8bMBU_Za z$Jk%~y0fEWOzBqU;(;mdBi*6;*9t)eR&8bwCQ?j+##(D*LwNvKtQ3rX4n5` z=$dB=m$CN(yd~YT#cA`7Uj*R6-*T4W{gfNx)tRbFMdJkaw<>=O!eAekLQFe54za0e zR5AO%Go~etxonQZfbf1a8;|Pc+;SQMPW(ZCe(H8Yu8;{rs8*A)%y*ok#%P9#cS~l} z*55^l91kBzNvfhs%A7YYPfpaFy|p}hU1mSdCD?qMu)hG!6>sqXdX<%_f{d> zcbR-1H)Zw1T}HLYj=k82RMGKW9>44(noK%new{7re~wJUgV!C z*g2nXTc8%;L}Ywg-)5lYcX<1L%>$-@(N(qOyb|zwN7Tn>MOlr`#*ApJ0QJcemX~B^ zzRAUGQ*@UCAp3_|nKTk0P75)%DRo?3B?$Xg*A%U-w5d$)>O@O2lL$v3$rb5%WQg%NA2F$a+PfGgk!uV#0=d=8=T? zH1f193lcZ46fl|We3{;}{oL1+YeifgC|N215NmC4&uhysK7dcetH~kR=?i?84<3s@ zSvMQ-$v0o%m8aQKcVo;lguJs%f@=0;Sr)SO?%+0SSMFdy(l(S>_&;XO+q^ z^JuV9ZFRg#Wa-U;wEgR0isjh6V77E;X3Vz{#d+F+mC&I^jBMQ%IDW2;=-pnPMHw-R zw$DWQA3sDXsURfrpr6HF9|M}wYMLU_=o`p|3u1`S5bYZ)$kZV+GdQF&Mp-F`&soc7 zQ!y^bm+2(FJRx7}#E^auzOE;J{8Xl9`uLI=Sj3%jy!nG#O7tNKRPdmYWhP{8(89uF zMXYODv9saVhBpG1q~pFj%G0lLsQ0Y>5f4v4TL9yC74Ou)_9$3ZZ^!LAWS~4U!!@St z0lr~HGu}6Ts)q`#JFp@kfEAlB4$kXHrQxqYz7_5iuzLwqvw2JQcW~%DA$heE?`9gyhqm-!lv@X3nwwRATC>}$W1Q2C z08DMV*Nx)9B&s&uPG&S^k$I7>Z^5_JA||Od!Y#JCgQcWQC8n0IWI2idYR0LW4u8KL z+!-)MCKb*L?AES|skqQ&;4hDBdQ7zQ-BB?;Ee&pBRe6TfkX(O74SQg&I0Z=vlk9Vr zq`vyu_0~f^es(uQ_yO&LbIP>l=D~dby{`(r^M~(-9HSIlb?sVfwK$YsGQEsh7I{6z zH-MYwy_@6yuOFPGB`~#c7=2wcZfFNJD(Nk|kyt?0C^Kaq-}M+-LGeGK5d824MB7 z8lZ#Hwp-5&H)tQ}Ii%Vy&`PcHEg;{O{^pygH9x;PYy4%=g|SrUJ|PO}f{IJ3&ev~D zm?OM{Kg0L0fL9^cG*MX(G|3;MYEyTmh)CFpL1}3&O^XeZQS9(l1GtS=&PO2G${H*- zXZ>XA@d{jyAlUWMC9IW2p{AUg898~ODAgLhPM&2Fz=J2T>-LJxRD|q>HkNvHFE|C% zeipu4DpH!n5Ab=%JgzWqU6;CaGhE6_e*m9OVVsKqW^>(&djhmsOHx~S3+g$+`+ zey~9M=p#^eUOO+k5~&+8{;&;{GAJJ&`fLGz$lQ_JR?CQ#En?Y^8+f`aW+l21jdx$B zZP`r!ydLl2%IMo@$;l(U?P=RpyowL&&IjCeW<7b+)y&Wq?J9X_&fLDST&Umh)GFc0 zBx74n{Q1ul^C>ZIm{8Fic0IIz-j@h3?XKcGY;rhJwSXw%TZKVE*qE4jMn};}bNuR1 z^#ZQkEKV`1rCGaM#Ov;OskB04N@yFlrHt(h%h0g4r*N5}_?I{$_qv57msZnaJM4KU z&gb`kQ#Y7o@}+1>!gZ{ZaA}D6fTPaPlLLw$UBlzXBF$J8C zrbv`Yb{CTy1dPPLV?m$B2>u+-X!h-eeTLK|_m_R*b1GMr-qvIq6tL%0vB9qlg)e~B zC7x@|!Ek5IY0!f@L|vPSVPMg!CbaOx*KNdp$ck|ed|5G9L(;ohVinH|C^pIm7+Tep zQ?i(%TCxMXS;JvB0b>WG%&(l=c5@>*gqaR>5}-N7$7`4Ghdt;0&vIjOzJX%8k?dD@J{=~JRe zA#RGkt7#kRcJhIekQl5ZfQ^|(mC@zZ@!TVM&XmYkm@jhF$#HSr_uXUF>|D~A#lYX5 zE5!%phIxBRB5tn5zNFqvf19R#LXFKP-+WdKw1FR_P#q{QV7l{_WggcUj@s%t*~(xi zDBTW~-3J%?zvVqcT1?(Z`kDi5xV3S=LL4=4xUglj8*&_cy%_T{pFUEL_zTox!1o+` z=HVH=(+l>Z7=@Mz1*X~~-FMJ~nwl9K(1UH24|mz1VtF*g*3**2g4W{u($Lc@HywM)=VsZ(UCo#RTzuF5LnkpIT>|cuvn&n26yXH(y3s84fGxN487=eUK0f%9 zPaS;~F*w;B6Y{R;h^2?HoOvghYmI=89}jk}0GEYC%MJde=Qr!pk-DPZa$3(#V|z!$ zno*e!`bTZy`$ejmz9ht4AVwO@UP&6&{iafgPU3>cr>pMUU;_3>G-7J+xplmJHTPq< zN@WLf&GmtBuzT7>nMRSF|J!-x!*rWZFTftSoNA7nZK?5%fiH>j1QaDXz52;zX7elK zl`&D~rCcc`j&mM&ob~c^$=FrS1jOTS4C&VyO%}taM{Bz4LvD-X!-}|pTW5t746^DF z&EkFOmvynLGr3cL)z>^4=H+^zA!*;dy_JCavoA73J=k&@& z%ydTHSKqW!ImAn6^_EfQ_>k@Ok&o*IXl=<;BKL%dAYZ+3w!dj7ng2Y)J+3G@N$}@4 zT<_VWb-w2vL}q|AAFF=Q+Dhx;htlZ7ySs^U^1a(Wq=Rn*Hyesw%qsi+QZvk(D6u1r zzGAHWg=a4WUL1h!W`dBCzHc*$GT7B$!o9)zXKSr}-Gw=P?6wn{M3+I6Dkr@Alv2a& zYo#-~T|mHMT=5ojP3pq_(Lk<^6Gsg}`KzS^G@3XUpXSxbaO(hE8^#7Y_fQTbvqj2W0XmW+2 z+{_*PloM*D?V`yrLwc}vuLq4>M-4t~rPr;E^)$UgnZttrQB!-0?hU;pt@r!!kywlp zcjls<&Mw!?2z5)7%yU~6EYNdlNuL8*Po|g^rK9*POlXTH!-$fAc9z-x9v!elFqQe> z9R8F#E*v$)<^Ir1TF+G?jqcU30XM_C~SRm^r zz{Xd$vS^C0GE^ICO7~zf9#abW`$y``l`s#pk3ZfAyv@=Vm$f#-q4|F!B_!vLI|{z% zqz!@ft#LTRR8;LwQx+4?E1p(~TYVYgI4mn?eTfhTQ-8?z!4I*{6}b&ZmEv{iW#}%T zhQ`H-Op9lE>x9F9j-slT`wNZEC5Efnpq-K*aTTJ9*8H{AW4|h}P5pM6cih7vtgx{~ zLP2cQz@I3kO=I@luU4sP9G!J((pDY6N@xg(FA^>`#kx3#J{bOWm zs_M+vHtZTUZGVY>)Ufsx-;MZIsEZTN@EdOOv(}G65#gAEueRim%7C%^E*W*(<{PD@ z1O_KdE8Y|dgr*Np0TjQ1*ig*E)+oH*0RK0?uM@?ipbGz8W+O@hoyUE9yyQsTnz zM>;Q;vu-REsd1rjny5U)aAz|8CLmOUx5e4mEuS5tO zF9jbmquga#{W%1+_Gt!M(sGoGV!|Q)d%ls!R%*j)d)Lu$4i<6E6g28g$CdznI(ZG= z>F*mGS(O>c-}JEOUXhVk*XREI)L5eLGT@1LgOqx^j5l>XssTkeOu{!sjp_Jy2}o0P zOzKj9kikOBnpIK{Gc2x?#t}V`49=P)-Jr^sE<0A8Rj->-;%wl`@zHewV^j4^X;HBU z=g%;$T734&)ns~oSs9HQb>b5hUdLw3<@z{cU}n}{XbkDtjI@%9Rr(R0p zD!1Qw!-(Zkr@NOVP7=x!E8Ys0u$nq01c(bCT8;DKBu%e3>$fP=PU8xzoZWPoO|NwSJ9I)aVM1E_5u(1&X}{LpnTbzu@a8I9)GUlo+QiOwlz@ zI1GExXzn1!=gH|Zcqzp~@3UO&G6nY=X`2VXy{*+Q@>FB%zVAswi6WLNcYV3amaIqS zVNpUW$Lo80T@d1O6#2?Gb;-y?9>}#>D zhK~$`Q`9drZ7Jwm#%J2k%qY1otKOpcvE`#8p{17RqJ;In1I&6RLDCevM+XYqW+%oQ zKvjR?2u#aWcy7SFh<+$uf{+n2&E&G_tov5cl`jyn$jEHk{Nm@6QxbGOSiKNs-h%!j zPsPaq(th_FHvSyM)9KpYaAj2dJ>L_4=`ZN2KDpX1s?3^<0brC#eT-(k9W29LnUxDV zaP>kaB4uCn{=-Pf>QMkN{k^`1jIP? zUEstQ%zejjnXR+VR?n3Dn}Vjy_H0T&%Z_t}lOr^!E35f{09MTro? zmOU}`)*aN2C=ueu>~+DGY2KYM9}r{fn?2*BW_TeUo*1V6d2h9&&6V`Lt+2!D)jDDL zFr`s(9VdSZA7EArIp_5*;q9Z4jwoP|09qyAL@N17R`2A{h-L0s40d6i`+j%9vu`j)OMIKhyEONFBG zt{L*lsfuwk%ujTg>Y#IpWJ$GY?mtRQL^1=mW@#+5rkxu}dKq8^985W0eW<7Yr7 z0tz9iVo(#gfn;TY<~aRFN!QN`7@V-J$w`}n^{53lcZ%7floRB820E%H?+NwIqP7FCo7IsUZemOR!tG@MI! zJTIL>^;1Jt;-}me9M)(CPWQfvuzh1-GYgl|>^I8?r>=!`khISzHSnz)(FqFIbY#c= z1sIx@&DIlfsL_$d%XF%bgoI=jvBrek7IT{!xx`so3x*Cg@FBrg>APH7vCjP4PzKRh6G)GrPs#gpuaLL2d{9OF(v~Qc{$=a^ zl1oELW=C_IWm8$*w-km?r!?pwlY_*%@?&+i$IZyId^d(3xO;?-Z)Ns{U>RfVYM5mCtNiu=T)wx{N5cH$rYLK~g$)#%PL!vaGG(6RYE*M4=m3^aNcN67~7S8-` zwsS3Vab*}kyv5>2g9dXNq&K<(1M%(&>Ss3NcG}X$G?$lZgsqT$(G`V3ulX$F7XI`-bKC%Q(<85?%E|t<*Kzi>FwWTYn#nKjT9riSb zyk2M>e_JXmKPU3=+7&8>(5pQ7l^GuF%VrexN%Yz(M~>oo?wJjIm808gMsjm8Y1uRA z8b=Z;N#uuN-9zQAGlEsD+_sRbS?anh!ezirI; zlr2u(-32m2n@-v(D;|qJ4GRiL3mRyr8)ro z7xK{khr>1wZgQ4XbK8%p?9AKry1?TN+ak^@(hjow6phpQD^J?0i(G?vP#kBl5llgO z;qY!rq5s(2b1FF|cg7YdcW@OX*5(T(Bm6+D(MF&6+CjZNf);^eF#s zzTsu}YD~Y{a830$7zF9L77~p3@{Tk4cFIZ3T16Rt%sXo^n>u5Q`U0Ht5S}yP=PlXf z;D}?dZqotWg*lp}#hwRNaA42F;u^7EquOswL zVIdQpf}{0uR2O0-x$MfhP}P`z<*OBTZ-b*SHH&2ENTT~fQ3Fo4CE>g_7!L!j`9`D- z8YF3u(dde3dGFfDzn^^R_lNC_YG=A_pm1@zMMg$O+1ZD0B`qalAuB1%47h0%mRaI2 z?C(EUyK?5sbynlV0kf&dQ*qC?DCGFmSZZro;F;{ctgb1k1<~t*lNFf5PPCCqwIKt~2%cAYzSglJE?Ck6s%X1xr&UyNycNL0xXfjR-pzW< zs<kSDN0% zUV0hzRa(5VVOmOab?PEr!g{%15bK4uF7MNq-WAh!!QwLVE_xCkouhLuQ{+u9`(7gp z_a5FY31knKF%v09oLLOJ{mb#CAWG|8UAp`+@sT{XrlS5^3_$+w?!!1|5U;w}aSv2X zcLXW#MtI8$WS`Rx_qnsSUj+kw&N8mci%d^FT(PV@aJ$W^>ynsF3wFV8j=wD>%*^`m z<=80y==%(C(H4yQ!gk95mOp4#M9!-)<~;hYbT_KHUKg9qzxYm5)LLl~FT|HT&Q>OG z#d43oWazeuB_556=F*y~W&;~EUI?D;&{ohb-<9vvOK`{lSb52Jx8O88V^ci>-lwsR zUJ5zKl<35v^*Ckw;ih~x)&fBTNFUFkxYnL@hhU34Zc<4-mU zJiPc*M^szz=?hCP4dAK`=2!zgQgH9Ps=aIQ# zP6wK=p1)7<_5*RSL*TwRB`(u=6J0Sf)RP8wTb6x3+Q*j0RR!y>l|7KK#OKa@cvDU{ zCL{2;MqUn9nc1P-CwYMDD1~f{k18aM<%b)wC$=7$$QjSr1$Q@Q$DjB%Qhv7i(Y%;F zQW2Fo7^iXF)&?^Vu)l1UlU=NUlVa=8HVS6a+ZHyutRu`BH(zf(++B$|j0ZKmM=$dp zI*_!%MM&PeiELwAv{AGT7d7nc>`W$)s(Na!Ep6gnRDw4KT?_OL{GnCwWnT6@6XFpf z)9)U)PSm_w6F}M-zqXt_nH5sw=a2mM_r6qothd4;=MXsu9kji? zk0(aunT!Bs+j;k*%SX=nLy*9zum6(cn+*p_7%Fh5GiKMB;DUL zZV7ot18nK&nVUQ~nYn@Q8}-zu^$+%UdreCk<&)RPKyC%fs?kXqG}xF3eZ2dn<%YOSG0!9Tl`xRqZ%wgs)N+H5P0Tz&l@! zn|RENw6A-~=en}C5fOplmyQ)-#ndD26f-pxk~jSr7pQ&Yqy`Db`Ul)nDbM{ul0vA81!5ZKP(n6^|l0UG!L^J&FS8I)WFtJkYj-zRCSvLBi?5X9iHm@l?urw6+qg7;c?6ty&kPW<#D_g z9TfeOlC$|`<%KNF>`6mL3yh*Iz0B>+5=9DTb;oR~U6tPsVs(0%uoGt--9J`zv`&QI z!h7!tHb=BOK3hbQSGB~JG$lDW7V7BITXIxhX=B}HJJ8HfrL1kuNe`E!6b0n-)h^OL zx~PUdz0)kHFKWZ{c22;-RhHZgE^O~kzgkXp9&dRd;SccRT$VnCweP6xLN0j^%V;%S%diW6H3e zuKAwajK6Y?r9C7Q{&Fv^sxnu?vbtc`iqP;N_v-FgQh&$%{n*fG&g;)oIaS(m1lA0XvT4c3TNe?k1951oRACALlXg@C5K8g{M2eX)pZymu6|g{E@R7fbAK= zc|kEziF~|i6B=%9Wi>zZGa(^Cu&bB0YXbywNgt#p&P?43EmVbU2)7?Exo$4)#4C_* zMyiXI9iN4NS1S3pW{Y%nu2m#i#<~e_lj`d{v9UW=`ttV^mS@-M?md)JB*dR8UkIJj z-3i_OMHt}g|JRex9L-u9BWb*maAP76^^87K)(*3?5PT1r>-4p7k}cXlG4 zJP{4DO_+T7d!Mf&6GBhe*70a(jsA{;*yh{D!70esQ1;K1fo^W4i4SK#fJyutB1m8K z{r|g9S4C{5Sk+y2R9}I~&i3xsc0syuldEg`2a^q~H=m<&5UP(~TL*bymSD>KQtYDt z%>4XxzoA0Esrvru{oJGZ#ZXEi9m+pnqjM7bqp5qsA~1pZuLWfD?|&^I|GQrB&u;$@ zeH%h#CuFnS`0eg$;TMGxvHhSBd0wCzIz%1(OS{+Pk&Y1mrTzKniJMsAZ{c3gZ{c28 zmABsVr#ZTgsK$S;$VVLE_t`wKm;H)HB4BWW&`HMp%O(0;i;zuj zCfNRSe*d4S{Ais`xf(|nnLi^$ejL1A-QKhMe(7@Pa-!$*wpZVgb%*C26qu~dvXS3m z|32(n`o2#SOpMa^Zk}de8ipBYdReZ@a`{{qUpNAojD-b5*EMB?&tcQZaUHZu(^=SJ z%KbNBkEu!SA|CkZ+%_>2gm~3)U%x)0pMJ@ZO>*y_PQYSkNM=?My0%qg;CUp+4gynr z6FXU|BXe3|<7ZQj&>l?`Gd2b-{_N79ernI|el>TDQfC#;Wj;u=zhH{gkVqB_40jVQ z3X}WFa~|7R8C3E*@wA~=Z>U|5^n1ur4+Z^*ayR^M7b?KA;qD*V=-WsM-w7tU2RA_T zY%`WQ=ol4P;i$w)!rCJ z>^Zsx3f`4tp3V4@Orfg|s-$307y7ab!@kw0ZrFFb$k~5-m_*1VN$q2N(udG-#}0{< z3q$iE`XzF&Ps@pgmzV(8kMFMLn5qvyiTzM@d1e}ZMekTp3Gi>X{$%77ZAJ3QEfsC zz=dxYG{J4`H0N(D(1v%WtOwxg9pct$W#CZ`BA~Z_^ks(6W&T0}%jTM*m#X4$cX;K1 zSd~Nas7+`pqtDu~4CL`vnJ~jn5~6nc;#eU@Qg}2 z=psYA;C0Z(i*avv1#(^2HqWmuSO1H>w+xFa-1mkpL_|?Q+5wT0mKKIkQo2F9yK87r zq>;{{rMnvjkj|kS1_g#8hVFhxaqqMDdC&8n=UmtOe0$a>Kge45y4QWLHUIywa@H8C z_3eyPg69iJ@zr78nH!u|j}r%|&5W@~u_QPc4z+5U0jCQ=2Y8mme^#Sz4AIqxi}BXl zshQO)D`h!D{ob#RP*CZi)am(H2auQbeEP>$|3~|k%}ojiVEcH+T6$|BLv$;IOpGi9 zR{JAg=F=qix;7kK?DgT-p2CoHny{kcNz3}E(nzaJmd_O#|E$*!^4EMCwpX^gHp<76 zOSQ$JGzT>zfDrz={xpwT0mm(|;M8ARK6y_57uZW>VSR*0u9=PGm4U(Tu^7KvWzXZgtjdthB1bK&G2%xNjJ_A) zxl(&9;M6pAA`j-pSzUNRFp_D`Q+-q>WcZ_`XYCp1AYJHO^LFN8RrusiiDPcK>xy|H z@~g#^2xeoO{dp^sntR^TDU!-YQwpBMt!r4r$ zBVKVA221}D0p9t>WROn5gLN}_b8^NRAB;P9AO93$_s%o$`$f=N)I<!_n7^I1LnWi9Qa&2TA07Ud;d~9z ze@z7(vYsnB^W-WMu&Z-9>Rhw>Z~*RXmrZu<&ZabWm%`@gwdJXn$))v8kWG`8b~>?L zFS;(y+TQfEp#syw^?Dn66fqhh^m(x_Dh}Q*jr9P_u?MsVp1}@!LHyVqn(16)&$ehk z?1-K!rA%L?!E{);9NYyJ99`16V#U*b28Oo-R^pcwl{4U{g>l||HF_8vLaqK^#U`cm z181s4npVqR3rKPTf_5pEPOJ1m*5!LOOT_k8gS~e!aIP6^W)Dwnr?ltBnH`lGpH8;wEl9wvqSxaLc**TU!xGgR2?ZzZ*t!kWkX zrDcE041tiO-gxXfdLFYGHADl+%||LOg3oG@Vsr_|yE)gGkG1u)-$nTn?m_rI8i|9K zUaVe^kstmKy7moDDiaYbU&Y2mRQnkc+12l)RITsI3=C{NqE$})&aCof4pblb zY=?J5>#NobpR0(TKJv%c#@dp{>ecgKtpiF$jW%Ci?G3c_J(GA88&UIy^zkrIWQmhS z`Vfd7KTIe%DgibjGFEgs&&Inl*7DMFnel8uOS)b5gj!v4!zH>)&v9JMhcsQ=uHTUZ z&2_7kgY$vXEDABCo4}%w=42;xou2W&_M|w}`j?a|I2|M^wNpdG(x6_X3CnRU`gZYY zD>L1Sq{0NKJ6(TS!_!U*_XK)BRcAU;u6F#TG5;f!4>^L#Cvh*u1!2l^Em7w1yY) z1+lYnFXRvE-Vk7sa5K#`>mD?UjmnX4BS_4SAs_-y>k6Fc4!L_LI~GxXZm zhR;v>xV@(@@=>X#YU!$1X$58cr@gIns}sIQH9s0*Rg7nAHsy4DW6=#+YwaT2?oev) zW}7*cbE~s$58%@)_|u2ormVEaHEQ?O~y8f;@|c%JtFqn@Xyy`Fzr+mCW5~o5*{i zlToA>^4KQ5_nwJlJHQ|RioV2We3ezTjej&_x?Q0w)7f(I1aJ}W#Up+e>$G1rl;L|W ziJyOno2E5lVPjsy|AmQieKDZ%yun1t0|1!QCt?XJ0b+vmO1kBdV{E%X`i{99~&GmYk04Z;&gJcNE)%i(4LP~6b zyH|5_pP&EswpvQiuhg-#5sYkqa2L}ZR zUBLn=_~*D7+4X3mX+iwLq__a-L0f93dJ~{^VeG~6KxwerRxQZ8NuP@4SK&R7kvsAl z;jb4qgE0d7R7>M{4##DYh{ZD=#9E2KZESxL5IE~cg~)Q4F%#<4fY_uI?|&JT37or` zq#x$fZS|M3fyo%M-YGR4$k#V z{xcP6T0WxkxJ2^HMS|@CSJ7gFdar5g%hPd{_I_ZE!e;oX)HdKct%Kr=gwrw&+V(6N zucYm6=&&@|6?^J-t90mbOnNx3z+!!j=BzA2;LyfoWxJ7IQLA_NSDo+nO#0{vX*U~k zU|YA{$GaJi_`C}$yiQNU|H$j7v@_}S=FN-@*>v%oLDZ(pcS&PZZrHmfr;){!S@tUl z;b9NwniifDDx2hzBm!`987isC!tHjmkC@Isl`jtseTov`Ra8`n)twT`5w!HW$8RCf z=1!512n*cp^E1fqdkSS@Px}2fb43*{?+?F4cI%5d?z)Vw5`qaw@4vp4P!ZDq_u`;q zv4MZpQr$^>^$)*O!tY%@^oyvfgq@wUR~>$`vuGlA`z-J9di=9634Kaamg!$$@{JeY z=XkckA75>gk`32@;(oE+@^ty1@uBj*bh@uDLPw41SSJu0@{zPa&_|SbBeU7f!(a?R zr(auJ$;rYg{Kp;^^%wu|T1lR)ii@37nw9NC-QRjD{P{nmR_l7K8(%?ch4a%rpQZm^ zkN35bt%P5mrGCP<^Yp9htBW_r|IlECJ(dZ8I5;-#qiEB=k3V{{+kgIBefZzc+Bgw5tcrP+(kvqmdL>eoU{ovDd8ij%1*54nr z{u6a18Tuq7Bq%t-$jIn798q?P$OV{G1Yzg$9gLmm+dPk-*eQ0_7dr}lJR81xFe&yB zCBu1G2x)#r!A~WrG{#`i_C`y{fj-wnNB!+wUj&%{MO4;JWd#5o6zeHeYajIA6Sq$$CH4q&Xa1f_|#69|0MkJ?6Nt56|B598L;=f zn!pR&>D|!V-NDdi+!=WxxoEdH`$^Qb6ME!L*lrSkZrn>+6*I(@Ew?=A2dY-Juz@wf zgJ@E96~9hRPoF*_$o=S5Z^brm~U^ zLEkAbL+%-05IB zU?3-}?cj!&j(s1rHcS51%;A#2h0iFA2l!_Ss)&1;S997?m3r>6kdhsDqyB4~Q0DMQ zY@;MIkx3T#_ucW_oG(H6WLx;?c)CwD%Ih(|v;WPL>RvQ7A)*a#xngLUFdtdXD^s0f zII=%TsAYBO^k>}h<()r&)KU;vpkOCNU(oia^1U>FUM0XgA+XBzY1b8>S5=njs2&Ws zlB*){a#DF!`DD6+#kS!tiq1ZA=t|G$?YA-6f}<5OLjuz@FPw<5=M?G-Bf&QB>oe*| z-r)sd{c#yWMS_Q0B!Sdi`@p_7eiId-4B(F_1j%l2Ek}HcsM&v|OLzl*Qc!L;GP8T5 z#0)DRnM!!!wO89tRk;Y;w&|4F99K%*SH?+tYGAR`8ei-{5l&xgX47-Ce=tr)Pw5Cg zbitkT`=NKhfTyU$Zrs$uCB6zEE+0q)jM12hcM%}ie1kz#DCOSEChf=j70R>T%2IRo zOSQ|g2kc6-Ha0!n&OEd&4Xaq5MN%eE4P4ezcjd%=9f!SE7$%)(@zE%6bLrM-0DHFw zyhJVyT612J#>Ria^IM=N>R6|89FnH94Gb-A9Ok_$j%^4h|$)HMQn! zH5STlzh_Lw=<}?p&mbzrYSu*6`^?pTbC>&TNlOF7x=7?V?VT5*;TuCE?MZ4ScAN zZ8&LPAwE~6F{a9TWqp7l-kQdF;XIL`+Dw`EQl=`oGTniy3i=e19)H-WD&86FnG7*r z7~O{aS>1%y!#P?Dax5;yzyY8o1iFkNC2%;wO+$s;!_m=YE_ZuKU!(Q1FLt3Yo`GurS6Jk_yi;4NILOnVjl%_|ndIgZ zEGae;H8P8~g87HSXVJu><1AF|$3)aihTb|HM|p=e2l*9K)vGB0dkqE^hG0qEC)Eo< zC-g1YF>5mJ(AXe6BQ!g2?sFl?2$JeXj^J;$udKViW-I-(Hyhtny_{ zGnuW@APk0POQ1xHE&_vToZna7w2z+&m;ndDKVG7ZfP2E3In;QU$$lQa;HYk{Qy6lE zKIv3ei$1?b--=#x_AVbMTd4rYoIMY zQb(p8=^nhoB@O3253OtWIOg4uJJ_PBUSR0C{&iuooV;3bf>rt54xro%u4YTq*heac z-ApMs7XJ*ea87}k5VAP@S%2TO+hI9Ze>m;A{8u?Jufe%Ze3>-%8U3i@%z*wYpO{1Y zuO`NGa@fbkisY_ zw3jWJ&jTP;Rn1;y8lfpG%?w_vADC%a#A?ci03#PyUpBe7J7B2{Y!4-FA+nsd>_IBg z_Qx?S@{^2q=gv15ITp`N>siYkJr}gnR0S_n4M9Ir1v25?Qok6la(TC&x(&9}d|5Zd zETIHy7_LIRB5|vAnE|s$cvaD1v&aEf3Ib}=c+7CJGaX2KaLYAR!>%vY&3`KL<|^C% zs@|a}Pq9xpH+JjT0_GVBODA_4iwynZ(N-=c$r&*|_Es94&{Fa1 zp)ruGk z=!8}AFMCO9yq&9Y%qcY9~*pfNn#KV3_+U+XxRi-4t3it!rx zLi%$+L~n3oYNx!!3n3W z7SKM|E<*5k$MKI0_|qja_$#1X0myAUER6PE%OcZXf2L`q>b%T13-72#u^X z8zsMFX^Gw3K$N~{#b#qB6Ja#-AHntaWq+}%T3b#LSRzCAp+2lc;*9yKYiCfaJs>|g zco0QJCUqCvx@qc^5spjJ>B-?>Sw-_N`ZUJzplqQTcz#YlRal$CAKT|34Y+cRI3vr- z$-zIyHR~V3>v_V906Lop_sAY-`f<>#4m8|`D|74j5N}psItdpSloXu?}th7?0AucC!mqW$KtS|2SYO#er zz&YnIQd;bgHB=lfF_EH)e8N~(g6PzjDP^S3Ti`)hDxyG_AXc$(pu;Z5KbMT|o5{sq zEJ&??cBnhCr{i_3`5wdoCDkcRZ1BW)Rg2glD`hXEc#}hVSE74z@=HUYy2f^y_@AA< zwttGY6*_pNB$qn2b&4e7;K)2|T`LMEc6H zB(m;aKKn4YWlY5K)~G;nic4Wc*P-u2HZJk?HY{ZPW6&r|d6h~qrBnUk@_oRn4!%RM z5i3!&X@L!;XpQ28g@pxJBMDnUQCf@9bC|R&I6IXe7#DJ%k71>4xp8rBuk--SmDETYI6kbpqokq1nIDubqFQ+X`=n7&{CS4` z;zf@4z~IC|0X6O*4Yq=vWIn}L+BiBpNI8a+_~c~%q2r(Vy%U2X)tP*9rNbk3#_CVu zx#$}mS+B0LfAw3U`{?ZK5-Xd~{@_DH;X&H~0~~9}$?<~0b&~Z2BU`@(uenCpg~fVD z=XE%NDN_wcbTEUGS`@k0uN1s&Glj$ZHDfdb zvVkzn#inm2v$j;nx+`;OS(C$5yN}JQ8OM(*jHwRC(ozo($zW8s5R0eVr4Oit6z}4r zlT-s{D5?m+%sGrb&uvyOQel{%omD1k)LEkUans+*Bz<4}d z-Ldm^>H+j1E9`r=+ri;oxOaphT5ij8984Q#vDNblgaV1xD2t-NcWV##5OUEZ|C}yi zy5@XROXrV$n~$bu-O{b7>Uf(O$r<|y!p-wJDg`;D!1Wa%J0d600AmA>>J&@ai8gl4 za!Mcp(upogK>9nv1WPbKPne}!ZsOxOU0X;e7Y0kM0xkYdO8H$ULy698%Zh$RaYpi* z$>BXyV+J`oj`iI1HQg zYQ<7d)xt#D(xfpVYQts-L;1Lw`e^Q6n-eJK*h6k({d{0mBDO?>4W*|E-x0WRifY~j zTa0R_xu0J}3(gp}ES^^Y)=9Yciat85W?x>Dsy#_QdU=i^xj|5%eBb;p6O(kJg@^-; zP^60ifnVdU24S$U$v-&D>7w>k3h7+fjM3_y*nN@tic9f~5+;>X5)7>zgH1xv0E)71 zh6WUTwULE|nJN~mu5h1|AC0j|*OjA8JtdSz?9pb>%Mt`y70jaFL37>(^*T<@!a1*f z%w#xjs@BMTG?jtUpP~*omnZJwWHuI=PK*>lK1ljTvrIr$NkPt%x^Z9r%0qYOhIo4` zNy)7?q~-?@s2U&9Tvb!_W=~0T1Q)BJ8po>Us?N1~9g&opm@2QSWmi}Z$>!Z0Jz4aK zjk%IUXQ?@GH%{c|Nc!~zy#e!(j0wX$h!bLL-Y7P-BF=CL)4r(svSekl;kMPeuwreb z@F6lUPb6tfUFTpdJDMh>ih7}W!SaXW2F&D|W}fy+XIpk6%C1W|%0b#(K&U+lnXuJY zZ!8Kp;;1%4-Y>H4YTJ*6X6{TRt()*4fE~ zS3*sjBM*0`ZJ1Usm9(cA;#F5QB`$X2*QvJR?v_5BWk6t$$|y#l$>X|(QV+$=I&Hsdu3$``mV{Z_QOS_rKt1gXqb+Z6d2>DI3LI;k3&p2 z#~1YYg^qsysp7wByTFqgzBlDA=q92sDyeAOADL@gH_O>VigMb=$A&^|N=$qC_56X} zODPiaAGATKaxRq<^X|%dV(uVkxe;>Oo}PR-)Gfzk80N*X-f}PKsxAI35!TG9w*<+E zX#1=$eh-3f9QHPqxLvn#`N0E&1A*I_ftW(c%c7had;uocUr# zd}B5XCbI-x0q?V}pEJ1b#ZDs@!zET`0v4LZWp4$zWZ%@0a~!#EIUiSxr~I&8JgZIM zhReu%g{8ljn>RiBHpTE%^={>Y4yDMHua8BW1yyVm5Fk4}fZ+m%a|{i>RTwP(vM@%P zQ>-ul0c3rbqjEPVYtZ8uhYTjX8iPe{n!C8T3avFOq^BF`&ze#L^$%O`g_M@8Uvb=f zsJNADT01V;Cuu#5-Xd6Fw6S(IYomkI7VM7M(Rb6(q-;`2aF?k>&)A#g^fb2im zM6bA7vh@013b-2_om6cmHYyP4MSqIuHry9=ZJa^2R+l!tg%mD4xb@Z|?Dyw0C36;1 zvNSMg&v{r{RD{H4lEKEmxvW~2;@sOErxX*;wkNpRvs#2gH|})3K)FAm<1nC_5e0{n zAO5`g2(t?PI-9^_(;YvLQ2fLe&Ml$WjQA6{d=?h}Oy}DC@E=T}AuAr>Ey#A~r|Rc) z2#q|lc`b7E%6Fq39UmEPKwsUGwmO z^5<2EEoU%1i@ojwt@!aAAYS{mk%>#zigUnA3&~EifcAS$TBk4vzPw**Cu*?5#o*qO z#%`IvZTNDl{{(Z?-NUPDdAUqL3K+=5d73Q{qoVQ%m5tY_pilDRY=}=X=Z!Im)=HG-9vuWsDKGzXU&*e@9^w#s%H7uCrqc zUy0)$*eq0SeO;^k4OtJC^5BxzUF^RpQr5<6fychf+}bmFJ#{*rXg7-%P%Db4b|})k z-<}zbt6k!5F>!yQ_V2jm^^i4YTy~GnPCHaIT#@cHzL|ymXMA=@4Qxuy{IJi^7iqdS z^Z*ebHa^3fa(FiIgeE+2M#-@-f0G2Wn{xuGV>t=jlPo+B%xV_ z^Wy_NDF>1?!_*rM$G~)*j&V(=ryAHk>!mwDdt3L<;-D4!xgo=D8&^Kq8<)k%wXcn< zk_OJjvtVJd{l~6kXT?{tfSn`HU?1-y^t94_=%Hofb4C=T1hGsBUZ@1ruc zVT7tbWGGx0(^$QGlcoR}iVq?e^aO+!Jz)nvm@~0|Jw@+yWw4M|;a1X)&B!V-FrA(b zjZkf4W8@weYY1|EKEe3S!0KoaUq?LzIszw?l;+0QJg*I#8@n%m^4-jD3ynHAdwHC9 zk7%rCHwnGIZg#5RCFgE*j|9)*MrE|$hpan26!Unw{i@|fTEY3N_aYuVu85A*pU|^K zJ8~af43S_nCp;1R#vtQh;hlH9Vmn0@&8femmQhEmDc9uJW1uw7F>d5d0-+*@QG3^W zdbpsaXZ)1T3AK2TS>N8yV*xVHV+wFb8scN1fo%kR03dT~_wb(JzC>eR*Vz!d>`r1@ z#JbG3uVMUnnfTNBDPPCp(8R@wxnL!U5y8J4L8Z{6bcnWsL||+^`BAyJf_Na8jlaGi z7aNHNPnSx8**4o)*p)EbO2TvfHWuS(_H#5#<8=&r(G{;HGdc9GFnL8aWlxan>|KUNsVSp7iomTuQgIxix)v*=&jDjIVt?wMO)01+RWD^1IjFv?Nd4W8L>B27kY* zgh^de!>KQch#TUF+fF(=5aKC5C&?e7!P@Prp`F|%b5Kgs`M24J0@#NG8d#49bOcTp zXtP_rcHgDynSjtq@GlRFhlja9)Dk+}dmnb+vU4Ebm2iWu$5k51oQw9C8|wz_?ZNP+ zW?PGx3O#x58Zjbzr z^)lzMFxBSU6W#eH_kYaR^Y0b@-7WMO`#XwH{@b3@x?G&_-Rf{B#y^;C2|oX%AER7Y zuL;eWF6nrdz6DDEeY_w2(yv%33*Ru6##dL;-rf*l{=XpU|4V}VLm8D+d+J{rdeucK z|8&kAG|H8q-E@c!qtU|Vs={RiVA9-hG#7UZNKl7x?tIdPLPrUV_g;MHES}^Pbd*M3 z>1h8FkPc7zh(ZTZpx!SD*DEc-uB1i=t@2ao`U!KyX926@v`>JR9l6JQZtjgOctEb= zrAq;r>;&fgZ`e_0@{bu;65m*U<$Z1K4db?fZpVR|6Q8s57Ia%hXqPsQd{R1S*a2=q z@A@!Bkt&!QPgA9(7MAaj-=j+aVi$=z{PpEQ$j|$w`w3aQrY@RW8G|GPtXOH#;(80{ zN?d9>H=DGYeB;oRd%(7q@gRfD$AH~6pq)>?eK?fcsdeDU9>R*V#2iGI zxTb!m!9T9)j1J*9xS|6w*B6j&7@1#{LP+x=IP!PAV>I=a8ua?@#msxd{F6&%XyB?- z^YVs6zYV-%o>|uBjK-8kMTHb(kBiGEMRV&QB?7UWGvLW+^eX$naF?ks>a(O5H#qim z>T`}yUV0zQ+CUhcNcLw~#+6qZJlY?dDDqPMdC?J$mEhwa7|R&oew|_yorE>H_T$R( z{y>!v<>h;*@*(ole$GCFHi_^R%8_d&Nb;%w5R*FZVyHdH ztvt?R2aU>3C6!UE5?R;LW~c|g53&xsi^9IPq!VpcjIMX1 zYYvi$f<=Io;^_R>`=SAz$q4OT*F>9QMhWqc%`-4qSGo73LDz$J+N?Jsw$TS!Qw-Gn zCNVVLVDi-FQ|BS6(VR+W)03IkTSP->ObJ^&V|C51*x=p*L<3-}c3ynb-wMZ(X6MaO zs-T4^KT^ksUO21QZ(@LMP*%#9p17DAlJ=EZps4P|>{Jw*6~-AYb@)P3+E#1L@P}pi zh{C>SO;4*?P~{T0ALU7-&)i8t=_Jr{dS4ZyVoa6 zWT#p%XA@qzYgaQV|E>31ztz5r7X~DET#wu3r=bj_xxERIzkD21{beN=N#(!(erU?IR`)}eLQ91+C0c@gPPcW}%|LHkZatHz~123AZxIMnqGthGv2Z6olkH#=3 zG!Z{jlH$(y^NE^NtY<5Xc_LYG(S`d*PfmUTwNnt{jN89jTtfro#hi>uHJY`eapL8& z5&Ii)W$&5IH?3ik?KS8J;)VKjoq;>%l8J2I3`S!yOZ4E-j?O`Fm=+kR#xVV|yusnh zTm4rXt!!N$y8ta-55Ll&L*zBl6wHq9 z3e3-d$HP6&9cQWo z6MY_mcfQ;sshB>_!VIcPByuD2#O{>!KK&Xk76<2OmFw+GEH<9g_F`_tp4BV!Ui>yQI!ogjOO-TqPMlf!nqZB{bL>4#A;L`Mb5imX2`1g? z!wgXeAQ)CbW1Ia!AG`fGHR*{c*l;q^2Lj2<7b_B z(?3N%3eK;jIXL{%CC?jFcCxq4!}^dH9rtzgxFdht->+C83NW#;ETIujB!zHaE^L$2 z#S*J&2z5^!wse6u;6v((J`h;-L`+Vyfam3y5b#*kzeNZwT8on7sG|o&H<#BKqN4hX z1GN`JgJ=Eji4@I}4-8mwIGxC$7P%Px+}wMnpN}i%4&mXE$T#6yswYv^Y#S&e8(QALvF1bP=>5sq2+Tt&f9q*oq{bF z^QsnM$dknZo$#GE*A5vR0A(s4Gz~wEm-}A&*I+1m;xl-yPq+rRR)sK$W%H`tPc5XW z>}yh}HP)&`_?^SU`mE+lW0EAp-f;=tY9!yn6T^C?I#{ZMvapLbEQRVu zvxKx>=$moA2`tp6SX+wE^eKlEl14osOOrcayainDylQ8Z(UyesA+=+&Gm?~h3_KM3 z0-q~HLZUaia(=qPR7la_HQr(joq|>ZuWfPhg+)?2g0RXC!iMdM=gD3wMt>))er{CN zxrv~8re8CWZhjWy}E8RGH|wb2)b zBW*;oxQVzSFY0IK=5Sw2h%3`oZka+anS}UKkA&I2(_& zEKOy|x`cwiTb>zkuCq$$)WBF}H5AIUMtB^?TcUpKpCgl@gh);xPI|7ui0^z7K+_mS zzhN7J6d%D);lH;OrSw6*u(&-s<-&+Bz($Zxq2GoRdzjkh-d}q8%d} zT*>|`1bAi7`u~N^9LL&h`Pq3d0yNT3#$V2*GJj?>ZvoSx{3;#Ak|-?l7$q*2c4ZCs zt@0glF1Fb*F^eq(UT9vCmj-{!tZo5#>(TAjFME$B|pG8FFE>e=acP*~M`+hO@*? z{Doq*VKPhPie~ie&|8{WW49=z$A{|)ocdIdq?Y`ucet@bYO)+Ni-IeB4p1ErJBlY_ zg=#jgddHV2!BW?-SNfY#E(6-&7^;Q|%cb&xMhovy|2S>OV8?P@UKMr53Q9a$zW6w(yMZx;0eyA_yxe5`HEr2Ne71h9dl_mBb9#1>65l7} z7jyvga`zijYO}nuY>IFx(II%NR!_l$cE2{4(a5=>F?}-&fm30Ga;K8NiglqoP{Lvk z%8K*iADijD{|o1^AY8=#m)}@-3!UYNiKrcN%2MqYcfD4!1cL620M^BIWmOFp_goiX zYob}lzrw%VuFc%!8K{b_?xW^ueB(@;D?)bs^Z@x$oX%-Fyqo%FnuSAmi)>MQ7=UpS zu8y%lpjG&Dg{B)*J*d2T+3>Z5d@@|AJgD2hR_ADZTtqX$WnyKq47y*WS#IBy6 z)uw^mHTvHw5C1JJNuT>1 z|IcK<_e7jcbC=l)9bLROZ>n731F4>!e^16MycYQPg1Rm!b=vJK{c|_k|3+TO?f-o% z7=->wgZ@Za*^ z((O0Nd|}$whhJ4zNgXyk?K}Jr{jQPH8ws>5rB4!ngvDV1>i<0VjqIEY-@_t{l)cF8 zZYjfeE`z&5NxCPN69`H8T{>W>h6%+#WzD9j%Bzch@pgE3{=IJhpU~0&il={vhlES{ ze58q2yFjxnpmB|^`a-&LGL3zEfw#xsj?+^xtlf!Vfu#lrVtMoBMVHS_6Y#Dr?XB)K zzNz*k(o?2wSkf`g3y$*W+ga0nLbUavQA^J8kOlNeGmWqWfj> z#E7fq#*g_z$q>zIT?cxJskQJoI-!X`y#;{=9klTn3 z+%0~@k~9%?^bKv7~?)Q@FiLD4ie?4?zdKKq`)(sU4KyUc89^KHfWkWS-H(=?B ziVlmeOFNm^h-kmpv2}Y&1B}_KaIF1)d5h?#@RwL{l5&G5DLy)O8$}YjtB}neQ02`f zRZbw_?%I-zjRHD#jMzNMSwk9$VY>u9Tm?-4Jh$ zzpbZV{g$puPLdXQxQ$&KjEtoD&c>2H)+*yuqoQl^WZ6Gz(M-WU|AjKp05QGlm1nm3 zR;ZghqF3*1TGgR9h(0pXjpaXR5jc0&SL3S=dM%`FZ?E{%yC=grBTQsRgDd+DC$FtD zHSRlsU(GB$=U*qVd{6!86K4c$x*0NLQX2&fJCE5IVZYGxXO4^CN_s8Fwq%#%5;@Hf zFfprfs%8gyXw^5lZyWDgY^FoAcxfJ%8tLJ=f4ceddQEYMW9unK>s824<%@^_ zq5Eqphmtj|u{5jJRPqMa=?jCv&u-U`&|Cknpgq4jN1P@)UDl)= zTVr=-%2?G>`p)WY3qj3Bda$hXQO)bv(UJ<6cq($X*n;$-75C>+$~idr0Yz94KPUltO2o7A{7n%;gy~yqYGzF5m`oM7tiioy#R*CNr<<|-s_mE3@nZ%`m25XJYxOLS(uA2>!NyVL5SI^ z-#r2W(1e$p9$f^*?$c-ra$3(F?H#`YjqVZ;jpzxO8LzSqQSHscsX1ptq6Rvmp4&7l z1o7RD{+qeMJ5vOqG&L2#|M%$O9}MR9Q_XRo0hsm!17!uunLWmc8-CP>Xtiwhp<6WZ z$AXbjNtRJms+;^?62o0TtHQ5SCfag1@`A%JKQ{alIT5?DzEw!wA*4o~+xb?m7c1A9 zgI=7&8WORtm|G@J7Xy_FGZMg(&`D`0+FS7aC_fNSS87hrjaOVl4xKMExL&uh4BU33 zNck?dCK?XSede^@xEPqHiP9=!As^k2o$!8_{gDf7!*kPe%$Z5&NJUCt#h86|2i7X4qZq+mBN#A={ruzi9UT z!wD<#({_pW}bS zpyOZK#{W0d3pHu~|9tJ0o}QJT$VXK-#Py=KeUR#^ya+-H0 zohU@aGol1Y=vy?DcfDtk0+E34k?a(1-Ky}MlIy!QgN)Iao;l^q9H##P9{f|e>BS3E{$88?RBgG?o@cD0pgXW<0Wy*;Mm>y!GsLFYcb&B zZKKdB+yd}UH2D#xFshxhZzi>RI1IoXm|?m9eckXVXEC~mSVAT*ov#EmviifQSrPJf zy}yIrvFp(YN0p-nantC8aC}N05#BKAAfh+=QW{+WATfpX2)&_+MvIGAe)F z^v31`zI>S{{{6JxnrksKtPUb66gI?$2#Kh+o_#@y(>6Xem8IPLPQkkTM}U=tDz>T6 zXrxg2*;dT~D)Fk~b;ndm@w>D4O3ClLWODG}#tOf5hG*YDGtr@bR3y&*!j7_DbBOAd zVP|4?Wqnl9GTBQ>3qzCKp2thaFU4_SaHpZ&YjSwhL1M@Vww>CMF)8_gs1cgDd}J;r z96A2Qtmm>kS?X}^ITF4)V$uO47w0(*UsijXlRD1d{L^NR?j0)d;~h5Mb5Jx!_|8T5 z9+^gQ!g3k4ZK{>K;tAG-RxirSlEK9(w6-!=p#wJgDe(oMe)*HNCpysh?JOEXfqr_Z zfM?dierVhCM9xE;9%%g2Ex8DG=#FFdEGn8DH#O{CoN~#K;kO3;P!iVVYiy1@L3aa~ zy_?#EUMdIPF_7P^8+z~=)k*Wdaa51a_Wl{n@xiN!Q}6Ou0%bMg!4)ZAJtZRKzKwSb zWkY+IO9G`hMtxr`ht;dvKLyrjoCjO&rPLH^w%r}h?W@miOt_k-`WQ9t6@TpXGyGtZ zY{r9_fb3Xlx9HZ8EY3K5u(z(uq{vIJ!^x?dCPW+PAQNK z3MBF8&5l1kQfg)x;E0hg#QtYOmcqrmIqxGfuDbPJ`)Gv*ve)dnogMJCI_v39Qgp6x z*Uk1gV1;77G{-nDYp-B^RuvEV2BZwqjblzdd!qWjK+Uer1kHCYQ%W(>*{vITa@+;S z)v%xd&A-Y^L(R4~4jDU*P_7N&bQS3CjhgLxeN`C>Jc7p3>1(?Iz`9W^dShJ0eoc^9 zKf;vm30c)@)eW_(trnLh=JAPcNgN}+CMxt;$ME9}(K(;nAa{ABUL)4aA&Q!@C)0M% zqS==%+W>q={lzyLf6BM_Xg}T@b>{g#ahd4{xT^15mHKX-$L0J7jRe;w zqfgVhi5kpEo1>kkhuH7CyTj>0AJ-I%5NAF)#*c(-Hzm%8I{KWy7^rgAL%pmYW(g)t zb%d?$to3ee2F4H*wRdy3S&gn1|GW+E6p&7(YoQR?eYtmI z-_@seo<pRi9$dZ8{8zZ5cG+UmUo*%Ni>yWp4v)nRTSjY=s~S)rfPQ#K5?EKcb{@ zsQ?TyYZj7*_f{XHcF9xt%rm7vnVJ<7UrnpaQUNAXT8_X`FdgCs34pkm5Fg%vV{C8| zSa$SvXbbxrg(}g8H*F1N0tiI$+tqYD(OUN8Bohe+EJCO9W<+Nf#G zJ4bwyHD`oXhqe*KeQQ_~oN4Ksj~>Yki=|g?GQ$%2lBV3}@_Ba%^He@qYOeaawrJ_j zRHoE)`h>cBw~P-;ahoeoC`o8B0zs_c?^yi=XO?geX_^pl`j0y@OZQiV)ldmpH|Jv0 zY5A@&%o|E3IN~m!;;^t7eDli;yL&$}G>m}rL@hHZpNM~u`F(LC2K@5o3=-&V2b*Xxz)(q%& zX_d=eOa>$F+-D{na&{jz3}|B^!5Gkfh_VmJ%u92;o^G?gKY!~XlqG*iHkc<$z1`QZ zhF{4zp$4wVmw?}|-N)CSzkG%!Wj)C2#9CRB5&<%^8~uJTfyztM;bf&{W~9*6ko443 zn2L@JlvP}B7fIR>D_-nRJU86zWDh+ac{%?qx#<64?=7R^>e_9=I3Wa2APE72I|O$K z5Zv7%!QH(mNN|T>g%|En!MzCX?(XjHMHi6wJGamM&K>7;_s_0h48|UNuf6t?XU%6m zbIxxYJ|}$)xm10<;sPSZWyhi$UAP*GMYLO0T264gMao3Cfso~~0>RC0CrJ^sj$Xp3 zZt4~rco?-hJ;f85$Z;Rs&LFDfk!YBlZG~+Ti8D;&SA)1{`NyB%Bwa5DVmPF}5PaWJ zam)qeR>;j6ONZ8jwER<9eVKnA<*Siwn~%(0ZsFv#I~2DNvvkB68S_b2j}RH)#Xa8D z6}BJz6(iAzEkoBgve-d$F_Q!D#p# zx0ZPUM|se^Hxc+RQ=ZJ1A1pK6TvFnuMowe)%4~+BnTI<5Q2H=d)}E+GPmbw-9PfEE z3`XAo`1)Mrf{aVDWfl|)&5Gx2c;uh7mm;r9X1eXz78njp*p~Je%}>OK`;-PY?sAoa z$TFfz!0$D(<}04=m<)yPoFCQ1JL3AiwE?>0=T5W3CrJNiX^VwecdvmAzc#teZT?w} zh4$}R48+FHtMQIr#l9p~OWOZ<&g2*i$>;nkw|jl+MKh#R)h9HjwhF@$0Ja9|5aq;N z!@UmQmknu;zc-sx?VM4V6FD}OKTAOS(4#8W2F8n)uyhRwIe2ZBb0VE(T7)B+WxL37 z(xCQfEZxmU&ZzpE_pJ<-iLGHB+JL=^2dj?^wtxlg%&huQhYo1t`zNv@?Yh-;c}XFt zpplbGGB$AfP0y;ReWSAKjN<1P$*YUEfslS%RTfAInfq$=FVz=j8@qnrg~z!9wJiNt zOCvR6M`|A-ut;u^2XlvO@XdVq;6lIdej^cqi3^PCiW-^a6RbzI_;aNarpUwV+Lm|3 z^|oeTDH|f1H!o2flPKYc_})IHfyAgPu$ANd<{_ARwE2Hat6qwl!_%re`G)EdU~K_v zY&v-pfW)x|mecV7Rb@5Y%;CXm1+6*K&qb|5|Eqiopl|Nh++G<)xD~N9VXU4W0^y(U zIC_$JqIY4LkiSRvHuW{+ufbq^KPuDsT zTJc{m7;3>3?kadHDII1|GZ*i!@Js~yGlkRBd7AS=TPxpS^&Jnx!T?dniByBS&oLpB zPgURGMCOdIV4nj=8KvmWw%At5koEZqLksnF)vH-O+}ksoS3Wd)Ex z=jv6Dx6RD4h88lCdMo3NV2stO-FAizu+LuNjfqZfcF2ENk39L3>G0>ld4#=Nd}h){ z8o$TCO#@FT zF(Qa@{W0HNd3%%ddc3baHL%v&1X)|l;tTDL-{1}T)uG1O@NexAqkMH|r!Py&bx47i zaAt)eZv)u16XU*W3kl-36T^il>IyXI(+g9g(KyvlSSvQ*MJ)uKwQ`V_BZc9Q8_jlq zSVoj|;{zV&&$Y)Cnh#`B6tA4R;{QQ=?TTMOGveUE-Y9|N3j?E_IGWzCaL4=bO3v3* z($z8I=Ia(=`Du*DZ1Vcy@%`Py7woE*Fxo+CJ|J$GP+zMPxs(0ANRtA69t013oXxG?WgltPTvl@ zvS9OY(-@zzxY{aI>olD&W(>Dt!w9A6_{3$aW4e*kZm)2E_j?*Pu20uojhvxArgEPQFC z;kO{!%it-wnL2@68{Ie(r9s=t>(X>Nx~Qxs4x|(=FWKlHrc5-tN5O8N^@qs6GZfS@L-!wGS zMq6CqY1wsE3D@7Fm|`zVLpR3?o&Zl6_FRaIuPd!ib<%uxp*R~-udeDI#5Y{}uW{U| zbpxC;I7sb&mTV>)g-JLrYH8CLyvnXALt%@Fgu7ahS4qlqP8r0ezEgd3+i`${sSLen zW4`_gD$3m?@1rtW<=~{!A|G)8?GEQk$}5iza`q_fPG!#+TaFFCP$ku`aZfvT+Kc?_ zQn6Qd!6Dq&-k%JJ`O{43IF+M2LOM2V>@1FH`;2sG#?7%VBxGcgU*9Ix-{xewp?`sL zt-1e)_)5#9_)b-K9GW|}ZuL|4>cEDdCzfQDF&Zgd>Lpu=749`UJ#y7a9yhl|5iThd zWFK;pn%$6z$+%}1#%O6H&*0O6tptQ5VuE9KD7n z@@R8My*$!$Q5YT9BLf!V5o8bs1ILiR-DEOu2Z;y>R&|<_&{B}7$?xEo#`CxSIg`Dq zCeb;NI{m7@zFYIljdc9B;2y0`-h~^5A>&>seT|FwH6+fec6xl(B}kDY z)*vlxyXyG<>5VA&NUv1j8v{LZR=KDZeLWTo`S={WQ@Q6u0n-e)5j$v6qU!tUO zDT>OU(N+CfAc>f`GRnA`Ev@D+3`}+PYD-&A9Lzx{64k2EEfi6?ToIa8bUn=!PVgQv zG)ze(`;kOluNCj>tjRbuV+)@z7ca-sGpCaoOxhG^SoU38lcqaGWI96^(bzeVO*sv8 zw+hJ^59R^{UiPfmt<7DQe!Rz*O#8sv`|8S)rxQ`>K?V#xv6dosG}{0T z%NZc+VBfQ=`iIdwZ2a$4f~?UJo6#9&M}@Z3gdAL4_(p(l<7#ACFHq3rS65X%aNW-yPNg3p~|NWJte%tRJ*AZm`b^ZL?BPP9$F!SYgC zr$TT|SZCodAum(;`-8Iow0+9ZJQz?K=?bPHbl~Y5TvMA#EI!_A@#5}!rBQcT8?+_J zf2z~YX}hgc*Woo)QC8h66&f)T50@klfJ>6|a58iqc#HCiwA!v9m+31**!bE zIFZ7)Z9#$tyL`J!n<=HcdcuK|U7(VjtHqFL+A@sGPYD$BE*ugc`!A6VV@9)X!j1&o zXa;sGHDeuDZ%GglEqAt#n6~45(z5u6C~!pZVge04I_kP|l+{m4)w7Rc-XCp{900q1 z*mvPF^FEQQ(Z75ck>5)5eqHNWa3s(l>b@(*)cO4HCHPsD-!T0b8vHZ3FYH8KG1!HhSD4#D3+LyZbr*W^ixxnS`!VTikyoFz?8KGq4`R#?u5Im(!Bdh>TFT zlGK>{zWaN5Wy~Xyb8Bsg?@$xmk9;&$427Ngnl%K!s@jkb)SfLi^O4)wdgSSlYHqS8 zh`&(W*m(W10U5oFdDhl+lzVSdF%j_5HY~nntZ><26;b>H>G&07ymEN<+ec1d^Ru01 zBcG<`DShYX7s}81zrS_fY#9Ru^(pgkK1G?l;`SHw3CFzLo<7kyiyoUPO-YGqW!Wb7kz5aXRfOsiX=yFRHl?*YHm9_9lK;XLtPyg0ahmuH{!i$wA3zz zRkf+ElUE%_$y}x@Kav)xL4_Aj23#$g&;sWWsnn}bY?I>krM%Ccs`4_1DtUimC93#h zl@=!fu(-K=Bplr4O3runc#23?<*8RKIZ7iA}{tEv;}{H-L@M)f#-9EMmg zl0%B=WHB`HH|PGS{>if)ZhxIKLHuCpp6;%`jxK3-cbmOkwx8Y2WW;~)gyDATcRx45 zmjVEWq6(6-=Q%}1S)px7A$|FS3l{~qL#t7Y%qRco+wJ%N2UlX@XUN|cKElKPA5;}E z!^<12ms;(o>-$I_p8g0A{Xy}Z{!a0{cVvl4EdJl?s7nSRn{IuO`J1VJ^q+P6bej<2 z6&JF`&H}oa{EiPdKaY}4A%7bt_nqQxf*b)Qr$bCo4FP3&QPFpKPh};g)0fzLgX4C- z974rP7r=kA`ukKLSS>y(hwAE>ozJ5K;ME&J&;BO-AEjx+F>Q$J19Rii9qZxmpa0Km zD|pJ8i1&^iP&uf>N;Z52W{J&{5iYnL9_V(DxL)g08G*9)9N-l*P+m*D7ZLF6zwE>o z-s%f4i_nfj-7J*A8XJnMjg;)J;HkRFP+mZxFDv)H=5!tFjx@Q64%uIp$?p&Hd^Jq$ zJPyz~ew$5=(wQ$S~_v>HF&uTlnK zRhcx5>PdZ5;4hzddCa}6$x-I$pxJFw4+_N!o+kQB9WV*bNUmd8DAB{OH-|t7i?Sl{ zO@1DE{+fVl)A&4)>uR^OgEaWnm*1+SmZVq1qmb!^Ic<8RBH9ghLKNi(oJgvak zVlyb`**Z-Wo$k9bv2SlGjCe?v@1MUH;JfNMwtccE*eA6>hf*e19W(kT1aF`;tNZAT zltz%GY}@abs2jV9dHC6{rW98qhoBVseJUv*v&L?~*smRf+Y50C^nYrHeOks5C%yg2 z4zEGf319s@9s8G;`@$(F5QA!OI)ghn5zhSf)-jwPN+@&fhc`~w;(P5{2kj5y4cK@3 zOwqv;C6k_NR`ov;`w6AS@v-J5Aw9aPDtf6-zcYFY&az1|V|W`#JL;y?O8k!Sx+XzTp~E>$_64DQ|aFIikGL(ujcI4i9~UY zP!CqUbdjcsu+Fp2p5ZMFsn(0y3rn-^)TPJS*Jg8% zF9O^R-o7lfOvB0|bl_XgU9r?8C-&LEmBXxD|A82^PTJcgR)X;+#*S2AM-0`Qc6?TIMEifGaM0|ke~-RyrE(Y7B{Q*sFc4g`{?$b9=fpdx z*3_qt>bNmcoQS7&Ep-F^aac<7Y(ufo76~WPz7W`!boh|d1+{YSD6UFe*?dhotPJR; zpRw&Y9zRB(eE#YFb>Sk?LLLsz4mxImbA-|?zn6PZOLGIHsT9(Ugc#q_bi_6Y!UXca zT~^my*yAd=n|Ijs)%&p7wGli1!L_N+_TF3l|4zMb7UWBQ%*gmS7U*ZL4 z$KYG9?w2QD3}e-KmJ5QNXL3Dav(d{ME5f@+6GeE+Y5IMQMe1n}6&fw}0%CUP#SnbM z&Y2FS%)Wl+?t_<;g3Z=2`MSN~jDn~P)l2#AD`53^&Mr?sDj$gSUrxk396o@&nrp(e z`_E9scmtDmWjvUey!j3w%CNDbrFR;35bwF8GXuoa&M8$*0@7xNSDXs2h%G7gT{?#~ z=kXSa215g1YvPC@PdS3g6Y+kxdw?ZU(Kwf=Dbw%TM^etN7f-;T-b;t1{?tSw}w?e;Jn zLj&p2OIg#w;>3>wW2z6ReZ5U_my0@7xl>OAAZI(h4s%C*`*s=qj^94!lNN!EDz+$J zqQlJDr?|A}sRP;x)6?@Ii>vlzF=rW?>{oor*WIU|GkT}e-+FsBo9lc4JEG90(0ipv z4elImbvD~9_Qa>eR7E)8OXID-Jv8W(qazwiVwj0)52TE;+4!uvA^MoWVAa7Z(xLZr zOK)0++@R%4I%txhSwZ*T>a<5(mGGK$W#Sf~ut;Nh)l5-gg!rGiY$LW^yJ8z(6R$Ud zmg7-7MT~?ZBtp34QlV__3+0}gJIggb!FZHC^~_7-FTP#_x6)JFD^k71cK&4fZp*9V zaC1qw^uJnS=h)8D4vp>N0yW!$6hrk;Nn-s;2xhPOc;UX6cfTK6!9SBV6R_w$!M<<) zw6Gg~fqgh*xK)IbM z*Ke1zNtCTqFDY#NYc{PSp-~o1h!TX1u?BKzi2qUTdt43t)owBrX&v4MA@@}gw(DEL3&>yhi(`3!06Ggk{h)Jx%0_czt z+n+pN(IP}DK0vNqrf}dD-OzPfY;q1u<<~l-FUvHK16YNB)9DG=Jz+kSb6u_;6Ek=| zD1WK*?l(>2+NpmTEalh1M)%}kU-LGAh|BsjZnmu}Kas1*W$(=~V8$u$46ZCWSLHgQ zPVXP^k@PRwr=?c@>D=r$b!$Q~FT`mawCzHG!reIZmo^G0l`#0jSHkei2@ zmW-dq1()ulO^Aw^ui6Imj!3Be?glGduer|dEQg+^cOhO_+>&Olofac8I!A00#gG^d`yJbwTZQ-I7Jp?hY zYyPv<&6iz$;cBl|3$|Ys9p1P{RAF!@F`>sFiJFoUj~=GepZ+^ZYyQ21&@#dXM=$Dx ziIKW%h=;C`9zW_oWw>~h&vw!#;QN_xD%CNF+6MoN{@=ChojIJ>N@$Nb)#QL5r0bH> zB^AhfKAQU{5kz`>d%F9aHkZTw`j)TYF-8ysP3tpegyNSkjDCv!%~K!Bi=E)5Jd~ z`!%B$S*Kdk57`1@xA*qmR~C>mtZMuSt&4g(US zfLa$uU1Z{o`GCWF-f(i+Jl}&%+xRoQ2cM%adDh?r<9fRblaq|Dvayj0%Q+>E`<}8o zo0ynjAJ5a^V7Iq%5!o14K-OKpg8S!W=R%bLGczL@5w>j(4h~|<_k(R0s$OBKom)jH zlo^6@s-u@{XCCfDZ-?T4C=$`WQUllQW(nk<> z;7wj-x@UhSxA-?0m+*c}{9EGuE$#17`rd$$KzS`cdwk zT<08RF6{^4P7@PlMpFDkrH$9C{eCwyUA=t+*Rl@dR9d`lyiJ+TkhrVE zZl{q2X@_AA<@-492jJ&FEgcc^Jng*cfV6obI5`7ckYBo3bgZ@5leHn^J;} z3mdmxS9ot1TbD?9s6e6ki*>x$EZ&{l7{PHX?`E#D5D2u$58WxN~JfB<}a zI?}ZetY_eu=HSK12*VdOMnY&_B@~vx?4(kbiu6h&7^a+W?Mg}?nc`))3MtBk^>u$- zq~MZ)`nl_QZl9qrv`YKr>YRD8a!;6ZPxbxLb*`(XMj=lm8!eYxDB6V~1!|z1JI&6? zemiO?70~;zbj-b;T=*^YOYe0ct!h`6gnA*))OqSj60wxYtO(yaE6{tK+gaBvtc2nI z%gxi8MDPIsdlihuEUm&QI3M3|ab42kcb`;6pQtLu4pz`R&lO&XI$6M`1_swXo3iM{ zRk>#sw|BQmND+?Fp0Pr*JTkN%b1THrKU=^#vBXRT9usLoyyteMlJMVCqOeoD9L%kq z$nS|w0hKn4wXzmWi(#>B+!_gj{Uz2Nc|_h|o@*9q+hWbc`s-l(&6s5x{8lb1!@a07 zV|PRv)uhripDt?1Z#%k_!qmYqta-_4!55)F(5{eYvzrd{>RlNyTw13 z((oM>pn-U2NElwzU}}!3Jq`|z;R1^gN-$rFhm4I7k0hkGM+Hzmcih2eA2>EkH~_um|!;`6;+CZ<@8{&%Fqa zOLWav8`0J4zCPFAtH($`mJ3R#SArf*&!K6_g(}9DU2REDA zPpk|dI?YJP{Id~V7rIv){)tSuNAhg_66Q)?^MvGf6ky$H#&mG_ijil)9X}o>xTGl* z@}A^nRxT~Mv8a?PQC}&id^FX|B<^Z*QEL=H`HjHEq+f0^Mp=Jah3~H0c{~Y#B~V_T zjrT2w8XNSqWV(8K>#DvOT!@HQS2LD)M%Qt94Oq!b|0USPt$V?EdcT7>=I=b;xq*8h zlIKhMsmK3{FX27QL)GQ4&wihI9FgtOsC}T|eyDbgFJGMgc(gTGokU`=#)r$&x!((J zVz2`enu{o2pXu}ZNf&%9>*B(NN0}Wl=2}U`N=Bn!GYE%6f~RJV0TtYH75F_B^c*iQ={P4EFdOXt7etR8Vb(ZovJ{WGaP2n5>EyqKT4eTlib>0c zW)aQ=M9c;GJM$Q<#)C(KR1bDqvALt%Jhsv+l%ebPP8Z!VASJk$ToNion%QYLI%A+$ zn8%5mbTz~IJxEK!`SxO0tiJU)=DMM^vOkUvtL_aZqp2jcZ)$7xHB04psPJ-J{&H1Q zpzfIa zNBKOlmy?VU%F*X-mhIhivcH5WVSQiyPaQH zD2%qpXEXA!+?8_E%{!}($C9M&lR!z)Y|N#gyiK8oxXScv9Lmt2=m(;)@B0*c61i$5 zao^nq5>|UNS!bn=Nbi}y%HY!&V~({en`zcEO%ah&knl8@)eTAx%rcX+P<{~|Csf?O ztCp-5%RG1K;>+dH=E@F&pMwSSd`kw9j+*>q zv<@OE9M-SU?`|UJLC%ZWVsxT>sgT>Q4Ic!A!|1KKP@1`LezQwQ6rhT__ zwHl@N7p9#w>puzPSM{$`kLkfvE&+qe`iQazCD)2#2oVx%>9NtWGGGjlNRm|A#Hh_H zr;VK$t>=Naaaslr^KqCyTOz!^Vg|!mgP=}ttalhSJld}^?KQ7GleFU;ieWuRiU^~9 ziG(2cZ{xNnNuDzMaBk{&<;%LQHJXL1Q{Rh0_F=_u zu-=2UFh5@Xv}1)Gj@{6K6uX+@`pkhD7d3sa`cM#U6b6Hh!y*jSg%%Vp6O1<+``M{R zNea}M)W(ZkTT9P^x%CV2gl{Va<+)Bts96}>0*Dld=OV{sOS_;^bds#D9*LL3 zSw~n{$8lwPBf>~+JT~82Y!5c7Hf&Re5|gxxO=fbPiA?3NdBDE~fJuw_O^t}7d~HmRhi{cQhV0Qf5sK-dQ@Zyr)5o)ZLXxh z2U$18MaVml_V|A!rIWh$DZPMYg|Cs|^p!tW{I(rVF;jv8ih~MHJFmwT82igNt=PY* zFmO(nsWzyAF(RR)4-4gshbg71=-~Gtp`VgSTvd(f;sZDXpmK~>hc;O>_k#c)uq(OGugeltSJk|DeSr0BZw-c)T=OUeax22u&YPyU zs$gIejdpU)R+l%2`_wlC&Y_E=)xUe)$~E{<0e13+h*>Uo&J{DsCGt3)y3|$piLRx3 z538)~ti*-h2I=v&Q~u&-3pHG8Lr^BEXChv|aP9zznc5M-@^U56_<;ZlQC!EqS&y)* zKt*i9vlA<2sA#U3lqMsKz9>$Gl*O_5UeuWbhty55an$*#mV zPK-z<_<571;yzu1y&>5aJ>PPuOC!6?6}KH4gGeN>njbE=|1Ls0qev{{cR5plihfko z@!Q)>c63Uudouvilq<}xcZGSWbkgyrD7QmlLo6>Fp%_J9nq|2!bH!+x;%w~)inf!J zf|J7Tmen7m)pNVX$|oOc^i=cP{NEnDw{oV-ff7@l zHU#AOL~HEX`hwgu4acG`Ph{y5s_vC?9_hXB9=}Vz+$yf+TEGUqcNg2n+OK9mtfp7J zODq;9iwIN=HEG{QP4x;A+}oUUE|fhT*-j2SfB-dQmd1g@!|Vs*(Y>$XhYC$_w*HBj z%H=hmeCJJ)U-mUu%`fbBZO-3kRJXD}S#IR+vj+u}H|L+Kq4Lq)+xD9`ipFBw1va~+ z3DqlalKpj0c!8U}%^X)j!RplnthR!s0;|)!U7=#`<0ArS#UU(C*s6jZ6nyU{vS__} zxlOHvsyFR)A#}D@4p?5e-a>n)<>VDE3H(HG8IN!!H~kS+>w;B{) zxv4{q6;mM8$k^q*H=nP<2L&BUKIw~Q9z~|-(@~J1@%i3)eZE#Nh zjwlY+TFgbyd6Kp`qg23j#>8U}+gvej%Jdnlf!+R(>&2jS}$Z8mOgOIdWN9#}1+g968*G zXeE|e3(C%XrC!1)H#$%uus5C*tuyV->r1L8wL9C^a_odM0gfK8m<~B12s^#Z!KsL$ zEjq1B1(Ce;zs%TtpTmtxM>~@f$v(t$q{2WZfw_E#bpDyK(tfeyG5?Oi^t<2g3*$Ws zaa|XEx{tih7Kch3h)3)Q40518Adl$LW#RRiFBhvN&}W*PikZN=1nslASH-aK+hZ5G zpqg1fz#yif7dMEyIbZbM24N`m@rM&vzua%e$KAxnuNEKC($YUrMe4{4JK%zMll4(J zV_vOc&<+{u2Kt)?2b(7TjfJBkB$lxtiRo%uwX1I|*3Et8ZpU(G;nJAP!b1>_ZH}e1 z0`La!$;`>big92Xmw5PUZ_L5rqQu%b9aeDJ{B$vk_$0+`UxhN*Rm?V}Y-0Mu81$;W zMO)8O#P77A`^K!Lar`|Uo!F%H*PG!;FLymWzlb?vL%9+fU5F|bZ=mgD#V0B14zJ+N zu=CIgw>JY`S9O|A*d$&bY8{|}aAz=Rd;FIPbrAHilaP8p^ z92*`1(T-^;&q0Ce8~0fDpBck{4llzU8w45h^2|ZM_I#7;QgbiWcI+LMK~lZAQxg*u zn)fG~$}5W*C@^O=;9V>S$9UkcoWU|t_!*}Fc6crhf&`$~fDBPVQc@4v;XeQpuLWCv zD7+SLxj&K>-XqBH7Sz;KqZQs~75?*)Y;tQ8{=BG%>=Dh^*HSHLilR&rN^o^eC@_BL zUS0DC7TJre9+;J#*2P2V-G389zU$OS|8b`LKSI&!WOM^lcX+et{cBrcmmMT)){*hO zZbYaBh={TVc9zB_X)WpnyS+TWzarXX3XZ9V^vUrH-%6e9nHH*T48KkQ$5qj@s~$P? zW`aKjB|s!CB$DTU(6LO!KM) z*OJ@f!T0xXJJs*wd77pT>A`1rtIX?%&izKjn_@L%Pdo0HzXzCQ`385^;fNk@N4>@^ z6#;57()1E?g;pd7)Foh9bed69rpb>Azp0G2s zPTRF(%d+j(GlH$ql)+lAtSm*77x!atx_;84KdHjLCw4CQ<0>KW?EQxcBKH~PwiP^% zvBSB&P2~s=Io5M zI&gG7eC+oiiqWZ^J?c~mi`y4lfcQ$Jk4@|oRZV&sGFMg#o zczXH>wpw<_Y!sY*tn92a*49Ldt6yF*%!nTmAZvki1S88_#-{da!&*7)B@Uxeb(f18 zwN>L!LH*$?qMYfNoBXiy5t7E`^ZKviwy#k_op14}$g2Ej67jGj+-9CP>>ThN)*U>? zqcnE9`&NH$@`GROFxL$7RIwa7dL~Ae_FJk%w#BBGew^{%WId`VpxL4)vIu`iu~q3U zos{qTmc7RRmo2RP#s$<|p$v;l{SUC0z!4BSFU!Z@x&Ti1+9J~O`0Z#J^07g@8IgrW z^5J2*<}Y?YXJ%b!jZMsBt)}Ne39Gg=`tiFXmQngTZ@Ba8v%-hvHoW_Dvoh;=I~WZY zvfWY`=Onv$bsN-|OekAk-kz(&m#4upa0-&`^OR+>jgQc_blx81LanpwNO8q-|_9|4H&hVE+^#NGm+Lp3@m-wb0cv zAmB*|!Y}@0g)P*po#$*=12Z)E?fQ&w1*hboW}1d7m0JqpY%C%3MWZz~{!rxZg?^N0 z(Y)B<)z4Ug_sPv~cIT6>$BnEM+DXv}Hg@m;DItQyhF@5dh#RZ-M=qd05Yg18WN;Og z1GeB^*v*KeoARzfkQ;05JCXkX%0 zFy32YjlA62T)WSm7dia{jR$4mjlxFDFP3o@O~N5qUh;~UbdFuzX=lXuo%ZLZpy&C{ z=H8G};#;kcg=ge!iAias-mjcRwecee4aer%px5UU_IXlrP^I{Rli z#_aUNz*s&;Ox`(?lb(+A_0TR`#es#U?)rcp5?9z@KN7t2=_k=0GwyKDY8wvj+nG6` zdMJy)CX{z8q5^{#{Yhr%HrW}5*KVCJ;%1pC&mSwx`=gTPz5fsZtj5f_dZD~(ab7fcU3Q>*oTRz7jr$?L*TvDG z`-q;tLzZskJkl~T28 z6uA1h>i3@?b0~f|g|qZQPR;X)?+Xc9MR<(aXrb?Oz`A-1<2zu}oC$ zqVG2gDJG!{!|T8Mfg5wMR6F|Ifriovtunpdp!MWk5UQC98wvF{F*zRVr5QN3z~;Q& z6^kmR5}l+iV|8uRaNzND0!+fCSj6Mf9Le0qwsl9nOeQIV3tPcMI#(1jr zzI|bhVi?v;0_vgN8yU-^q(o2>=7y{DHSeS=nU7dsiz zbNLl+3_HgfH!jP-|4s;KzQWjZ*hSfFCWeF9LMVCm&XBme-0E}Mj9@W5~v>Neroc)Ikh=mzvxCxZmp#joOl8|ZZUki zgxMmS>GV;BnT9lHx7kVXavBk_D!0h8Q#IIOqT(=PgQ^Tvqr75Z5DX+87K|9o7K=AN z_q#f+^GMo=3OB!=zjw`?Ha90W;tiAga2m?Y*qCwE+SXd2ino(3bs>hsrJPQu6^R=e z1`9p+ld=BsT~?eWKfu|^R3~m@YPjX4t)q)lg`W)jJM3p!{j$2=1=@zpen&JPyWfl@ z{p@7zQq;Gc%|!_ik6~A;mgZs7=@p)9ePQIHTACiaIbV@>73b=s4eL#phsR8UvV(6t z+FGe+_8UqHtxcr!xjJBrCOn)N88(LRGG53|S4PoW>Bm}(_yZLKRS+yY6zX1aU@j_qZ zy62c2D|yS4p}b6*IpKfEK{-x}^cM~$nzIy24>N4wZM%SZoha+JWY^qc6-W~_+4*=I zU%Rp^1`V=qvt_>4JNTBFgLLXf;>3m9uO4A%7p*gk6X*4_INWR{zFXIhU+r|%zC`$? zI*5mrlGr9{R=tZ_MK0>1*k&6iqLl8eBx4{nNfVQk$7p;z$W{#h*Ct;?)JPuLG3OT|jnxGQ^OJHH~p)!jL8e%F)=)E56Q z0L>1eav=+JfRi+W0F{&lNUKhMs!hP&b#mwdO$rCA-}U;n;$Ef(QtbN_ZrQmW#xH0N z%Nuh`1=i){(rB$O-aV_x-Q|-#;<;pDq~ZB()Dsl7_()PkZYTlR+DbDM7GFg;NnpDR z9?C%K#Z74O-zGPU;SOZdoqfZKBp0p6AuZZDS*#;#W>#wF>#ps|coQnr9WYtJ&@mlA zer4jU(-S0egV22sd%=MP+JA5|6<4scflEL?t4-=L8&i{Xv)2LT33lv-%^RbIZ>=-= zotn(LYKK+*Ek}g**m;}?As_N6)ypVb_DM|iH^ecK6c*sw9xJN4=aNT~Xp@}Ip)Zjz zhNGee&1XkxbF8cV1E-sWg3-LfbCDJuH#*%FCvNk_hByx_s#ZJVp_?UHg@`lUHDk}G zGbZ9u$+h)!`zJ(+N_4XdX!^C5_uep>3iUU|?eUL5GZ?X>TBu`F0mS!*@M+y=aNv$n zUsVE<@8#Eie)66j!KS5ebhz1Beze^%3x_Sfh+cl*|0RRXUOA;vx86wEaB!@Sm)XL% zfZT{fSKH#DaAp4z6hxHxaz5Oo7?`ZrtmnkDtX~kaAiEANE{FC~l03O`JUA2we#gu1 zDqAkxJjg0+y5|ZJb9YuS({U;1YA7(BRZH4>$4vWm*K{WQ1LzhOwzf)IMqNeEJ9b-VwOQ2@$td6YdSL{6MPmn>>t*zkP+#{`)db$GiMviHedpoui*o)% zdJu!mkvECSu{Fq;QD^R)tz^(tsB-pbY4bX~F5}BS3>anucIJllb#IeypPLv)d3#bA zrx-QCcj7~9`$jAzY1%6Sa?{0bL&Vp{*Mx9ea;@n25a}e?P&@ISVsUwB8e^(jkEsoj` zOcG)0yskYc(T}JBxyB{OUfm&jo3a^(x^V=SKGnhyi2$TRQFUUhmhF+}Ky2X0tH|SZ z4uP4*XxH+SwBPQQAviembi9s#ct7jeG@k9s%R5-aYzW-@&?J-mZ^wY22I+-W`O$?D z?)?+qDtCvNFGxo3h-@=)WwH|z6EvGqADEKrfBkFzTdc(FtK}$55Uq9r)UN8Z`to#V zN{kGC=SPorc{n*a5dcwe?g5-(K#>@i`#s2&_u0-{#gQ=+2ps#ev)LW*?UmYib$9ij z8ON>D9dTym=Jw%%2CX28s6xrX&q5X_pf>GlSH+<5Ixaw^oekkb!P&H~4mstEbR8AS z7g;)faM~wn0gn*`YBw=6;%#+*I_vjn{;lwUo3Y;EW^lC6(PlTB`0iPcj0hfBo~d^% z&l`A?6AbRLno>4;HuF>8w4+Ye`|JLjDA_%P_wNat|MhJwFgqP#J0s`y=JmDz?eBxSrInTxJduWr$s9sv*%CX?EK-7(fZZawExOd9k zO);pjrSIAi+jZaAW%nXZz4QK&KLDZ0H#`IS>-lfp{Xfd&&rk!DpZwi3(}>mU@DE-iy}Ar$w9H#K;72x!{X0{wx92N}#Su2Ik$B^GQWlMdx~bN!(FzR1nNFVK=0Tr8lF>j-r4^Cnt%%%`OQ~W+EsJy`WqSJ6XX<4Us`Bd z7D_!7b|dk9I1fpvlEO2FC={BPyxn-B{e0muHe{er^4(2L-JrZ(+U0z7&wJlqjq8d} zT$8bAWycp{gD84Xz~6UO?ZI!i#hJl2Bi|h>gga-jfWUDP(MdmnA(-zB;c0cfP3Td`}70AI^HcqxF z&4{q6*#V)|I2v6I|4BzIbr>F=Ioj)8w^s{NrA1C5k(O5-Zzgxo$ji350=Rdr6P`px z2%0X*Qwh~^4B6kt^*FEGIt;xXZn=hxxu@l?_lEMI!=@Q+era`F0v0nj8%nT)c1e&) zD9(?iO_2Z+8cakAfFDcqKD-6I@~lC%!&ia5ninV0m1>+W$GBV+Hdb_+1%vs?6LfaH zmrNv(L*=pG&n;iS-CU`4-P4}yqqcp|b_5?Ai+&ZksBTrxfQ5_0ytr*|$y-PRUL3(z zm3@ZO|5e*}M>XAbUHUwVC&Bh&Cbnk`^_UH`YTxYN9xa?2ON0nZ+{0_?*G8j zx=`2m-dPNyR#TdQIw6&CBUp1~8iE0d;v{$9{Jwt>0oy`!?`x$-^AXPf6yCZrhL>w2vt_XPH# zr8R?eW7T1`^D=GE*JAHI)miRsMpOKg0HvGLp~bOH^EVW(dmXjnc)v)6tGJpyzeNmReUyMPzYXTL8@0dv7ab zZdpH>N|?Lc^cY-rs?s3Hr<9SRYkHhK>sbtqICf3!Bs{zU=&M5Rg_F`IWSCxXVB@c@ zUhAJ4G^J1X!Ym{6ug{nFn6_c!k^m#yduBMFTqZ=#r^Nt7rGrR(?Q^b{`?!EYhEB2BtfQP_!U3jJm4k!mNNFMSY``}4vR zikh;BG<9mN!&6(iqRaf7Q);wK_H}&8c(1ZtAv%(bkh^QTptn>%LTBskjJECbniWq9 zVIh^BN!N`k!}zWFf>cN~3gCwh4b{`G7PjBv0oKiAkTR-rvz_Yu@_8&=>_tqT<11QJ z>L|F>4eYRZEHKyyoY)EazT2}iXs9xp>#()(l8B_7nXhV{@?TXC6vbcOuhnVRr>2b^Iz#HjWhL5a=0fEy}F| znVw17VrTnDkKOAEvoG4|m9l)y%dmnM#D? zKgD)4Ji;wwCEZn1_t4>DYg{`PCOZO;TA(bG@=a}b(JI?V4b z1?sRJ%Jm?@=Ug)km_B2WcbZ6FNK|)9d{RdDX&79@Lw5!m__7@r(sj;b=%(Yoy+YJq z-qPLT7--GOS#@|o^`z#7LE~EGH;<6f z*gP^WG40ircaaJA<8clb#=(naG6c*~Axvf)M?6af7gLGeP_ZN-e)&7WS#U*3 z0DA*1$xEXnr>K?|hwQB+d&99}qdmZ@u8^wpxeqi)Y=OQd>TGWf8?zoFmMgoftpE0&H9MeoDpg#0q1243>iy6j5Y%2UI1 z;@8#^aP?Y)HfQ%17D$G)3t5C{C^OvJdZ)$S2pue!Ib@Cyw%2&y7k7g`qz~n)uynV# z5BIP0wQVTlEb?#`_sF^-)z9WGQ@9a1)zd>A-yDx+NZq`tdB2~*FCZN}DJF+8`hQ3g ziqq>fII}hMOS0M*^kyq)3*Vpa6TD23%)x;F(IaweJD0gR-?0VOmUC)+RVK=h=~sS0 zG_A7xx#T=d^n(-a=rFV_R6$!)&o%8g{`6C51XEm*QD?;wm34_hi8}O~w(C_Gi8VY^ zm$`98ZPV(aN(#~|+`6mvdoh;@Y3ro)o2lO^Bdk%k7`!I2{ZVk+$`gJL|m$Q@C{@jgxfPybwy_+HtH z#pO_Z8Pr(mqG#|8#dp{-F_6A70|cRD)f|x_FP;@JkEAMIW=>EOg>HBSj5S++wUt<0 z>3)U8@#8oTQ1`mL^mKRUgA-F+6*^mVtaYFJy}i(x2QrWwmK3?RnR~_$XM0 z#Gfliv0*1SPj_7Cm0ko{-VN{j`1v#YjDc0p8_53P9!8CnHPd#80x>e#Jj75$<1w%* zd-#z@TfeSOL672VO;=bvlapQCZsO2o8ExsWt+mbHR-s>^xfqK6yU2xy%DG#<9Tyhz zuC5!O?Rv)e;kZhKrezw5iW*m2A0h)INI8%J*~bH5;VcU~rD`l*=iS#jjW057t_Ttb zTH-@*=v@?=@a+jy;xN{J4f7^3{vNSi)ND?tfi0xV7v!4Xq`q?xl~i;I# z&z4J8Bmx+!;Ii9zRFm%dwN7DwwSaFOJf~Q{a@S$>M^lsbHBRqv=V++fqp5?)Tawt6C1Ulo3T+i5^kf#dzj$)>o|1?%Czl%iOrcYwCi$|0W!2ev}M<&to~yIY>;EQS+MCZilLB0~y!b0R%zDR~g-m^vq~ zQ_jh!%;n+HmbAZskIi%0jK!oB3YB%^&~mM-r63o8xNk&!nBkAjb#<+Fq%LD^iBA&> z@Bn(|I%y|dd(S?Nzjvu*`7>AI0o!RqH#*D} zzc$y;KQj}nV0q^%*YvyR%p1w`&pV1cN;5Hg`a@>$VWuwau%D|kXgt1i>>KgID+8!V z_(&9P12@}fIsDkq_B63>>unc2#PpV8V#Z*-A8$)=vN^Teb$xegN@8$!*AA4wspCEs zou->agoK~}mEB5n z5qf?RnfXQuhaO-(I}plBaJ8k6=mV?_pBwI;Vg+18=v$USd@3#ru2KQ>h=E*Fu~jka zhg}Hgs^)4_eZ7NbuGh6@h{ENX5nCMTL6thkC#<_{kfl+Oqo?{s&)op5 zfFSvzooRg#fAryC4Yo?N`-UN~>KD^Ic*4^tbIT}FW9?g)4()%(2QW|bIi^#iqAC%38ZLM^D~-0!@Q6b&ij6PknXwt@GW=9Lv* zRyQA)6^9V=>6gm(mTf;hS*!kwG8`5wpKHT4C3JKaJGT?s)H`?D+5ncv_sa8ENWPlW zuT855GJMZG@v5k+KRKuv#M?ahx^FqW=HAP_F0>)E^44C9(gV}I({;hqzKt+30DJ7_ zDfYk>y0`U2L-*Zd<_+R}6&=^V1(?rp&Tq|H^g%psw?4AG5#p*3Qb_KyNBYz~74~V5ETJbs!w;2t5zrPL=^5L=G%mTyZx&o>WNg}&$LQX~o zy|6P2r>Z*LT`}rW)`Xq4yQYxHWXhcaSO2dGiLPH;j}clsG#gH2ngV_c0zM|?)8T~4 zTgpuXW%2YSs1!8f`+^OR^r^I1%DL}cl^*r1%qg*3?DkczzNd3q-JuvCXlZGi0gNOg zW&}Le8!m{9JiDRsxrP|g7SFo@=uD=qqx9mKHs2YlwN(Go6YW0XG*@~az~d3__HvQS zzhHwWCwHM}KSKqWl;eMvV(bW(fsDd%B7iC=hJ?h9Tw`Ai{}7C3S# zdne^v=vapEYz=J5RE%KkM0#F(?d;>xR#mv51r=r`zRFhqTSVwRWa*NXY4DRq1>B&| zx0&x!jylN1v6!G#_t;pZH1%eGKG5SPKXmP{warbAPTV@vt0fMj2{YZsTMUhTYri6P z&YMd4>4YUIO za2dec)ZRj*RL0DS_a7a)-m6E}UG1Pm1$!skkW0JDqclg##@77igUvQ!mV=VtWvn~z z$vajLe*bWXlYlky7M#o~eer+Rh!HK2_@41!f_b=!;i=IRsHkQtRLj;DHhTvXbtpZb{i#WCems>C?`+#>}Q3JpT~NOIDXnV2JGc-+A4qRrr0 zGn)-+Q@x`ps@xkh#*i@7`l+^dfWEg)REi} zy++GPxjfQDwHJ5UI7o=3*Sua?9V;hj(&iu09KEf9e-*$0@J+S@)j{p_w56aMO|s?O zBek>0($GmU@4x--y*#a6i?;*Yc;9;|AfWCl)kq4Kc>@*sQ6VB2UElIfxEBJOu?WH616okQhvWN-H%^VOlXRKJh5nxDCYm3xv{;UWM~<3ZckD

>AN2gpk z1u^N$p5{&9(TUugkx`=cOd{-EkNZVqyS_O%?avm(s>axhU0Q)(9Qjo5GP*j;nS`KJ z^Ii33oi!#UMcy<>{vjosy|rdA+ao7MzIn}-k=}Ux-e%xi7#D}QmnvcL5h2lQ{M2_m z?x0DGa#z6k0QKw_fUcx$Ir7S+A02d7XEOU6^m)v^iybR-RfqXFLO9b0UZnh zaLv`MjD9!gXlr_CK29S%neuE_26;QUpDK z+1`!=|A~KZj7GHyBSsDTL`k{b-lc*>yvA&IHDC8jV6S@nkJ;X;AQ^i4SN{k{|L0Wl zTa}6#!K|+C1NiNFu}J=lJneMr+Ogy8jQPAj+~2RnuWpM#uTy@fD$J1k&2Jz3K9taZ z)WnyU{~xQo|AV#0?YRGo1s@jxapv|^RY6SU9CEZk+s;XGxH}X3>`3t`F@FkrGw{_7 zgKx$z$Z1Hq;R@DltfTOhU;O^H@r9r2g(cNP?fs4+R_OvJsIcTGz&h9Q4$1wZI604j z0ybTk4C&=s<1kgipx9#^;j2-E=&Gl8&&+DkLp0wogBE$F)&j|K9Z4gXxx2;OC}Hj; zm-+B&jIO@v+NE35DM=q!{pfSx+nnYxScO#gp zrF6-^Gj0R^v!TV$BD*RxGbpCkj5S9u+xhNyVGCZh`nrQ5gahVn9z6g#YOx5?lm9Lu z_k(nkr>icg%I-=XSZ6=blnZQ-2n_IWCF(hxXilt#{rc;N_&5cxU5U0jLG~N^c@6t+ zi$iW({vxkB40NzV4J00O!ShYN6%f4N5+AY&D+alKoqqw7FO!p|4uG%=>_Kk!AO01o zvcAov#vd}go%K!FP*7iEjD3SeNX1TZgN#296hq9#Pm+9?>wMQu$ck4@zYcAhUFt}? zm6>deKQ~I!KA3^(+j!BuIQS`v-&kDKs+Hs9J_XcW%8`JK-@YfzRgWC8I&t2&{S=r9 z95CG#m~VG&A=O`;Y?>hh=4KBFc?dZPnI4{uXiH=F>aa;l_3)bR-H7&?9fsa+1@6c! z7=w?Hn%k#+pOI-6-a^ZriX0b{X8 zIKOsjh}`)0_2*~o?COQ2YufovoKTBtcRUoxQ!CYagWn{r?!)ml(O%vzBZpwm=Vblf zP;OdVwk={}${9uEmtbu%K6kj#{WF;$c7*%+#iYJql(uW+x4%4Zy6k*|dUi5R5_nv` zh=fhfBNUE){w##YVUPV@hW8+eA)CH{K3B~9Uh(eXoZc0t4<=wGUcl|5_bn8)-;jB@ z$-1AAtKiS-5XN|hUveYqUgcP!1uJ669Qzv+z4%RvxCL0wMJ&=cFYHf3*c_N~XM3Bk zQcciR>UnDb(v!{J4^R!|NT2>eruy+?DFE%j>GOph1#PGO?{wo|{pSy#%;A%qfajHq zoZfDtuE#TpXZJJ8*?TVJSWj-xgH+KWn@_NH2)SS$?Df|B8vn?;$l8^(t4oiE_etDj z6btRHgG6cXBp9h&d`6nMepp*Ggxw#^Nd$<%;^yf1JONdor7l6sjRNYcK(DC!&#l>^ z;pwAl8C29)7yF<02F5*aPF$7{g1z!%`rYSElM5%;!ajfD!yTL0IFGwLg1#QadhPT! z&06g~KaR-I)BUb5zdZ=arMdM~_^kZ^5nhB*AzlSl%eWRyV?R63*0dQd%QDx!kZubc`&9;%= z{3rf&5hqUuOs%-nD^7G<^&?QOyqO2X{C`XhPVxL)Kr_5&-g>2WLiAEMPmcfvmv1_9 zf*df*yL<5EYp6m(qNB*;F)ZxR?*rdNJQSNsH;0C|k~I%#X1-@~aFna&4y|F1?PztEdIFnHB;Eo;+U zdcYR^Q^rx#2UDM}GGZz`3x%(_*bP4pE`8o+@U$;>9}rR#O4oTsJIW#rKPG0E!4`Jx zYRq;dJjuQPkS8XE{al#FFq!nz2Cv!V=T@1ac6kjmePqx1-?}9iU4*}=MBT5i!~8h> zBtC2K0IR|o1|*q^hhbnjHJ|E@oQ08dK-`yQ8{O`1mZu1Jk7E`7B*!I%0oq^mW={Tp zQ&6m}zfvK&vXgs0loXLY2pE-lBR9xzyO!dMJzqGL3&B}`r>>+nk6!aIaK<7ry_i^A=Ub*#KX|6 z#_5um%Z?3+HgW6utY@Q8->dn4F?add>Xa;(rn<{H{qpBZN2PobaMtaPOPVKr+IK6Kx3VZtob6CB~{sliV-$$ z?KbAyd?-p|D+}U-=nm{BKIdy+O>d;p%eY$G_b3ww!~&ajmLzmIYcBQb=8r%(n*w;} zi0_ET#=olyI-!p;AJ3f`b zl9)nQ#R;|km#}x-KT@>y|KX&U|I`2Sziemr`$PYo6m5Ld-$g(GiZOR%dU*RkT0d@m zNMg)>^L$*()pd&-a{djKXBHT%ET+mu|L1M`MQM-ehjyXsJ9S9ZQ{1S{kjWOmH)a5Lk@qyFr=o; ze{R^-E_g;?{vWCFoHMa^B$P;+qdi2WCchJIk+ACN=e@SlY{y8Gm}i_mNq8+}u{E-uj2xIVcWSIU92fGqYOIA7Ir z+f(gNXwYpxyuI7Y8)S%Kbzcm9QvpqPs`0H2v2|-4fu-pQRS`~&BgIh}^Sai0OazEo z(Uo-AQ8M`&;{)%4%^3O6?gUqT3KOXIn>1*5LWH5TX9``KR1;I^dyH9Sf4!d7_TNkc z^c$SaA%yM{Sgy@JyV58sqw#8`65h#tPc4q~x1;xKRy`5G2lK^~b5}@B)YP4<{@BPw z*<&@O_m;y;{k#`XMoZ=x?y~z)I#~_@EQUSpU#d}5eyOQ-7yPH&I=S)x0#m40cpg`N zf)@pvvgci_eh=OYP-WF_kZ{Bc_MBwi`F4q!d>hE;q_J7kNdTg2xWuC>Di#6Vn+fbM zb^TfLPMb$Lc`K=2lV&Eb;lXfdhss9BxKmXU{_ZQv9$*cP@0E9*?n{m ztp4oX)gvfg>ZYrcw?AskOrzMjioGu|kkc!`;GC3&LS);`>HQ4z=_lQXM_iSngYFMLVboJQ?73ImXt9+}f01}9F-!moT7j?&3ltcC_e^`eT?nbd7Zb>8bW zvm)VrkiVnp-pJ7pSf_%^_S#N!&>_CZiMlyEHeFu=cq$b}o(6-&nZ+4mm4eTy;MyI5 zHCpP$9=)`qF1BABIbaV02|1h0$nJmGMCmk7aLwpFvPiEqyNkPexnW{jUqK3H6B=1l za80l6CIdYy>rSyv61pPJ{q@eGcfyM%m%Xy4htLVmod1h|gvp`^>S8%^1I`9uY33RO z0bf7HPIn}ddAl1?i!exrwd?Z$eoe$r|`^G*W zg_pl0Px$()?HH$2&*WnY)-buA?dRZbF0*P`kda6G^Dqrxm*eC&cns>qJnbn~GN8e7 zBCg5&yysjq{u}b0Unr}rdOFCl{a?}sI;Qbv*~R(@%vW>kOx+8+dZ3s@I=rG~Gh{_w6%DSG9IL}vTRZ5C6|U(R!>Dq(M!lX$<&Ir(b1TxJoj~KVw#e7cpgEQCN)O!9 z(enK9-i+Z_@ozY|DE)$CRf`KNo5*yL)-tW64OgJDrjz|v0w#tnU5HZ=mIU1Oc-?^*ZzsUKFFN>Mw8(C_^59ceSuc z$gt}8#P`}JPEh090u6!DdtYKy;LGc0Nyj<7mv)&h0QH88lkLJsZ#)av7ZyKU?SwQz zpYp1wT*PJ&1nUBmsrX`?dmM{R^mN&*WTqXmK{3|$xw;uH&PjreapCgH*G2uMf6?G- zYaZ=HTSfwC``cCK+IH6W8X_5IFX}FKsZ*P)4x>r*Z89dPj>7bw0W>L$3w?Jcnq|lf zEMcgzDCwFd(O=OL`DQ4-nJcf3i6few%goKG;nJ&fU2J*hF*!ArWSBwkV8*gOMM?8B zaolwT+x#jq(Td3|S$21zJd|d7#9_5tdRaq@_c(}RSFc>jLV=%g3!#JAVAbe$)_&tw z#T}zI%YTK6BP5UOKKINF)gaQrM z%}0bfWdS3RQeOtRoxL_p=^(}S%Mt0676EVvJ>~c@ub!XL#TLz=p=aT~1VI3kn`&3b zZou}XuTK}`DZNYHfbzNG#d}ZQQoDDR_(^cKk9>|UweB!p60cxZ9D)5x;kvv0efTF@AwLtV^*W<3q@k>#y4ZoQsv&5r;zk15eHv8MIZKh0sH zsE{++=tq32ji-j*x0bxQFE(5Ok4<59aZpw$<8QbKaqL6nTR)7<17Dw6YMKbe+LRRNOwsvjbr*aAwYcJrKF^9o-qo3i!iJjb{4z)P_h-JCf9|C+IQ zH~)-WKiC5Rd)DC+f%bgt*xYuc$!;7Q?*A()(2-(S`C{CIf!FJ!ax__yi3v9k2SBth z!px8*w3Ww>T#Gisb`{jp(&@zi}E46ZsZIjtLVjw%~Mbuf7 zvUBHTVa%r1P6jKJrj&xz`rSCka6|;S-PhKQwcg3;8L;7Y> zixtJ`0&?|1{Ua}dpz~4X zBER7m{cP06lm2XpUR((>C0n3nWq}uuKK{GdP!mZG5}3c=Rjx_6JqVG&Fu}^VF5wCn z@{paW+f--Iwh>vuQBYk3gB+Cp!jWiSq?LGgs#1@ange1{T-uhi0(n9zlg1%f&zG50 zo>A2m)+QD2i=|GGr_GM^DdtM=r)s!%HGji7E!c`^^pH=}E>k(yY;G`@1RB_}PaYwj z6c4E3TwPUvfCPLh6=1aLn(zM`w$8?n~TaeGXzj!7!h@h9`V>@hx8=NXs=KCEThO=&`gmJugXaNYqeE zG5cC*T&fBuj(hYIu+m4un9ncR9Z22Mj9ZIk$=wt*X_Y*S*9Ot01mIVra= z4KctivL+;(jvUK{FOXh;u_%$<5Kv97$;SdcZK;Mn@}G@u;4(9ZT6d-Bepm(?6f@YM z%7p!eBy*eJml#S3l&NscnwSmQaOMwl>)7_V)+VGD%|1@YER;v}ano(zq(pOPRuVmXE`73d0;I!73ZAu!%dnTR942uR%*=@b*pijxnP9TX8 z%!}SOPh!Hx8x74L?9#ahzxF}6O>1wfI-eE3Q@iTh+DYadO*|Y@n6>%PTLVqi^DhKY zf-FKgm1&6ddI}afkCV%u-mife6m4?6J?-cM8t{_*DxJTR8??CM#~(9tMzck(8Trn? zhQ^lJs3pv{Mf``r>s6t+5mX9ApHsj3Gi#+eQwCkpe5A3>b6Ku;$CAy5n?B#VwLu~e z)G@be)KghxOLHJgf%`;)>fLbW0HCpD%y51sRYQAWv!dj4kU+7adG2r&?Rc3c?yr6J za3}VPJyPXn{4F+T-DM-;>5&tq2Ggl@z)$__VfQbc`Ns^Nn?5=zMJlqB(MS$%4_HFu zU+SljSMT6DQJre+W5(ae6>ne%|;k6neo+KSVkQgeB^5_-cr zaqLxFZMf@ZjHZA(Sg+ZxB8lC7tBC5fz!bLLgba@xxBU`Qqkj{aOb04{%;W$(8`Y7r z)PyJ^N&8dBJ_qA1wFvdR$8leZU-d{GqKtJ(@T?6Mkg+}J3>x#8xOqPI>^20D@djXu zlA$ggPCx`a6cCX~jH_>Rlv`>WlgBY4D%P{a6IVS!sm?`6q*tY2Lq}(OVAEu;W^41b zfwN7w%veXFjm1G;x6x;H$$IJCbC}SlkqwVH((TE-aG*u1pr&_6es;rfuk)vwSA26( zwm3sa6(MW9)hcG8dFh#Hah@9WVJ1l1(h;OG{4JvN_y_N5?#P$Ejx;F#0AE`UxdjM{7ILIjvoJ zc?#uvk5u*&uuW=_ifcWMWy{9>w^a>mh!weWd?XKC=~}m4vZiTmI+F84>Wg=9X3iqn zDhs$BHgI@F-y7{2WrD^s3=k+bSyM~Ul-20%AGb;BuS(`GfpasiZ?00>R9Op~aJ9}| z&Z)D63z!o}dq@{urY?WS1#$V@%O7khCW}Wh$HqY6isiQVwkzX0lH4)a!5{Q~*#3Kt2+%*feG<+#Z@&+vIRc+BWsw&Hy z^k&b4xvTcbBIj|@iFD_hnzll87-$k^p2`dD5h8avRZe-p5nqfNaQT9Yi{FzSGIO-{ zDkwpk#!ZETUMip4g0Fr6twoKT)c&2+pzR(z$RkTr21Km*6c~ zSBm#FM2)CYVb}BgcTB9_PnO>z6OeRzvw8A}l_PM7RoE|zaNYZK#l5Ls$mrnE-8{9T zIijy40hK-KxtpuXO{UP@PbFg~>0M0x6)aUf`;|HQRgHFGj~I;QG5dzIk1Zvk;<76s zA4)9AAHWHZ^o@?km%u8AdvpRGdRSnOX@Q_Lo_esPj zXy$#5-Ne+=JK#n_;@rf&8>%e=FLnPe@ESLAdEP%dv(9W&D$e@0l;)VL{qu3R<;UTb z%1SkkxYBNu8MDxr>b9`f=mah@JWbuJzVhnFd99p80iBWCb1WT~Xt>gZx^2-a7nAX1 zh!wQQb&zas!=RyO8*hZ_%f!q^6H#(%9IiyaN-^Db*U|Iz%0@;Os77DQ8YDX10ak46 zux!!wrF#Z=^`n9N7X@xA788OKrZCW7(8KtNqT=V*WeD4y$nDuSW}ts$%bUoG52!Rc#ahjO zvAnCa$VrN7LqigOB(Y)R9fsydfu}jG%p&r@2~-0ru68Pi%WkKXGcR5MnA`U0_^;wE z6Q+d1gK=!FvYFG;N$bW+BSL0D?Kqeuo-Awj{U4`Ee(VgpPTL?a>0t$Hl;eXOm<7UFA=;M1(;_xKS(Y;)mmX@3f@mw3PEk zYn@-Fc~y$m^?Xvl+n?SCQDxI7$vS7Os4n@aR&#%?cS1wbJWZ^>{-{r;qP-A0tpJ|$ zAZU+EGUx2G4xUhN=9dM3-fe2yjhCn!_w^C^-~lDIM=#apm%d!9V%Zhm-0kD!^)VP9 z)L$hdG~b#i`J0x*gfYHGpwia)5b4yDEWQXscthe1>~h)7GxGyBNnvm9U_U!SfI;AlnO4BTPbngFou` zTO*a7#TQGvCdb{|IJv$cP3&Z1{nB&F(9I0Z{AeK^SGF`TG6tj^QLEN!f_B8GhXagWn zq5gRPyb7^V3wFx40GT7-&QKuQ>E;syu>~6M@3|Cc<0Ga5)VQFzd=-mWlThDe`#2q= zmw&+1PSqwZ(WXB019FCxxO!2p(@CW$g6$Zn97$L`n9m*QN;Z{tUzLYzMJSj_f+Cr*fOOUWPcnzE=B zXlTpRGm+QS9t%5-)Z%nK-8Amm{)iXI2{%ice_9Im;g>D@JVKLrBKa%(+&|b*PCS8a z)VS}(Ats)#?JzR#YlKyv;s}i{H4`P)59Pq&ZG{9NB-kjqNX3xN(&Y;tYBUR0a}-QY zww7YK%+>ETuB28G9Nzl_{rs^+w5zr3JSL>-|Adrgs*CAuNEPX;5e~ABaq^-9v*tgN z4}YGHrURbh=0(&fbBm=7FpJa?zMYWfktet5Lf2Ro&$7$-D!nR-6a6#*8Rjo>?~(y7 zPSG(jhONr#+?Ab7OAy>85t;A7z%*j;+56;Ug(v7e_vz`HggI?v3xLnV+q> zf@T&s%5_JUF$SBfYK6qKLZTnaD~q@!W;$)3mY}ELj5X2i_rlC}#h3NWI0uOgUu(Lt z*+$;88|=bvXZG}a`ZWQ=6wbU&g+`5CvFX znsw@Cohwd$ip;XC$y?v&I)37G$4o2;HBeu8VYEQLTD60WS3PJb;I`s',string(pipeline().parameters.BatchCount)),'',string(pipeline().parameters.Item))", + "type": "Expression" + } + } + } + ], + "parameters": { + "TaskObject": { + "type": "object", + "defaultValue": { + "TaskInstanceId": 75, + "TaskMasterId": 12, + "TaskStatus": "Untried", + "TaskType": "SQL Database to Azure Storage", + "Enabled": 1, + "ExecutionUid": "2c5924ee-b855-4d2b-bb7e-4f5dde4c4dd3", + "NumberOfRetries": 111, + "DegreeOfCopyParallelism": 1, + "KeyVaultBaseUrl": "https://adsgofastkeyvault.vault.azure.net/", + "ScheduleMasterId": 2, + "TaskGroupConcurrency": 10, + "TaskGroupPriority": 0, + "Source": { + "Type": "Azure SQL", + "Database": { + "SystemName": "adsgofastdatakakeaccelsqlsvr.database.windows.net", + "Name": "AWSample", + "AuthenticationType": "MSI" + }, + "Extraction": { + "Type": "Table", + "FullOrIncremental": "Full", + "IncrementalType": null, + "TableSchema": "SalesLT", + "TableName": "SalesOrderHeader" + } + }, + "Target": { + "Type": "Azure Blob", + "StorageAccountName": "https://adsgofastdatalakeaccelst.blob.core.windows.net", + "StorageAccountContainer": "datalakeraw", + "StorageAccountAccessMethod": "MSI", + "RelativePath": "/AwSample/SalesLT/SalesOrderHeader/2020/7/9/14/12/", + "DataFileName": "SalesLT.SalesOrderHeader.parquet", + "SchemaFileName": "SalesLT.SalesOrderHeader", + "FirstRowAsHeader": null, + "SheetName": null, + "SkipLineCount": null, + "MaxConcorrentConnections": null + }, + "DataFactory": { + "Id": 1, + "Name": "adsgofastdatakakeacceladf", + "ResourceGroup": "AdsGoFastDataLakeAccel", + "SubscriptionId": "035a1364-f00d-48e2-b582-4fe125905ee3", + "ADFPipeline": "AZ_SQL_AZ_Storage_Parquet_@GF{IR}" + } + } + }, + "Mapping": { + "type": "object" + }, + "BatchCount": { + "type": "int" + }, + "Item": { + "type": "int" + } + }, + "variables": { + "SQLStatement1": { + "type": "String" + } + }, + "folder": { + "name": "ADS Go Fast/Data Movement/@GF{IR}/Components" + }, + "annotations": [], + "lastPublishTime": "2020-07-29T02:39:55Z" + }, + "type": "Microsoft.DataFactory/factories/pipelines" +} \ No newline at end of file diff --git a/solution/DataFactory/Templates/pipeline/AZ_SQL_Post_Copy_{IR}.json b/solution/DataFactory/Templates/pipeline/AZ_SQL_Post_Copy_{IR}.json new file mode 100644 index 00000000..42662bd4 --- /dev/null +++ b/solution/DataFactory/Templates/pipeline/AZ_SQL_Post_Copy_{IR}.json @@ -0,0 +1,556 @@ +{ + "name": "AZ_SQL_Post_Copy_@GF{IR}", + "properties": { + "activities": [ + { + "name": "If Exist PostCopySQL", + "type": "IfCondition", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "expression": { + "value": "@not(empty(pipeline().parameters.TaskObject.Target.PostCopySQL))", + "type": "Expression" + }, + "ifTrueActivities": [ + { + "name": "Run PostCopySQL", + "type": "Lookup", + "dependsOn": [], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "source": { + "type": "AzureSqlSource", + "sqlReaderQuery": { + "value": "@pipeline().parameters.TaskObject.Target.PostCopySQL", + "type": "Expression" + }, + "queryTimeout": "02:00:00", + "partitionOption": "None" + }, + "dataset": { + "referenceName": "AzureSqlTable_@GF{IR}", + "type": "DatasetReference", + "parameters": { + "Schema": { + "value": "@pipeline().parameters.TaskObject.Target.StagingTableSchema", + "type": "Expression" + }, + "Table": { + "value": "@pipeline().parameters.TaskObject.Target.StagingTableName", + "type": "Expression" + }, + "Server": { + "value": "@pipeline().parameters.TaskObject.Target.Database.SystemName", + "type": "Expression" + }, + "Database": { + "value": "@pipeline().parameters.TaskObject.Target.Database.Name", + "type": "Expression" + } + } + } + } + }, + { + "name": "AF Log - Run PostCopySQL Failed", + "type": "ExecutePipeline", + "dependsOn": [ + { + "activity": "Run PostCopySQL", + "dependencyConditions": [ + "Failed" + ] + } + ], + "userProperties": [], + "typeProperties": { + "pipeline": { + "referenceName": "AZ_Function_Generic", + "type": "PipelineReference" + }, + "waitOnCompletion": false, + "parameters": { + "Body": { + "value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":1,\"LogSource\":\"ADF\",\"ActivityType\":\"Run PostCopySQL\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"EndDateTimeOffSet\":\"', string(utcnow()), '\",\"Comment\":\"', string(activity('Run PostCopySQL').error.message), '\",\"Status\":\"Failed\"}'))", + "type": "Expression" + }, + "FunctionName": "Log", + "Method": "Post" + } + } + } + ] + } + }, + { + "name": "If Exist MergeSQL", + "type": "IfCondition", + "dependsOn": [ + { + "activity": "If Exist PostCopySQL", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "userProperties": [], + "typeProperties": { + "expression": { + "value": "@not(empty(pipeline().parameters.TaskObject.Target.MergeSQL))", + "type": "Expression" + }, + "ifTrueActivities": [ + { + "name": "Run MergeSQL", + "type": "Lookup", + "dependsOn": [], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "source": { + "type": "AzureSqlSource", + "sqlReaderQuery": { + "value": "@pipeline().parameters.TaskObject.Target.MergeSQL", + "type": "Expression" + }, + "queryTimeout": "02:00:00", + "partitionOption": "None" + }, + "dataset": { + "referenceName": "AzureSqlTable_@GF{IR}", + "type": "DatasetReference", + "parameters": { + "Schema": { + "value": "@pipeline().parameters.TaskObject.Target.StagingTableSchema", + "type": "Expression" + }, + "Table": { + "value": "@pipeline().parameters.TaskObject.Target.StagingTableName", + "type": "Expression" + }, + "Server": { + "value": "@pipeline().parameters.TaskObject.Target.Database.SystemName", + "type": "Expression" + }, + "Database": { + "value": "@pipeline().parameters.TaskObject.Target.Database.Name", + "type": "Expression" + } + } + } + } + }, + { + "name": "AF Log - Run MergeSQL Failed", + "type": "ExecutePipeline", + "dependsOn": [ + { + "activity": "Run MergeSQL", + "dependencyConditions": [ + "Failed" + ] + } + ], + "userProperties": [], + "typeProperties": { + "pipeline": { + "referenceName": "AZ_Function_Generic", + "type": "PipelineReference" + }, + "waitOnCompletion": false, + "parameters": { + "Body": { + "value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":1,\"LogSource\":\"ADF\",\"ActivityType\":\"Run MergeSQL\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"EndDateTimeOffSet\":\"', string(utcnow()), '\",\"Comment\":\"', string(activity('Run MergeSQL').error.message), '\",\"Status\":\"Failed\"}'))", + "type": "Expression" + }, + "FunctionName": "Log", + "Method": "Post" + } + } + } + ] + } + }, + { + "name": "If AutoGenerateMerge", + "type": "IfCondition", + "dependsOn": [ + { + "activity": "If Exist PostCopySQL", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "userProperties": [], + "typeProperties": { + "expression": { + "value": "@bool(pipeline().parameters.TaskObject.Target.AutoGenerateMerge)", + "type": "Expression" + }, + "ifTrueActivities": [ + { + "name": "Run MergeStatement", + "type": "Lookup", + "dependsOn": [ + { + "activity": "AF Get Merge Statement", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "source": { + "type": "AzureSqlSource", + "sqlReaderQuery": { + "value": "@activity('AF Get Merge Statement').output.MergeStatement", + "type": "Expression" + }, + "queryTimeout": "02:00:00", + "partitionOption": "None" + }, + "dataset": { + "referenceName": "AzureSqlTable_@GF{IR}", + "type": "DatasetReference", + "parameters": { + "Schema": { + "value": "@pipeline().parameters.TaskObject.Target.StagingTableSchema", + "type": "Expression" + }, + "Table": { + "value": "@pipeline().parameters.TaskObject.Target.StagingTableName", + "type": "Expression" + }, + "Server": { + "value": "@pipeline().parameters.TaskObject.Target.Database.SystemName", + "type": "Expression" + }, + "Database": { + "value": "@pipeline().parameters.TaskObject.Target.Database.Name", + "type": "Expression" + } + } + }, + "firstRowOnly": false + } + }, + { + "name": "AF Get Information Schema SQL Stage", + "type": "AzureFunctionActivity", + "dependsOn": [], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "functionName": "GetInformationSchemaSQL", + "method": "POST", + "body": { + "value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"TableSchema\":\"', string(pipeline().parameters.TaskObject.Target.StagingTableSchema), '\",\"TableName\":\"', string(pipeline().parameters.TaskObject.Target.StagingTableName),'\"}'))", + "type": "Expression" + } + }, + "linkedServiceName": { + "referenceName": "AzureFunctionAdsGoFastDataLakeAccelFunApp", + "type": "LinkedServiceReference" + } + }, + { + "name": "AF Get Merge Statement", + "type": "AzureFunctionActivity", + "dependsOn": [ + { + "activity": "Lookup Get Metadata Stage", + "dependencyConditions": [ + "Succeeded" + ] + }, + { + "activity": "Lookup Get Metadata Target", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "functionName": "GetSQLMergeStatement", + "method": "POST", + "body": { + "value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId),'\",\"TargetTableSchema\":\"',string(pipeline().parameters.TaskObject.Target.TableSchema),'\",\"TargetTableName\":\"',string(pipeline().parameters.TaskObject.Target.TableName),'\",\"StagingTableSchema\":\"',string(pipeline().parameters.TaskObject.Target.StagingTableSchema),'\",\"StagingTableName\":\"',string(pipeline().parameters.TaskObject.Target.StagingTableName),'\",\"Stage\":', string(activity('Lookup Get Metadata Stage').output.value), ',\"Target\":', string(activity('Lookup Get Metadata Target').output.value),'}'))", + "type": "Expression" + } + }, + "linkedServiceName": { + "referenceName": "AzureFunctionAdsGoFastDataLakeAccelFunApp", + "type": "LinkedServiceReference" + } + }, + { + "name": "Lookup Get Metadata Stage", + "type": "Lookup", + "dependsOn": [ + { + "activity": "AF Get Information Schema SQL Stage", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "source": { + "type": "AzureSqlSource", + "sqlReaderQuery": { + "value": "@activity('AF Get Information Schema SQL Stage').output.InformationSchemaSQL", + "type": "Expression" + }, + "queryTimeout": "02:00:00", + "partitionOption": "None" + }, + "dataset": { + "referenceName": "AzureSqlTable_@GF{IR}", + "type": "DatasetReference", + "parameters": { + "Schema": { + "value": "@pipeline().parameters.TaskObject.Target.StagingTableSchema", + "type": "Expression" + }, + "Table": { + "value": "@pipeline().parameters.TaskObject.Target.StagingTableName", + "type": "Expression" + }, + "Server": { + "value": "@pipeline().parameters.TaskObject.Target.Database.SystemName", + "type": "Expression" + }, + "Database": { + "value": "@pipeline().parameters.TaskObject.Target.Database.Name", + "type": "Expression" + } + } + }, + "firstRowOnly": false + } + }, + { + "name": "Lookup Get Metadata Target", + "type": "Lookup", + "dependsOn": [ + { + "activity": "AF Get Information Schema SQL Target", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "source": { + "type": "AzureSqlSource", + "sqlReaderQuery": { + "value": "@activity('AF Get Information Schema SQL Target').output.InformationSchemaSQL", + "type": "Expression" + }, + "queryTimeout": "02:00:00", + "partitionOption": "None" + }, + "dataset": { + "referenceName": "AzureSqlTable_@GF{IR}", + "type": "DatasetReference", + "parameters": { + "Schema": { + "value": "@pipeline().parameters.TaskObject.Target.TableSchema", + "type": "Expression" + }, + "Table": { + "value": "@pipeline().parameters.TaskObject.Target.TableName", + "type": "Expression" + }, + "Server": { + "value": "@pipeline().parameters.TaskObject.Target.Database.SystemName", + "type": "Expression" + }, + "Database": { + "value": "@pipeline().parameters.TaskObject.Target.Database.Name", + "type": "Expression" + } + } + }, + "firstRowOnly": false + } + }, + { + "name": "AF Get Information Schema SQL Target", + "type": "AzureFunctionActivity", + "dependsOn": [], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "functionName": "GetInformationSchemaSQL", + "method": "POST", + "body": { + "value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"TableSchema\":\"', string(pipeline().parameters.TaskObject.Target.TableSchema), '\",\"TableName\":\"', string(pipeline().parameters.TaskObject.Target.TableName),'\"}'))", + "type": "Expression" + } + }, + "linkedServiceName": { + "referenceName": "AzureFunctionAdsGoFastDataLakeAccelFunApp", + "type": "LinkedServiceReference" + } + }, + { + "name": "AF Log - Run AutoMerge Failed", + "type": "ExecutePipeline", + "dependsOn": [ + { + "activity": "Run MergeStatement", + "dependencyConditions": [ + "Failed" + ] + } + ], + "userProperties": [], + "typeProperties": { + "pipeline": { + "referenceName": "AZ_Function_Generic", + "type": "PipelineReference" + }, + "waitOnCompletion": false, + "parameters": { + "Body": { + "value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":1,\"LogSource\":\"ADF\",\"ActivityType\":\"Run AutoMerge\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"EndDateTimeOffSet\":\"', string(utcnow()), '\",\"Comment\":\"', string(activity('Run MergeStatement').error.message), '\",\"Status\":\"Failed\"}'))", + "type": "Expression" + }, + "FunctionName": "Log", + "Method": "Post" + } + } + }, + { + "name": "AF Log - Run Lookup Get Metadata Target Failed", + "type": "ExecutePipeline", + "dependsOn": [ + { + "activity": "Lookup Get Metadata Target", + "dependencyConditions": [ + "Failed" + ] + } + ], + "userProperties": [], + "typeProperties": { + "pipeline": { + "referenceName": "AZ_Function_Generic", + "type": "PipelineReference" + }, + "waitOnCompletion": false, + "parameters": { + "Body": { + "value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":1,\"LogSource\":\"ADF\",\"ActivityType\":\"Lookup Get Metadata Target\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"EndDateTimeOffSet\":\"', string(utcnow()), '\",\"Comment\":\"', string(activity('Lookup Get Metadata Target').error.message), '\",\"Status\":\"Failed\"}'))", + "type": "Expression" + }, + "FunctionName": "Log", + "Method": "Post" + } + } + }, + { + "name": "AF Log - Lookup Get Metadata Stage Failed", + "type": "ExecutePipeline", + "dependsOn": [ + { + "activity": "Lookup Get Metadata Stage", + "dependencyConditions": [ + "Failed" + ] + } + ], + "userProperties": [], + "typeProperties": { + "pipeline": { + "referenceName": "AZ_Function_Generic", + "type": "PipelineReference" + }, + "waitOnCompletion": false, + "parameters": { + "Body": { + "value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":1,\"LogSource\":\"ADF\",\"ActivityType\":\"Lookup Get Metadata Stage\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"EndDateTimeOffSet\":\"', string(utcnow()), '\",\"Comment\":\"', string(activity('Lookup Get Metadata Stage').error.message), '\",\"Status\":\"Failed\"}'))", + "type": "Expression" + }, + "FunctionName": "Log", + "Method": "Post" + } + } + } + ] + } + } + ], + "parameters": { + "TaskObject": { + "type": "object" + } + }, + "folder": { + "name": "ADS Go Fast/Data Movement/@GF{IR}/Common" + }, + "annotations": [], + "lastPublishTime": "2020-08-04T13:09:30Z" + }, + "type": "Microsoft.DataFactory/factories/pipelines" +} \ No newline at end of file diff --git a/solution/DataFactory/Templates/pipeline/AZ_SQL_StoredProcedure_{IR}.json b/solution/DataFactory/Templates/pipeline/AZ_SQL_StoredProcedure_{IR}.json new file mode 100644 index 00000000..8ab8dc44 --- /dev/null +++ b/solution/DataFactory/Templates/pipeline/AZ_SQL_StoredProcedure_{IR}.json @@ -0,0 +1,92 @@ +{ + "name": "AZ_SQL_StoredProcedure_@GF{IR}", + "properties": { + "activities": [ + { + "name": "Execute Stored Procedure", + "type": "Lookup", + "dependsOn": [], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "source": { + "type": "AzureSqlSource", + "sqlReaderQuery": { + "value": "@pipeline().parameters.TaskObject.Source.Execute.StoredProcedure", + "type": "Expression" + }, + "queryTimeout": "02:00:00", + "partitionOption": "None" + }, + "dataset": { + "referenceName": "AzureSqlTable_@GF{IR}", + "type": "DatasetReference", + "parameters": { + "Schema": { + "value": "@pipeline().parameters.TaskObject.Source.Extraction.TableSchema", + "type": "Expression" + }, + "Table": { + "value": "@pipeline().parameters.TaskObject.Source.Extraction.TableName", + "type": "Expression" + }, + "Server": { + "value": "@pipeline().parameters.TaskObject.Source.Database.SystemName", + "type": "Expression" + }, + "Database": { + "value": "@pipeline().parameters.TaskObject.Source.Database.Name", + "type": "Expression" + } + } + } + } + }, + { + "name": "Pipeline AF Log - Lookup Failed", + "type": "ExecutePipeline", + "dependsOn": [ + { + "activity": "Execute Stored Procedure", + "dependencyConditions": [ + "Failed" + ] + } + ], + "userProperties": [], + "typeProperties": { + "pipeline": { + "referenceName": "AZ_Function_Generic", + "type": "PipelineReference" + }, + "waitOnCompletion": false, + "parameters": { + "Body": { + "value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":1,\"LogSource\":\"ADF\",\"ActivityType\":\"Execute Stored Procedure\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"EndDateTimeOffSet\":\"', string(utcnow()), '\",\"Comment\":\"', string(activity('Execute Stored Procedure').error.message), '\",\"Status\":\"Failed\"}'))", + "type": "Expression" + }, + "FunctionName": "Log", + "Method": "Post" + } + } + } + ], + "parameters": { + "TaskObject": { + "type": "object" + } + }, + "folder": { + "name": "ADS Go Fast/Data Movement/@GF{IR}" + }, + "annotations": [], + "lastPublishTime": "2020-08-06T09:21:51Z" + }, + "type": "Microsoft.DataFactory/factories/pipelines" +} \ No newline at end of file diff --git a/solution/DataFactory/pipeline/OnP-SQL-Watermark-Chunk-OnP-SH-IR.json b/solution/DataFactory/Templates/pipeline/AZ_SQL_Watermark_Chunk_{IR}.json similarity index 92% rename from solution/DataFactory/pipeline/OnP-SQL-Watermark-Chunk-OnP-SH-IR.json rename to solution/DataFactory/Templates/pipeline/AZ_SQL_Watermark_Chunk_{IR}.json index d719375d..1bb652bd 100644 --- a/solution/DataFactory/pipeline/OnP-SQL-Watermark-Chunk-OnP-SH-IR.json +++ b/solution/DataFactory/Templates/pipeline/AZ_SQL_Watermark_Chunk_{IR}.json @@ -1,5 +1,5 @@ { - "name": "OnP-SQL-Watermark-Chunk-OnP-SH-IR", + "name": "AZ_SQL_Watermark_Chunk_@GF{IR}", "properties": { "activities": [ { @@ -16,7 +16,7 @@ "userProperties": [], "typeProperties": { "pipeline": { - "referenceName": "AZ-Function-Generic", + "referenceName": "AZ_Function_Generic", "type": "PipelineReference" }, "waitOnCompletion": false, @@ -43,13 +43,13 @@ "isSequential": true, "activities": [ { - "name": "Execute OnP-SQL-Watermark", + "name": "Execute AZ_SQL_Watermark", "type": "ExecutePipeline", "dependsOn": [], "userProperties": [], "typeProperties": { "pipeline": { - "referenceName": "OnP-SQL-Watermark-OnP-SH-IR", + "referenceName": "AZ_SQL_Watermark_@GF{IR}", "type": "PipelineReference" }, "waitOnCompletion": true, @@ -130,7 +130,7 @@ "Name": "adsgofastdatakakeacceladf", "ResourceGroup": "AdsGoFastDataLakeAccel", "SubscriptionId": "035a1364-f00d-48e2-b582-4fe125905ee3", - "ADFPipeline": "AZ-SQL-AZ-Storage-Parquet-SH-IR" + "ADFPipeline": "AZ_SQL_AZ_Storage_Parquet_@GF{IR}" } } }, @@ -145,10 +145,10 @@ } }, "folder": { - "name": "ADS Go Fast/Data Movement/OnPrem SH IR/Components" + "name": "ADS Go Fast/Data Movement/@GF{IR}/Components" }, "annotations": [], - "lastPublishTime": "2020-08-04T07:52:08Z" + "lastPublishTime": "2020-07-29T02:39:55Z" }, "type": "Microsoft.DataFactory/factories/pipelines" } \ No newline at end of file diff --git a/solution/DataFactory/Templates/pipeline/AZ_SQL_Watermark_{IR}.json b/solution/DataFactory/Templates/pipeline/AZ_SQL_Watermark_{IR}.json new file mode 100644 index 00000000..b3fb3e58 --- /dev/null +++ b/solution/DataFactory/Templates/pipeline/AZ_SQL_Watermark_{IR}.json @@ -0,0 +1,636 @@ +{ + "name": "AZ_SQL_Watermark_@GF{IR}", + "properties": { + "activities": [ + { + "name": "Switch Storage Type", + "type": "Switch", + "dependsOn": [ + { + "activity": "Set SQLStatement", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "userProperties": [], + "typeProperties": { + "on": { + "value": "@pipeline().parameters.TaskObject.Target.Type", + "type": "Expression" + }, + "cases": [ + { + "value": "ADLS", + "activities": [ + { + "name": "Copy Azure SQL to ADLS", + "type": "Copy", + "dependsOn": [ + { + "activity": "Pipeline AF Log - Azure SQL to ADLS Start", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "source": { + "type": "AzureSqlSource", + "sqlReaderQuery": { + "value": "@variables('SQLStatement')", + "type": "Expression" + }, + "queryTimeout": "02:00:00", + "partitionOption": "None" + }, + "sink": { + "type": "ParquetSink", + "storeSettings": { + "type": "AzureBlobFSWriteSettings" + } + }, + "enableStaging": false, + "parallelCopies": { + "value": "@pipeline().parameters.TaskObject.DegreeOfCopyParallelism", + "type": "Expression" + }, + "translator": { + "value": "@pipeline().parameters.Mapping", + "type": "Expression" + } + }, + "inputs": [ + { + "referenceName": "AzureSqlTable_@GF{IR}", + "type": "DatasetReference", + "parameters": { + "Schema": { + "value": "@pipeline().parameters.TaskObject.Source.Extraction.TableSchema", + "type": "Expression" + }, + "Table": { + "value": "@pipeline().parameters.TaskObject.Source.Extraction.TableName", + "type": "Expression" + }, + "Server": { + "value": "@pipeline().parameters.TaskObject.Source.Database.SystemName", + "type": "Expression" + }, + "Database": { + "value": "@pipeline().parameters.TaskObject.Source.Database.Name", + "type": "Expression" + } + } + } + ], + "outputs": [ + { + "referenceName": "ADLSParquet_@GF{IR}", + "type": "DatasetReference", + "parameters": { + "RelativePath": { + "value": "@pipeline().parameters.TaskObject.Target.RelativePath", + "type": "Expression" + }, + "FileName": { + "value": "@replace(pipeline().parameters.TaskObject.Target.DataFileName,'.parquet',concat('.chunk_', string(pipeline().parameters.Item),'.parquet'))", + "type": "Expression" + }, + "StorageAccountEndpoint": { + "value": "@pipeline().parameters.TaskObject.Target.StorageAccountName", + "type": "Expression" + }, + "StorageAccountContainerName": { + "value": "@pipeline().parameters.TaskObject.Target.StorageAccountContainer", + "type": "Expression" + } + } + } + ] + }, + { + "name": "Pipeline AF Log - Azure SQL to ADLS Failed", + "type": "ExecutePipeline", + "dependsOn": [ + { + "activity": "Copy Azure SQL to ADLS", + "dependencyConditions": [ + "Failed" + ] + } + ], + "userProperties": [], + "typeProperties": { + "pipeline": { + "referenceName": "AZ_Function_Generic", + "type": "PipelineReference" + }, + "waitOnCompletion": false, + "parameters": { + "Body": { + "value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":1,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy Azure SQL to ADLS\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"EndDateTimeOffSet\":\"', string(utcnow()), '\",\"Comment\":\"', string(activity('Copy Azure SQL to ADLS').error.message), '\",\"Status\":\"Failed\"}'))", + "type": "Expression" + }, + "FunctionName": "Log", + "Method": "Post" + } + } + }, + { + "name": "AF Persist Parquet Metadata - ADLS", + "type": "AzureFunctionActivity", + "dependsOn": [ + { + "activity": "Get Parquet Metadata ADLS", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "functionName": "TaskExecutionSchemaFile", + "method": "POST", + "body": { + "value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"StorageAccountName\":\"', string(pipeline().parameters.TaskObject.Target.StorageAccountName), '\",\"StorageAccountContainer\":\"', string(pipeline().parameters.TaskObject.Target.StorageAccountContainer), '\",\"RelativePath\":\"', string(pipeline().parameters.TaskObject.Target.RelativePath), '\",\"SchemaFileName\":\"', string(pipeline().parameters.TaskObject.Target.SchemaFileName), '\",\"SourceType\":\"', string(pipeline().parameters.TaskObject.Source.Type), '\",\"TargetType\":\"', string(pipeline().parameters.TaskObject.Target.Type), '\",\"Data\":',string(activity('Get Parquet Metadata ADLS').output),',\"MetadataType\":\"Parquet\"}'))", + "type": "Expression" + } + }, + "linkedServiceName": { + "referenceName": "AzureFunctionAdsGoFastDataLakeAccelFunApp", + "type": "LinkedServiceReference" + } + }, + { + "name": "Get Parquet Metadata ADLS", + "type": "GetMetadata", + "dependsOn": [ + { + "activity": "Copy Azure SQL to ADLS", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "dataset": { + "referenceName": "ADLSParquet_@GF{IR}", + "type": "DatasetReference", + "parameters": { + "RelativePath": { + "value": "@pipeline().parameters.TaskObject.Target.RelativePath", + "type": "Expression" + }, + "FileName": { + "value": "@replace(pipeline().parameters.TaskObject.Target.DataFileName,'.parquet',concat('.chunk_', string(pipeline().parameters.Item),'.parquet'))", + "type": "Expression" + }, + "StorageAccountEndpoint": { + "value": "@pipeline().parameters.TaskObject.Target.StorageAccountName", + "type": "Expression" + }, + "StorageAccountContainerName": { + "value": "@pipeline().parameters.TaskObject.Target.StorageAccountContainer", + "type": "Expression" + } + } + }, + "fieldList": [ + "structure" + ], + "storeSettings": { + "type": "AzureBlobFSReadSettings", + "recursive": true + } + } + }, + { + "name": "Pipeline AF Log - Azure SQL to ADLS Start", + "type": "ExecutePipeline", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "pipeline": { + "referenceName": "AZ_Function_Generic", + "type": "PipelineReference" + }, + "waitOnCompletion": false, + "parameters": { + "Body": { + "value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":3,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy Azure SQL to ADLS\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"Status\":\"Started\"}'))", + "type": "Expression" + }, + "FunctionName": "Log", + "Method": "Post" + } + } + }, + { + "name": "Pipeline AF Log - Azure SQL ADLS to Succeed", + "type": "ExecutePipeline", + "dependsOn": [ + { + "activity": "Copy Azure SQL to ADLS", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "userProperties": [], + "typeProperties": { + "pipeline": { + "referenceName": "AZ_Function_Generic", + "type": "PipelineReference" + }, + "waitOnCompletion": false, + "parameters": { + "Body": { + "value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":1,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy Azure SQL to ADLS\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"EndDateTimeOffSet\":\"', string(utcnow()), '\",\"RowsInserted\":\"', string(activity('Copy Azure SQL to ADLS').output.rowsCopied), '\",\"Comment\":\"\",\"Status\":\"Complete\"}'))", + "type": "Expression" + }, + "FunctionName": "Log", + "Method": "Post" + } + } + } + ] + }, + { + "value": "Azure Blob", + "activities": [ + { + "name": "Copy Azure SQL to Blob", + "type": "Copy", + "dependsOn": [ + { + "activity": "Pipeline AF Log - Azure SQL to Blob Start", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "source": { + "type": "AzureSqlSource", + "sqlReaderQuery": { + "value": "@variables('SQLStatement')", + "type": "Expression" + }, + "queryTimeout": "02:00:00", + "partitionOption": "None" + }, + "sink": { + "type": "ParquetSink", + "storeSettings": { + "type": "AzureBlobStorageWriteSettings" + } + }, + "enableStaging": false, + "parallelCopies": { + "value": "@pipeline().parameters.TaskObject.DegreeOfCopyParallelism", + "type": "Expression" + }, + "translator": { + "value": "@pipeline().parameters.Mapping", + "type": "Expression" + } + }, + "inputs": [ + { + "referenceName": "AzureSqlTable_@GF{IR}", + "type": "DatasetReference", + "parameters": { + "Schema": { + "value": "@pipeline().parameters.TaskObject.Source.Extraction.TableSchema", + "type": "Expression" + }, + "Table": { + "value": "@pipeline().parameters.TaskObject.Source.Extraction.TableName", + "type": "Expression" + }, + "Server": { + "value": "@pipeline().parameters.TaskObject.Source.Database.SystemName", + "type": "Expression" + }, + "Database": { + "value": "@pipeline().parameters.TaskObject.Source.Database.Name", + "type": "Expression" + } + } + } + ], + "outputs": [ + { + "referenceName": "BlobParquet_@GF{IR}", + "type": "DatasetReference", + "parameters": { + "RelativePath": { + "value": "@pipeline().parameters.TaskObject.Target.RelativePath", + "type": "Expression" + }, + "FileName": { + "value": "@replace(pipeline().parameters.TaskObject.Target.DataFileName,'.parquet',concat('.chunk_', string(pipeline().parameters.Item),'.parquet'))", + "type": "Expression" + }, + "StorageAccountEndpoint": { + "value": "@pipeline().parameters.TaskObject.Target.StorageAccountName", + "type": "Expression" + }, + "StorageAccountContainerName": { + "value": "@pipeline().parameters.TaskObject.Target.StorageAccountContainer", + "type": "Expression" + } + } + } + ] + }, + { + "name": "Pipeline AF Log - Azure SQL to Blob Failed", + "type": "ExecutePipeline", + "dependsOn": [ + { + "activity": "Copy Azure SQL to Blob", + "dependencyConditions": [ + "Failed" + ] + } + ], + "userProperties": [], + "typeProperties": { + "pipeline": { + "referenceName": "AZ_Function_Generic", + "type": "PipelineReference" + }, + "waitOnCompletion": false, + "parameters": { + "Body": { + "value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":1,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy Azure SQL to Blob\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"EndDateTimeOffSet\":\"', string(utcnow()), '\",\"Comment\":\"', string(activity('Copy Azure SQL to Blob').error.message), '\",\"Status\":\"Failed\"}'))", + "type": "Expression" + }, + "FunctionName": "Log", + "Method": "Post" + } + } + }, + { + "name": "AF Persist Parquet Metadata - Blob", + "type": "AzureFunctionActivity", + "dependsOn": [ + { + "activity": "Get Parquet Metadata Blob", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "functionName": "TaskExecutionSchemaFile", + "method": "POST", + "body": { + "value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"StorageAccountName\":\"', string(pipeline().parameters.TaskObject.Target.StorageAccountName), '\",\"StorageAccountContainer\":\"', string(pipeline().parameters.TaskObject.Target.StorageAccountContainer), '\",\"RelativePath\":\"', string(pipeline().parameters.TaskObject.Target.RelativePath), '\",\"SchemaFileName\":\"', string(pipeline().parameters.TaskObject.Target.SchemaFileName), '\",\"SourceType\":\"', string(pipeline().parameters.TaskObject.Source.Type), '\",\"TargetType\":\"', string(pipeline().parameters.TaskObject.Target.Type), '\",\"Data\":',string(activity('Get Parquet Metadata Blob').output),',\"MetadataType\":\"Parquet\"}'))", + "type": "Expression" + } + }, + "linkedServiceName": { + "referenceName": "AzureFunctionAdsGoFastDataLakeAccelFunApp", + "type": "LinkedServiceReference" + } + }, + { + "name": "Get Parquet Metadata Blob", + "type": "GetMetadata", + "dependsOn": [ + { + "activity": "Copy Azure SQL to Blob", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "dataset": { + "referenceName": "BlobParquet_@GF{IR}", + "type": "DatasetReference", + "parameters": { + "RelativePath": { + "value": "@pipeline().parameters.TaskObject.Target.RelativePath", + "type": "Expression" + }, + "FileName": { + "value": "@replace(pipeline().parameters.TaskObject.Target.DataFileName,'.parquet',concat('.chunk_', string(pipeline().parameters.Item),'.parquet'))", + "type": "Expression" + }, + "StorageAccountEndpoint": { + "value": "@pipeline().parameters.TaskObject.Target.StorageAccountName", + "type": "Expression" + }, + "StorageAccountContainerName": { + "value": "@pipeline().parameters.TaskObject.Target.StorageAccountContainer", + "type": "Expression" + } + } + }, + "fieldList": [ + "structure" + ], + "storeSettings": { + "type": "AzureBlobStorageReadSettings", + "recursive": true + } + } + }, + { + "name": "Pipeline AF Log - Azure SQL to Blob Start", + "type": "ExecutePipeline", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "pipeline": { + "referenceName": "AZ_Function_Generic", + "type": "PipelineReference" + }, + "waitOnCompletion": false, + "parameters": { + "Body": { + "value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":3,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy Azure SQL to Blob\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"Status\":\"Started\"}'))", + "type": "Expression" + }, + "FunctionName": "Log", + "Method": "Post" + } + } + }, + { + "name": "Pipeline AF Log - Azure SQL Blob to Succeed", + "type": "ExecutePipeline", + "dependsOn": [ + { + "activity": "Copy Azure SQL to Blob", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "userProperties": [], + "typeProperties": { + "pipeline": { + "referenceName": "AZ_Function_Generic", + "type": "PipelineReference" + }, + "waitOnCompletion": false, + "parameters": { + "Body": { + "value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":1,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy Azure SQL to Blob\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"EndDateTimeOffSet\":\"', string(utcnow()), '\",\"RowsInserted\":\"', string(activity('Copy Azure SQL to Blob').output.rowsCopied), '\",\"Comment\":\"\",\"Status\":\"Complete\"}'))", + "type": "Expression" + }, + "FunctionName": "Log", + "Method": "Post" + } + } + } + ] + } + ] + } + }, + { + "name": "Set SQLStatement", + "type": "SetVariable", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "variableName": "SQLStatement", + "value": { + "value": "@replace(replace(replace(pipeline().parameters.TaskObject.Source.Extraction.SQLStatement,'',string(pipeline().parameters.BatchCount)),'',string(pipeline().parameters.Item)),'',pipeline().parameters.NewWaterMark)", + "type": "Expression" + } + } + } + ], + "parameters": { + "TaskObject": { + "type": "object", + "defaultValue": { + "TaskInstanceId": 75, + "TaskMasterId": 12, + "TaskStatus": "Untried", + "TaskType": "SQL Database to Azure Storage", + "Enabled": 1, + "ExecutionUid": "2c5924ee-b855-4d2b-bb7e-4f5dde4c4dd3", + "NumberOfRetries": 111, + "DegreeOfCopyParallelism": 1, + "KeyVaultBaseUrl": "https://adsgofastkeyvault.vault.azure.net/", + "ScheduleMasterId": 2, + "TaskGroupConcurrency": 10, + "TaskGroupPriority": 0, + "Source": { + "Type": "Azure SQL", + "Database": { + "SystemName": "adsgofastdatakakeaccelsqlsvr.database.windows.net", + "Name": "AWSample", + "AuthenticationType": "MSI" + }, + "Extraction": { + "Type": "Table", + "FullOrIncremental": "Full", + "IncrementalType": null, + "TableSchema": "SalesLT", + "TableName": "SalesOrderHeader" + } + }, + "Target": { + "Type": "Azure Blob", + "StorageAccountName": "https://adsgofastdatalakeaccelst.blob.core.windows.net", + "StorageAccountContainer": "datalakeraw", + "StorageAccountAccessMethod": "MSI", + "RelativePath": "/AwSample/SalesLT/SalesOrderHeader/2020/7/9/14/12/", + "DataFileName": "SalesLT.SalesOrderHeader.parquet", + "SchemaFileName": "SalesLT.SalesOrderHeader", + "FirstRowAsHeader": null, + "SheetName": null, + "SkipLineCount": null, + "MaxConcorrentConnections": null + }, + "DataFactory": { + "Id": 1, + "Name": "adsgofastdatakakeacceladf", + "ResourceGroup": "AdsGoFastDataLakeAccel", + "SubscriptionId": "035a1364-f00d-48e2-b582-4fe125905ee3", + "ADFPipeline": "AZ_SQL_AZ_Storage_Parquet_@GF{IR}" + } + } + }, + "Mapping": { + "type": "object" + }, + "NewWaterMark": { + "type": "string" + }, + "Item": { + "type": "int" + }, + "BatchCount": { + "type": "int" + } + }, + "variables": { + "SQLStatement": { + "type": "String" + } + }, + "folder": { + "name": "ADS Go Fast/Data Movement/@GF{IR}/Components" + }, + "annotations": [], + "lastPublishTime": "2020-07-29T02:39:55Z" + }, + "type": "Microsoft.DataFactory/factories/pipelines" +} \ No newline at end of file diff --git a/solution/DataFactory/pipeline/AZ-Storage-Binary-AZ-Storage-Binary-AZ-IR.json b/solution/DataFactory/Templates/pipeline/AZ_Storage_Binary_AZ_Storage_Binary_{IR}.json similarity index 60% rename from solution/DataFactory/pipeline/AZ-Storage-Binary-AZ-Storage-Binary-AZ-IR.json rename to solution/DataFactory/Templates/pipeline/AZ_Storage_Binary_AZ_Storage_Binary_{IR}.json index 590842c1..f0285478 100644 --- a/solution/DataFactory/pipeline/AZ-Storage-Binary-AZ-Storage-Binary-AZ-IR.json +++ b/solution/DataFactory/Templates/pipeline/AZ_Storage_Binary_AZ_Storage_Binary_{IR}.json @@ -1,5 +1,5 @@ { - "name": "AZ-Storage-Binary-AZ-Storage-Binary-AZ-IR", + "name": "AZ_Storage_Binary_AZ_Storage_Binary_@GF{IR}", "properties": { "activities": [ { @@ -14,7 +14,7 @@ }, "cases": [ { - "value": "AzureBlobAzureBlob", + "value": "Azure BlobAzure Blob", "activities": [ { "name": "Copy Blob to Blob", @@ -40,7 +40,18 @@ "type": "BinarySource", "storeSettings": { "type": "AzureBlobStorageReadSettings", - "recursive": true + "recursive": { + "value": "@pipeline().parameters.TaskObject.Source.Recursively", + "type": "Expression" + }, + "wildcardFileName": { + "value": "@pipeline().parameters.TaskObject.Source.DataFileName", + "type": "Expression" + }, + "deleteFilesAfterCompletion": { + "value": "@pipeline().parameters.TaskObject.Source.DeleteAfterCompletion", + "type": "Expression" + } }, "formatSettings": { "type": "BinaryReadSettings" @@ -49,7 +60,8 @@ "sink": { "type": "BinarySink", "storeSettings": { - "type": "AzureBlobStorageWriteSettings" + "type": "AzureBlobStorageWriteSettings", + "copyBehavior": "PreserveHierarchy" } }, "enableStaging": false, @@ -60,7 +72,7 @@ }, "inputs": [ { - "referenceName": "BlobBinary_AZ_IR", + "referenceName": "BlobBinary_@GF{IR}", "type": "DatasetReference", "parameters": { "StorageAccountEndpoint": { @@ -84,7 +96,7 @@ ], "outputs": [ { - "referenceName": "BlobBinary_AZ_IR", + "referenceName": "BlobBinary_@GF{IR}", "type": "DatasetReference", "parameters": { "StorageAccountEndpoint": { @@ -107,6 +119,34 @@ } ] }, + { + "name": "Pipeline AF Log - Blob to Blob Failed", + "type": "ExecutePipeline", + "dependsOn": [ + { + "activity": "Copy Blob to Blob", + "dependencyConditions": [ + "Failed" + ] + } + ], + "userProperties": [], + "typeProperties": { + "pipeline": { + "referenceName": "AZ_Function_Generic", + "type": "PipelineReference" + }, + "waitOnCompletion": false, + "parameters": { + "Body": { + "value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":1,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy Blob to Blob\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"EndDateTimeOffSet\":\"', string(utcnow()), '\",\"Comment\":\"', string(activity('Copy Blob to Blob').error.message), '\",\"Status\":\"Failed\"}'))", + "type": "Expression" + }, + "FunctionName": "Log", + "Method": "Post" + } + } + }, { "name": "Pipeline AF Log - Blob to Blob Start", "type": "ExecutePipeline", @@ -114,17 +154,17 @@ "userProperties": [], "typeProperties": { "pipeline": { - "referenceName": "AZ-Function-Generic", + "referenceName": "AZ_Function_Generic", "type": "PipelineReference" }, "waitOnCompletion": false, "parameters": { "Body": { - "value": "[\n{\n \"TaskInstanceId\":@{pipeline().parameters.TaskObject.TaskInstanceId},\n \"ExecutionUid\":@{pipeline().parameters.TaskObject.ExecutionUid},\n \"RunId\":@{pipeline().RunId}, \n \"StartTime\":@{utcnow()},\n \"Status\":\"Processing\",\n \"Activity\":\"Copy Blob to Blob\",\n \"Comments\":\"\"\n}\n]", + "value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":3,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy Blob to Blob\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"Status\":\"Started\"}'))", "type": "Expression" }, "FunctionName": "Log", - "Method": "Put" + "Method": "Post" } } }, @@ -142,45 +182,17 @@ "userProperties": [], "typeProperties": { "pipeline": { - "referenceName": "AZ-Function-Generic", + "referenceName": "AZ_Function_Generic", "type": "PipelineReference" }, "waitOnCompletion": false, "parameters": { "Body": { - "value": "[\n{\n \"TaskInstanceId\":@{pipeline().parameters.TaskObject.TaskInstanceId},\n \"ExecutionUid\":@{pipeline().parameters.TaskObject.ExecutionUid},\n \"RunId\":@{pipeline().RunId}, \n \"EndTime\":@{utcnow()},\n \"Status\":\"Completed\",\n \"Activity\":\"Copy Blob to Blob\",\n \"Comments\":\"\"\n}\n]", + "value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":1,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy Blob to Blob\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"EndDateTimeOffSet\":\"', string(utcnow()), '\",\"RowsInserted\":\"', string(activity('Copy Blob to Blob').output.filesWritten), '\",\"Comment\":\"\",\"Status\":\"Complete\"}'))", "type": "Expression" }, "FunctionName": "Log", - "Method": "Put" - } - } - }, - { - "name": "Pipeline AF Log - Blob to Blob Failed", - "type": "ExecutePipeline", - "dependsOn": [ - { - "activity": "Copy Blob to Blob", - "dependencyConditions": [ - "Failed" - ] - } - ], - "userProperties": [], - "typeProperties": { - "pipeline": { - "referenceName": "AZ-Function-Generic", - "type": "PipelineReference" - }, - "waitOnCompletion": false, - "parameters": { - "Body": { - "value": "[\n{\n \"TaskInstanceId\":@{pipeline().parameters.TaskObject.TaskInstanceId},\n \"ExecutionUid\":@{pipeline().parameters.TaskObject.ExecutionUid},\n \"RunId\":@{pipeline().RunId}, \n \"EndTime\":@{utcnow()},\n \"Status\":\"Failed\",\n \"Activity\":\"Copy Blob to Blob\",\n \"Comments\":@{activity('Copy Blob to Blob').errors}\n}\n]", - "type": "Expression" - }, - "FunctionName": "Log", - "Method": "Put" + "Method": "Post" } } } @@ -213,8 +225,18 @@ "type": "BinarySource", "storeSettings": { "type": "AzureBlobFSReadSettings", - "recursive": true, - "deleteFilesAfterCompletion": true + "recursive": { + "value": "@pipeline().parameters.TaskObject.Source.Recursively", + "type": "Expression" + }, + "wildcardFileName": { + "value": "@pipeline().parameters.TaskObject.Source.DataFileName", + "type": "Expression" + }, + "deleteFilesAfterCompletion": { + "value": "@pipeline().parameters.TaskObject.Source.DeleteAfterCompletion", + "type": "Expression" + } }, "formatSettings": { "type": "BinaryReadSettings" @@ -223,7 +245,8 @@ "sink": { "type": "BinarySink", "storeSettings": { - "type": "AzureBlobFSWriteSettings" + "type": "AzureBlobFSWriteSettings", + "copyBehavior": "PreserveHierarchy" } }, "enableStaging": false, @@ -234,7 +257,7 @@ }, "inputs": [ { - "referenceName": "ADLSBinary_AZ_IR", + "referenceName": "ADLSBinary_@GF{IR}", "type": "DatasetReference", "parameters": { "StorageAccountEndpoint": { @@ -258,7 +281,7 @@ ], "outputs": [ { - "referenceName": "ADLSBinary_AZ_IR", + "referenceName": "ADLSBinary_@GF{IR}", "type": "DatasetReference", "parameters": { "StorageAccountEndpoint": { @@ -281,6 +304,34 @@ } ] }, + { + "name": "Pipeline AF Log - ADLS to ADLS Failed", + "type": "ExecutePipeline", + "dependsOn": [ + { + "activity": "Copy ADLS to ADLS", + "dependencyConditions": [ + "Failed" + ] + } + ], + "userProperties": [], + "typeProperties": { + "pipeline": { + "referenceName": "AZ_Function_Generic", + "type": "PipelineReference" + }, + "waitOnCompletion": false, + "parameters": { + "Body": { + "value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":1,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy ADLS to ADLS\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"EndDateTimeOffSet\":\"', string(utcnow()), '\",\"Comment\":\"', string(activity('Copy ADLS to ADLS').error.message), '\",\"Status\":\"Failed\"}'))", + "type": "Expression" + }, + "FunctionName": "Log", + "Method": "Post" + } + } + }, { "name": "Pipeline AF Log - ADLS to ADLS Start", "type": "ExecutePipeline", @@ -288,17 +339,17 @@ "userProperties": [], "typeProperties": { "pipeline": { - "referenceName": "AZ-Function-Generic", + "referenceName": "AZ_Function_Generic", "type": "PipelineReference" }, "waitOnCompletion": false, "parameters": { "Body": { - "value": "[\n{\n \"TaskInstanceId\":@{pipeline().parameters.TaskObject.TaskInstanceId},\n \"ExecutionUid\":@{pipeline().parameters.TaskObject.ExecutionUid},\n \"RunId\":@{pipeline().RunId}, \n \"StartTime\":@{utcnow()},\n \"Status\":\"Processing\",\n \"Activity\":\"Copy ADLS to ADLS\",\n \"Comments\":\"\"\n}\n]", + "value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":3,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy ADLS to ADLS\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"Status\":\"Started\"}'))", "type": "Expression" }, "FunctionName": "Log", - "Method": "Put" + "Method": "Post" } } }, @@ -316,52 +367,24 @@ "userProperties": [], "typeProperties": { "pipeline": { - "referenceName": "AZ-Function-Generic", + "referenceName": "AZ_Function_Generic", "type": "PipelineReference" }, "waitOnCompletion": false, "parameters": { "Body": { - "value": "[\n{\n \"TaskInstanceId\":@{pipeline().parameters.TaskObject.TaskInstanceId},\n \"ExecutionUid\":@{pipeline().parameters.TaskObject.ExecutionUid},\n \"RunId\":@{pipeline().RunId}, \n \"EndTime\":@{utcnow()},\n \"Status\":\"Completed\",\n \"Activity\":\"Copy ADLS to ADLS\",\n \"Comments\":\"\"\n}\n]", + "value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":1,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy ADLS to ADLS\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"EndDateTimeOffSet\":\"', string(utcnow()), '\",\"RowsInserted\":\"', string(activity('Copy ADLS to ADLS').output.filesWritten), '\",\"Comment\":\"\",\"Status\":\"Complete\"}'))", "type": "Expression" }, "FunctionName": "Log", - "Method": "Put" - } - } - }, - { - "name": "Pipeline AF Log - ADLS to ADLS Failed", - "type": "ExecutePipeline", - "dependsOn": [ - { - "activity": "Copy ADLS to ADLS", - "dependencyConditions": [ - "Failed" - ] - } - ], - "userProperties": [], - "typeProperties": { - "pipeline": { - "referenceName": "AZ-Function-Generic", - "type": "PipelineReference" - }, - "waitOnCompletion": false, - "parameters": { - "Body": { - "value": "[\n{\n \"TaskInstanceId\":@{pipeline().parameters.TaskObject.TaskInstanceId},\n \"ExecutionUid\":@{pipeline().parameters.TaskObject.ExecutionUid},\n \"RunId\":@{pipeline().RunId}, \n \"EndTime\":@{utcnow()},\n \"Status\":\"Failed\",\n \"Activity\":\"Copy ADLS to ADLS\",\n \"Comments\":@{activity('Copy ADLS to ADLS').errors}\n}\n]", - "type": "Expression" - }, - "FunctionName": "Log", - "Method": "Put" + "Method": "Post" } } } ] }, { - "value": "AzureBlobADLS", + "value": "Azure BlobADLS", "activities": [ { "name": "Copy Blob to ADLS", @@ -387,7 +410,18 @@ "type": "BinarySource", "storeSettings": { "type": "AzureBlobStorageReadSettings", - "recursive": true + "recursive": { + "value": "@pipeline().parameters.TaskObject.Source.Recursively", + "type": "Expression" + }, + "wildcardFileName": { + "value": "@pipeline().parameters.TaskObject.Source.DataFileName", + "type": "Expression" + }, + "deleteFilesAfterCompletion": { + "value": "@pipeline().parameters.TaskObject.Source.DeleteAfterCompletion", + "type": "Expression" + } }, "formatSettings": { "type": "BinaryReadSettings" @@ -396,7 +430,8 @@ "sink": { "type": "BinarySink", "storeSettings": { - "type": "AzureBlobFSWriteSettings" + "type": "AzureBlobFSWriteSettings", + "copyBehavior": "PreserveHierarchy" } }, "enableStaging": false, @@ -407,7 +442,7 @@ }, "inputs": [ { - "referenceName": "BlobBinary_AZ_IR", + "referenceName": "BlobBinary_@GF{IR}", "type": "DatasetReference", "parameters": { "StorageAccountEndpoint": { @@ -431,7 +466,7 @@ ], "outputs": [ { - "referenceName": "ADLSBinary_AZ_IR", + "referenceName": "ADLSBinary_@GF{IR}", "type": "DatasetReference", "parameters": { "StorageAccountEndpoint": { @@ -454,6 +489,34 @@ } ] }, + { + "name": "Pipeline AF Log - Blob to ADLS Failed", + "type": "ExecutePipeline", + "dependsOn": [ + { + "activity": "Copy Blob to ADLS", + "dependencyConditions": [ + "Failed" + ] + } + ], + "userProperties": [], + "typeProperties": { + "pipeline": { + "referenceName": "AZ_Function_Generic", + "type": "PipelineReference" + }, + "waitOnCompletion": false, + "parameters": { + "Body": { + "value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":1,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy Blob to ADLS\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"EndDateTimeOffSet\":\"', string(utcnow()), '\",\"Comment\":\"', string(activity('Copy Blob to ADLS').error.message), '\",\"Status\":\"Failed\"}'))", + "type": "Expression" + }, + "FunctionName": "Log", + "Method": "Post" + } + } + }, { "name": "Pipeline AF Log - Blob to ADLS Start", "type": "ExecutePipeline", @@ -461,17 +524,17 @@ "userProperties": [], "typeProperties": { "pipeline": { - "referenceName": "AZ-Function-Generic", + "referenceName": "AZ_Function_Generic", "type": "PipelineReference" }, "waitOnCompletion": false, "parameters": { "Body": { - "value": "[\n{\n \"TaskInstanceId\":@{pipeline().parameters.TaskObject.TaskInstanceId},\n \"ExecutionUid\":@{pipeline().parameters.TaskObject.ExecutionUid},\n \"RunId\":@{pipeline().RunId}, \n \"StartTime\":@{utcnow()},\n \"Status\":\"Processing\",\n \"Activity\":\"Copy Blob to ADLS\",\n \"Comments\":\"\"\n}\n]", + "value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":3,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy Blob to ADLS\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"Status\":\"Started\"}'))", "type": "Expression" }, "FunctionName": "Log", - "Method": "Put" + "Method": "Post" } } }, @@ -489,52 +552,24 @@ "userProperties": [], "typeProperties": { "pipeline": { - "referenceName": "AZ-Function-Generic", + "referenceName": "AZ_Function_Generic", "type": "PipelineReference" }, "waitOnCompletion": false, "parameters": { "Body": { - "value": "[\n{\n \"TaskInstanceId\":@{pipeline().parameters.TaskObject.TaskInstanceId},\n \"ExecutionUid\":@{pipeline().parameters.TaskObject.ExecutionUid},\n \"RunId\":@{pipeline().RunId}, \n \"EndTime\":@{utcnow()},\n \"Status\":\"Completed\",\n \"Activity\":\"Copy Blob to ADLS\",\n \"Comments\":\"\"\n}\n]", + "value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":1,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy Blob to ADLS\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"EndDateTimeOffSet\":\"', string(utcnow()), '\",\"RowsInserted\":\"', string(activity('Copy Blob to ADLS').output.filesWritten), '\",\"Comment\":\"\",\"Status\":\"Complete\"}'))", "type": "Expression" }, "FunctionName": "Log", - "Method": "Put" - } - } - }, - { - "name": "Pipeline AF Log - Blob to ADLS Failed", - "type": "ExecutePipeline", - "dependsOn": [ - { - "activity": "Copy Blob to ADLS", - "dependencyConditions": [ - "Failed" - ] - } - ], - "userProperties": [], - "typeProperties": { - "pipeline": { - "referenceName": "AZ-Function-Generic", - "type": "PipelineReference" - }, - "waitOnCompletion": false, - "parameters": { - "Body": { - "value": "[\n{\n \"TaskInstanceId\":@{pipeline().parameters.TaskObject.TaskInstanceId},\n \"ExecutionUid\":@{pipeline().parameters.TaskObject.ExecutionUid},\n \"RunId\":@{pipeline().RunId}, \n \"EndTime\":@{utcnow()},\n \"Status\":\"Failed\",\n \"Activity\":\"Copy Blob to ADLS\",\n \"Comments\":@{activity('Copy Blob to ADLS').errors}\n}\n]", - "type": "Expression" - }, - "FunctionName": "Log", - "Method": "Put" + "Method": "Post" } } } ] }, { - "value": "ADLSAzureBlob", + "value": "ADLSAzure Blob", "activities": [ { "name": "Copy ADLS to Blob", @@ -560,8 +595,18 @@ "type": "BinarySource", "storeSettings": { "type": "AzureBlobFSReadSettings", - "recursive": true, - "deleteFilesAfterCompletion": true + "recursive": { + "value": "@pipeline().parameters.TaskObject.Source.Recursively", + "type": "Expression" + }, + "wildcardFileName": { + "value": "@pipeline().parameters.TaskObject.Source.DataFileName", + "type": "Expression" + }, + "deleteFilesAfterCompletion": { + "value": "@pipeline().parameters.TaskObject.Source.DeleteAfterCompletion", + "type": "Expression" + } }, "formatSettings": { "type": "BinaryReadSettings" @@ -570,7 +615,8 @@ "sink": { "type": "BinarySink", "storeSettings": { - "type": "AzureBlobStorageWriteSettings" + "type": "AzureBlobStorageWriteSettings", + "copyBehavior": "PreserveHierarchy" } }, "enableStaging": false, @@ -581,7 +627,7 @@ }, "inputs": [ { - "referenceName": "ADLSBinary_AZ_IR", + "referenceName": "ADLSBinary_@GF{IR}", "type": "DatasetReference", "parameters": { "StorageAccountEndpoint": { @@ -605,7 +651,7 @@ ], "outputs": [ { - "referenceName": "BlobBinary_AZ_IR", + "referenceName": "BlobBinary_@GF{IR}", "type": "DatasetReference", "parameters": { "StorageAccountEndpoint": { @@ -628,6 +674,34 @@ } ] }, + { + "name": "Pipeline AF Log - ADLS to Blob Failed", + "type": "ExecutePipeline", + "dependsOn": [ + { + "activity": "Copy ADLS to Blob", + "dependencyConditions": [ + "Failed" + ] + } + ], + "userProperties": [], + "typeProperties": { + "pipeline": { + "referenceName": "AZ_Function_Generic", + "type": "PipelineReference" + }, + "waitOnCompletion": false, + "parameters": { + "Body": { + "value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":1,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy ADLS to Blob\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"EndDateTimeOffSet\":\"', string(utcnow()), '\",\"Comment\":\"', string(activity('Copy ADLS to Blob').error.message), '\",\"Status\":\"Failed\"}'))", + "type": "Expression" + }, + "FunctionName": "Log", + "Method": "Post" + } + } + }, { "name": "Pipeline AF Log - ADLS to Blob Start", "type": "ExecutePipeline", @@ -635,17 +709,17 @@ "userProperties": [], "typeProperties": { "pipeline": { - "referenceName": "AZ-Function-Generic", + "referenceName": "AZ_Function_Generic", "type": "PipelineReference" }, "waitOnCompletion": false, "parameters": { "Body": { - "value": "[\n{\n \"TaskInstanceId\":@{pipeline().parameters.TaskObject.TaskInstanceId},\n \"ExecutionUid\":@{pipeline().parameters.TaskObject.ExecutionUid},\n \"RunId\":@{pipeline().RunId}, \n \"StartTime\":@{utcnow()},\n \"Status\":\"Processing\",\n \"Activity\":\"Copy ADLS to Blob\",\n \"Comments\":\"\"\n}\n]", + "value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":3,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy ADLS to Blob\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"Status\":\"Started\"}'))", "type": "Expression" }, "FunctionName": "Log", - "Method": "Put" + "Method": "Post" } } }, @@ -663,45 +737,17 @@ "userProperties": [], "typeProperties": { "pipeline": { - "referenceName": "AZ-Function-Generic", + "referenceName": "AZ_Function_Generic", "type": "PipelineReference" }, "waitOnCompletion": false, "parameters": { "Body": { - "value": "[\n{\n \"TaskInstanceId\":@{pipeline().parameters.TaskObject.TaskInstanceId},\n \"ExecutionUid\":@{pipeline().parameters.TaskObject.ExecutionUid},\n \"RunId\":@{pipeline().RunId}, \n \"EndTime\":@{utcnow()},\n \"Status\":\"Completed\",\n \"Activity\":\"Copy ADLS to Blob\",\n \"Comments\":\"\"\n}\n]", + "value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":1,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy ADLS to Blob\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"EndDateTimeOffSet\":\"', string(utcnow()), '\",\"RowsInserted\":\"', string(activity('Copy ADLS to Blob').output.filesWritten), '\",\"Comment\":\"\",\"Status\":\"Complete\"}'))", "type": "Expression" }, "FunctionName": "Log", - "Method": "Put" - } - } - }, - { - "name": "Pipeline AF Log - ADLS to Blob Failed", - "type": "ExecutePipeline", - "dependsOn": [ - { - "activity": "Copy ADLS to Blob", - "dependencyConditions": [ - "Failed" - ] - } - ], - "userProperties": [], - "typeProperties": { - "pipeline": { - "referenceName": "AZ-Function-Generic", - "type": "PipelineReference" - }, - "waitOnCompletion": false, - "parameters": { - "Body": { - "value": "[\n{\n \"TaskInstanceId\":@{pipeline().parameters.TaskObject.TaskInstanceId},\n \"ExecutionUid\":@{pipeline().parameters.TaskObject.ExecutionUid},\n \"RunId\":@{pipeline().RunId}, \n \"EndTime\":@{utcnow()},\n \"Status\":\"Failed\",\n \"Activity\":\"Copy ADLS to Blob\",\n \"Comments\":@{activity('Copy ADLS to Blob').errors}\n}\n]", - "type": "Expression" - }, - "FunctionName": "Log", - "Method": "Put" + "Method": "Post" } } } @@ -714,14 +760,14 @@ "parameters": { "TaskObject": { "type": "object", - "defaultValue": "[ { \"TaskInstanceId\": 2, \"TaskMasterId\": 1, \"TaskStatus\": \"Untried\", \"TaskType\": \"RelationalDataExtractToDataLake\", \"Enabled\": true, \"ExecutionUid\": 1, \"Target\": { \"Database\": { \"Name\": \"MetadataDB\", \"SystemName\" : \"adsgofastdatakakeaccelsqlsvr.database.windows.net\", \"Type\": \"AzureSQL\", \"AuthenticationType\": \"MSI\", \"UsernameKeyVaultSecretName\": \"\", \"PasswordKeyVaultSecretName\": \"\" }, \"Extraction\": { \"Type\": \"Table\", \"FullOrIncremental\": true, \"IncrementalType\": null, \"TableSchema\" : \"stage\", \"TableName\": \"Customer\" } }, \"Source\": { \"StorageAccountName\": \"https://adsgofastdatalakeaccelst.dfs.core.windows.net\", \"Type\" : \"ADLS\", \"StorageAccountContainer\": \"Container\", \"StorageAccountContainerName\": \"datalakelanding\", \"StorageAccountAccessMethod\": \"MSI\", \"StorageAccountSASUriKeyVaultSecretName\": null, \"RelativePath\": \"/Unprocessed/adsgofastdatakakeaccelsqlsvr/AWSample/SalesLT/2020/06/08/17/\", \"DataFileName\": \"Customer_Data.parquet\", \"SchemaFileName\": \"Customer_Schema.json\" }, \"DataFactory\": { \"Id\": 1, \"Name\": \"adsgofastdatakakeacceladf\", \"ResourceGroup\": \"AdsGoFastDataLakeAccel\", \"SubscriptionId\": \"035a1364-f00d-48e2-b582-4fe125905ee3\", } } ] " + "defaultValue": "[ { \"TaskInstanceId\": 2, \"TaskMasterId\": 1, \"TaskStatus\": \"Untried\", \"TaskType\": \"AzureStorageToAzureStorage_@GF{IR}\", \"Enabled\": true, \"ExecutionUid\": 1, \"KeyVaultBaseUrl\": \"https://adsgofastkeyvault.vault.azure.net/\", \"Source\": { \"StorageAccountName\": \"https://adsgofastdatalakeadls.dfs.core.windows.net\", \"Type\": \"ADLS\", \"StorageAccountContainer\": \"datalakelanding\", \"StorageAccountAccessMethod\": \"MSI\", \"StorageAccountSASUriKeyVaultSecretName\": null, \"RelativePath\": \"/Unprocessed/adsgofastdatakakeaccelsqlsvr/AWSample/SalesLT/2020/06/08/17/\", \"DataFileName\": \"Customer_Data.parquet\", \"SchemaFileName\": \"Customer_Schema.json\" }, \"Target\": { \"StorageAccountName\": \"https://adsgofastdatalakeadls.dfs.core.windows.net\", \"Type\": \"ADLS\", \"StorageAccountContainer\": \"datalakelanding\", \"StorageAccountAccessMethod\": \"MSI\", \"StorageAccountSASUriKeyVaultSecretName\": null, \"RelativePath\": \"/Processed/adsgofastdatakakeaccelsqlsvr/AWSample/SalesLT/2020/06/08/17/\", \"DataFileName\": \"Customer_Data.parquet\", \"SchemaFileName\": \"Customer_Schema.json\" }, \"DataFactory\": { \"Id\": 1, \"Name\": \"adsgofastdatakakeacceladf\", \"ResourceGroup\": \"AdsGoFastDataLakeAccel\", \"SubscriptionId\": \"035a1364-f00d-48e2-b582-4fe125905ee3\", } } ]" } }, "folder": { - "name": "ADS Go Fast/Data Movement/Auto Resolve IR" + "name": "ADS Go Fast/Data Movement/@GF{IR}" }, "annotations": [], - "lastPublishTime": "2020-07-23T08:15:14Z" + "lastPublishTime": "2020-08-05T04:14:00Z" }, "type": "Microsoft.DataFactory/factories/pipelines" } \ No newline at end of file diff --git a/solution/DataFactory/Templates/pipeline/AZ_Storage_CSV_AZ_SQL_{IR}.json b/solution/DataFactory/Templates/pipeline/AZ_Storage_CSV_AZ_SQL_{IR}.json new file mode 100644 index 00000000..21352919 --- /dev/null +++ b/solution/DataFactory/Templates/pipeline/AZ_Storage_CSV_AZ_SQL_{IR}.json @@ -0,0 +1,460 @@ +{ + "name": "AZ_Storage_CSV_AZ_SQL_@GF{IR}", + "properties": { + "activities": [ + { + "name": "Switch Storage Type", + "type": "Switch", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "on": { + "value": "@pipeline().parameters.TaskObject.Source.Type", + "type": "Expression" + }, + "cases": [ + { + "value": "ADLS", + "activities": [ + { + "name": "Copy ADLS to Azure SQL", + "type": "Copy", + "dependsOn": [ + { + "activity": "Pipeline AF Log - ADLS to Azure SQL Start", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "source": { + "type": "DelimitedTextSource", + "storeSettings": { + "type": "AzureBlobFSReadSettings", + "maxConcurrentConnections": { + "value": "@pipeline().parameters.TaskObject.Source.MaxConcorrentConnections", + "type": "Expression" + }, + "recursive": true, + "wildcardFolderPath": { + "value": "@pipeline().parameters.TaskObject.Source.RelativePath", + "type": "Expression" + }, + "wildcardFileName": { + "value": "@pipeline().parameters.TaskObject.Source.DataFileName", + "type": "Expression" + }, + "enablePartitionDiscovery": false + }, + "formatSettings": { + "type": "DelimitedTextReadSettings", + "skipLineCount": { + "value": "@pipeline().parameters.TaskObject.Source.SkipLineCount", + "type": "Expression" + } + } + }, + "sink": { + "type": "AzureSqlSink", + "preCopyScript": { + "value": "@{pipeline().parameters.TaskObject.Target.PreCopySQL}", + "type": "Expression" + }, + "tableOption": "autoCreate", + "disableMetricsCollection": false + }, + "enableStaging": false, + "parallelCopies": { + "value": "@pipeline().parameters.TaskObject.DegreeOfCopyParallelism", + "type": "Expression" + }, + "translator": { + "value": "@pipeline().parameters.TaskObject.Target.DynamicMapping", + "type": "Expression" + } + }, + "inputs": [ + { + "referenceName": "ADLSDelimitedText_@GF{IR}", + "type": "DatasetReference", + "parameters": { + "RelativePath": { + "value": "@pipeline().parameters.TaskObject.Source.RelativePath", + "type": "Expression" + }, + "FileName": { + "value": "@pipeline().parameters.TaskObject.Source.DataFileName", + "type": "Expression" + }, + "StorageAccountEndpoint": { + "value": "@pipeline().parameters.TaskObject.Source.StorageAccountName", + "type": "Expression" + }, + "StorageAccountContainerName": { + "value": "@pipeline().parameters.TaskObject.Source.StorageAccountContainer", + "type": "Expression" + }, + "FirstRownAsHeader": { + "value": "@pipeline().parameters.TaskObject.Source.FirstRowAsHeader", + "type": "Expression" + } + } + } + ], + "outputs": [ + { + "referenceName": "AzureSqlTable_@GF{IR}", + "type": "DatasetReference", + "parameters": { + "Schema": { + "value": "@pipeline().parameters.TaskObject.Target.StagingTableSchema", + "type": "Expression" + }, + "Table": { + "value": "@pipeline().parameters.TaskObject.Target.StagingTableName", + "type": "Expression" + }, + "Server": { + "value": "@pipeline().parameters.TaskObject.Target.Database.SystemName", + "type": "Expression" + }, + "Database": { + "value": "@pipeline().parameters.TaskObject.Target.Database.Name", + "type": "Expression" + } + } + } + ] + }, + { + "name": "Pipeline AF Log - ADLS to Azure SQL Failed", + "type": "ExecutePipeline", + "dependsOn": [ + { + "activity": "Copy ADLS to Azure SQL", + "dependencyConditions": [ + "Failed" + ] + } + ], + "userProperties": [], + "typeProperties": { + "pipeline": { + "referenceName": "AZ_Function_Generic", + "type": "PipelineReference" + }, + "waitOnCompletion": false, + "parameters": { + "Body": { + "value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":1,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy ADLS to Azure SQL\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"EndDateTimeOffSet\":\"', string(utcnow()), '\",\"Comment\":\"', string(activity('Copy ADLS to Azure SQL').error.message), '\",\"Status\":\"Failed\"}'))", + "type": "Expression" + }, + "FunctionName": "Log", + "Method": "Post" + } + } + }, + { + "name": "Pipeline AF Log - ADLS to Azure SQL Start", + "type": "ExecutePipeline", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "pipeline": { + "referenceName": "AZ_Function_Generic", + "type": "PipelineReference" + }, + "waitOnCompletion": false, + "parameters": { + "Body": { + "value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":3,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy ADLS to Azure SQL\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"Status\":\"Started\"}'))", + "type": "Expression" + }, + "FunctionName": "Log", + "Method": "Post" + } + } + }, + { + "name": "Pipeline AF Log - ADLS to Azure SQL Succeed", + "type": "ExecutePipeline", + "dependsOn": [ + { + "activity": "Copy ADLS to Azure SQL", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "userProperties": [], + "typeProperties": { + "pipeline": { + "referenceName": "AZ_Function_Generic", + "type": "PipelineReference" + }, + "waitOnCompletion": false, + "parameters": { + "Body": { + "value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":1,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy ADLS to Azure SQL\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"EndDateTimeOffSet\":\"', string(utcnow()), '\",\"RowsInserted\":\"', string(activity('Copy ADLS to Azure SQL').output.rowsCopied), '\",\"Comment\":\"\",\"Status\":\"Complete\"}'))", + "type": "Expression" + }, + "FunctionName": "Log", + "Method": "Post" + } + } + } + ] + }, + { + "value": "Azure Blob", + "activities": [ + { + "name": "Copy Blob to Azure SQL", + "type": "Copy", + "dependsOn": [ + { + "activity": "Pipeline AF Log - BLOB to Azure SQL Start", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "source": { + "type": "DelimitedTextSource", + "storeSettings": { + "type": "AzureBlobStorageReadSettings", + "maxConcurrentConnections": { + "value": "@pipeline().parameters.TaskObject.Source.MaxConcorrentConnections", + "type": "Expression" + }, + "recursive": true, + "wildcardFolderPath": { + "value": "@pipeline().parameters.TaskObject.Source.RelativePath", + "type": "Expression" + }, + "wildcardFileName": { + "value": "@pipeline().parameters.TaskObject.Source.DataFileName", + "type": "Expression" + }, + "enablePartitionDiscovery": false + }, + "formatSettings": { + "type": "DelimitedTextReadSettings", + "skipLineCount": { + "value": "@pipeline().parameters.TaskObject.Source.SkipLineCount", + "type": "Expression" + } + } + }, + "sink": { + "type": "AzureSqlSink", + "preCopyScript": { + "value": "@{pipeline().parameters.TaskObject.Target.PreCopySQL}", + "type": "Expression" + }, + "tableOption": "autoCreate", + "disableMetricsCollection": false + }, + "enableStaging": false, + "parallelCopies": { + "value": "@pipeline().parameters.TaskObject.DegreeOfCopyParallelism", + "type": "Expression" + }, + "translator": { + "value": "@pipeline().parameters.TaskObject.Target.DynamicMapping", + "type": "Expression" + } + }, + "inputs": [ + { + "referenceName": "BlobDelimitedText_@GF{IR}", + "type": "DatasetReference", + "parameters": { + "RelativePath": { + "value": "@pipeline().parameters.TaskObject.Source.RelativePath", + "type": "Expression" + }, + "FileName": { + "value": "@pipeline().parameters.TaskObject.Source.DataFileName", + "type": "Expression" + }, + "StorageAccountEndpoint": { + "value": "@pipeline().parameters.TaskObject.Source.StorageAccountName", + "type": "Expression" + }, + "StorageAccountContainerName": { + "value": "@pipeline().parameters.TaskObject.Source.StorageAccountContainer", + "type": "Expression" + }, + "FirstRowAsHeader": { + "value": "@pipeline().parameters.TaskObject.Source.FirstRowAsHeader", + "type": "Expression" + } + } + } + ], + "outputs": [ + { + "referenceName": "AzureSqlTable_@GF{IR}", + "type": "DatasetReference", + "parameters": { + "Schema": { + "value": "@pipeline().parameters.TaskObject.Target.StagingTableSchema", + "type": "Expression" + }, + "Table": { + "value": "@pipeline().parameters.TaskObject.Target.StagingTableName", + "type": "Expression" + }, + "Server": { + "value": "@pipeline().parameters.TaskObject.Target.Database.SystemName", + "type": "Expression" + }, + "Database": { + "value": "@pipeline().parameters.TaskObject.Target.Database.Name", + "type": "Expression" + } + } + } + ] + }, + { + "name": "Pipeline AF Log - BLOB to Azure SQL Failed", + "type": "ExecutePipeline", + "dependsOn": [ + { + "activity": "Copy Blob to Azure SQL", + "dependencyConditions": [ + "Failed" + ] + } + ], + "userProperties": [], + "typeProperties": { + "pipeline": { + "referenceName": "AZ_Function_Generic", + "type": "PipelineReference" + }, + "waitOnCompletion": false, + "parameters": { + "Body": { + "value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":1,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy Blob to Azure SQL\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"EndDateTimeOffSet\":\"', string(utcnow()), '\",\"Comment\":\"', string(activity('Copy Blob to Azure SQL').error.message), '\",\"Status\":\"Failed\"}'))", + "type": "Expression" + }, + "FunctionName": "Log", + "Method": "Post" + } + } + }, + { + "name": "Pipeline AF Log - BLOB to Azure SQL Start", + "type": "ExecutePipeline", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "pipeline": { + "referenceName": "AZ_Function_Generic", + "type": "PipelineReference" + }, + "waitOnCompletion": false, + "parameters": { + "Body": { + "value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":3,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy Blob to Azure SQL\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"Status\":\"Started\"}'))", + "type": "Expression" + }, + "FunctionName": "Log", + "Method": "Post" + } + } + }, + { + "name": "Pipeline AF Log - Blob to Azure SQL Succeed", + "type": "ExecutePipeline", + "dependsOn": [ + { + "activity": "Copy Blob to Azure SQL", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "userProperties": [], + "typeProperties": { + "pipeline": { + "referenceName": "AZ_Function_Generic", + "type": "PipelineReference" + }, + "waitOnCompletion": false, + "parameters": { + "Body": { + "value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":1,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy Blob to Azure SQL\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"EndDateTimeOffSet\":\"', string(utcnow()), '\",\"RowsInserted\":\"', string(activity('Copy Blob to Azure SQL').output.rowsCopied), '\",\"Comment\":\"\",\"Status\":\"Complete\"}'))", + "type": "Expression" + }, + "FunctionName": "Log", + "Method": "Post" + } + } + } + ] + } + ] + } + }, + { + "name": "Execute AZ_SQL_Post-Copy", + "type": "ExecutePipeline", + "dependsOn": [ + { + "activity": "Switch Storage Type", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "userProperties": [], + "typeProperties": { + "pipeline": { + "referenceName": "AZ_SQL_Post_Copy_@GF{IR}", + "type": "PipelineReference" + }, + "waitOnCompletion": true, + "parameters": { + "TaskObject": { + "value": "@pipeline().parameters.TaskObject", + "type": "Expression" + } + } + } + } + ], + "parameters": { + "TaskObject": { + "type": "object" + } + }, + "folder": { + "name": "ADS Go Fast/Data Movement/@GF{IR}" + }, + "annotations": [], + "lastPublishTime": "2020-07-29T09:43:40Z" + }, + "type": "Microsoft.DataFactory/factories/pipelines" +} \ No newline at end of file diff --git a/solution/DataFactory/Templates/pipeline/AZ_Storage_Excel_AZ_SQL_{IR}.json b/solution/DataFactory/Templates/pipeline/AZ_Storage_Excel_AZ_SQL_{IR}.json new file mode 100644 index 00000000..3658273e --- /dev/null +++ b/solution/DataFactory/Templates/pipeline/AZ_Storage_Excel_AZ_SQL_{IR}.json @@ -0,0 +1,420 @@ +{ + "name": "AZ_Storage_Excel_AZ_SQL_@GF{IR}", + "properties": { + "activities": [ + { + "name": "Switch Storage Type", + "type": "Switch", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "on": { + "value": "@pipeline().parameters.TaskObject.Source.Type", + "type": "Expression" + }, + "cases": [ + { + "value": "ADLS", + "activities": [ + { + "name": "Copy ADLS to Azure SQL", + "type": "Copy", + "dependsOn": [ + { + "activity": "Pipeline AF Log - ADLS to Azure SQL Start", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "source": { + "type": "ExcelSource", + "storeSettings": { + "type": "AzureBlobFSReadSettings", + "recursive": true + } + }, + "sink": { + "type": "AzureSqlSink", + "preCopyScript": { + "value": "@{pipeline().parameters.TaskObject.Target.PreCopySQL}", + "type": "Expression" + }, + "tableOption": "autoCreate", + "disableMetricsCollection": false + }, + "enableStaging": false, + "parallelCopies": { + "value": "@pipeline().parameters.TaskObject.DegreeOfCopyParallelism", + "type": "Expression" + } + }, + "inputs": [ + { + "referenceName": "ADLSExcel_@GF{IR}", + "type": "DatasetReference", + "parameters": { + "RelativePath": { + "value": "@pipeline().parameters.TaskObject.Source.RelativePath", + "type": "Expression" + }, + "FileName": { + "value": "@pipeline().parameters.TaskObject.Source.DataFileName", + "type": "Expression" + }, + "StorageAccountEndpoint": { + "value": "@pipeline().parameters.TaskObject.Source.StorageAccountName", + "type": "Expression" + }, + "StorageAccountContainerName": { + "value": "@pipeline().parameters.TaskObject.Source.StorageAccountContainer", + "type": "Expression" + }, + "SheetName": { + "value": "@pipeline().parameters.TaskObject.Source.SheetName", + "type": "Expression" + }, + "FirstRowAsHeader": { + "value": "@pipeline().parameters.TaskObject.Source.FirstRowAsHeader", + "type": "Expression" + } + } + } + ], + "outputs": [ + { + "referenceName": "AzureSqlTable_@GF{IR}", + "type": "DatasetReference", + "parameters": { + "Schema": { + "value": "@pipeline().parameters.TaskObject.Target.StagingTableSchema", + "type": "Expression" + }, + "Table": { + "value": "@pipeline().parameters.TaskObject.Target.StagingTableName", + "type": "Expression" + }, + "Server": { + "value": "@pipeline().parameters.TaskObject.Target.Database.SystemName", + "type": "Expression" + }, + "Database": { + "value": "@pipeline().parameters.TaskObject.Target.Database.Name", + "type": "Expression" + } + } + } + ] + }, + { + "name": "Pipeline AF Log - ADLS to Azure SQL Failed", + "type": "ExecutePipeline", + "dependsOn": [ + { + "activity": "Copy ADLS to Azure SQL", + "dependencyConditions": [ + "Failed" + ] + } + ], + "userProperties": [], + "typeProperties": { + "pipeline": { + "referenceName": "AZ_Function_Generic", + "type": "PipelineReference" + }, + "waitOnCompletion": false, + "parameters": { + "Body": { + "value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":1,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy ADLS to Azure SQL\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"EndDateTimeOffSet\":\"', string(utcnow()), '\",\"Comment\":\"', string(activity('Copy ADLS to Azure SQL').error.message), '\",\"Status\":\"Failed\"}'))", + "type": "Expression" + }, + "FunctionName": "Log", + "Method": "Post" + } + } + }, + { + "name": "Pipeline AF Log - ADLS to Azure SQL Start", + "type": "ExecutePipeline", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "pipeline": { + "referenceName": "AZ_Function_Generic", + "type": "PipelineReference" + }, + "waitOnCompletion": false, + "parameters": { + "Body": { + "value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":3,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy ADLS to Azure SQL\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"Status\":\"Started\"}'))", + "type": "Expression" + }, + "FunctionName": "Log", + "Method": "Post" + } + } + }, + { + "name": "Pipeline AF Log - ADLS to Azure SQL Succeed", + "type": "ExecutePipeline", + "dependsOn": [ + { + "activity": "Copy ADLS to Azure SQL", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "userProperties": [], + "typeProperties": { + "pipeline": { + "referenceName": "AZ_Function_Generic", + "type": "PipelineReference" + }, + "waitOnCompletion": false, + "parameters": { + "Body": { + "value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":1,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy ADLS to Azure SQL\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"EndDateTimeOffSet\":\"', string(utcnow()), '\",\"RowsInserted\":\"', string(activity('Copy ADLS to Azure SQL').output.rowsCopied), '\",\"Comment\":\"\",\"Status\":\"Complete\"}'))", + "type": "Expression" + }, + "FunctionName": "Log", + "Method": "Post" + } + } + } + ] + }, + { + "value": "Azure Blob", + "activities": [ + { + "name": "Copy Blob to Azure SQL", + "type": "Copy", + "dependsOn": [ + { + "activity": "Pipeline AF Log - BLOB to Azure SQL Start", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "source": { + "type": "ExcelSource", + "storeSettings": { + "type": "AzureBlobStorageReadSettings", + "recursive": true + } + }, + "sink": { + "type": "AzureSqlSink", + "preCopyScript": { + "value": "@{pipeline().parameters.TaskObject.Target.PreCopySQL}", + "type": "Expression" + }, + "tableOption": "autoCreate", + "disableMetricsCollection": false + }, + "enableStaging": false, + "parallelCopies": { + "value": "@pipeline().parameters.TaskObject.DegreeOfCopyParallelism", + "type": "Expression" + } + }, + "inputs": [ + { + "referenceName": "BlobExcel_@GF{IR}", + "type": "DatasetReference", + "parameters": { + "RelativePath": { + "value": "@pipeline().parameters.TaskObject.Source.RelativePath", + "type": "Expression" + }, + "FileName": { + "value": "@pipeline().parameters.TaskObject.Source.DataFileName", + "type": "Expression" + }, + "StorageAccountEndpoint": { + "value": "@pipeline().parameters.TaskObject.Source.StorageAccountName", + "type": "Expression" + }, + "StorageAccountContainerName": { + "value": "@pipeline().parameters.TaskObject.Source.StorageAccountContainer", + "type": "Expression" + }, + "SheetName": { + "value": "@pipeline().parameters.TaskObject.Source.SheetName", + "type": "Expression" + }, + "FirstRowAsHeader": { + "value": "@pipeline().parameters.TaskObject.Source.FirstRowAsHeader", + "type": "Expression" + } + } + } + ], + "outputs": [ + { + "referenceName": "AzureSqlTable_@GF{IR}", + "type": "DatasetReference", + "parameters": { + "Schema": { + "value": "@pipeline().parameters.TaskObject.Target.StagingTableSchema", + "type": "Expression" + }, + "Table": { + "value": "@pipeline().parameters.TaskObject.Target.StagingTableName", + "type": "Expression" + }, + "Server": { + "value": "@pipeline().parameters.TaskObject.Target.Database.SystemName", + "type": "Expression" + }, + "Database": { + "value": "@pipeline().parameters.TaskObject.Target.Database.Name", + "type": "Expression" + } + } + } + ] + }, + { + "name": "Pipeline AF Log - BLOB to Azure SQL Failed", + "type": "ExecutePipeline", + "dependsOn": [ + { + "activity": "Copy Blob to Azure SQL", + "dependencyConditions": [ + "Failed" + ] + } + ], + "userProperties": [], + "typeProperties": { + "pipeline": { + "referenceName": "AZ_Function_Generic", + "type": "PipelineReference" + }, + "waitOnCompletion": false, + "parameters": { + "Body": { + "value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":1,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy Blob to Azure SQL\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"EndDateTimeOffSet\":\"', string(utcnow()), '\",\"Comment\":\"', string(activity('Copy Blob to Azure SQL').error.message), '\",\"Status\":\"Failed\"}'))", + "type": "Expression" + }, + "FunctionName": "Log", + "Method": "Post" + } + } + }, + { + "name": "Pipeline AF Log - BLOB to Azure SQL Start", + "type": "ExecutePipeline", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "pipeline": { + "referenceName": "AZ_Function_Generic", + "type": "PipelineReference" + }, + "waitOnCompletion": false, + "parameters": { + "Body": { + "value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":3,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy Blob to Azure SQL\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"Status\":\"Started\"}'))", + "type": "Expression" + }, + "FunctionName": "Log", + "Method": "Post" + } + } + }, + { + "name": "Pipeline AF Log - Blob to Azure SQL Succeed", + "type": "ExecutePipeline", + "dependsOn": [ + { + "activity": "Copy Blob to Azure SQL", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "userProperties": [], + "typeProperties": { + "pipeline": { + "referenceName": "AZ_Function_Generic", + "type": "PipelineReference" + }, + "waitOnCompletion": false, + "parameters": { + "Body": { + "value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":1,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy Blob to Azure SQL\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"EndDateTimeOffSet\":\"', string(utcnow()), '\",\"RowsInserted\":\"', string(activity('Copy Blob to Azure SQL').output.rowsCopied), '\",\"Comment\":\"\",\"Status\":\"Complete\"}'))", + "type": "Expression" + }, + "FunctionName": "Log", + "Method": "Post" + } + } + } + ] + } + ] + } + }, + { + "name": "Execute AZ_SQL_Post-Copy", + "type": "ExecutePipeline", + "dependsOn": [ + { + "activity": "Switch Storage Type", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "userProperties": [], + "typeProperties": { + "pipeline": { + "referenceName": "AZ_SQL_Post_Copy_@GF{IR}", + "type": "PipelineReference" + }, + "waitOnCompletion": true, + "parameters": { + "TaskObject": { + "value": "@pipeline().parameters.TaskObject", + "type": "Expression" + } + } + } + } + ], + "parameters": { + "TaskObject": { + "type": "object" + } + }, + "folder": { + "name": "ADS Go Fast/Data Movement/@GF{IR}" + }, + "annotations": [], + "lastPublishTime": "2020-07-24T01:51:29Z" + }, + "type": "Microsoft.DataFactory/factories/pipelines" +} \ No newline at end of file diff --git a/solution/DataFactory/Templates/pipeline/AZ_Storage_Excel_AZ_Storage_CSV_{IR}.json b/solution/DataFactory/Templates/pipeline/AZ_Storage_Excel_AZ_Storage_CSV_{IR}.json new file mode 100644 index 00000000..dd943b22 --- /dev/null +++ b/solution/DataFactory/Templates/pipeline/AZ_Storage_Excel_AZ_Storage_CSV_{IR}.json @@ -0,0 +1,406 @@ +{ + "name": "AZ_Storage_Excel_AZ_Storage_CSV_@GF{IR}", + "properties": { + "activities": [ + { + "name": "Switch Storage Type", + "type": "Switch", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "on": { + "value": "@pipeline().parameters.TaskObject.Source.Type", + "type": "Expression" + }, + "cases": [ + { + "value": "ADLS", + "activities": [ + { + "name": "Copy ADLS to ADLS", + "type": "Copy", + "dependsOn": [ + { + "activity": "Pipeline AF Log - ADLS to ADLS Start", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "source": { + "type": "ExcelSource", + "storeSettings": { + "type": "AzureBlobFSReadSettings", + "recursive": true + } + }, + "sink": { + "type": "DelimitedTextSink", + "storeSettings": { + "type": "AzureBlobFSWriteSettings" + }, + "formatSettings": { + "type": "DelimitedTextWriteSettings", + "quoteAllText": true, + "fileExtension": ".txt" + } + }, + "enableStaging": false, + "parallelCopies": { + "value": "@pipeline().parameters.TaskObject.DegreeOfCopyParallelism", + "type": "Expression" + } + }, + "inputs": [ + { + "referenceName": "ADLSExcel_@GF{IR}", + "type": "DatasetReference", + "parameters": { + "RelativePath": { + "value": "@pipeline().parameters.TaskObject.Source.RelativePath", + "type": "Expression" + }, + "FileName": { + "value": "@pipeline().parameters.TaskObject.Source.DataFileName", + "type": "Expression" + }, + "StorageAccountEndpoint": { + "value": "@pipeline().parameters.TaskObject.Source.StorageAccountName", + "type": "Expression" + }, + "StorageAccountContainerName": { + "value": "@pipeline().parameters.TaskObject.Source.StorageAccountContainer", + "type": "Expression" + }, + "SheetName": { + "value": "@pipeline().parameters.TaskObject.Source.SheetName", + "type": "Expression" + }, + "FirstRowAsHeader": { + "value": "@pipeline().parameters.TaskObject.Source.FirstRowAsHeader", + "type": "Expression" + } + } + } + ], + "outputs": [ + { + "referenceName": "ADLSDelimitedText_@GF{IR}", + "type": "DatasetReference", + "parameters": { + "RelativePath": { + "value": "@pipeline().parameters.TaskObject.Target.RelativePath", + "type": "Expression" + }, + "FileName": { + "value": "@pipeline().parameters.TaskObject.Target.DataFileName", + "type": "Expression" + }, + "StorageAccountEndpoint": { + "value": "@pipeline().parameters.TaskObject.Target.StorageAccountName", + "type": "Expression" + }, + "StorageAccountContainerName": { + "value": "@pipeline().parameters.TaskObject.Target.StorageAccountContainer", + "type": "Expression" + }, + "FirstRownAsHeader": { + "value": "@pipeline().parameters.TaskObject.Target.FirstRowAsHeader", + "type": "Expression" + } + } + } + ] + }, + { + "name": "Pipeline AF Log - ADLS to ADLS Failed", + "type": "ExecutePipeline", + "dependsOn": [ + { + "activity": "Copy ADLS to ADLS", + "dependencyConditions": [ + "Failed" + ] + } + ], + "userProperties": [], + "typeProperties": { + "pipeline": { + "referenceName": "AZ_Function_Generic", + "type": "PipelineReference" + }, + "waitOnCompletion": false, + "parameters": { + "Body": { + "value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":1,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy ADLS to ADLS\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"EndDateTimeOffSet\":\"', string(utcnow()), '\",\"Comment\":\"', string(activity('Copy ADLS to ADLS').error.message), '\",\"Status\":\"Failed\"}'))", + "type": "Expression" + }, + "FunctionName": "Log", + "Method": "Post" + } + } + }, + { + "name": "Pipeline AF Log - ADLS to ADLS Start", + "type": "ExecutePipeline", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "pipeline": { + "referenceName": "AZ_Function_Generic", + "type": "PipelineReference" + }, + "waitOnCompletion": false, + "parameters": { + "Body": { + "value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":3,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy ADLS to ADLS\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"Status\":\"Started\"}'))", + "type": "Expression" + }, + "FunctionName": "Log", + "Method": "Post" + } + } + }, + { + "name": "Pipeline AF Log - ADLS to ADLS Succeed", + "type": "ExecutePipeline", + "dependsOn": [ + { + "activity": "Copy ADLS to ADLS", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "userProperties": [], + "typeProperties": { + "pipeline": { + "referenceName": "AZ_Function_Generic", + "type": "PipelineReference" + }, + "waitOnCompletion": false, + "parameters": { + "Body": { + "value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":1,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy ADLS to ADLS\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"EndDateTimeOffSet\":\"', string(utcnow()), '\",\"RowsInserted\":\"', string(activity('Copy ADLS to ADLS').output.rowsCopied), '\",\"Comment\":\"\",\"Status\":\"Complete\"}'))", + "type": "Expression" + }, + "FunctionName": "Log", + "Method": "Post" + } + } + } + ] + }, + { + "value": "Azure Blob", + "activities": [ + { + "name": "Copy Blob to Blob", + "type": "Copy", + "dependsOn": [ + { + "activity": "Pipeline AF Log - BLOB to Blob Start", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "source": { + "type": "ExcelSource", + "storeSettings": { + "type": "AzureBlobStorageReadSettings", + "recursive": true + } + }, + "sink": { + "type": "DelimitedTextSink", + "storeSettings": { + "type": "AzureBlobStorageWriteSettings" + }, + "formatSettings": { + "type": "DelimitedTextWriteSettings", + "quoteAllText": true, + "fileExtension": ".txt" + } + }, + "enableStaging": false, + "parallelCopies": { + "value": "@pipeline().parameters.TaskObject.DegreeOfCopyParallelism", + "type": "Expression" + } + }, + "inputs": [ + { + "referenceName": "BlobExcel_@GF{IR}", + "type": "DatasetReference", + "parameters": { + "RelativePath": { + "value": "@pipeline().parameters.TaskObject.Source.RelativePath", + "type": "Expression" + }, + "FileName": { + "value": "@pipeline().parameters.TaskObject.Source.DataFileName", + "type": "Expression" + }, + "StorageAccountEndpoint": { + "value": "@pipeline().parameters.TaskObject.Source.StorageAccountName", + "type": "Expression" + }, + "StorageAccountContainerName": { + "value": "@pipeline().parameters.TaskObject.Source.StorageAccountContainer", + "type": "Expression" + }, + "SheetName": { + "value": "@pipeline().parameters.TaskObject.Source.SheetName", + "type": "Expression" + }, + "FirstRowAsHeader": { + "value": "@pipeline().parameters.TaskObject.Source.FirstRowAsHeader", + "type": "Expression" + } + } + } + ], + "outputs": [ + { + "referenceName": "BlobDelimitedText_@GF{IR}", + "type": "DatasetReference", + "parameters": { + "RelativePath": { + "value": "@pipeline().parameters.TaskObject.Target.RelativePath", + "type": "Expression" + }, + "FileName": { + "value": "@pipeline().parameters.TaskObject.Target.DataFileName", + "type": "Expression" + }, + "StorageAccountEndpoint": { + "value": "@pipeline().parameters.TaskObject.Target.StorageAccountName", + "type": "Expression" + }, + "StorageAccountContainerName": { + "value": "@pipeline().parameters.TaskObject.Target.StorageAccountContainer", + "type": "Expression" + }, + "FirstRowAsHeader": { + "value": "@pipeline().parameters.TaskObject.Target.FirstRowAsHeader", + "type": "Expression" + } + } + } + ] + }, + { + "name": "Pipeline AF Log - BLOB to Azure SQL Failed", + "type": "ExecutePipeline", + "dependsOn": [ + { + "activity": "Copy Blob to Blob", + "dependencyConditions": [ + "Failed" + ] + } + ], + "userProperties": [], + "typeProperties": { + "pipeline": { + "referenceName": "AZ_Function_Generic", + "type": "PipelineReference" + }, + "waitOnCompletion": false, + "parameters": { + "Body": { + "value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":1,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy Blob to Blob\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"EndDateTimeOffSet\":\"', string(utcnow()), '\",\"Comment\":\"', string(activity('Copy Blob to Blob').error.message), '\",\"Status\":\"Failed\"}'))", + "type": "Expression" + }, + "FunctionName": "Log", + "Method": "Post" + } + } + }, + { + "name": "Pipeline AF Log - BLOB to Blob Start", + "type": "ExecutePipeline", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "pipeline": { + "referenceName": "AZ_Function_Generic", + "type": "PipelineReference" + }, + "waitOnCompletion": false, + "parameters": { + "Body": { + "value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":3,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy Blob to Blob\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"Status\":\"Started\"}'))", + "type": "Expression" + }, + "FunctionName": "Log", + "Method": "Post" + } + } + }, + { + "name": "Pipeline AF Log - Blob to Azure SQL Succeed", + "type": "ExecutePipeline", + "dependsOn": [ + { + "activity": "Copy Blob to Blob", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "userProperties": [], + "typeProperties": { + "pipeline": { + "referenceName": "AZ_Function_Generic", + "type": "PipelineReference" + }, + "waitOnCompletion": false, + "parameters": { + "Body": { + "value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":1,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy Blob to Blob\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"EndDateTimeOffSet\":\"', string(utcnow()), '\",\"RowsInserted\":\"', string(activity('Copy Blob to Blob').output.rowsCopied), '\",\"Comment\":\"\",\"Status\":\"Complete\"}'))", + "type": "Expression" + }, + "FunctionName": "Log", + "Method": "Post" + } + } + } + ] + } + ] + } + } + ], + "parameters": { + "TaskObject": { + "type": "object" + } + }, + "folder": { + "name": "ADS Go Fast/Data Movement/@GF{IR}" + }, + "annotations": [], + "lastPublishTime": "2020-08-05T08:58:04Z" + }, + "type": "Microsoft.DataFactory/factories/pipelines" +} \ No newline at end of file diff --git a/solution/DataFactory/Templates/pipeline/AZ_Storage_JSON_AZ_SQL_{IR}.json b/solution/DataFactory/Templates/pipeline/AZ_Storage_JSON_AZ_SQL_{IR}.json new file mode 100644 index 00000000..f16dd48f --- /dev/null +++ b/solution/DataFactory/Templates/pipeline/AZ_Storage_JSON_AZ_SQL_{IR}.json @@ -0,0 +1,408 @@ +{ + "name": "AZ_Storage_JSON_AZ_SQL_@GF{IR}", + "properties": { + "activities": [ + { + "name": "Switch Storage Type", + "type": "Switch", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "on": { + "value": "@pipeline().parameters.TaskObject.Source.Type", + "type": "Expression" + }, + "cases": [ + { + "value": "ADLS", + "activities": [ + { + "name": "Copy ADLS to Azure SQL", + "type": "Copy", + "dependsOn": [ + { + "activity": "Pipeline AF Log - ADLS to Azure SQL Start", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "source": { + "type": "JsonSource", + "storeSettings": { + "type": "AzureBlobFSReadSettings", + "recursive": true + }, + "formatSettings": { + "type": "JsonReadSettings" + } + }, + "sink": { + "type": "AzureSqlSink", + "preCopyScript": { + "value": "@{pipeline().parameters.TaskObject.Target.PreCopySQL}", + "type": "Expression" + }, + "disableMetricsCollection": false + }, + "enableStaging": false, + "parallelCopies": { + "value": "@pipeline().parameters.TaskObject.DegreeOfCopyParallelism", + "type": "Expression" + } + }, + "inputs": [ + { + "referenceName": "ADLSJson_@GF{IR}", + "type": "DatasetReference", + "parameters": { + "RelativePath": { + "value": "@pipeline().parameters.TaskObject.Source.RelativePath", + "type": "Expression" + }, + "FileName": { + "value": "@pipeline().parameters.TaskObject.Source.DataFileName", + "type": "Expression" + }, + "StorageAccountEndpoint": { + "value": "@pipeline().parameters.TaskObject.Source.StorageAccountName", + "type": "Expression" + }, + "StorageAccountContainerName": { + "value": "@pipeline().parameters.TaskObject.Source.StorageAccountContainer", + "type": "Expression" + } + } + } + ], + "outputs": [ + { + "referenceName": "AzureSqlTable_@GF{IR}", + "type": "DatasetReference", + "parameters": { + "Schema": { + "value": "@pipeline().parameters.TaskObject.Target.StagingTableSchema", + "type": "Expression" + }, + "Table": { + "value": "@pipeline().parameters.TaskObject.Target.StagingTableName", + "type": "Expression" + }, + "Server": { + "value": "@pipeline().parameters.TaskObject.Target.Database.SystemName", + "type": "Expression" + }, + "Database": { + "value": "@pipeline().parameters.TaskObject.Target.Database.Name", + "type": "Expression" + } + } + } + ] + }, + { + "name": "Pipeline AF Log - ADLS to Azure SQL Failed", + "type": "ExecutePipeline", + "dependsOn": [ + { + "activity": "Copy ADLS to Azure SQL", + "dependencyConditions": [ + "Failed" + ] + } + ], + "userProperties": [], + "typeProperties": { + "pipeline": { + "referenceName": "AZ_Function_Generic", + "type": "PipelineReference" + }, + "waitOnCompletion": false, + "parameters": { + "Body": { + "value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":1,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy ADLS to Azure SQL\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"EndDateTimeOffSet\":\"', string(utcnow()), '\",\"Comment\":\"', string(activity('Copy ADLS to Azure SQL').error.message), '\",\"Status\":\"Failed\"}'))", + "type": "Expression" + }, + "FunctionName": "Log", + "Method": "Post" + } + } + }, + { + "name": "Pipeline AF Log - ADLS to Azure SQL Start", + "type": "ExecutePipeline", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "pipeline": { + "referenceName": "AZ_Function_Generic", + "type": "PipelineReference" + }, + "waitOnCompletion": false, + "parameters": { + "Body": { + "value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":3,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy ADLS to Azure SQL\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"Status\":\"Started\"}'))", + "type": "Expression" + }, + "FunctionName": "Log", + "Method": "Post" + } + } + }, + { + "name": "Pipeline AF Log - ADLS to Azure SQL Succeed", + "type": "ExecutePipeline", + "dependsOn": [ + { + "activity": "Copy ADLS to Azure SQL", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "userProperties": [], + "typeProperties": { + "pipeline": { + "referenceName": "AZ_Function_Generic", + "type": "PipelineReference" + }, + "waitOnCompletion": false, + "parameters": { + "Body": { + "value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":1,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy ADLS to Azure SQL\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"EndDateTimeOffSet\":\"', string(utcnow()), '\",\"RowsInserted\":\"', string(activity('Copy ADLS to Azure SQL').output.rowsCopied), '\",\"Comment\":\"\",\"Status\":\"Complete\"}'))", + "type": "Expression" + }, + "FunctionName": "Log", + "Method": "Post" + } + } + } + ] + }, + { + "value": "Azure Blob", + "activities": [ + { + "name": "Copy Blob to Azure SQL", + "type": "Copy", + "dependsOn": [ + { + "activity": "Pipeline AF Log - BLOB to Azure SQL Start", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "source": { + "type": "JsonSource", + "storeSettings": { + "type": "AzureBlobStorageReadSettings", + "recursive": true + }, + "formatSettings": { + "type": "JsonReadSettings" + } + }, + "sink": { + "type": "AzureSqlSink", + "preCopyScript": { + "value": "@{pipeline().parameters.TaskObject.Target.PreCopySQL}", + "type": "Expression" + }, + "disableMetricsCollection": false + }, + "enableStaging": false, + "parallelCopies": { + "value": "@pipeline().parameters.TaskObject.DegreeOfCopyParallelism", + "type": "Expression" + } + }, + "inputs": [ + { + "referenceName": "BlobJson_@GF{IR}", + "type": "DatasetReference", + "parameters": { + "RelativePath": { + "value": "@pipeline().parameters.TaskObject.Source.RelativePath", + "type": "Expression" + }, + "FileName": { + "value": "@pipeline().parameters.TaskObject.Source.DataFileName", + "type": "Expression" + }, + "StorageAccountEndpoint": { + "value": "@pipeline().parameters.TaskObject.Source.StorageAccountName", + "type": "Expression" + }, + "StorageAccountContainerName": { + "value": "@pipeline().parameters.TaskObject.Source.StorageAccountContainer", + "type": "Expression" + } + } + } + ], + "outputs": [ + { + "referenceName": "AzureSqlTable_@GF{IR}", + "type": "DatasetReference", + "parameters": { + "Schema": { + "value": "@pipeline().parameters.TaskObject.Target.StagingTableSchema", + "type": "Expression" + }, + "Table": { + "value": "@pipeline().parameters.TaskObject.Target.StagingTableName", + "type": "Expression" + }, + "Server": { + "value": "@pipeline().parameters.TaskObject.Target.Database.SystemName", + "type": "Expression" + }, + "Database": { + "value": "@pipeline().parameters.TaskObject.Target.Database.Name", + "type": "Expression" + } + } + } + ] + }, + { + "name": "Pipeline AF Log - BLOB to Azure SQL Failed", + "type": "ExecutePipeline", + "dependsOn": [ + { + "activity": "Copy Blob to Azure SQL", + "dependencyConditions": [ + "Failed" + ] + } + ], + "userProperties": [], + "typeProperties": { + "pipeline": { + "referenceName": "AZ_Function_Generic", + "type": "PipelineReference" + }, + "waitOnCompletion": false, + "parameters": { + "Body": { + "value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":1,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy Blob to Azure SQL\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"EndDateTimeOffSet\":\"', string(utcnow()), '\",\"Comment\":\"', string(activity('Copy Blob to Azure SQL').error.message), '\",\"Status\":\"Failed\"}'))", + "type": "Expression" + }, + "FunctionName": "Log", + "Method": "Post" + } + } + }, + { + "name": "Pipeline AF Log - BLOB to Azure SQL Start", + "type": "ExecutePipeline", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "pipeline": { + "referenceName": "AZ_Function_Generic", + "type": "PipelineReference" + }, + "waitOnCompletion": false, + "parameters": { + "Body": { + "value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":3,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy Blob to Azure SQL\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"Status\":\"Started\"}'))", + "type": "Expression" + }, + "FunctionName": "Log", + "Method": "Post" + } + } + }, + { + "name": "Pipeline AF Log - Blob to Azure SQL Succeed", + "type": "ExecutePipeline", + "dependsOn": [ + { + "activity": "Copy Blob to Azure SQL", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "userProperties": [], + "typeProperties": { + "pipeline": { + "referenceName": "AZ_Function_Generic", + "type": "PipelineReference" + }, + "waitOnCompletion": false, + "parameters": { + "Body": { + "value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":1,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy Blob to Azure SQL\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"EndDateTimeOffSet\":\"', string(utcnow()), '\",\"RowsInserted\":\"', string(activity('Copy Blob to Azure SQL').output.rowsCopied), '\",\"Comment\":\"\",\"Status\":\"Complete\"}'))", + "type": "Expression" + }, + "FunctionName": "Log", + "Method": "Post" + } + } + } + ] + } + ] + } + }, + { + "name": "Execute AZ_SQL_Post-Copy", + "type": "ExecutePipeline", + "dependsOn": [ + { + "activity": "Switch Storage Type", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "userProperties": [], + "typeProperties": { + "pipeline": { + "referenceName": "AZ_SQL_Post_Copy_@GF{IR}", + "type": "PipelineReference" + }, + "waitOnCompletion": true, + "parameters": { + "TaskObject": { + "value": "@pipeline().parameters.TaskObject", + "type": "Expression" + } + } + } + } + ], + "parameters": { + "TaskObject": { + "type": "object" + } + }, + "folder": { + "name": "ADS Go Fast/Data Movement/@GF{IR}" + }, + "annotations": [], + "lastPublishTime": "2020-07-29T05:19:06Z" + }, + "type": "Microsoft.DataFactory/factories/pipelines" +} \ No newline at end of file diff --git a/solution/DataFactory/Templates/pipeline/AZ_Storage_Parquet_AZ_SQL_{IR}.json b/solution/DataFactory/Templates/pipeline/AZ_Storage_Parquet_AZ_SQL_{IR}.json new file mode 100644 index 00000000..31739c04 --- /dev/null +++ b/solution/DataFactory/Templates/pipeline/AZ_Storage_Parquet_AZ_SQL_{IR}.json @@ -0,0 +1,494 @@ +{ + "name": "AZ_Storage_Parquet_AZ_SQL_@GF{IR}", + "properties": { + "activities": [ + { + "name": "Switch Storage Type", + "type": "Switch", + "dependsOn": [ + { + "activity": "AF Get Mapping", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "userProperties": [], + "typeProperties": { + "on": { + "value": "@pipeline().parameters.TaskObject.Source.Type", + "type": "Expression" + }, + "cases": [ + { + "value": "ADLS", + "activities": [ + { + "name": "Copy ADLS to Azure SQL", + "type": "Copy", + "dependsOn": [ + { + "activity": "Pipeline AF Log - ADLS to Azure SQL Start", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "source": { + "type": "ParquetSource", + "storeSettings": { + "type": "AzureBlobFSReadSettings", + "recursive": false, + "wildcardFolderPath": { + "value": "@pipeline().parameters.TaskObject.Source.RelativePath", + "type": "Expression" + }, + "wildcardFileName": "*.parquet", + "enablePartitionDiscovery": false + } + }, + "sink": { + "type": "AzureSqlSink", + "preCopyScript": { + "value": "@{pipeline().parameters.TaskObject.Target.PreCopySQL}", + "type": "Expression" + }, + "disableMetricsCollection": false + }, + "enableStaging": false, + "parallelCopies": { + "value": "@pipeline().parameters.TaskObject.DegreeOfCopyParallelism", + "type": "Expression" + }, + "translator": { + "value": "@activity('AF Get Mapping').output.value", + "type": "Expression" + } + }, + "inputs": [ + { + "referenceName": "ADLSParquet_@GF{IR}", + "type": "DatasetReference", + "parameters": { + "RelativePath": { + "value": "@pipeline().parameters.TaskObject.Source.RelativePath", + "type": "Expression" + }, + "FileName": { + "value": "@pipeline().parameters.TaskObject.Source.DataFileName", + "type": "Expression" + }, + "StorageAccountEndpoint": { + "value": "@pipeline().parameters.TaskObject.Source.StorageAccountName", + "type": "Expression" + }, + "StorageAccountContainerName": { + "value": "@pipeline().parameters.TaskObject.Source.StorageAccountContainer", + "type": "Expression" + } + } + } + ], + "outputs": [ + { + "referenceName": "AzureSqlTable_@GF{IR}", + "type": "DatasetReference", + "parameters": { + "Schema": { + "value": "@pipeline().parameters.TaskObject.Target.StagingTableSchema", + "type": "Expression" + }, + "Table": { + "value": "@pipeline().parameters.TaskObject.Target.StagingTableName", + "type": "Expression" + }, + "Server": { + "value": "@pipeline().parameters.TaskObject.Target.Database.SystemName", + "type": "Expression" + }, + "Database": { + "value": "@pipeline().parameters.TaskObject.Target.Database.Name", + "type": "Expression" + } + } + } + ] + }, + { + "name": "Pipeline AF Log - ADLS to Azure SQL Failed", + "type": "ExecutePipeline", + "dependsOn": [ + { + "activity": "Copy ADLS to Azure SQL", + "dependencyConditions": [ + "Failed" + ] + } + ], + "userProperties": [], + "typeProperties": { + "pipeline": { + "referenceName": "AZ_Function_Generic", + "type": "PipelineReference" + }, + "waitOnCompletion": false, + "parameters": { + "Body": { + "value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":1,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy ADLS to Azure SQL\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"EndDateTimeOffSet\":\"', string(utcnow()), '\",\"Comment\":\"', string(activity('Copy ADLS to Azure SQL').error.message), '\",\"Status\":\"Failed\"}'))", + "type": "Expression" + }, + "FunctionName": "Log", + "Method": "Post" + } + } + }, + { + "name": "Pipeline AF Log - ADLS to Azure SQL Start", + "type": "ExecutePipeline", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "pipeline": { + "referenceName": "AZ_Function_Generic", + "type": "PipelineReference" + }, + "waitOnCompletion": false, + "parameters": { + "Body": { + "value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":3,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy ADLS to Azure SQL\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"Status\":\"Started\"}'))", + "type": "Expression" + }, + "FunctionName": "Log", + "Method": "Post" + } + } + }, + { + "name": "Pipeline AF Log - ADLS to Azure SQL Succeed", + "type": "ExecutePipeline", + "dependsOn": [ + { + "activity": "Copy ADLS to Azure SQL", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "userProperties": [], + "typeProperties": { + "pipeline": { + "referenceName": "AZ_Function_Generic", + "type": "PipelineReference" + }, + "waitOnCompletion": false, + "parameters": { + "Body": { + "value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":1,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy ADLS to Azure SQL\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"EndDateTimeOffSet\":\"', string(utcnow()), '\",\"RowsInserted\":\"', string(activity('Copy ADLS to Azure SQL').output.rowsCopied), '\",\"Comment\":\"\",\"Status\":\"Complete\"}'))", + "type": "Expression" + }, + "FunctionName": "Log", + "Method": "Post" + } + } + } + ] + }, + { + "value": "Azure Blob", + "activities": [ + { + "name": "Copy Blob to Azure SQL", + "type": "Copy", + "dependsOn": [ + { + "activity": "Pipeline AF Log - BLOB to Azure SQL Start", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "source": { + "type": "ParquetSource", + "storeSettings": { + "type": "AzureBlobStorageReadSettings", + "recursive": false, + "wildcardFolderPath": { + "value": "@pipeline().parameters.TaskObject.Source.RelativePath", + "type": "Expression" + }, + "wildcardFileName": "*.parquet", + "enablePartitionDiscovery": false + } + }, + "sink": { + "type": "AzureSqlSink", + "preCopyScript": { + "value": "@{pipeline().parameters.TaskObject.Target.PreCopySQL}", + "type": "Expression" + }, + "disableMetricsCollection": false + }, + "enableStaging": false, + "parallelCopies": { + "value": "@pipeline().parameters.TaskObject.DegreeOfCopyParallelism", + "type": "Expression" + }, + "translator": { + "value": "@activity('AF Get Mapping').output.value", + "type": "Expression" + } + }, + "inputs": [ + { + "referenceName": "BlobParquet_@GF{IR}", + "type": "DatasetReference", + "parameters": { + "RelativePath": { + "value": "@pipeline().parameters.TaskObject.Source.RelativePath", + "type": "Expression" + }, + "FileName": { + "value": "@pipeline().parameters.TaskObject.Source.DataFileName", + "type": "Expression" + }, + "StorageAccountEndpoint": { + "value": "@pipeline().parameters.TaskObject.Source.StorageAccountName", + "type": "Expression" + }, + "StorageAccountContainerName": { + "value": "@pipeline().parameters.TaskObject.Source.StorageAccountContainer", + "type": "Expression" + } + } + } + ], + "outputs": [ + { + "referenceName": "AzureSqlTable_@GF{IR}", + "type": "DatasetReference", + "parameters": { + "Schema": { + "value": "@pipeline().parameters.TaskObject.Target.StagingTableSchema", + "type": "Expression" + }, + "Table": { + "value": "@pipeline().parameters.TaskObject.Target.StagingTableName", + "type": "Expression" + }, + "Server": { + "value": "@pipeline().parameters.TaskObject.Target.Database.SystemName", + "type": "Expression" + }, + "Database": { + "value": "@pipeline().parameters.TaskObject.Target.Database.Name", + "type": "Expression" + } + } + } + ] + }, + { + "name": "Pipeline AF Log - BLOB to Azure SQL Failed", + "type": "ExecutePipeline", + "dependsOn": [ + { + "activity": "Copy Blob to Azure SQL", + "dependencyConditions": [ + "Failed" + ] + } + ], + "userProperties": [], + "typeProperties": { + "pipeline": { + "referenceName": "AZ_Function_Generic", + "type": "PipelineReference" + }, + "waitOnCompletion": false, + "parameters": { + "Body": { + "value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":1,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy Blob to Azure SQL\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"EndDateTimeOffSet\":\"', string(utcnow()), '\",\"Comment\":\"', string(activity('Copy Blob to Azure SQL').error.message), '\",\"Status\":\"Failed\"}'))", + "type": "Expression" + }, + "FunctionName": "Log", + "Method": "Post" + } + } + }, + { + "name": "Pipeline AF Log - BLOB to Azure SQL Start", + "type": "ExecutePipeline", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "pipeline": { + "referenceName": "AZ_Function_Generic", + "type": "PipelineReference" + }, + "waitOnCompletion": false, + "parameters": { + "Body": { + "value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":3,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy Blob to Azure SQL\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"Status\":\"Started\"}'))", + "type": "Expression" + }, + "FunctionName": "Log", + "Method": "Post" + } + } + }, + { + "name": "Pipeline AF Log - Blob to Azure SQL Succeed", + "type": "ExecutePipeline", + "dependsOn": [ + { + "activity": "Copy Blob to Azure SQL", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "userProperties": [], + "typeProperties": { + "pipeline": { + "referenceName": "AZ_Function_Generic", + "type": "PipelineReference" + }, + "waitOnCompletion": false, + "parameters": { + "Body": { + "value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":1,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy Blob to Azure SQL\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"EndDateTimeOffSet\":\"', string(utcnow()), '\",\"RowsInserted\":\"', string(activity('Copy Blob to Azure SQL').output.rowsCopied), '\",\"Comment\":\"\",\"Status\":\"Complete\"}'))", + "type": "Expression" + }, + "FunctionName": "Log", + "Method": "Post" + } + } + } + ] + } + ] + } + }, + { + "name": "If Auto Create Table", + "type": "IfCondition", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "expression": { + "value": "@bool(pipeline().parameters.TaskObject.Target.AutoCreateTable)", + "type": "Expression" + }, + "ifTrueActivities": [ + { + "name": "Execute Create Table", + "type": "ExecutePipeline", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "pipeline": { + "referenceName": "AZ_SQL_Create_Table_@GF{IR}", + "type": "PipelineReference" + }, + "waitOnCompletion": true, + "parameters": { + "TaskObject": { + "value": "@pipeline().parameters.TaskObject", + "type": "Expression" + } + } + } + } + ] + } + }, + { + "name": "Execute AZ_SQL_Post-Copy", + "type": "ExecutePipeline", + "dependsOn": [ + { + "activity": "Switch Storage Type", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "userProperties": [], + "typeProperties": { + "pipeline": { + "referenceName": "AZ_SQL_Post_Copy_@GF{IR}", + "type": "PipelineReference" + }, + "waitOnCompletion": true, + "parameters": { + "TaskObject": { + "value": "@pipeline().parameters.TaskObject", + "type": "Expression" + } + } + } + }, + { + "name": "AF Get Mapping", + "type": "AzureFunctionActivity", + "dependsOn": [ + { + "activity": "If Auto Create Table", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "functionName": "GetSourceTargetMapping", + "method": "POST", + "body": { + "value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"StorageAccountName\":\"', string(pipeline().parameters.TaskObject.Source.StorageAccountName), '\",\"StorageAccountContainer\":\"', string(pipeline().parameters.TaskObject.Source.StorageAccountContainer), '\",\"RelativePath\":\"', string(pipeline().parameters.TaskObject.Source.RelativePath), '\",\"SchemaFileName\":\"', string(pipeline().parameters.TaskObject.Source.SchemaFileName), '\",\"SourceType\":\"', string(pipeline().parameters.TaskObject.Source.Type), '\",\"TargetType\":\"', string(pipeline().parameters.TaskObject.Target.Type), '\",\"MetadataType\":\"Parquet\"}'))", + "type": "Expression" + } + }, + "linkedServiceName": { + "referenceName": "AzureFunctionAdsGoFastDataLakeAccelFunApp", + "type": "LinkedServiceReference" + } + } + ], + "parameters": { + "TaskObject": { + "type": "object" + } + }, + "folder": { + "name": "ADS Go Fast/Data Movement/@GF{IR}" + }, + "annotations": [], + "lastPublishTime": "2020-08-05T03:16:41Z" + }, + "type": "Microsoft.DataFactory/factories/pipelines" +} \ No newline at end of file diff --git a/solution/DataFactory/pipeline/Create-Task-Master-AZ-SQL-OnP-SH-IR.json b/solution/DataFactory/Templates/pipeline/Create_Task_Master_AZ_SQL_{IR}.json similarity index 90% rename from solution/DataFactory/pipeline/Create-Task-Master-AZ-SQL-OnP-SH-IR.json rename to solution/DataFactory/Templates/pipeline/Create_Task_Master_AZ_SQL_{IR}.json index 32bb0e15..75a345c5 100644 --- a/solution/DataFactory/pipeline/Create-Task-Master-AZ-SQL-OnP-SH-IR.json +++ b/solution/DataFactory/Templates/pipeline/Create_Task_Master_AZ_SQL_{IR}.json @@ -1,5 +1,5 @@ { - "name": "Create-Task-Master-AZ-SQL-OnP-SH-IR", + "name": "Create-Task_MASTER_AZ_SQL_@GF{IR}", "properties": { "activities": [ { @@ -16,7 +16,7 @@ "userProperties": [], "typeProperties": { "pipeline": { - "referenceName": "AZ-Function-Generic", + "referenceName": "AZ_Function_Generic", "type": "PipelineReference" }, "waitOnCompletion": false, @@ -37,7 +37,7 @@ "userProperties": [], "typeProperties": { "pipeline": { - "referenceName": "AZ-Function-Generic", + "referenceName": "AZ_Function_Generic", "type": "PipelineReference" }, "waitOnCompletion": false, @@ -65,7 +65,7 @@ "userProperties": [], "typeProperties": { "pipeline": { - "referenceName": "AZ-Function-Generic", + "referenceName": "AZ_Function_Generic", "type": "PipelineReference" }, "waitOnCompletion": false, @@ -109,25 +109,17 @@ "partitionOption": "None" }, "dataset": { - "referenceName": "SqlServerTable_OnPrem_SH_IR", + "referenceName": "AzureSqlTable_@GF{IR}", "type": "DatasetReference", "parameters": { - "TableSchema": { + "Schema": { "value": "@pipeline().parameters.TaskObject.Source.Extraction.TableSchema", "type": "Expression" }, - "TableName": { + "Table": { "value": "@pipeline().parameters.TaskObject.Source.Extraction.TableName", "type": "Expression" }, - "KeyVaultBaseUrl": { - "value": "@pipeline().parameters.TaskObject.KeyVaultBaseUrl", - "type": "Expression" - }, - "PasswordSecret": { - "value": "@pipeline().parameters.TaskObject.Source.Database.PasswordKeyVaultSecretName", - "type": "Expression" - }, "Server": { "value": "@pipeline().parameters.TaskObject.Source.Database.SystemName", "type": "Expression" @@ -135,10 +127,6 @@ "Database": { "value": "@pipeline().parameters.TaskObject.Source.Database.Name", "type": "Expression" - }, - "UserName": { - "value": "@pipeline().parameters.TaskObject.Source.Database.Username", - "type": "Expression" } } }, @@ -248,7 +236,7 @@ } }, "folder": { - "name": "ADS Go Fast/Create Task Master/OnPrem SH IR" + "name": "ADS Go Fast/Create Task Master/@GF{IR}" }, "annotations": [], "lastPublishTime": "2020-08-04T13:09:29Z" diff --git a/solution/DataFactory/pipeline/GEN-File-Binary-AZ-Storage-Binary-OnP-SH-IR.json b/solution/DataFactory/Templates/pipeline/GEN_File_Binary_AZ_Storage_Binary_{IR}.json similarity index 73% rename from solution/DataFactory/pipeline/GEN-File-Binary-AZ-Storage-Binary-OnP-SH-IR.json rename to solution/DataFactory/Templates/pipeline/GEN_File_Binary_AZ_Storage_Binary_{IR}.json index 29f14703..de431320 100644 --- a/solution/DataFactory/pipeline/GEN-File-Binary-AZ-Storage-Binary-OnP-SH-IR.json +++ b/solution/DataFactory/Templates/pipeline/GEN_File_Binary_AZ_Storage_Binary_{IR}.json @@ -1,9 +1,9 @@ { - "name": "GEN-File-Binary-AZ-Storage-Binary-OnP-SH-IR", + "name": "GEN_File_Binary_AZ_Storage_Binary_@GF{IR}", "properties": { "activities": [ { - "name": "Switch Azure Storage", + "name": "Switch Storage Type", "type": "Switch", "dependsOn": [], "userProperties": [], @@ -40,18 +40,8 @@ "type": "BinarySource", "storeSettings": { "type": "FileServerReadSettings", - "recursive": { - "value": "@pipeline().parameters.TaskObject.Source.Recursively", - "type": "Expression" - }, - "wildcardFileName": { - "value": "@pipeline().parameters.TaskObject.Source.DataFileName", - "type": "Expression" - }, - "deleteFilesAfterCompletion": { - "value": "@pipeline().parameters.TaskObject.Source.DeleteAfterCompletion", - "type": "Expression" - } + "recursive": true, + "deleteFilesAfterCompletion": true }, "formatSettings": { "type": "BinaryReadSettings" @@ -60,8 +50,7 @@ "sink": { "type": "BinarySink", "storeSettings": { - "type": "AzureBlobFSWriteSettings", - "copyBehavior": "PreserveHierarchy" + "type": "AzureBlobFSWriteSettings" } }, "enableStaging": false, @@ -72,7 +61,7 @@ }, "inputs": [ { - "referenceName": "FileBinary_OnPrem_SH_IR", + "referenceName": "FileBinary_@GF{IR}", "type": "DatasetReference", "parameters": { "Host": { @@ -88,11 +77,11 @@ "type": "Expression" }, "Directory": { - "value": "@pipeline().parameters.TaskObject.Source.RelativePath", + "value": "@pipeline().parameters.TaskObject.Source.Directory", "type": "Expression" }, "File": { - "value": "@pipeline().parameters.TaskObject.Source.DataFileName", + "value": "@pipeline().parameters.TaskObject.Source.File", "type": "Expression" }, "KeyVaultBaseUrl": { @@ -104,21 +93,21 @@ ], "outputs": [ { - "referenceName": "ADLSBinary_OnPrem_SH_IR", + "referenceName": "ADLSBinary_@GF{IR}", "type": "DatasetReference", "parameters": { "StorageAccountEndpoint": { "value": "@pipeline().parameters.TaskObject.Target.StorageAccountName", "type": "Expression" }, - "FileSystem": { - "value": "@pipeline().parameters.TaskObject.Target.StorageAccountContainer", - "type": "Expression" - }, "Directory": { "value": "@pipeline().parameters.TaskObject.Target.RelativePath", "type": "Expression" }, + "FileSystem": { + "value": "@pipeline().parameters.TaskObject.Target.StorageAccountContainer", + "type": "Expression" + }, "File": { "value": "@pipeline().parameters.TaskObject.Target.DataFileName", "type": "Expression" @@ -127,6 +116,34 @@ } ] }, + { + "name": "Pipeline AF Log - File to ADLS Failed", + "type": "ExecutePipeline", + "dependsOn": [ + { + "activity": "Copy File to ADLS", + "dependencyConditions": [ + "Failed" + ] + } + ], + "userProperties": [], + "typeProperties": { + "pipeline": { + "referenceName": "AZ_Function_Generic", + "type": "PipelineReference" + }, + "waitOnCompletion": false, + "parameters": { + "Body": { + "value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":1,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy File to ADLS\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"EndDateTimeOffSet\":\"', string(utcnow()), '\",\"Comment\":\"', string(activity('Copy File to ADLS').error.message), '\",\"Status\":\"Failed\"}'))", + "type": "Expression" + }, + "FunctionName": "Log", + "Method": "Post" + } + } + }, { "name": "Pipeline AF Log - File to ADLS Start", "type": "ExecutePipeline", @@ -134,13 +151,13 @@ "userProperties": [], "typeProperties": { "pipeline": { - "referenceName": "AZ-Function-Generic", + "referenceName": "AZ_Function_Generic", "type": "PipelineReference" }, "waitOnCompletion": false, "parameters": { "Body": { - "value": "[\n{\n \"TaskInstanceId\":@{pipeline().parameters.TaskObject.TaskInstanceId},\n \"ExecutionUid\":@{pipeline().parameters.TaskObject.ExecutionUid},\n \"RunId\":@{pipeline().RunId}, \n \"StartTime\":@{utcnow()},\n \"Status\":\"Processing\",\n \"Activity\":\"Copy File to ADLS\",\n \"Comments\":\"\"\n}\n]", + "value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":3,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy File to ADLS\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"Status\":\"Started\"}'))", "type": "Expression" }, "FunctionName": "Log", @@ -162,41 +179,13 @@ "userProperties": [], "typeProperties": { "pipeline": { - "referenceName": "AZ-Function-Generic", + "referenceName": "AZ_Function_Generic", "type": "PipelineReference" }, "waitOnCompletion": false, "parameters": { "Body": { - "value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":1,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy File to ADLS\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"EndDateTimeOffSet\":\"', string(utcnow()), '\",\"RowsInserted\":\"', string(activity('Copy File to ADLS').output.filesWritten), '\",\"Comment\":\"\",\"Status\":\"Complete\"}'))", - "type": "Expression" - }, - "FunctionName": "Log", - "Method": "Post" - } - } - }, - { - "name": "Pipeline AF Log - File to ADLS Failed", - "type": "ExecutePipeline", - "dependsOn": [ - { - "activity": "Copy File to ADLS", - "dependencyConditions": [ - "Failed" - ] - } - ], - "userProperties": [], - "typeProperties": { - "pipeline": { - "referenceName": "AZ-Function-Generic", - "type": "PipelineReference" - }, - "waitOnCompletion": false, - "parameters": { - "Body": { - "value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":1,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy File to ADLS\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"EndDateTimeOffSet\":\"', string(utcnow()), '\",\"Comment\":\"', string(activity('Copy File to ADLS').error.message), '\",\"Status\":\"Failed\"}'))", + "value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":1,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy File to ADLS\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"EndDateTimeOffSet\":\"', string(utcnow()), '\",\"RowsInserted\":\"', string(activity('Copy File to ADLS').output.rowsCopied), '\",\"Comment\":\"\",\"Status\":\"Complete\"}'))", "type": "Expression" }, "FunctionName": "Log", @@ -233,18 +222,8 @@ "type": "BinarySource", "storeSettings": { "type": "FileServerReadSettings", - "recursive": { - "value": "@pipeline().parameters.TaskObject.Source.Recursively", - "type": "Expression" - }, - "wildcardFileName": { - "value": "@pipeline().parameters.TaskObject.Source.DataFileName", - "type": "Expression" - }, - "deleteFilesAfterCompletion": { - "value": "@pipeline().parameters.TaskObject.Source.DeleteAfterCompletion", - "type": "Expression" - } + "recursive": true, + "deleteFilesAfterCompletion": true }, "formatSettings": { "type": "BinaryReadSettings" @@ -253,8 +232,7 @@ "sink": { "type": "BinarySink", "storeSettings": { - "type": "AzureBlobStorageWriteSettings", - "copyBehavior": "PreserveHierarchy" + "type": "AzureBlobStorageWriteSettings" } }, "enableStaging": false, @@ -265,7 +243,7 @@ }, "inputs": [ { - "referenceName": "FileBinary_OnPrem_SH_IR", + "referenceName": "FileBinary_@GF{IR}", "type": "DatasetReference", "parameters": { "Host": { @@ -281,11 +259,11 @@ "type": "Expression" }, "Directory": { - "value": "@pipeline().parameters.TaskObject.Source.RelativePath", + "value": "@pipeline().parameters.TaskObject.Source.Directory", "type": "Expression" }, "File": { - "value": "@pipeline().parameters.TaskObject.Source.DataFileName", + "value": "@pipeline().parameters.TaskObject.Source.File", "type": "Expression" }, "KeyVaultBaseUrl": { @@ -297,7 +275,7 @@ ], "outputs": [ { - "referenceName": "BlobBinary_OnPrem_SH_IR", + "referenceName": "BlobBinary_@GF{IR}", "type": "DatasetReference", "parameters": { "StorageAccountEndpoint": { @@ -320,6 +298,34 @@ } ] }, + { + "name": "Pipeline AF Log - File to Blob Failed", + "type": "ExecutePipeline", + "dependsOn": [ + { + "activity": "Copy File to Blob", + "dependencyConditions": [ + "Failed" + ] + } + ], + "userProperties": [], + "typeProperties": { + "pipeline": { + "referenceName": "AZ_Function_Generic", + "type": "PipelineReference" + }, + "waitOnCompletion": false, + "parameters": { + "Body": { + "value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":1,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy File to Blob\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"EndDateTimeOffSet\":\"', string(utcnow()), '\",\"Comment\":\"', string(activity('Copy File to Blob').error.message), '\",\"Status\":\"Failed\"}'))", + "type": "Expression" + }, + "FunctionName": "Log", + "Method": "Post" + } + } + }, { "name": "Pipeline AF Log - File to Blob Start", "type": "ExecutePipeline", @@ -327,13 +333,13 @@ "userProperties": [], "typeProperties": { "pipeline": { - "referenceName": "AZ-Function-Generic", + "referenceName": "AZ_Function_Generic", "type": "PipelineReference" }, "waitOnCompletion": false, "parameters": { "Body": { - "value": "[\n{\n \"TaskInstanceId\":@{pipeline().parameters.TaskObject.TaskInstanceId},\n \"ExecutionUid\":@{pipeline().parameters.TaskObject.ExecutionUid},\n \"RunId\":@{pipeline().RunId}, \n \"StartTime\":@{utcnow()},\n \"Status\":\"Processing\",\n \"Activity\":\"Copy File to Blob\",\n \"Comments\":\"\"\n}\n]", + "value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":3,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy File to Blob\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"Status\":\"Started\"}'))", "type": "Expression" }, "FunctionName": "Log", @@ -355,41 +361,13 @@ "userProperties": [], "typeProperties": { "pipeline": { - "referenceName": "AZ-Function-Generic", + "referenceName": "AZ_Function_Generic", "type": "PipelineReference" }, "waitOnCompletion": false, "parameters": { "Body": { - "value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":1,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy File to Blob\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"EndDateTimeOffSet\":\"', string(utcnow()), '\",\"RowsInserted\":\"', string(activity('Copy File to Blob').output.filesWritten), '\",\"Comment\":\"\",\"Status\":\"Complete\"}'))", - "type": "Expression" - }, - "FunctionName": "Log", - "Method": "Post" - } - } - }, - { - "name": "Pipeline AF Log - File to Blob Failed", - "type": "ExecutePipeline", - "dependsOn": [ - { - "activity": "Copy File to Blob", - "dependencyConditions": [ - "Failed" - ] - } - ], - "userProperties": [], - "typeProperties": { - "pipeline": { - "referenceName": "AZ-Function-Generic", - "type": "PipelineReference" - }, - "waitOnCompletion": false, - "parameters": { - "Body": { - "value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":1,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy File to Blob\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"EndDateTimeOffSet\":\"', string(utcnow()), '\",\"Comment\":\"', string(activity('Copy File to Blob').error.message), '\",\"Status\":\"Failed\"}'))", + "value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":1,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy File to Blob\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"EndDateTimeOffSet\":\"', string(utcnow()), '\",\"RowsInserted\":\"', string(activity('Copy File to Blob').output.rowsCopied), '\",\"Comment\":\"\",\"Status\":\"Complete\"}'))", "type": "Expression" }, "FunctionName": "Log", @@ -406,14 +384,14 @@ "parameters": { "TaskObject": { "type": "object", - "defaultValue": "[ { \"TaskInstanceId\": 2, \"TaskMasterId\": 1, \"TaskStatus\": \"Untried\", \"TaskType\": \"FiletoAzureStorage-OnPrem-SH-IR\", \"Enabled\": true, \"ExecutionUid\": 1, \"KeyVaultBaseUrl\": \"https://adsgofastkeyvault.vault.azure.net/\", \"Source\": { \"Type\": \"File\", \"Host\": \"D:\\\\dataingestion\\\\\", \"UserId\": \"AzureUser\", \"PasswordKeyVaultSecretName\": \"adsgofast-onpre-file-password\", \"Directory\": \"AdventureWorks2017/2020/06/15/14/22/\", \"File\": \"Sales.SalesOrderDetails.parquet\" }, \"Target\": { \"StorageAccountName\": \"https://adsgofastdatalakeadls.dfs.core.windows.net\", \"Type\": \"ADLS\", \"StorageAccountContainer\": \"datalakelanding\", \"StorageAccountAccessMethod\": \"MSI\", \"StorageAccountSASUriKeyVaultSecretName\": null, \"RelativePath\": \"/Unprocessed/adsgofastdatakakeaccelsqlsvr/AWSample/SalesLT/2020/06/08/17/\", \"DataFileName\": \"Customer_Data.parquet\", \"SchemaFileName\": \"Customer_Schema.json\" }, \"DataFactory\": { \"Id\": 1, \"Name\": \"adsgofastdatakakeacceladf\", \"ResourceGroup\": \"AdsGoFastDataLakeAccel\", \"SubscriptionId\": \"035a1364-f00d-48e2-b582-4fe125905ee3\", } } ]" + "defaultValue": " [ { \"TaskInstanceId\": 2, \"TaskMasterId\": 1, \"TaskStatus\": \"Untried\", \"TaskType\": \"FiletoAzureStorage-Azure_@GF{IR}\", \"Enabled\": true, \"ExecutionUid\": 1, \"KeyVaultBaseUrl\": \"https://adsgofastkeyvault.vault.azure.net/\", \"Source\": { \"Type\": \"File\", \"Host\": \"\\\\\\\\10.2.0.4\\\\D\\\\dataingestion\\\\\", \"UserId\": \"adsgofastonprem\\\\AzureUser\", \"PasswordKeyVaultSecretName\": \"adsgofast-onpre_File_password\", \"Directory\": \"AdventureWorks2017/2020/06/15/14/22/\", \"File\": \"Sales.SalesOrderDetails.parquet\" }, \"Target\": { \"StorageAccountName\": \"https://adsgofastdatalakeaccelst.blob.core.windows.net\", \"Type\": \"Blob\", \"StorageAccountContainer\": \"datalakelanding\", \"StorageAccountAccessMethod\": \"MSI\", \"StorageAccountSASUriKeyVaultSecretName\": null, \"RelativePath\": \"/Unprocessed/adsgofastdatakakeaccelsqlsvr/AWSample/SalesLT/2020/06/08/17/\", \"DataFileName\": \"Customer_Data.parquet\", \"SchemaFileName\": \"Customer_Schema.json\" }, \"DataFactory\": { \"Id\": 1, \"Name\": \"adsgofastdatakakeacceladf\", \"ResourceGroup\": \"AdsGoFastDataLakeAccel\", \"SubscriptionId\": \"035a1364-f00d-48e2-b582-4fe125905ee3\", } } ]" } }, "folder": { - "name": "ADS Go Fast/Data Movement/OnPrem SH IR" + "name": "ADS Go Fast/Data Movement/@GF{IR}" }, "annotations": [], - "lastPublishTime": "2020-08-04T02:49:50Z" + "lastPublishTime": "2020-07-24T01:51:29Z" }, "type": "Microsoft.DataFactory/factories/pipelines" } \ No newline at end of file diff --git a/solution/DataFactory/Templates/pipeline/Master_{IR}.json b/solution/DataFactory/Templates/pipeline/Master_{IR}.json new file mode 100644 index 00000000..e1dfe675 --- /dev/null +++ b/solution/DataFactory/Templates/pipeline/Master_{IR}.json @@ -0,0 +1,470 @@ +{ + "name": "Master", + "properties": { + "activities": [ + { + "name": "Switch Pipelines", + "type": "Switch", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "on": { + "value": "@pipeline().parameters.TaskObject.DataFactory.ADFPipeline", + "type": "Expression" + }, + "cases": [ + { + "value": "AZ_SQL_AZ_Storage_Parquet_@GF{IR}", + "activities": [ + { + "name": "Execute AZ_SQL_AZ_Storage_Parquet_@GF{IR}", + "type": "ExecutePipeline", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "pipeline": { + "referenceName": "AZ_SQL_AZ_Storage_Parquet_@GF{IR}", + "type": "PipelineReference" + }, + "waitOnCompletion": true, + "parameters": { + "TaskObject": { + "value": "@pipeline().parameters.TaskObject", + "type": "Expression" + } + } + } + } + ] + }, + { + "value": "AZ_Storage_Binary_AZ_Storage_Binary_@GF{IR}", + "activities": [ + { + "name": "Execute AZ_Storage_Binary_AZ_Storage_Binary_@GF{IR}", + "type": "ExecutePipeline", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "pipeline": { + "referenceName": "AZ_Storage_Binary_AZ_Storage_Binary_@GF{IR}", + "type": "PipelineReference" + }, + "waitOnCompletion": true, + "parameters": { + "TaskObject": { + "value": "@pipeline().parameters.TaskObject", + "type": "Expression" + } + } + } + } + ] + }, + { + "value": "AZ_Storage_CSV_AZ_SQL_@GF{IR}", + "activities": [ + { + "name": "Execute AZ_Storage_CSV_AZ_SQL_@GF{IR}", + "type": "ExecutePipeline", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "pipeline": { + "referenceName": "AZ_Storage_CSV_AZ_SQL_@GF{IR}", + "type": "PipelineReference" + }, + "waitOnCompletion": true, + "parameters": { + "TaskObject": { + "value": "@pipeline().parameters.TaskObject", + "type": "Expression" + } + } + } + } + ] + }, + { + "value": "AZ_Storage_Excel_AZ_SQL_@GF{IR}", + "activities": [ + { + "name": "Execute AZ_Storage_Excel_AZ_SQL_@GF{IR}", + "type": "ExecutePipeline", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "pipeline": { + "referenceName": "AZ_Storage_Excel_AZ_SQL_@GF{IR}", + "type": "PipelineReference" + }, + "waitOnCompletion": true, + "parameters": { + "TaskObject": { + "value": "@pipeline().parameters.TaskObject", + "type": "Expression" + } + } + } + } + ] + }, + { + "value": "AZ_Storage_JSON_AZ_SQL_@GF{IR}", + "activities": [ + { + "name": "Execute AZ_Storage_JSON_AZ_SQL_@GF{IR}", + "type": "ExecutePipeline", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "pipeline": { + "referenceName": "AZ_Storage_JSON_AZ_SQL_@GF{IR}", + "type": "PipelineReference" + }, + "waitOnCompletion": true, + "parameters": { + "TaskObject": { + "value": "@pipeline().parameters.TaskObject", + "type": "Expression" + } + } + } + } + ] + }, + { + "value": "AZ_Storage_Parquet_AZ_SQL_@GF{IR}", + "activities": [ + { + "name": "Execute AZ_Storage_Parquet_AZ_SQL_@GF{IR}", + "type": "ExecutePipeline", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "pipeline": { + "referenceName": "AZ_Storage_Parquet_AZ_SQL_@GF{IR}", + "type": "PipelineReference" + }, + "waitOnCompletion": true, + "parameters": { + "TaskObject": { + "value": "@pipeline().parameters.TaskObject", + "type": "Expression" + } + } + } + } + ] + }, + { + "value": "GEN_File_Binary_AZ_Storage_Binary_@GF{IR}", + "activities": [ + { + "name": "Execute GEN_File_Binary_AZ_Storage_Binary_@GF{IR}", + "type": "ExecutePipeline", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "pipeline": { + "referenceName": "GEN_File_Binary_AZ_Storage_Binary_@GF{IR}", + "type": "PipelineReference" + }, + "waitOnCompletion": true, + "parameters": { + "TaskObject": { + "value": "@pipeline().parameters.TaskObject", + "type": "Expression" + } + } + } + } + ] + }, + { + "value": "SH_SQL_AZ_Storage_Parquet_@GF{IR}", + "activities": [ + { + "name": "Execute SH_SQL_AZ_Storage_Parquet_@GF{IR}", + "type": "ExecutePipeline", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "pipeline": { + "referenceName": "SH_SQL_AZ_Storage_Parquet_@GF{IR}", + "type": "PipelineReference" + }, + "waitOnCompletion": true, + "parameters": { + "TaskObject": { + "value": "@pipeline().parameters.TaskObject", + "type": "Expression" + } + } + } + } + ] + }, + { + "value": "SH_SQL_GEN_File_Parquet_@GF{IR}", + "activities": [ + { + "name": "Execute SH_SQL_GEN_File_Parquet_@GF{IR}", + "type": "ExecutePipeline", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "pipeline": { + "referenceName": "SH_SQL_GEN_File_Parquet_@GF{IR}", + "type": "PipelineReference" + }, + "waitOnCompletion": true, + "parameters": { + "TaskObject": { + "value": "@pipeline().parameters.TaskObject", + "type": "Expression" + } + } + } + } + ] + }, + { + "value": "Create-Task_MASTER_AZ_SQL_@GF{IR}", + "activities": [ + { + "name": "Execute Create-Task_MASTER_AZ_SQL_@GF{IR}", + "type": "ExecutePipeline", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "pipeline": { + "referenceName": "Create-Task_MASTER_AZ_SQL_@GF{IR}", + "type": "PipelineReference" + }, + "waitOnCompletion": true, + "parameters": { + "TaskObject": { + "value": "@pipeline().parameters.TaskObject", + "type": "Expression" + } + } + } + } + ] + }, + { + "value": "AZ_Storage_Excel_AZ_Storage_CSV_@GF{IR}", + "activities": [ + { + "name": "Execute AZ_Storage_Excel_AZ_Storage_CSV_@GF{IR}", + "type": "ExecutePipeline", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "pipeline": { + "referenceName": "AZ_Storage_Excel_AZ_Storage_CSV_@GF{IR}", + "type": "PipelineReference" + }, + "waitOnCompletion": true, + "parameters": { + "TaskObject": { + "value": "@pipeline().parameters.TaskObject", + "type": "Expression" + } + } + } + } + ] + }, + { + "value": "AZ_SQL_StoredProcedure_@GF{IR}", + "activities": [ + { + "name": "Execute AZ_SQL_StoredProcedure_@GF{IR}", + "type": "ExecutePipeline", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "pipeline": { + "referenceName": "AZ_SQL_StoredProcedure_@GF{IR}", + "type": "PipelineReference" + }, + "waitOnCompletion": true, + "parameters": { + "TaskObject": { + "value": "@pipeline().parameters.TaskObject", + "type": "Expression" + } + } + } + } + ] + } + ], + "defaultActivities": [ + { + "name": "Pipeline AF Log - Invalid Pipeline", + "type": "ExecutePipeline", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "pipeline": { + "referenceName": "AZ_Function_Generic", + "type": "PipelineReference" + }, + "waitOnCompletion": false, + "parameters": { + "Body": { + "value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":1,\"LogSource\":\"ADF\",\"ActivityType\":\"Data-Movement-Master\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"EndDateTimeOffSet\":\"', string(utcnow()), '\",\"Comment\":\"Invalid TaskType: ', string(pipeline().parameters.TaskObject.TaskType), '\",\"Status\":\"Failed\",\"NumberOfRetries\":\"', string(pipeline().parameters.TaskObject.NumberOfRetries),'\"}'))", + "type": "Expression" + }, + "FunctionName": "Log", + "Method": "Post" + } + } + }, + { + "name": "Fail Due to Invalid Pipeline", + "type": "Fail", + "dependsOn": [ + { + "activity": "Pipeline AF Log - Invalid Pipeline", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "userProperties": [], + "typeProperties": { + "message": { + "value": "@concat('Invalid Pipeline:',pipeline().parameters.TaskObject.DataFactory.ADFPipeline)", + "type": "Expression" + }, + "errorCode": "500" + } + } + ] + } + }, + { + "name": "Pipeline AF Log - Failed", + "type": "ExecutePipeline", + "dependsOn": [ + { + "activity": "Switch Pipelines", + "dependencyConditions": [ + "Failed" + ] + } + ], + "userProperties": [], + "typeProperties": { + "pipeline": { + "referenceName": "AZ_Function_Generic", + "type": "PipelineReference" + }, + "waitOnCompletion": true, + "parameters": { + "Body": { + "value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":1,\"LogSource\":\"ADF\",\"ActivityType\":\"Data-Movement-Master\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"EndDateTimeOffSet\":\"', string(utcnow()), '\",\"Comment\":\"', string(activity('Switch Pipelines').error.message), '\",\"Status\":\"Failed\",\"NumberOfRetries\":\"', string(pipeline().parameters.TaskObject.NumberOfRetries),'\"}'))", + "type": "Expression" + }, + "FunctionName": "Log", + "Method": "Post" + } + } + }, + { + "name": "Pipeline AF Log - Succeed", + "type": "ExecutePipeline", + "dependsOn": [ + { + "activity": "Switch Pipelines", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "userProperties": [], + "typeProperties": { + "pipeline": { + "referenceName": "AZ_Function_Generic", + "type": "PipelineReference" + }, + "waitOnCompletion": true, + "parameters": { + "Body": { + "value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":1,\"LogSource\":\"ADF\",\"ActivityType\":\"Data-Movement-Master\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"EndDateTimeOffSet\":\"', string(utcnow()), '\",\"Comment\":\"\",\"Status\":\"Complete\",\"NumberOfRetries\":\"', string(pipeline().parameters.TaskObject.NumberOfRetries),'\"}'))", + "type": "Expression" + }, + "FunctionName": "Log", + "Method": "Post" + } + } + } + ], + "parameters": { + "TaskObject": { + "type": "object", + "defaultValue": { + "TaskInstanceId": 865, + "TaskMasterId": 102, + "TaskStatus": "Untried", + "TaskType": "SQL Database to Azure Storage", + "Enabled": 1, + "ExecutionUid": "d954cfee-44c1-419f-83d5-51f75eaf809c", + "NumberOfRetries": 0, + "DegreeOfCopyParallelism": 1, + "KeyVaultBaseUrl": "https://adsgofastkeyvault.vault.azure.net/", + "ScheduleMasterId": 2, + "TaskGroupConcurrency": 10, + "TaskGroupPriority": 0, + "Source": { + "Type": "SQL Server", + "Database": { + "SystemName": "adsgofast-onpre", + "Name": "AdventureWorks2017", + "AuthenticationType": "SQLAuth", + "Username": "sqladfir", + "PasswordKeyVaultSecretName": "adsgofast-onpre-sqladfir-password" + }, + "Extraction": { + "Type": "Table", + "FullOrIncremental": "Full", + "IncrementalType": null, + "TableSchema": "Sales", + "TableName": "SalesOrderHeader" + } + }, + "Target": { + "Type": "Azure Blob", + "StorageAccountName": "https://adsgofastdatalakeaccelst.blob.core.windows.net", + "StorageAccountContainer": "datalakeraw", + "StorageAccountAccessMethod": "MSI", + "RelativePath": "AdventureWorks2017/Sales/SalesOrderHeader/2020/7/23/9/0/", + "DataFileName": "Sales.SalesOrderHeader.parquet", + "SchemaFileName": "Sales.SalesOrderHeader.json", + "FirstRowAsHeader": null, + "SheetName": null, + "SkipLineCount": null, + "MaxConcorrentConnections": null + }, + "DataFactory": { + "Id": 1, + "Name": "adsgofastdatakakeacceladf", + "ResourceGroup": "AdsGoFastDataLakeAccel", + "SubscriptionId": "035a1364-f00d-48e2-b582-4fe125905ee3", + "ADFPipeline": "OnP_SQL_AZ_Storage_Parquet_@GF{IR}" + } + } + } + }, + "folder": { + "name": "ADS Go Fast" + }, + "annotations": [], + "lastPublishTime": "2020-08-06T06:27:14Z" + }, + "type": "Microsoft.DataFactory/factories/pipelines" +} \ No newline at end of file diff --git a/solution/DataFactory/pipeline/OnP-SQL-AZ-Storage-Parquet-OnP-SH-IR.json b/solution/DataFactory/Templates/pipeline/SH_SQL_AZ_Storage_Parquet_{IR}.json similarity index 87% rename from solution/DataFactory/pipeline/OnP-SQL-AZ-Storage-Parquet-OnP-SH-IR.json rename to solution/DataFactory/Templates/pipeline/SH_SQL_AZ_Storage_Parquet_{IR}.json index 0b0eb15e..55a4164c 100644 --- a/solution/DataFactory/pipeline/OnP-SQL-AZ-Storage-Parquet-OnP-SH-IR.json +++ b/solution/DataFactory/Templates/pipeline/SH_SQL_AZ_Storage_Parquet_{IR}.json @@ -1,5 +1,5 @@ { - "name": "OnP-SQL-AZ-Storage-Parquet-OnP-SH-IR", + "name": "SH_SQL_AZ_Storage_Parquet_@GF{IR}", "properties": { "activities": [ { @@ -16,7 +16,7 @@ "userProperties": [], "typeProperties": { "pipeline": { - "referenceName": "AZ-Function-Generic", + "referenceName": "AZ_Function_Generic", "type": "PipelineReference" }, "waitOnCompletion": false, @@ -117,7 +117,7 @@ "partitionOption": "None" }, "dataset": { - "referenceName": "SqlServerTable_OnPrem_SH_IR", + "referenceName": "SqlServerTable_@GF{IR}", "type": "DatasetReference", "parameters": { "TableSchema": { @@ -175,13 +175,13 @@ "value": "Full", "activities": [ { - "name": "Execute OnP-SQL-Full-Load-OnP-SH-IR", + "name": "Execute SH_SQL_Full_Load_@GF{IR}", "type": "ExecutePipeline", "dependsOn": [], "userProperties": [], "typeProperties": { "pipeline": { - "referenceName": "OnP-SQL-Full-Load-OnP-SH-IR", + "referenceName": "SH_SQL_Full_Load_@GF{IR}", "type": "PipelineReference" }, "waitOnCompletion": true, @@ -205,7 +205,7 @@ "value": "Watermark", "activities": [ { - "name": "Execute OnP-SQL-Watermark-OnP-SH-IR", + "name": "Execute SH_SQL_Watermark_@GF{IR}", "type": "ExecutePipeline", "dependsOn": [ { @@ -218,7 +218,7 @@ "userProperties": [], "typeProperties": { "pipeline": { - "referenceName": "OnP-SQL-Watermark-OnP-SH-IR", + "referenceName": "SH_SQL_Watermark_@GF{IR}", "type": "PipelineReference" }, "waitOnCompletion": true, @@ -263,7 +263,7 @@ "partitionOption": "None" }, "dataset": { - "referenceName": "SqlServerTable_OnPrem_SH_IR", + "referenceName": "SqlServerTable_@GF{IR}", "type": "DatasetReference", "parameters": { "TableSchema": { @@ -302,10 +302,10 @@ ] }, { - "value": "Full-Chunk", + "value": "Full_Chunk", "activities": [ { - "name": "Execute OnP-SQL-Full-Load-Chunk-OnP-SH-IR", + "name": "Execute SH_SQL_Full_Load_Chunk_@GF{IR}", "type": "ExecutePipeline", "dependsOn": [ { @@ -318,7 +318,7 @@ "userProperties": [], "typeProperties": { "pipeline": { - "referenceName": "OnP-SQL-Full-Load-Chunk-OnP-SH-IR", + "referenceName": "SH_SQL_Full_Load_Chunk_@GF{IR}", "type": "PipelineReference" }, "waitOnCompletion": true, @@ -361,7 +361,7 @@ "partitionOption": "None" }, "dataset": { - "referenceName": "SqlServerTable_OnPrem_SH_IR", + "referenceName": "SqlServerTable_@GF{IR}", "type": "DatasetReference", "parameters": { "TableSchema": { @@ -400,10 +400,10 @@ ] }, { - "value": "Watermark-Chunk", + "value": "Watermark_Chunk", "activities": [ { - "name": "Execute OnP-SQL-Watermark-Chunk-OnP-SH-IR", + "name": "Execute SH_SQL_Watermark_Chunk_@GF{IR}", "type": "ExecutePipeline", "dependsOn": [ { @@ -416,7 +416,7 @@ "userProperties": [], "typeProperties": { "pipeline": { - "referenceName": "OnP-SQL-Watermark-Chunk-OnP-SH-IR", + "referenceName": "SH_SQL_Watermark_Chunk_@GF{IR}", "type": "PipelineReference" }, "waitOnCompletion": true, @@ -463,7 +463,7 @@ "partitionOption": "None" }, "dataset": { - "referenceName": "SqlServerTable_OnPrem_SH_IR", + "referenceName": "SqlServerTable_@GF{IR}", "type": "DatasetReference", "parameters": { "TableSchema": { @@ -508,11 +508,11 @@ "parameters": { "TaskObject": { "type": "object", - "defaultValue": " [ { \"TaskInstanceId\": 2, \"TaskMasterId\": 1, \"TaskStatus\": \"Untried\", \"TaskType\": \"SQLtoAzureStorage-OnPrem-SH-IR\", \"Enabled\": true, \"ExecutionUid\": 1, \"KeyVaultBaseUrl\": \"https://adsgofastkeyvault.vault.azure.net/\", \"Source\": { \"Type\": \"MsSqlServer\", \"Database\": { \"Name\": \"AdventureWorks2017\", \"SystemName\": \"adsgofast-onpre\", \"Type\": \"MsSqlServer\", \"AuthenticationType\": \"SQLAuth\", \"Username\": \"sqladfir\", \"PasswordKeyVaultSecretName\": \"adsgofast-onpre-sqladfir-password\" }, \"Extraction\": { \"Type\": \"Table\", \"FullOrIncremental\": true, \"IncrementalType\": null, \"TableSchema\": \"Sales\", \"TableName\": \"SalesOrderDetail\" } }, \"Target\": { \"StorageAccountName\": \"https://adsgofastdatalakeadls.dfs.core.windows.net\", \"Type\": \"ADLS\", \"StorageAccountContainer\": \"datalakelanding\", \"StorageAccountAccessMethod\": \"MSI\", \"StorageAccountSASUriKeyVaultSecretName\": null, \"RelativePath\": \"/Unprocessed/adsgofastdatakakeaccelsqlsvr/AWSample/SalesLT/2020/06/08/17/\", \"DataFileName\": \"Customer_Data.parquet\", \"SchemaFileName\": \"Customer_Schema.json\" }, \"DataFactory\": { \"Id\": 1, \"Name\": \"adsgofastdatakakeacceladf\", \"ResourceGroup\": \"AdsGoFastDataLakeAccel\", \"SubscriptionId\": \"035a1364-f00d-48e2-b582-4fe125905ee3\", } } ]" + "defaultValue": " [ { \"TaskInstanceId\": 2, \"TaskMasterId\": 1, \"TaskStatus\": \"Untried\", \"TaskType\": \"SQLtoAzureStorage_@GF{IR}\", \"Enabled\": true, \"ExecutionUid\": 1, \"KeyVaultBaseUrl\": \"https://adsgofastkeyvault.vault.azure.net/\", \"Source\": { \"Type\": \"MsSqlServer\", \"Database\": { \"Name\": \"AdventureWorks2017\", \"SystemName\": \"adsgofast-onpre\", \"Type\": \"MsSqlServer\", \"AuthenticationType\": \"SQLAuth\", \"Username\": \"sqladfir\", \"PasswordKeyVaultSecretName\": \"adsgofast-onpre-sqladfir-password\" }, \"Extraction\": { \"Type\": \"Table\", \"FullOrIncremental\": true, \"IncrementalType\": null, \"TableSchema\": \"Sales\", \"TableName\": \"SalesOrderDetail\" } }, \"Target\": { \"StorageAccountName\": \"https://adsgofastdatalakeadls.dfs.core.windows.net\", \"Type\": \"ADLS\", \"StorageAccountContainer\": \"datalakelanding\", \"StorageAccountAccessMethod\": \"MSI\", \"StorageAccountSASUriKeyVaultSecretName\": null, \"RelativePath\": \"/Unprocessed/adsgofastdatakakeaccelsqlsvr/AWSample/SalesLT/2020/06/08/17/\", \"DataFileName\": \"Customer_Data.parquet\", \"SchemaFileName\": \"Customer_Schema.json\" }, \"DataFactory\": { \"Id\": 1, \"Name\": \"adsgofastdatakakeacceladf\", \"ResourceGroup\": \"AdsGoFastDataLakeAccel\", \"SubscriptionId\": \"035a1364-f00d-48e2-b582-4fe125905ee3\", } } ]" } }, "folder": { - "name": "ADS Go Fast/Data Movement/OnPrem SH IR" + "name": "ADS Go Fast/Data Movement/@GF{IR}" }, "annotations": [], "lastPublishTime": "2020-08-04T12:40:45Z" diff --git a/solution/DataFactory/Templates/pipeline/SH_SQL_Full_Load_Chunk_{IR}.json b/solution/DataFactory/Templates/pipeline/SH_SQL_Full_Load_Chunk_{IR}.json new file mode 100644 index 00000000..27351385 --- /dev/null +++ b/solution/DataFactory/Templates/pipeline/SH_SQL_Full_Load_Chunk_{IR}.json @@ -0,0 +1,118 @@ +{ + "name": "SH_SQL_Full_Load_Chunk_@GF{IR}", + "properties": { + "activities": [ + { + "name": "ForEach Chunk", + "type": "ForEach", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "items": { + "value": "@range(1, pipeline().parameters.BatchCount)", + "type": "Expression" + }, + "isSequential": true, + "activities": [ + { + "name": "Execute SH_SQL_Full_Load", + "type": "ExecutePipeline", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "pipeline": { + "referenceName": "SH_SQL_Full_Load_@GF{IR}", + "type": "PipelineReference" + }, + "waitOnCompletion": true, + "parameters": { + "TaskObject": { + "value": "@pipeline().parameters.TaskObject", + "type": "Expression" + }, + "Mapping": { + "value": "@pipeline().parameters.Mapping", + "type": "Expression" + }, + "BatchCount": { + "value": "@pipeline().parameters.BatchCount", + "type": "Expression" + }, + "Item": { + "value": "@item()", + "type": "Expression" + } + } + } + } + ] + } + } + ], + "parameters": { + "TaskObject": { + "type": "object", + "defaultValue": { + "TaskInstanceId": 75, + "TaskMasterId": 12, + "TaskStatus": "Untried", + "TaskType": "SQL Database to Azure Storage", + "Enabled": 1, + "ExecutionUid": "2c5924ee-b855-4d2b-bb7e-4f5dde4c4dd3", + "NumberOfRetries": 111, + "DegreeOfCopyParallelism": 1, + "KeyVaultBaseUrl": "https://adsgofastkeyvault.vault.azure.net/", + "ScheduleMasterId": 2, + "TaskGroupConcurrency": 10, + "TaskGroupPriority": 0, + "Source": { + "Type": "Azure SQL", + "Database": { + "SystemName": "adsgofastdatakakeaccelsqlsvr.database.windows.net", + "Name": "AWSample", + "AuthenticationType": "MSI" + }, + "Extraction": { + "Type": "Table", + "FullOrIncremental": "Full", + "IncrementalType": null, + "TableSchema": "SalesLT", + "TableName": "SalesOrderHeader" + } + }, + "Target": { + "Type": "Azure Blob", + "StorageAccountName": "https://adsgofastdatalakeaccelst.blob.core.windows.net", + "StorageAccountContainer": "datalakeraw", + "StorageAccountAccessMethod": "MSI", + "RelativePath": "/AwSample/SalesLT/SalesOrderHeader/2020/7/9/14/12/", + "DataFileName": "SalesLT.SalesOrderHeader.parquet", + "SchemaFileName": "SalesLT.SalesOrderHeader", + "FirstRowAsHeader": null, + "SheetName": null, + "SkipLineCount": null, + "MaxConcorrentConnections": null + }, + "DataFactory": { + "Id": 1, + "Name": "adsgofastdatakakeacceladf", + "ResourceGroup": "AdsGoFastDataLakeAccel", + "SubscriptionId": "035a1364-f00d-48e2-b582-4fe125905ee3", + "ADFPipeline": "SH-AZ_Storage_Parquet_@GF{IR}" + } + } + }, + "Mapping": { + "type": "object" + }, + "BatchCount": { + "type": "int" + } + }, + "folder": { + "name": "ADS Go Fast/Data Movement/@GF{IR}/Components" + }, + "annotations": [] + }, + "type": "Microsoft.DataFactory/factories/pipelines" +} diff --git a/solution/DataFactory/pipeline/OnP-SQL-Full-Load-OnP-SH-IR.json b/solution/DataFactory/Templates/pipeline/SH_SQL_Full_Load_{IR}.json similarity index 95% rename from solution/DataFactory/pipeline/OnP-SQL-Full-Load-OnP-SH-IR.json rename to solution/DataFactory/Templates/pipeline/SH_SQL_Full_Load_{IR}.json index 7e4daf11..cabc67fc 100644 --- a/solution/DataFactory/pipeline/OnP-SQL-Full-Load-OnP-SH-IR.json +++ b/solution/DataFactory/Templates/pipeline/SH_SQL_Full_Load_{IR}.json @@ -1,5 +1,5 @@ { - "name": "OnP-SQL-Full-Load-OnP-SH-IR", + "name": "SH_SQL_Full_Load_@GF{IR}", "properties": { "activities": [ { @@ -62,8 +62,7 @@ "value": "@variables('SQLStatement')", "type": "Expression" }, - "queryTimeout": "02:00:00", - "partitionOption": "None" + "queryTimeout": "02:00:00" }, "sink": { "type": "ParquetSink", @@ -83,7 +82,7 @@ }, "inputs": [ { - "referenceName": "SqlServerTable_OnPrem_SH_IR", + "referenceName": "SqlServerTable_@GF{IR}", "type": "DatasetReference", "parameters": { "TableSchema": { @@ -119,7 +118,7 @@ ], "outputs": [ { - "referenceName": "ADLSParquet_OnPrem_SH_IR", + "referenceName": "ADLSParquet_@GF{IR}", "type": "DatasetReference", "parameters": { "RelativePath": { @@ -149,7 +148,7 @@ "userProperties": [], "typeProperties": { "pipeline": { - "referenceName": "AZ-Function-Generic", + "referenceName": "AZ_Function_Generic", "type": "PipelineReference" }, "waitOnCompletion": false, @@ -216,7 +215,7 @@ "userProperties": [], "typeProperties": { "dataset": { - "referenceName": "ADLSParquet_SH_IR", + "referenceName": "ADLSParquet_@GF{IR}", "type": "DatasetReference", "parameters": { "RelativePath": { @@ -260,7 +259,7 @@ "userProperties": [], "typeProperties": { "pipeline": { - "referenceName": "AZ-Function-Generic", + "referenceName": "AZ_Function_Generic", "type": "PipelineReference" }, "waitOnCompletion": false, @@ -288,7 +287,7 @@ "userProperties": [], "typeProperties": { "pipeline": { - "referenceName": "AZ-Function-Generic", + "referenceName": "AZ_Function_Generic", "type": "PipelineReference" }, "waitOnCompletion": false, @@ -333,8 +332,7 @@ "value": "@variables('SQLStatement')", "type": "Expression" }, - "queryTimeout": "02:00:00", - "partitionOption": "None" + "queryTimeout": "02:00:00" }, "sink": { "type": "ParquetSink", @@ -354,7 +352,7 @@ }, "inputs": [ { - "referenceName": "SqlServerTable_OnPrem_SH_IR", + "referenceName": "SqlServerTable_@GF{IR}", "type": "DatasetReference", "parameters": { "TableSchema": { @@ -390,7 +388,7 @@ ], "outputs": [ { - "referenceName": "BlobParquet_OnPrem_SH_IR", + "referenceName": "BlobParquet_@GF{IR}", "type": "DatasetReference", "parameters": { "RelativePath": { @@ -420,7 +418,7 @@ "userProperties": [], "typeProperties": { "pipeline": { - "referenceName": "AZ-Function-Generic", + "referenceName": "AZ_Function_Generic", "type": "PipelineReference" }, "waitOnCompletion": false, @@ -487,7 +485,7 @@ "userProperties": [], "typeProperties": { "dataset": { - "referenceName": "BlobParquet_SH_IR", + "referenceName": "BlobParquet_@GF{IR}", "type": "DatasetReference", "parameters": { "RelativePath": { @@ -531,7 +529,7 @@ "userProperties": [], "typeProperties": { "pipeline": { - "referenceName": "AZ-Function-Generic", + "referenceName": "AZ_Function_Generic", "type": "PipelineReference" }, "waitOnCompletion": false, @@ -559,7 +557,7 @@ "userProperties": [], "typeProperties": { "pipeline": { - "referenceName": "AZ-Function-Generic", + "referenceName": "AZ_Function_Generic", "type": "PipelineReference" }, "waitOnCompletion": false, @@ -628,7 +626,7 @@ "Name": "adsgofastdatakakeacceladf", "ResourceGroup": "AdsGoFastDataLakeAccel", "SubscriptionId": "035a1364-f00d-48e2-b582-4fe125905ee3", - "ADFPipeline": "AZ-SQL-AZ-Storage-Parquet-SH-IR" + "ADFPipeline": "AZ_SQL_AZ_Storage_Parquet_@GF{IR}" } } }, @@ -648,10 +646,9 @@ } }, "folder": { - "name": "ADS Go Fast/Data Movement/OnPrem SH IR/Components" + "name": "ADS Go Fast/Data Movement/@GF{IR}/Components" }, - "annotations": [], - "lastPublishTime": "2020-08-05T00:59:22Z" + "annotations": [] }, "type": "Microsoft.DataFactory/factories/pipelines" -} \ No newline at end of file +} diff --git a/solution/DataFactory/Templates/pipeline/SH_SQL_GEN_File_Parquet_{IR}.json b/solution/DataFactory/Templates/pipeline/SH_SQL_GEN_File_Parquet_{IR}.json new file mode 100644 index 00000000..92db9e87 --- /dev/null +++ b/solution/DataFactory/Templates/pipeline/SH_SQL_GEN_File_Parquet_{IR}.json @@ -0,0 +1,364 @@ +{ + "name": "SH_SQL_GEN_File_Parquet_@GF{IR}", + "properties": { + "activities": [ + { + "name": "Copy SQL to File", + "type": "Copy", + "dependsOn": [ + { + "activity": "Pipeline AF Log - SQL to File Start", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "source": { + "type": "SqlServerSource", + "queryTimeout": "02:00:00", + "partitionOption": "None" + }, + "sink": { + "type": "ParquetSink", + "storeSettings": { + "type": "FileServerWriteSettings" + } + }, + "enableStaging": false, + "parallelCopies": { + "value": "@pipeline().parameters.TaskObject.DegreeOfCopyParallelism", + "type": "Expression" + }, + "translator": { + "value": "@activity('AF Persist Metadata and Get Mapping').output.value", + "type": "Expression" + } + }, + "inputs": [ + { + "referenceName": "SqlServerTable_@GF{IR}", + "type": "DatasetReference", + "parameters": { + "TableSchema": { + "value": "@pipeline().parameters.TaskObject.Source.Extraction.TableSchema", + "type": "Expression" + }, + "TableName": { + "value": "@pipeline().parameters.TaskObject.Source.Extraction.TableName", + "type": "Expression" + }, + "KeyVaultBaseUrl": { + "value": "@pipeline().parameters.TaskObject.KeyVaultBaseUrl", + "type": "Expression" + }, + "PasswordSecret": { + "value": "@pipeline().parameters.TaskObject.Source.Database.PasswordKeyVaultSecretName", + "type": "Expression" + }, + "Server": { + "value": "@pipeline().parameters.TaskObject.Source.Database.SystemName", + "type": "Expression" + }, + "Database": { + "value": "@pipeline().parameters.TaskObject.Source.Database.Name", + "type": "Expression" + }, + "UserName": { + "value": "@pipeline().parameters.TaskObject.Source.Database.Username", + "type": "Expression" + } + } + } + ], + "outputs": [ + { + "referenceName": "FileParquet_@GF{IR}", + "type": "DatasetReference", + "parameters": { + "Host": { + "value": "@pipeline().parameters.TaskObject.Target.Host", + "type": "Expression" + }, + "UserId": { + "value": "@pipeline().parameters.TaskObject.Target.UserId", + "type": "Expression" + }, + "Secret": { + "value": "@pipeline().parameters.TaskObject.Target.PasswordKeyVaultSecretName", + "type": "Expression" + }, + "Directory": { + "value": "@pipeline().parameters.TaskObject.Target.Directory", + "type": "Expression" + }, + "File": { + "value": "@pipeline().parameters.TaskObject.Target.File", + "type": "Expression" + }, + "KeyVaultBaseUrl": { + "value": "@pipeline().parameters.TaskObject.KeyVaultBaseUrl", + "type": "Expression" + } + } + } + ] + }, + { + "name": "Pipeline AF Log - SQL to File Start", + "type": "ExecutePipeline", + "dependsOn": [ + { + "activity": "AF Persist Metadata and Get Mapping", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "userProperties": [], + "typeProperties": { + "pipeline": { + "referenceName": "AZ_Function_Generic", + "type": "PipelineReference" + }, + "waitOnCompletion": false, + "parameters": { + "Body": { + "value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":3,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy SQL Server to File\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"Status\":\"Started\"}'))", + "type": "Expression" + }, + "FunctionName": "Log", + "Method": "Post" + } + } + }, + { + "name": "Pipeline AF Log - SQL to File Succeed", + "type": "ExecutePipeline", + "dependsOn": [ + { + "activity": "Copy SQL to File", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "userProperties": [], + "typeProperties": { + "pipeline": { + "referenceName": "AZ_Function_Generic", + "type": "PipelineReference" + }, + "waitOnCompletion": false, + "parameters": { + "Body": { + "value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":1,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy SQL to File\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"EndDateTimeOffSet\":\"', string(utcnow()), '\",\"RowsInserted\":\"', string(activity('Copy SQL to File').output.rowsCopied), '\",\"Comment\":\"\",\"Status\":\"Complete\"}'))", + "type": "Expression" + }, + "FunctionName": "Log", + "Method": "Post" + } + } + }, + { + "name": "Pipeline AF Log - SQL to File Failed", + "type": "ExecutePipeline", + "dependsOn": [ + { + "activity": "Copy SQL to File", + "dependencyConditions": [ + "Failed" + ] + } + ], + "userProperties": [], + "typeProperties": { + "pipeline": { + "referenceName": "AZ_Function_Generic", + "type": "PipelineReference" + }, + "waitOnCompletion": false, + "parameters": { + "Body": { + "value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":1,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy SQL Server to File Share\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"EndDateTimeOffSet\":\"', string(utcnow()), '\",\"Comment\":\"', string(activity('Copy SQL to File').error.message), '\",\"Status\":\"Failed\"}'))", + "type": "Expression" + }, + "FunctionName": "Log", + "Method": "Post" + } + } + }, + { + "name": "AF Log - Get Metadata Failed", + "type": "ExecutePipeline", + "dependsOn": [ + { + "activity": "Lookup Get SQL Metadata", + "dependencyConditions": [ + "Failed" + ] + } + ], + "userProperties": [], + "typeProperties": { + "pipeline": { + "referenceName": "AZ_Function_Generic", + "type": "PipelineReference" + }, + "waitOnCompletion": false, + "parameters": { + "Body": { + "value": "[\n {\n \"TaskInstanceId\":@{pipeline().parameters.TaskObject.TaskInstanceId},\n \"ExecutionUid\":@{pipeline().parameters.TaskObject.ExecutionUid},\n \"RunId\":@{pipeline().RunId}, \n \"EndTime\":@{utcnow()},\n \"Status\":\"Failed\",\n \"Activity\":\"Get Metadata\",\n \"Comments\":@{activity('Lookup Get SQL Metadata').errors}\n }\n]", + "type": "Expression" + }, + "FunctionName": "Log", + "Method": "Post" + } + } + }, + { + "name": "AF Get Information Schema SQL", + "type": "AzureFunctionActivity", + "dependsOn": [], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "functionName": "GetInformationSchemaSQL", + "method": "POST", + "body": { + "value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"TableSchema\":\"', string(pipeline().parameters.TaskObject.Source.Extraction.TableSchema), '\",\"TableName\":\"', string(pipeline().parameters.TaskObject.Source.Extraction.TableName),'\"}'))", + "type": "Expression" + } + }, + "linkedServiceName": { + "referenceName": "AzureFunctionAdsGoFastDataLakeAccelFunApp", + "type": "LinkedServiceReference" + } + }, + { + "name": "AF Persist Metadata and Get Mapping", + "type": "AzureFunctionActivity", + "dependsOn": [ + { + "activity": "Lookup Get SQL Metadata", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "functionName": "TaskExecutionSchemaFile", + "method": "POST", + "body": { + "value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"StorageAccountName\":\"', string(pipeline().parameters.TaskObject.Target.StorageAccountName), '\",\"StorageAccountContainer\":\"', string(pipeline().parameters.TaskObject.Target.StorageAccountContainer), '\",\"RelativePath\":\"', string(pipeline().parameters.TaskObject.Target.RelativePath), '\",\"SchemaFileName\":\"', string(pipeline().parameters.TaskObject.Target.SchemaFileName), '\",\"SourceType\":\"', string(pipeline().parameters.TaskObject.Source.Type), '\",\"TargetType\":\"', string(pipeline().parameters.TaskObject.Target.Type), '\",\"Data\":',string(activity('Lookup Get SQL Metadata').output),',\"MetadataType\":\"SQL\"}'))", + "type": "Expression" + } + }, + "linkedServiceName": { + "referenceName": "AzureFunctionAdsGoFastDataLakeAccelFunApp", + "type": "LinkedServiceReference" + } + }, + { + "name": "Lookup Get SQL Metadata", + "type": "Lookup", + "dependsOn": [ + { + "activity": "AF Get Information Schema SQL", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "source": { + "type": "SqlServerSource", + "sqlReaderQuery": { + "value": "@activity('AF Get Information Schema SQL').output.InformationSchemaSQL", + "type": "Expression" + }, + "queryTimeout": "02:00:00", + "partitionOption": "None" + }, + "dataset": { + "referenceName": "SqlServerTable_@GF{IR}", + "type": "DatasetReference", + "parameters": { + "TableSchema": { + "value": "@pipeline().parameters.TaskObject.Source.Extraction.TableSchema", + "type": "Expression" + }, + "TableName": { + "value": "@pipeline().parameters.TaskObject.Source.Extraction.TableName", + "type": "Expression" + }, + "KeyVaultBaseUrl": { + "value": "@pipeline().parameters.TaskObject.KeyVaultBaseUrl", + "type": "Expression" + }, + "PasswordSecret": { + "value": "@pipeline().parameters.TaskObject.Source.Database.PasswordKeyVaultSecretName", + "type": "Expression" + }, + "Server": { + "value": "@pipeline().parameters.TaskObject.Source.Database.SystemName", + "type": "Expression" + }, + "Database": { + "value": "@pipeline().parameters.TaskObject.Source.Database.Name", + "type": "Expression" + }, + "UserName": { + "value": "@pipeline().parameters.TaskObject.Source.Database.Username", + "type": "Expression" + } + } + }, + "firstRowOnly": false + } + } + ], + "parameters": { + "TaskObject": { + "type": "object", + "defaultValue": "[ { \"TaskInstanceId\": 2, \"TaskMasterId\": 1, \"TaskStatus\": \"Untried\", \"TaskType\": \"RelationalDataExtractToDataLake\", \"Enabled\": true, \"ExecutionUid\": 1, \"KeyVaultBaseUrl\" : \"https://adsgofastkeyvault.vault.azure.net/\", \"Source\": { \"Type\":\"MsSqlServer\", \"Database\": { \"Name\": \"AdventureWorks2017\", \"SystemName\": \"adsgofast-onpre\", \"Type\": \"MsSqlServer\", \"AuthenticationType\": \"SQLAuth\", \"Username\": \"sqladfir\", \"PasswordKeyVaultSecretName\": \"adsgofast-onpre-sqladfir-password\" }, \"Extraction\": { \"Type\": \"Table\", \"FullOrIncremental\": true, \"IncrementalType\": null, \"TableSchema\": \"Sales\", \"TableName\": \"SalesOrderDetail\" } }, \"Target\": { \"Type\" : \"File\", \"Host\": \"D:\\\\dataingestion\\\\\", \"UserId\": \"AzureUser\", \"PasswordKeyVaultSecretName\": \"adsgofast-onpre_File_password\", \"Directory\": \"AdventureWorks2017/2020/06/15/14/22/\", \"File\": \"Sales.SalesOrderDetails.parquet\" }, \"DataFactory\": { \"Id\": 1, \"Name\": \"adsgofastdatakakeacceladf\", \"ResourceGroup\": \"AdsGoFastDataLakeAccel\", \"SubscriptionId\": \"035a1364-f00d-48e2-b582-4fe125905ee3\", } } ]" + } + }, + "folder": { + "name": "ADS Go Fast/Data Movement/@GF{IR}" + }, + "annotations": [], + "lastPublishTime": "2020-07-27T03:24:22Z" + }, + "type": "Microsoft.DataFactory/factories/pipelines" +} \ No newline at end of file diff --git a/solution/DataFactory/Templates/pipeline/SH_SQL_Watermark_Chunk_{IR}.json b/solution/DataFactory/Templates/pipeline/SH_SQL_Watermark_Chunk_{IR}.json new file mode 100644 index 00000000..97fcbe2e --- /dev/null +++ b/solution/DataFactory/Templates/pipeline/SH_SQL_Watermark_Chunk_{IR}.json @@ -0,0 +1,153 @@ +{ + "name": "SH_SQL_Watermark_Chunk_@GF{IR}", + "properties": { + "activities": [ + { + "name": "AF Set New Watermark", + "type": "ExecutePipeline", + "dependsOn": [ + { + "activity": "ForEach Chunk", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "userProperties": [], + "typeProperties": { + "pipeline": { + "referenceName": "AZ_Function_Generic", + "type": "PipelineReference" + }, + "waitOnCompletion": false, + "parameters": { + "Body": { + "value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"TaskMasterId\":\"', string(pipeline().parameters.TaskObject.TaskMasterId),'\",\"TaskMasterWaterMarkColumnType\":\"', string(pipeline().parameters.TaskObject.Source.Extraction.IncrementalColumnType),'\",\"WaterMarkValue\":\"', string(pipeline().parameters.NewWatermark), '\"}'))", + "type": "Expression" + }, + "FunctionName": "WaterMark", + "Method": "Post" + } + } + }, + { + "name": "ForEach Chunk", + "type": "ForEach", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "items": { + "value": "@range(1, pipeline().parameters.BatchCount)", + "type": "Expression" + }, + "isSequential": true, + "activities": [ + { + "name": "Execute SH_SQL_Watermark", + "type": "ExecutePipeline", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "pipeline": { + "referenceName": "SH_SQL_Watermark_@GF{IR}", + "type": "PipelineReference" + }, + "waitOnCompletion": true, + "parameters": { + "TaskObject": { + "value": "@pipeline().parameters.TaskObject", + "type": "Expression" + }, + "Mapping": { + "value": "@pipeline().parameters.Mapping", + "type": "Expression" + }, + "NewWaterMark": { + "value": "@pipeline().parameters.NewWatermark", + "type": "Expression" + }, + "Item": { + "value": "@item()", + "type": "Expression" + }, + "BatchCount": { + "value": "@pipeline().parameters.BatchCount", + "type": "Expression" + } + } + } + } + ] + } + } + ], + "parameters": { + "TaskObject": { + "type": "object", + "defaultValue": { + "TaskInstanceId": 75, + "TaskMasterId": 12, + "TaskStatus": "Untried", + "TaskType": "SQL Database to Azure Storage", + "Enabled": 1, + "ExecutionUid": "2c5924ee-b855-4d2b-bb7e-4f5dde4c4dd3", + "NumberOfRetries": 111, + "DegreeOfCopyParallelism": 1, + "KeyVaultBaseUrl": "https://adsgofastkeyvault.vault.azure.net/", + "ScheduleMasterId": 2, + "TaskGroupConcurrency": 10, + "TaskGroupPriority": 0, + "Source": { + "Type": "Azure SQL", + "Database": { + "SystemName": "adsgofastdatakakeaccelsqlsvr.database.windows.net", + "Name": "AWSample", + "AuthenticationType": "MSI" + }, + "Extraction": { + "Type": "Table", + "FullOrIncremental": "Full", + "IncrementalType": null, + "TableSchema": "SalesLT", + "TableName": "SalesOrderHeader" + } + }, + "Target": { + "Type": "Azure Blob", + "StorageAccountName": "https://adsgofastdatalakeaccelst.blob.core.windows.net", + "StorageAccountContainer": "datalakeraw", + "StorageAccountAccessMethod": "MSI", + "RelativePath": "/AwSample/SalesLT/SalesOrderHeader/2020/7/9/14/12/", + "DataFileName": "SalesLT.SalesOrderHeader.parquet", + "SchemaFileName": "SalesLT.SalesOrderHeader", + "FirstRowAsHeader": null, + "SheetName": null, + "SkipLineCount": null, + "MaxConcorrentConnections": null + }, + "DataFactory": { + "Id": 1, + "Name": "adsgofastdatakakeacceladf", + "ResourceGroup": "AdsGoFastDataLakeAccel", + "SubscriptionId": "035a1364-f00d-48e2-b582-4fe125905ee3", + "ADFPipeline": "AZ_SQL_AZ_Storage_Parquet_@GF{IR}" + } + } + }, + "Mapping": { + "type": "object" + }, + "NewWatermark": { + "type": "string" + }, + "BatchCount": { + "type": "int" + } + }, + "folder": { + "name": "ADS Go Fast/Data Movement/@GF{IR}/Components" + }, + "annotations": [] + }, + "type": "Microsoft.DataFactory/factories/pipelines" +} diff --git a/solution/DataFactory/pipeline/OnP-SQL-Watermark-OnP-SH-IR.json b/solution/DataFactory/Templates/pipeline/SH_SQL_Watermark_{IR}.json similarity index 95% rename from solution/DataFactory/pipeline/OnP-SQL-Watermark-OnP-SH-IR.json rename to solution/DataFactory/Templates/pipeline/SH_SQL_Watermark_{IR}.json index f02e6210..503ae5a8 100644 --- a/solution/DataFactory/pipeline/OnP-SQL-Watermark-OnP-SH-IR.json +++ b/solution/DataFactory/Templates/pipeline/SH_SQL_Watermark_{IR}.json @@ -1,5 +1,5 @@ { - "name": "OnP-SQL-Watermark-OnP-SH-IR", + "name": "SH_SQL_Watermark_@GF{IR}", "properties": { "activities": [ { @@ -62,8 +62,7 @@ "value": "@variables('SQLStatement')", "type": "Expression" }, - "queryTimeout": "02:00:00", - "partitionOption": "None" + "queryTimeout": "02:00:00" }, "sink": { "type": "ParquetSink", @@ -83,7 +82,7 @@ }, "inputs": [ { - "referenceName": "SqlServerTable_OnPrem_SH_IR", + "referenceName": "SqlServerTable_@GF{IR}", "type": "DatasetReference", "parameters": { "TableSchema": { @@ -119,7 +118,7 @@ ], "outputs": [ { - "referenceName": "ADLSParquet_OnPrem_SH_IR", + "referenceName": "ADLSParquet_@GF{IR}", "type": "DatasetReference", "parameters": { "RelativePath": { @@ -149,7 +148,7 @@ "userProperties": [], "typeProperties": { "pipeline": { - "referenceName": "AZ-Function-Generic", + "referenceName": "AZ_Function_Generic", "type": "PipelineReference" }, "waitOnCompletion": false, @@ -216,7 +215,7 @@ "userProperties": [], "typeProperties": { "dataset": { - "referenceName": "ADLSParquet_SH_IR", + "referenceName": "ADLSParquet_@GF{IR}", "type": "DatasetReference", "parameters": { "RelativePath": { @@ -260,7 +259,7 @@ "userProperties": [], "typeProperties": { "pipeline": { - "referenceName": "AZ-Function-Generic", + "referenceName": "AZ_Function_Generic", "type": "PipelineReference" }, "waitOnCompletion": false, @@ -288,7 +287,7 @@ "userProperties": [], "typeProperties": { "pipeline": { - "referenceName": "AZ-Function-Generic", + "referenceName": "AZ_Function_Generic", "type": "PipelineReference" }, "waitOnCompletion": false, @@ -329,8 +328,7 @@ "typeProperties": { "source": { "type": "SqlServerSource", - "queryTimeout": "02:00:00", - "partitionOption": "None" + "queryTimeout": "02:00:00" }, "sink": { "type": "ParquetSink", @@ -350,7 +348,7 @@ }, "inputs": [ { - "referenceName": "SqlServerTable_OnPrem_SH_IR", + "referenceName": "SqlServerTable_@GF{IR}", "type": "DatasetReference", "parameters": { "TableSchema": { @@ -386,7 +384,7 @@ ], "outputs": [ { - "referenceName": "BlobParquet_OnPrem_SH_IR", + "referenceName": "BlobParquet_@GF{IR}", "type": "DatasetReference", "parameters": { "RelativePath": { @@ -416,7 +414,7 @@ "userProperties": [], "typeProperties": { "pipeline": { - "referenceName": "AZ-Function-Generic", + "referenceName": "AZ_Function_Generic", "type": "PipelineReference" }, "waitOnCompletion": false, @@ -483,7 +481,7 @@ "userProperties": [], "typeProperties": { "dataset": { - "referenceName": "BlobParquet_SH_IR", + "referenceName": "BlobParquet_@GF{IR}", "type": "DatasetReference", "parameters": { "RelativePath": { @@ -527,7 +525,7 @@ "userProperties": [], "typeProperties": { "pipeline": { - "referenceName": "AZ-Function-Generic", + "referenceName": "AZ_Function_Generic", "type": "PipelineReference" }, "waitOnCompletion": false, @@ -555,7 +553,7 @@ "userProperties": [], "typeProperties": { "pipeline": { - "referenceName": "AZ-Function-Generic", + "referenceName": "AZ_Function_Generic", "type": "PipelineReference" }, "waitOnCompletion": false, @@ -624,7 +622,7 @@ "Name": "adsgofastdatakakeacceladf", "ResourceGroup": "AdsGoFastDataLakeAccel", "SubscriptionId": "035a1364-f00d-48e2-b582-4fe125905ee3", - "ADFPipeline": "AZ-SQL-AZ-Storage-Parquet-SH-IR" + "ADFPipeline": "AZ_SQL_AZ_Storage_Parquet_@GF{IR}" } } }, @@ -647,10 +645,9 @@ } }, "folder": { - "name": "ADS Go Fast/Data Movement/OnPrem SH IR/Components" + "name": "ADS Go Fast/Data Movement/@GF{IR}/Components" }, - "annotations": [], - "lastPublishTime": "2020-07-29T02:39:55Z" + "annotations": [] }, "type": "Microsoft.DataFactory/factories/pipelines" -} \ No newline at end of file +} diff --git a/solution/DataFactory/Templates/trigger/Trigger on File End.json b/solution/DataFactory/Templates/trigger/Trigger on File End.json new file mode 100644 index 00000000..7d4ea584 --- /dev/null +++ b/solution/DataFactory/Templates/trigger/Trigger on File End.json @@ -0,0 +1,18 @@ +{ + "name": "Trigger on File End", + "properties": { + "annotations": [], + "runtimeState": "Stopped", + "pipelines": [], + "type": "BlobEventsTrigger", + "typeProperties": { + "blobPathBeginsWith": "/datalakelanding/blobs/Unprocessed/", + "blobPathEndsWith": "final.end", + "ignoreEmptyBlobs": false, + "scope": "/subscriptions/035a1364-f00d-48e2-b582-4fe125905ee3/resourceGroups/AdsGoFastDataLakeAccel/providers/Microsoft.Storage/storageAccounts/adsgofastdatalakeaccelst", + "events": [ + "Microsoft.Storage.BlobCreated" + ] + } + } +} \ No newline at end of file diff --git a/solution/DataFactory/dataset/ADLSBinary_OnPrem_SH_IR.json b/solution/DataFactory/dataset/ADLSBinary_OnPrem_SH_IR.json deleted file mode 100644 index abbedc33..00000000 --- a/solution/DataFactory/dataset/ADLSBinary_OnPrem_SH_IR.json +++ /dev/null @@ -1,52 +0,0 @@ -{ - "name": "ADLSBinary_OnPrem_SH_IR", - "properties": { - "linkedServiceName": { - "referenceName": "GenericAzureDataLakeStorage_OnPrem_Net", - "type": "LinkedServiceReference", - "parameters": { - "StorageAccountEndpoint": { - "value": "@dataset().StorageAccountEndpoint", - "type": "Expression" - } - } - }, - "parameters": { - "StorageAccountEndpoint": { - "type": "string" - }, - "FileSystem": { - "type": "string" - }, - "Directory": { - "type": "string" - }, - "File": { - "type": "string" - } - }, - "folder": { - "name": "ADS Go Fast/Generic/OnPrem SH IR" - }, - "annotations": [], - "type": "Binary", - "typeProperties": { - "location": { - "type": "AzureBlobFSLocation", - "fileName": { - "value": "@dataset().File", - "type": "Expression" - }, - "folderPath": { - "value": "@dataset().Directory", - "type": "Expression" - }, - "fileSystem": { - "value": "@dataset().FileSystem", - "type": "Expression" - } - } - } - }, - "type": "Microsoft.DataFactory/factories/datasets" -} \ No newline at end of file diff --git a/solution/DataFactory/dataset/ADLSParquet_OnPrem_SH_IR.json b/solution/DataFactory/dataset/ADLSParquet_OnPrem_SH_IR.json deleted file mode 100644 index 2a097fa9..00000000 --- a/solution/DataFactory/dataset/ADLSParquet_OnPrem_SH_IR.json +++ /dev/null @@ -1,54 +0,0 @@ -{ - "name": "ADLSParquet_OnPrem_SH_IR", - "properties": { - "linkedServiceName": { - "referenceName": "GenericAzureDataLakeStorage_OnPrem_Net", - "type": "LinkedServiceReference", - "parameters": { - "StorageAccountEndpoint": { - "value": "@dataset().StorageAccountEndpoint", - "type": "Expression" - } - } - }, - "parameters": { - "RelativePath": { - "type": "string" - }, - "FileName": { - "type": "string" - }, - "StorageAccountEndpoint": { - "type": "string" - }, - "StorageAccountContainerName": { - "type": "string" - } - }, - "folder": { - "name": "ADS Go Fast/Generic/OnPrem SH IR" - }, - "annotations": [], - "type": "Parquet", - "typeProperties": { - "location": { - "type": "AzureBlobFSLocation", - "fileName": { - "value": "@dataset().FileName", - "type": "Expression" - }, - "folderPath": { - "value": "@dataset().RelativePath", - "type": "Expression" - }, - "fileSystem": { - "value": "@dataset().StorageAccountContainerName", - "type": "Expression" - } - }, - "compressionCodec": "gzip" - }, - "schema": [] - }, - "type": "Microsoft.DataFactory/factories/datasets" -} \ No newline at end of file diff --git a/solution/DataFactory/dataset/BlobBinary_OnPrem_SH_IR.json b/solution/DataFactory/dataset/BlobBinary_OnPrem_SH_IR.json deleted file mode 100644 index 5735be2f..00000000 --- a/solution/DataFactory/dataset/BlobBinary_OnPrem_SH_IR.json +++ /dev/null @@ -1,52 +0,0 @@ -{ - "name": "BlobBinary_OnPrem_SH_IR", - "properties": { - "linkedServiceName": { - "referenceName": "GenericBlobStorage_OnPrem_Net", - "type": "LinkedServiceReference", - "parameters": { - "StorageAccountEndpoint": { - "value": "@dataset().StorageAccountEndpoint", - "type": "Expression" - } - } - }, - "parameters": { - "StorageAccountEndpoint": { - "type": "string" - }, - "FileSystem": { - "type": "string" - }, - "Directory": { - "type": "string" - }, - "File": { - "type": "string" - } - }, - "folder": { - "name": "ADS Go Fast/Generic/OnPrem SH IR" - }, - "annotations": [], - "type": "Binary", - "typeProperties": { - "location": { - "type": "AzureBlobStorageLocation", - "fileName": { - "value": "@dataset().File", - "type": "Expression" - }, - "folderPath": { - "value": "@dataset().Directory", - "type": "Expression" - }, - "container": { - "value": "@dataset().FileSystem", - "type": "Expression" - } - } - } - }, - "type": "Microsoft.DataFactory/factories/datasets" -} \ No newline at end of file diff --git a/solution/DataFactory/dataset/BlobParquet_OnPrem_SH_IR.json b/solution/DataFactory/dataset/BlobParquet_OnPrem_SH_IR.json deleted file mode 100644 index 691d915c..00000000 --- a/solution/DataFactory/dataset/BlobParquet_OnPrem_SH_IR.json +++ /dev/null @@ -1,54 +0,0 @@ -{ - "name": "BlobParquet_OnPrem_SH_IR", - "properties": { - "linkedServiceName": { - "referenceName": "GenericBlobStorage_OnPrem_Net", - "type": "LinkedServiceReference", - "parameters": { - "StorageAccountEndpoint": { - "value": "@dataset().StorageAccountEndpoint", - "type": "Expression" - } - } - }, - "parameters": { - "RelativePath": { - "type": "string" - }, - "FileName": { - "type": "string" - }, - "StorageAccountEndpoint": { - "type": "string" - }, - "StorageAccountContainerName": { - "type": "string" - } - }, - "folder": { - "name": "ADS Go Fast/Generic/OnPrem SH IR" - }, - "annotations": [], - "type": "Parquet", - "typeProperties": { - "location": { - "type": "AzureBlobStorageLocation", - "fileName": { - "value": "@dataset().FileName", - "type": "Expression" - }, - "folderPath": { - "value": "@dataset().RelativePath", - "type": "Expression" - }, - "container": { - "value": "@dataset().StorageAccountContainerName", - "type": "Expression" - } - }, - "compressionCodec": "gzip" - }, - "schema": [] - }, - "type": "Microsoft.DataFactory/factories/datasets" -} \ No newline at end of file diff --git a/solution/DataFactory/integrationRuntime/SelfHostedIntegrationRuntime-Azure-VNET.json b/solution/DataFactory/integrationRuntime/SelfHostedIntegrationRuntime-Azure-VNET.json deleted file mode 100644 index 6687cdec..00000000 --- a/solution/DataFactory/integrationRuntime/SelfHostedIntegrationRuntime-Azure-VNET.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "name": "SelfHostedIntegrationRuntime-Azure-VNET", - "properties": { - "type": "SelfHosted" - } -} \ No newline at end of file diff --git a/solution/DataFactory/integrationRuntime/SelfHostedIntegrationRuntime-OnPem-Net.json b/solution/DataFactory/integrationRuntime/SelfHostedIntegrationRuntime-OnPem-Net.json deleted file mode 100644 index e8ff489e..00000000 --- a/solution/DataFactory/integrationRuntime/SelfHostedIntegrationRuntime-OnPem-Net.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "name": "SelfHostedIntegrationRuntime-OnPem-Net", - "properties": { - "type": "SelfHosted" - } -} \ No newline at end of file diff --git a/solution/DataFactory/linkedService/GenericAzureDataLakeStorage.json b/solution/DataFactory/linkedService/GenericAzureDataLakeStorage.json deleted file mode 100644 index 2123a6c5..00000000 --- a/solution/DataFactory/linkedService/GenericAzureDataLakeStorage.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "name": "GenericAzureDataLakeStorage", - "type": "Microsoft.DataFactory/factories/linkedservices", - "properties": { - "type": "AzureBlobFS", - "parameters": { - "StorageAccountEndpoint": { - "type": "String", - "defaultValue": "" - } - }, - "typeProperties": { - "url": "@{linkedService().StorageAccountEndpoint}" - }, - "annotations": [] - } -} \ No newline at end of file diff --git a/solution/DataFactory/linkedService/GenericBlobStorage.json b/solution/DataFactory/linkedService/GenericBlobStorage.json deleted file mode 100644 index 0d3d26d1..00000000 --- a/solution/DataFactory/linkedService/GenericBlobStorage.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "name": "GenericBlobStorage", - "type": "Microsoft.DataFactory/factories/linkedservices", - "properties": { - "type": "AzureBlobStorage", - "parameters": { - "StorageAccountEndpoint": { - "type": "String", - "defaultValue": "" - } - }, - "typeProperties": { - "serviceEndpoint": "@{linkedService().StorageAccountEndpoint}" - }, - "description": "Test Description", - "annotations": [] - } -} \ No newline at end of file diff --git a/solution/DataFactory/pipeline/AZ-SQL-AZ-Storage-Parquet-AZ-IR.json b/solution/DataFactory/pipeline/AZ-SQL-AZ-Storage-Parquet-AZ-IR.json deleted file mode 100644 index fa55ff25..00000000 --- a/solution/DataFactory/pipeline/AZ-SQL-AZ-Storage-Parquet-AZ-IR.json +++ /dev/null @@ -1,745 +0,0 @@ -{ - "name": "AZ-SQL-AZ-Storage-Parquet-AZ-IR", - "properties": { - "activities": [ - { - "name": "Switch Storage Type", - "type": "Switch", - "dependsOn": [ - { - "activity": "AF Persist Metadata and Get Mapping", - "dependencyConditions": [ - "Succeeded" - ] - } - ], - "userProperties": [], - "typeProperties": { - "on": { - "value": "@pipeline().parameters.TaskObject.Source.Type", - "type": "Expression" - }, - "cases": [ - { - "value": "ADLS", - "activities": [ - { - "name": "Copy SQL to ADLS", - "type": "Copy", - "dependsOn": [ - { - "activity": "Pipeline AF Log - Azure SQL to ADLS Start", - "dependencyConditions": [ - "Succeeded" - ] - } - ], - "policy": { - "timeout": "7.00:00:00", - "retry": 0, - "retryIntervalInSeconds": 30, - "secureOutput": false, - "secureInput": false - }, - "userProperties": [], - "typeProperties": { - "source": { - "type": "AzureSqlSource", - "queryTimeout": "02:00:00", - "partitionOption": "None" - }, - "sink": { - "type": "ParquetSink", - "storeSettings": { - "type": "AzureBlobFSWriteSettings" - } - }, - "enableStaging": false, - "parallelCopies": { - "value": "@pipeline().parameters.TaskObject.DegreeOfCopyParallelism", - "type": "Expression" - }, - "translator": { - "value": "@activity('AF Persist Metadata and Get Mapping').output.value", - "type": "Expression" - } - }, - "inputs": [ - { - "referenceName": "AzureSqlTable_AZ_IR", - "type": "DatasetReference", - "parameters": { - "Schema": { - "value": "@pipeline().parameters.TaskObject.Source.Extraction.TableSchema", - "type": "Expression" - }, - "Table": { - "value": "@pipeline().parameters.TaskObject.Source.Extraction.TableName", - "type": "Expression" - }, - "Server": { - "value": "@pipeline().parameters.TaskObject.Source.Database.SystemName", - "type": "Expression" - }, - "Database": { - "value": "@pipeline().parameters.TaskObject.Source.Database.Name", - "type": "Expression" - } - } - } - ], - "outputs": [ - { - "referenceName": "ADLSParquet_AZ_IR", - "type": "DatasetReference", - "parameters": { - "RelativePath": { - "value": "@pipeline().parameters.TaskObject.Target.RelativePath", - "type": "Expression" - }, - "FileName": { - "value": "@pipeline().parameters.TaskObject.Target.DataFileName", - "type": "Expression" - }, - "StorageAccountEndpoint": { - "value": "@pipeline().parameters.TaskObject.Target.StorageAccountName", - "type": "Expression" - }, - "StorageAccountContainerName": { - "value": "@pipeline().parameters.TaskObject.Target.StorageAccountContainer", - "type": "Expression" - } - } - } - ] - }, - { - "name": "AF Persist Parquet Metadata - ADLS", - "type": "AzureFunctionActivity", - "dependsOn": [ - { - "activity": "Get Parquet Metadata ADLS", - "dependencyConditions": [ - "Succeeded" - ] - } - ], - "policy": { - "timeout": "7.00:00:00", - "retry": 0, - "retryIntervalInSeconds": 30, - "secureOutput": false, - "secureInput": false - }, - "userProperties": [], - "typeProperties": { - "functionName": "TaskExecutionSchemaFile", - "method": "POST", - "body": { - "value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"StorageAccountName\":\"', string(pipeline().parameters.TaskObject.Target.StorageAccountName), '\",\"StorageAccountContainer\":\"', string(pipeline().parameters.TaskObject.Target.StorageAccountContainer), '\",\"RelativePath\":\"', string(pipeline().parameters.TaskObject.Target.RelativePath), '\",\"SchemaFileName\":\"', string(pipeline().parameters.TaskObject.Target.SchemaFileName), '\",\"SourceType\":\"', string(pipeline().parameters.TaskObject.Source.Type), '\",\"TargetType\":\"', string(pipeline().parameters.TaskObject.Target.Type), '\",\"Data\":',string(activity('Get Parquet Metadata ADLS').output),',\"MetadataType\":\"Parquet\"}'))", - "type": "Expression" - } - }, - "linkedServiceName": { - "referenceName": "AzureFunctionAdsGoFastDataLakeAccelFunApp", - "type": "LinkedServiceReference" - } - }, - { - "name": "Get Parquet Metadata ADLS", - "type": "GetMetadata", - "dependsOn": [ - { - "activity": "Copy SQL to ADLS", - "dependencyConditions": [ - "Succeeded" - ] - } - ], - "policy": { - "timeout": "7.00:00:00", - "retry": 0, - "retryIntervalInSeconds": 30, - "secureOutput": false, - "secureInput": false - }, - "userProperties": [], - "typeProperties": { - "dataset": { - "referenceName": "ADLSParquet_AZ_IR", - "type": "DatasetReference", - "parameters": { - "RelativePath": { - "value": "@pipeline().parameters.TaskObject.Target.RelativePath", - "type": "Expression" - }, - "FileName": { - "value": "@pipeline().parameters.TaskObject.Target.DataFileName", - "type": "Expression" - }, - "StorageAccountEndpoint": { - "value": "@pipeline().parameters.TaskObject.Target.StorageAccountName", - "type": "Expression" - }, - "StorageAccountContainerName": { - "value": "@pipeline().parameters.TaskObject.Target.StorageAccountContainer", - "type": "Expression" - } - } - }, - "fieldList": [ - "structure" - ], - "storeSettings": { - "type": "AzureBlobFSReadSettings", - "recursive": true - } - } - }, - { - "name": "Pipeline AF Log - Azure SQL to ADLS Failed", - "type": "ExecutePipeline", - "dependsOn": [ - { - "activity": "Copy SQL to ADLS", - "dependencyConditions": [ - "Failed" - ] - } - ], - "userProperties": [], - "typeProperties": { - "pipeline": { - "referenceName": "AZ-Function-Generic", - "type": "PipelineReference" - }, - "waitOnCompletion": false, - "parameters": { - "Body": { - "value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":1,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy Azure SQL to ADLS\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"EndDateTimeOffSet\":\"', string(utcnow()), '\",\"Comment\":\"', string(activity('Copy SQL to ADLS').error.message), '\",\"Status\":\"Failed\"}'))", - "type": "Expression" - }, - "FunctionName": "Log", - "Method": "Post" - } - } - }, - { - "name": "Pipeline AF Log - Azure SQL to ADLS Start", - "type": "ExecutePipeline", - "dependsOn": [], - "userProperties": [], - "typeProperties": { - "pipeline": { - "referenceName": "AZ-Function-Generic", - "type": "PipelineReference" - }, - "waitOnCompletion": false, - "parameters": { - "Body": { - "value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":3,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy Azure SQL to ADLS\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"Status\":\"Started\"}'))", - "type": "Expression" - }, - "FunctionName": "Log", - "Method": "Post" - } - } - }, - { - "name": "Pipeline AF Log - Azure SQL to ADLS Succeed", - "type": "ExecutePipeline", - "dependsOn": [ - { - "activity": "AF Persist Parquet Metadata - ADLS", - "dependencyConditions": [ - "Succeeded" - ] - } - ], - "userProperties": [], - "typeProperties": { - "pipeline": { - "referenceName": "AZ-Function-Generic", - "type": "PipelineReference" - }, - "waitOnCompletion": false, - "parameters": { - "Body": { - "value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":1,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy Azure SQL to ADLS\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"EndDateTimeOffSet\":\"', string(utcnow()), '\",\"Comment\":\"\",\"Status\":\"Complete\"}'))", - "type": "Expression" - }, - "FunctionName": "Log", - "Method": "Post" - } - } - } - ] - }, - { - "value": "Azure Blob", - "activities": [ - { - "name": "Copy SQL to Blob", - "type": "Copy", - "dependsOn": [ - { - "activity": "Pipeline AF Log - SQL to Blob Start", - "dependencyConditions": [ - "Succeeded" - ] - } - ], - "policy": { - "timeout": "7.00:00:00", - "retry": 0, - "retryIntervalInSeconds": 30, - "secureOutput": false, - "secureInput": false - }, - "userProperties": [], - "typeProperties": { - "source": { - "type": "AzureSqlSource", - "queryTimeout": "02:00:00", - "partitionOption": "None" - }, - "sink": { - "type": "ParquetSink", - "storeSettings": { - "type": "AzureBlobStorageWriteSettings" - } - }, - "enableStaging": false, - "parallelCopies": { - "value": "@pipeline().parameters.TaskObject.DegreeOfCopyParallelism", - "type": "Expression" - }, - "translator": { - "value": "@activity('AF Persist Metadata and Get Mapping').output.value", - "type": "Expression" - } - }, - "inputs": [ - { - "referenceName": "AzureSqlTable_AZ_IR", - "type": "DatasetReference", - "parameters": { - "Schema": { - "value": "@pipeline().parameters.TaskObject.Source.Extraction.TableSchema", - "type": "Expression" - }, - "Table": { - "value": "@pipeline().parameters.TaskObject.Source.Extraction.TableName", - "type": "Expression" - }, - "Server": { - "value": "@pipeline().parameters.TaskObject.Source.Database.SystemName", - "type": "Expression" - }, - "Database": { - "value": "@pipeline().parameters.TaskObject.Source.Database.Name", - "type": "Expression" - } - } - } - ], - "outputs": [ - { - "referenceName": "BlobParquet_AZ_IR", - "type": "DatasetReference", - "parameters": { - "RelativePath": { - "value": "@pipeline().parameters.TaskObject.Target.RelativePath", - "type": "Expression" - }, - "FileName": { - "value": "@pipeline().parameters.TaskObject.Target.DataFileName", - "type": "Expression" - }, - "StorageAccountEndpoint": { - "value": "@pipeline().parameters.TaskObject.Target.StorageAccountName", - "type": "Expression" - }, - "StorageAccountContainerName": { - "value": "@pipeline().parameters.TaskObject.Target.StorageAccountContainer", - "type": "Expression" - } - } - } - ] - }, - { - "name": "Pipeline AF Log - SQL to Blob Start", - "type": "ExecutePipeline", - "dependsOn": [], - "userProperties": [], - "typeProperties": { - "pipeline": { - "referenceName": "AZ-Function-Generic", - "type": "PipelineReference" - }, - "waitOnCompletion": false, - "parameters": { - "Body": { - "value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":3,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy Azure SQL to Blob\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"Status\":\"Started\"}'))", - "type": "Expression" - }, - "FunctionName": "Log", - "Method": "Put" - } - } - }, - { - "name": "AF Persist Parquet Metadata - Blob", - "type": "AzureFunctionActivity", - "dependsOn": [ - { - "activity": "Get Parquet Metadata Blob", - "dependencyConditions": [ - "Succeeded" - ] - } - ], - "policy": { - "timeout": "7.00:00:00", - "retry": 0, - "retryIntervalInSeconds": 30, - "secureOutput": false, - "secureInput": false - }, - "userProperties": [], - "typeProperties": { - "functionName": "TaskExecutionSchemaFile", - "method": "POST", - "body": { - "value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"StorageAccountName\":\"', string(pipeline().parameters.TaskObject.Target.StorageAccountName), '\",\"StorageAccountContainer\":\"', string(pipeline().parameters.TaskObject.Target.StorageAccountContainer), '\",\"RelativePath\":\"', string(pipeline().parameters.TaskObject.Target.RelativePath), '\",\"SchemaFileName\":\"', string(pipeline().parameters.TaskObject.Target.SchemaFileName), '\",\"SourceType\":\"', string(pipeline().parameters.TaskObject.Source.Type), '\",\"TargetType\":\"', string(pipeline().parameters.TaskObject.Target.Type), '\",\"Data\":',string(activity('Get Parquet Metadata Blob').output),',\"MetadataType\":\"Parquet\"}'))", - "type": "Expression" - } - }, - "linkedServiceName": { - "referenceName": "AzureFunctionAdsGoFastDataLakeAccelFunApp", - "type": "LinkedServiceReference" - } - }, - { - "name": "Get Parquet Metadata Blob", - "type": "GetMetadata", - "dependsOn": [ - { - "activity": "Copy SQL to Blob", - "dependencyConditions": [ - "Succeeded" - ] - } - ], - "policy": { - "timeout": "7.00:00:00", - "retry": 0, - "retryIntervalInSeconds": 30, - "secureOutput": false, - "secureInput": false - }, - "userProperties": [], - "typeProperties": { - "dataset": { - "referenceName": "BlobParquet_AZ_IR", - "type": "DatasetReference", - "parameters": { - "RelativePath": { - "value": "@pipeline().parameters.TaskObject.Target.RelativePath", - "type": "Expression" - }, - "FileName": { - "value": "@pipeline().parameters.TaskObject.Target.DataFileName", - "type": "Expression" - }, - "StorageAccountEndpoint": { - "value": "@pipeline().parameters.TaskObject.Target.StorageAccountName", - "type": "Expression" - }, - "StorageAccountContainerName": { - "value": "@pipeline().parameters.TaskObject.Target.StorageAccountContainer", - "type": "Expression" - } - } - }, - "fieldList": [ - "structure" - ], - "storeSettings": { - "type": "AzureBlobStorageReadSettings", - "recursive": true - } - } - }, - { - "name": "Pipeline AF Log - Azure SQL to Blob Failed", - "type": "ExecutePipeline", - "dependsOn": [ - { - "activity": "Copy SQL to Blob", - "dependencyConditions": [ - "Failed" - ] - } - ], - "userProperties": [], - "typeProperties": { - "pipeline": { - "referenceName": "AZ-Function-Generic", - "type": "PipelineReference" - }, - "waitOnCompletion": false, - "parameters": { - "Body": { - "value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":1,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy Azure SQL to Blob\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"EndDateTimeOffSet\":\"', string(utcnow()), '\",\"Comment\":\"', string(activity('Copy SQL to Blob').error.message), '\",\"Status\":\"Failed\"}'))", - "type": "Expression" - }, - "FunctionName": "Log", - "Method": "Post" - } - } - }, - { - "name": "Pipeline AF Log - Azure SQL to Blob Succeed", - "type": "ExecutePipeline", - "dependsOn": [ - { - "activity": "AF Persist Parquet Metadata - Blob", - "dependencyConditions": [ - "Succeeded" - ] - } - ], - "userProperties": [], - "typeProperties": { - "pipeline": { - "referenceName": "AZ-Function-Generic", - "type": "PipelineReference" - }, - "waitOnCompletion": false, - "parameters": { - "Body": { - "value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":1,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy Azure SQL to Blob\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"EndDateTimeOffSet\":\"', string(utcnow()), '\",\"Comment\":\"\",\"Status\":\"Complete\"}'))", - "type": "Expression" - }, - "FunctionName": "Log", - "Method": "Post" - } - } - } - ] - } - ] - } - }, - { - "name": "AF Log - Get Metadata Failed", - "type": "ExecutePipeline", - "dependsOn": [ - { - "activity": "Lookup Get SQL Metadata", - "dependencyConditions": [ - "Failed" - ] - } - ], - "userProperties": [], - "typeProperties": { - "pipeline": { - "referenceName": "AZ-Function-Generic", - "type": "PipelineReference" - }, - "waitOnCompletion": false, - "parameters": { - "Body": { - "value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":1,\"LogSource\":\"ADF\",\"ActivityType\":\"Get Metadata\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"EndDateTimeOffSet\":\"', string(utcnow()), '\",\"Comment\":\"', string(activity('Lookup Get SQL Metadata').error.message), '\",\"Status\":\"Failed\"}'))", - "type": "Expression" - }, - "FunctionName": "Log", - "Method": "Post" - } - } - }, - { - "name": "AF Log - Get Metadata Start", - "type": "ExecutePipeline", - "dependsOn": [], - "userProperties": [], - "typeProperties": { - "pipeline": { - "referenceName": "AZ-Function-Generic", - "type": "PipelineReference" - }, - "waitOnCompletion": false, - "parameters": { - "Body": { - "value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":3,\"LogSource\":\"ADF\",\"ActivityType\":\"Get Metadata\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"Status\":\"Started\"}'))", - "type": "Expression" - }, - "FunctionName": "Log", - "Method": "Post" - } - } - }, - { - "name": "AF Log - Get Metadata Succeed", - "type": "ExecutePipeline", - "dependsOn": [ - { - "activity": "AF Persist Metadata and Get Mapping", - "dependencyConditions": [ - "Succeeded" - ] - } - ], - "userProperties": [], - "typeProperties": { - "pipeline": { - "referenceName": "AZ-Function-Generic", - "type": "PipelineReference" - }, - "waitOnCompletion": false, - "parameters": { - "Body": { - "value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":1,\"LogSource\":\"ADF\",\"ActivityType\":\"Get Metadata\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"EndDateTimeOffSet\":\"', string(utcnow()), '\",\"Comment\":\"\",\"Status\":\"Complete\"}'))", - "type": "Expression" - }, - "FunctionName": "Log", - "Method": "Post" - } - } - }, - { - "name": "Lookup Get SQL Metadata", - "type": "Lookup", - "dependsOn": [ - { - "activity": "AF Get Information Schema SQL", - "dependencyConditions": [ - "Succeeded" - ] - } - ], - "policy": { - "timeout": "7.00:00:00", - "retry": 0, - "retryIntervalInSeconds": 30, - "secureOutput": false, - "secureInput": false - }, - "userProperties": [], - "typeProperties": { - "source": { - "type": "AzureSqlSource", - "sqlReaderQuery": { - "value": "@activity('AF Get Information Schema SQL').output.InformationSchemaSQL", - "type": "Expression" - }, - "queryTimeout": "02:00:00", - "partitionOption": "None" - }, - "dataset": { - "referenceName": "AzureSqlTable_AZ_IR", - "type": "DatasetReference", - "parameters": { - "Schema": { - "value": "@pipeline().parameters.TaskObject.Source.Extraction.TableSchema", - "type": "Expression" - }, - "Table": { - "value": "@pipeline().parameters.TaskObject.Source.Extraction.TableName", - "type": "Expression" - }, - "Server": { - "value": "@pipeline().parameters.TaskObject.Source.Database.SystemName", - "type": "Expression" - }, - "Database": { - "value": "@pipeline().parameters.TaskObject.Source.Database.Name", - "type": "Expression" - } - } - }, - "firstRowOnly": false - } - }, - { - "name": "AF Get Information Schema SQL", - "type": "AzureFunctionActivity", - "dependsOn": [ - { - "activity": "AF Log - Get Metadata Start", - "dependencyConditions": [ - "Succeeded" - ] - } - ], - "policy": { - "timeout": "7.00:00:00", - "retry": 0, - "retryIntervalInSeconds": 30, - "secureOutput": false, - "secureInput": false - }, - "userProperties": [], - "typeProperties": { - "functionName": "GetInformationSchemaSQL", - "method": "POST", - "body": { - "value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"TableSchema\":\"', string(pipeline().parameters.TaskObject.Source.Extraction.TableSchema), '\",\"TableName\":\"', string(pipeline().parameters.TaskObject.Source.Extraction.TableName),'\"}'))", - "type": "Expression" - } - }, - "linkedServiceName": { - "referenceName": "AzureFunctionAdsGoFastDataLakeAccelFunApp", - "type": "LinkedServiceReference" - } - }, - { - "name": "AF Persist Metadata and Get Mapping", - "type": "AzureFunctionActivity", - "dependsOn": [ - { - "activity": "Lookup Get SQL Metadata", - "dependencyConditions": [ - "Succeeded" - ] - } - ], - "policy": { - "timeout": "7.00:00:00", - "retry": 0, - "retryIntervalInSeconds": 30, - "secureOutput": false, - "secureInput": false - }, - "userProperties": [], - "typeProperties": { - "functionName": "TaskExecutionSchemaFile", - "method": "POST", - "body": { - "value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"StorageAccountName\":\"', string(pipeline().parameters.TaskObject.Target.StorageAccountName), '\",\"StorageAccountContainer\":\"', string(pipeline().parameters.TaskObject.Target.StorageAccountContainer), '\",\"RelativePath\":\"', string(pipeline().parameters.TaskObject.Target.RelativePath), '\",\"SchemaFileName\":\"', string(pipeline().parameters.TaskObject.Target.SchemaFileName), '\",\"SourceType\":\"', string(pipeline().parameters.TaskObject.Source.Type), '\",\"TargetType\":\"', string(pipeline().parameters.TaskObject.Target.Type), '\",\"Data\":',string(activity('Lookup Get SQL Metadata').output),',\"MetadataType\":\"SQL\"}'))", - "type": "Expression" - } - }, - "linkedServiceName": { - "referenceName": "AzureFunctionAdsGoFastDataLakeAccelFunApp", - "type": "LinkedServiceReference" - } - } - ], - "parameters": { - "TaskObject": { - "type": "object", - "defaultValue": "[ { \"TaskInstanceId\": 2, \"TaskMasterId\": 1, \"TaskStatus\": \"Untried\", \"TaskType\": \"RelationalDataExtractToDataLake\", \"Enabled\": true, \"ExecutionUid\": 1, \"Source\": { \"Database\": { \"Name\": \"AWSample\", \"SystemName\" : \"adsgofastdatakakeaccelsqlsvr.database.windows.net\", \"Type\": \"AzureSQL\", \"AuthenticationType\": \"MSI\", \"UsernameKeyVaultSecretName\": \"\", \"PasswordKeyVaultSecretName\": \"\" }, \"Extraction\": { \"Type\": \"Table\", \"FullOrIncremental\": true, \"IncrementalType\": null, \"TableSchema\" : \"SalesLT\", \"TableName\": \"Customer\" } }, \"Target\": { \"StorageAccountName\": \"https://adsgofastdatalakeaccelst.dfs.core.windows.net\", \"StorageAccountContainer\": \"Container\", \"StorageAccountContainerName\": \"datalakelanding\", \"StorageAccountAccessMethod\": \"MSI\", \"StorageAccountSASUriKeyVaultSecretName\": null, \"RelativePath\": \"/Unprocessed/adsgofastdatakakeaccelsqlsvr/AWSample/SalesLT/2020/06/08/17/\", \"DataFileName\": \"Customer_Data.parquet\", \"SchemaFileName\": \"Customer_Schema.json\" }, \"DataFactory\": { \"Id\": 1, \"Name\": \"adsgofastdatakakeacceladf\", \"ResourceGroup\": \"AdsGoFastDataLakeAccel\", \"SubscriptionId\": \"035a1364-f00d-48e2-b582-4fe125905ee3\", } } ] " - } - }, - "folder": { - "name": "ADS Go Fast/Data Movement/Auto Resolve IR" - }, - "annotations": [], - "lastPublishTime": "2020-07-24T02:37:15Z" - }, - "type": "Microsoft.DataFactory/factories/pipelines" -} \ No newline at end of file diff --git a/solution/DataFactory/pipeline/AZ-Storage-Parquet-AZ-SQL-AZ-IR.json b/solution/DataFactory/pipeline/AZ-Storage-Parquet-AZ-SQL-AZ-IR.json deleted file mode 100644 index 47cc250d..00000000 --- a/solution/DataFactory/pipeline/AZ-Storage-Parquet-AZ-SQL-AZ-IR.json +++ /dev/null @@ -1,376 +0,0 @@ -{ - "name": "AZ-Storage-Parquet-AZ-SQL-AZ-IR", - "properties": { - "activities": [ - { - "name": "Switch Storage Type", - "type": "Switch", - "dependsOn": [], - "userProperties": [], - "typeProperties": { - "on": { - "value": "@pipeline().parameters.TaskObject.Source.Type", - "type": "Expression" - }, - "cases": [ - { - "value": "ADLS", - "activities": [ - { - "name": "Copy ADLS to Azure SQL", - "type": "Copy", - "dependsOn": [ - { - "activity": "Pipeline AF Log - ADLS to Azure SQL Start", - "dependencyConditions": [ - "Succeeded" - ] - } - ], - "policy": { - "timeout": "7.00:00:00", - "retry": 0, - "retryIntervalInSeconds": 30, - "secureOutput": false, - "secureInput": false - }, - "userProperties": [], - "typeProperties": { - "source": { - "type": "ParquetSource", - "storeSettings": { - "type": "AzureBlobFSReadSettings", - "recursive": true - } - }, - "sink": { - "type": "AzureSqlSink", - "preCopyScript": { - "value": "@{pipeline().parameters.TaskObject.Target.PreCopySQL}", - "type": "Expression" - }, - "disableMetricsCollection": false - }, - "enableStaging": false, - "parallelCopies": { - "value": "@pipeline().parameters.TaskObject.DegreeOfCopyParallelism", - "type": "Expression" - } - }, - "inputs": [ - { - "referenceName": "ADLSParquet_AZ_IR", - "type": "DatasetReference", - "parameters": { - "RelativePath": { - "value": "@pipeline().parameters.TaskObject.Target.RelativePath", - "type": "Expression" - }, - "FileName": { - "value": "@pipeline().parameters.TaskObject.Target.DataFileName", - "type": "Expression" - }, - "StorageAccountEndpoint": { - "value": "@pipeline().parameters.TaskObject.Target.StorageAccountName", - "type": "Expression" - }, - "StorageAccountContainerName": { - "value": "@pipeline().parameters.TaskObject.Target.StorageAccountContainer", - "type": "Expression" - } - } - } - ], - "outputs": [ - { - "referenceName": "AzureSqlTable_AZ_IR", - "type": "DatasetReference", - "parameters": { - "Schema": { - "value": "@pipeline().parameters.TaskObject.Source.StagingTableSchema", - "type": "Expression" - }, - "Table": { - "value": "@pipeline().parameters.TaskObject.Source.StagingTableName", - "type": "Expression" - }, - "Server": { - "value": "@pipeline().parameters.TaskObject.Source.Database.SystemName", - "type": "Expression" - }, - "Database": { - "value": "@pipeline().parameters.TaskObject.Source.Database.Name", - "type": "Expression" - } - } - } - ] - }, - { - "name": "Pipeline AF Log - ADLS to Azure SQL Start", - "type": "ExecutePipeline", - "dependsOn": [], - "userProperties": [], - "typeProperties": { - "pipeline": { - "referenceName": "AZ-Function-Generic", - "type": "PipelineReference" - }, - "waitOnCompletion": false, - "parameters": { - "Body": { - "value": "[\n{\n \"TaskInstanceId\":@{pipeline().parameters.TaskObject.TaskInstanceId},\n \"ExecutionUid\":@{pipeline().parameters.TaskObject.ExecutionUid},\n \"RunId\":@{pipeline().RunId}, \n \"StartTime\":@{utcnow()},\n \"Status\":\"Processing\",\n \"Activity\":\"Copy ADLS to Azure SQL\",\n \"Comments\":\"\"\n}\n]", - "type": "Expression" - }, - "FunctionName": "Log", - "Method": "Put" - } - } - }, - { - "name": "Pipeline AF Log - ADLS to Azure SQL Succeed", - "type": "ExecutePipeline", - "dependsOn": [ - { - "activity": "Copy ADLS to Azure SQL", - "dependencyConditions": [ - "Succeeded" - ] - } - ], - "userProperties": [], - "typeProperties": { - "pipeline": { - "referenceName": "AZ-Function-Generic", - "type": "PipelineReference" - }, - "waitOnCompletion": false, - "parameters": { - "Body": { - "value": "[\n{\n \"TaskInstanceId\":@{pipeline().parameters.TaskObject.TaskInstanceId},\n \"ExecutionUid\":@{pipeline().parameters.TaskObject.ExecutionUid},\n \"RunId\":@{pipeline().RunId}, \n \"EndTime\":@{utcnow()},\n \"Status\":\"Completed\",\n \"Activity\":\"Copy ADLS to Azure SQL\",\n \"Comments\":\"\"\n}\n]", - "type": "Expression" - }, - "FunctionName": "Log", - "Method": "Put" - } - } - }, - { - "name": "Pipeline AF Log - ADLS to Azure SQL Failed", - "type": "ExecutePipeline", - "dependsOn": [ - { - "activity": "Copy ADLS to Azure SQL", - "dependencyConditions": [ - "Failed" - ] - } - ], - "userProperties": [], - "typeProperties": { - "pipeline": { - "referenceName": "AZ-Function-Generic", - "type": "PipelineReference" - }, - "waitOnCompletion": false, - "parameters": { - "Body": { - "value": "[\n{\n \"TaskInstanceId\":@{pipeline().parameters.TaskObject.TaskInstanceId},\n \"ExecutionUid\":@{pipeline().parameters.TaskObject.ExecutionUid},\n \"RunId\":@{pipeline().RunId}, \n \"EndTime\":@{utcnow()},\n \"Status\":\"Failed\",\n \"Activity\":\"Copy ADLS to Azure SQL\",\n \"Comments\":@{activity('Copy ADLS to Azure SQL').errors}\n}\n]", - "type": "Expression" - }, - "FunctionName": "Log", - "Method": "Put" - } - } - } - ] - }, - { - "value": "Azure Blob", - "activities": [ - { - "name": "Copy Blob to Azure SQL", - "type": "Copy", - "dependsOn": [ - { - "activity": "Pipeline AF Log - Blob to Azure SQL Start", - "dependencyConditions": [ - "Succeeded" - ] - } - ], - "policy": { - "timeout": "7.00:00:00", - "retry": 0, - "retryIntervalInSeconds": 30, - "secureOutput": false, - "secureInput": false - }, - "userProperties": [], - "typeProperties": { - "source": { - "type": "ParquetSource", - "storeSettings": { - "type": "AzureBlobStorageReadSettings", - "recursive": true - } - }, - "sink": { - "type": "AzureSqlSink", - "preCopyScript": { - "value": "@{pipeline().parameters.TaskObject.Target.PreCopySQL}", - "type": "Expression" - }, - "disableMetricsCollection": false - }, - "enableStaging": false, - "parallelCopies": { - "value": "@pipeline().parameters.TaskObject.DegreeOfCopyParallelism", - "type": "Expression" - } - }, - "inputs": [ - { - "referenceName": "BlobParquet_AZ_IR", - "type": "DatasetReference", - "parameters": { - "RelativePath": { - "value": "@pipeline().parameters.TaskObject.Target.RelativePath", - "type": "Expression" - }, - "FileName": { - "value": "@pipeline().parameters.TaskObject.Target.DataFileName", - "type": "Expression" - }, - "StorageAccountEndpoint": { - "value": "@pipeline().parameters.TaskObject.Target.StorageAccountName", - "type": "Expression" - }, - "StorageAccountContainerName": { - "value": "@pipeline().parameters.TaskObject.Target.StorageAccountContainer", - "type": "Expression" - } - } - } - ], - "outputs": [ - { - "referenceName": "AzureSqlTable_AZ_IR", - "type": "DatasetReference", - "parameters": { - "Schema": { - "value": "@pipeline().parameters.TaskObject.Source.StagingTableSchema", - "type": "Expression" - }, - "Table": { - "value": "@pipeline().parameters.TaskObject.Source.StagingTableName", - "type": "Expression" - }, - "Server": { - "value": "@pipeline().parameters.TaskObject.Source.Database.SystemName", - "type": "Expression" - }, - "Database": { - "value": "@pipeline().parameters.TaskObject.Source.Database.Name", - "type": "Expression" - } - } - } - ] - }, - { - "name": "Pipeline AF Log - Blob to Azure SQL Start", - "type": "ExecutePipeline", - "dependsOn": [], - "userProperties": [], - "typeProperties": { - "pipeline": { - "referenceName": "AZ-Function-Generic", - "type": "PipelineReference" - }, - "waitOnCompletion": false, - "parameters": { - "Body": { - "value": "[\n{\n \"TaskInstanceId\":@{pipeline().parameters.TaskObject.TaskInstanceId},\n \"ExecutionUid\":@{pipeline().parameters.TaskObject.ExecutionUid},\n \"RunId\":@{pipeline().RunId}, \n \"StartTime\":@{utcnow()},\n \"Status\":\"Processing\",\n \"Activity\":\"Copy Blob to Azure SQL\",\n \"Comments\":\"\"\n}\n]", - "type": "Expression" - }, - "FunctionName": "Log", - "Method": "Put" - } - } - }, - { - "name": "Pipeline AF Log - BLOB to Azure SQL Succeed", - "type": "ExecutePipeline", - "dependsOn": [ - { - "activity": "Copy Blob to Azure SQL", - "dependencyConditions": [ - "Succeeded" - ] - } - ], - "userProperties": [], - "typeProperties": { - "pipeline": { - "referenceName": "AZ-Function-Generic", - "type": "PipelineReference" - }, - "waitOnCompletion": false, - "parameters": { - "Body": { - "value": "[\n{\n \"TaskInstanceId\":@{pipeline().parameters.TaskObject.TaskInstanceId},\n \"ExecutionUid\":@{pipeline().parameters.TaskObject.ExecutionUid},\n \"RunId\":@{pipeline().RunId}, \n \"EndTime\":@{utcnow()},\n \"Status\":\"Completed\",\n \"Activity\":\"Copy Blob to Azure SQL\",\n \"Comments\":\"\"\n}\n]", - "type": "Expression" - }, - "FunctionName": "Log", - "Method": "Put" - } - } - }, - { - "name": "Pipeline AF Log - BLOB to Azure SQL Failed", - "type": "ExecutePipeline", - "dependsOn": [ - { - "activity": "Copy Blob to Azure SQL", - "dependencyConditions": [ - "Failed" - ] - } - ], - "userProperties": [], - "typeProperties": { - "pipeline": { - "referenceName": "AZ-Function-Generic", - "type": "PipelineReference" - }, - "waitOnCompletion": false, - "parameters": { - "Body": { - "value": "[\n{\n \"TaskInstanceId\":@{pipeline().parameters.TaskObject.TaskInstanceId},\n \"ExecutionUid\":@{pipeline().parameters.TaskObject.ExecutionUid},\n \"RunId\":@{pipeline().RunId}, \n \"EndTime\":@{utcnow()},\n \"Status\":\"Failed\",\n \"Activity\":\"Copy Blob to Azure SQL\",\n \"Comments\":@{activity('Copy Blob to Azure SQL').errors}\n}\n]", - "type": "Expression" - }, - "FunctionName": "Log", - "Method": "Put" - } - } - } - ] - } - ] - } - } - ], - "parameters": { - "TaskObject": { - "type": "object" - } - }, - "folder": { - "name": "ADS Go Fast/Data Movement/Auto Resolve IR" - }, - "annotations": [], - "lastPublishTime": "2020-07-23T08:15:14Z" - }, - "type": "Microsoft.DataFactory/factories/pipelines" -} \ No newline at end of file diff --git a/solution/Database/ADSGoFastDbUp/AdsGoFastDbUp/0-1-0-beta/A-Journaled/006-SampleTasks_TaskMaster.sql b/solution/Database/ADSGoFastDbUp/AdsGoFastDbUp/0-1-0-beta/A-Journaled/006-SampleTasks_TaskMaster.sql index 6814daee..975d57f3 100644 --- a/solution/Database/ADSGoFastDbUp/AdsGoFastDbUp/0-1-0-beta/A-Journaled/006-SampleTasks_TaskMaster.sql +++ b/solution/Database/ADSGoFastDbUp/AdsGoFastDbUp/0-1-0-beta/A-Journaled/006-SampleTasks_TaskMaster.sql @@ -23,8 +23,7 @@ INSERT [dbo].[TaskMaster] ([TaskMasterId], [TaskMasterName], [TaskTypeId], [Task "AutoCreateTable": "False", "TableSchema": "dbo", "TableName": "AUFinancialLicenses", - "PreCopySQL": "IF OBJECT_ID(''dbo.stg_AUFinancialLicenses'') IS NOT NULL - Truncate Table dbo.stg_AUFinancialLicenses", + "PreCopySQL": "IF OBJECT_ID(''dbo.stg_AUFinancialLicenses'') IS NOT NULL \r\n Truncate Table dbo.stg_AUFinancialLicenses", "PostCopySQL": "", "MergeSQL": "", "AutoGenerateMerge": "False" @@ -49,8 +48,7 @@ INSERT [dbo].[TaskMaster] ([TaskMasterId], [TaskMasterName], [TaskTypeId], [Task "StagingTableSchema": "dbo", "StagingTableName": "stg_NYTaxiYellowTripData", "AutoCreateTable": "False", - "PreCopySQL": "IF OBJECT_ID(''dbo.stg_NYTaxiYellowTripData'') IS NOT NULL - Truncate Table dbo.stg_NYTaxiYellowTripData", + "PreCopySQL": "IF OBJECT_ID(''dbo.stg_NYTaxiYellowTripData'') IS NOT NULL \r\n Truncate Table dbo.stg_NYTaxiYellowTripData", "PostCopySQL": "", "MergeSQL": "", "AutoGenerateMerge": "True", @@ -75,8 +73,7 @@ INSERT [dbo].[TaskMaster] ([TaskMasterId], [TaskMasterName], [TaskTypeId], [Task "StagingTableSchema": "dbo", "StagingTableName": "stg_BrisbHospRegDeaths", "AutoCreateTable": "False", - "PreCopySQL": "IF OBJECT_ID(''dbo.stg_BrisbHospRegDeaths'') IS NOT NULL - Truncate Table dbo.stg_BrisbHospRegDeaths", + "PreCopySQL": "IF OBJECT_ID(''dbo.stg_BrisbHospRegDeaths'') IS NOT NULL \r\n Truncate Table dbo.stg_BrisbHospRegDeaths", "PostCopySQL": "", "MergeSQL": "", "AutoGenerateMerge": "True" diff --git a/solution/Database/ADSGoFastDbUp/AdsGoFastDbUp/0-1-0-beta/A-Journaled/008-SampleTasks_SourceAndTargetSystems.sql b/solution/Database/ADSGoFastDbUp/AdsGoFastDbUp/0-1-0-beta/A-Journaled/008-SampleTasks_SourceAndTargetSystems.sql index f65006f9..84a5d127 100644 --- a/solution/Database/ADSGoFastDbUp/AdsGoFastDbUp/0-1-0-beta/A-Journaled/008-SampleTasks_SourceAndTargetSystems.sql +++ b/solution/Database/ADSGoFastDbUp/AdsGoFastDbUp/0-1-0-beta/A-Journaled/008-SampleTasks_SourceAndTargetSystems.sql @@ -5,146 +5,78 @@ -----------------------------------------------------------------------*/ + SET IDENTITY_INSERT [dbo].[SourceAndTargetSystems] ON GO -INSERT [dbo].[SourceAndTargetSystems] ([SystemId], [SystemName], [SystemType], [SystemDescription], [SystemServer], [SystemAuthType], [SystemUserName], [SystemSecretName], [SystemKeyVaultBaseUrl], [SystemJSON], [ActiveYN]) VALUES (1, N'AWSample-adsgofastdatakakeaccelsqlsvr', N'Azure SQL', N'AWSample Dev', N'adsgofastdatakakeaccelsqlsvr.database.windows.net', N'MSI', NULL, NULL, N'https://adsgofastkeyvault.vault.azure.net/', N' +INSERT [dbo].[SourceAndTargetSystems] ([SystemId], [SystemName], [SystemType], [SystemDescription], [SystemServer], [SystemAuthType], [SystemUserName], [SystemSecretName], [SystemKeyVaultBaseUrl], [SystemJSON], [ActiveYN], [IntegrationRuntime]) VALUES (1, N'Sample - Azure SQL Server ', N'Azure SQL', N'Sample Azure SQL Server Source', N'adsgofastdatakakeaccelsqlsvr.database.windows.net', N'MSI', NULL, NULL, N'https://adsgofastkeyvault.vault.azure.net/', N' { "Database" : "AWSample" } -', 1) +', 1, NULL) GO -INSERT [dbo].[SourceAndTargetSystems] ([SystemId], [SystemName], [SystemType], [SystemDescription], [SystemServer], [SystemAuthType], [SystemUserName], [SystemSecretName], [SystemKeyVaultBaseUrl], [SystemJSON], [ActiveYN]) VALUES (2, N'Staging-adsgofastdatakakeaccelsqlsvr', N'Azure SQL', N'Staging Dev', N'adsgofastdatakakeaccelsqlsvr.database.windows.net', N'MSI', NULL, NULL, N'https://adsgofastkeyvault.vault.azure.net/', N' { "Database" : "Staging" }', 1) +INSERT [dbo].[SourceAndTargetSystems] ([SystemId], [SystemName], [SystemType], [SystemDescription], [SystemServer], [SystemAuthType], [SystemUserName], [SystemSecretName], [SystemKeyVaultBaseUrl], [SystemJSON], [ActiveYN], [IntegrationRuntime]) VALUES (2, N'Sample - Exploration SQL', N'Azure SQL', N'Sample - Exploration SQL', N'adsgofastdatakakeaccelsqlsvr.database.windows.net', N'MSI', NULL, NULL, N'https://adsgofastkeyvault.vault.azure.net/', N' { "Database" : "Staging" }', 1, NULL) GO -INSERT [dbo].[SourceAndTargetSystems] ([SystemId], [SystemName], [SystemType], [SystemDescription], [SystemServer], [SystemAuthType], [SystemUserName], [SystemSecretName], [SystemKeyVaultBaseUrl], [SystemJSON], [ActiveYN]) VALUES (3, N'datalakeraw-adsgofastdatalakeaccelst', N'Azure Blob', N'datalakeraw Dev', N'https://adsgofastdatalakeaccelst.blob.core.windows.net', N'MSI', NULL, NULL, N'https://adsgofastkeyvault.vault.azure.net/', N' +INSERT [dbo].[SourceAndTargetSystems] ([SystemId], [SystemName], [SystemType], [SystemDescription], [SystemServer], [SystemAuthType], [SystemUserName], [SystemSecretName], [SystemKeyVaultBaseUrl], [SystemJSON], [ActiveYN], [IntegrationRuntime]) VALUES (3, N'Sample - DataLake Raw (Blob)', N'Azure Blob', N'Sample - DataLake Raw', N'https://adsgofastdatalakeaccelst.blob.core.windows.net', N'MSI', NULL, NULL, N'https://adsgofastkeyvault.vault.azure.net/', N' { "Container" : "datalakeraw" } -', 1) +', 1, NULL) GO -INSERT [dbo].[SourceAndTargetSystems] ([SystemId], [SystemName], [SystemType], [SystemDescription], [SystemServer], [SystemAuthType], [SystemUserName], [SystemSecretName], [SystemKeyVaultBaseUrl], [SystemJSON], [ActiveYN]) VALUES (4, N'datalakeraw-adsgofastdatalakeadls', N'ADLS', N'datalakeraw Dev ADLS Gen2', N'https://adsgofastdatalakeadls.dfs.core.windows.net', N'MSI', NULL, NULL, N'https://adsgofastkeyvault.vault.azure.net/', N' +INSERT [dbo].[SourceAndTargetSystems] ([SystemId], [SystemName], [SystemType], [SystemDescription], [SystemServer], [SystemAuthType], [SystemUserName], [SystemSecretName], [SystemKeyVaultBaseUrl], [SystemJSON], [ActiveYN], [IntegrationRuntime]) VALUES (4, N'Sample - DataLake Raw (ADLS)', N'ADLS', N'Sample - DataLake Raw (ADLS)', N'https://adsgofastdatalakeadls.dfs.core.windows.net', N'MSI', NULL, NULL, N'https://adsgofastkeyvault.vault.azure.net/', N' { "Container" : "datalakeraw" } -', 1) +', 1, NULL) GO -INSERT [dbo].[SourceAndTargetSystems] ([SystemId], [SystemName], [SystemType], [SystemDescription], [SystemServer], [SystemAuthType], [SystemUserName], [SystemSecretName], [SystemKeyVaultBaseUrl], [SystemJSON], [ActiveYN]) VALUES (5, N'adsgofastonpremadfir-dataingestion', N'File', N'DataIngestion File', N'\\\\adsgofastonpremadfir\\D$\\dataingestion\\', N'WindowsAuth', N'AzureUser', N'adsgofast-onpre-file-password', N'https://adsgofastkeyvault.vault.azure.net/', NULL, 1) +INSERT [dbo].[SourceAndTargetSystems] ([SystemId], [SystemName], [SystemType], [SystemDescription], [SystemServer], [SystemAuthType], [SystemUserName], [SystemSecretName], [SystemKeyVaultBaseUrl], [SystemJSON], [ActiveYN], [IntegrationRuntime]) VALUES (5, N'Sample - On Premise Fileshare Source', N'File', N'Sample - On Premise Fileshare Source', N'\\\\adsgofastonpremadfir\\D$\\dataingestion\\', N'WindowsAuth', N'AzureUser', N'adsgofast-onpre-file-password', N'https://adsgofastkeyvault.vault.azure.net/', NULL, 1, NULL) GO -INSERT [dbo].[SourceAndTargetSystems] ([SystemId], [SystemName], [SystemType], [SystemDescription], [SystemServer], [SystemAuthType], [SystemUserName], [SystemSecretName], [SystemKeyVaultBaseUrl], [SystemJSON], [ActiveYN]) VALUES (6, N'AdventureWorks2017-adsgofast-onpre', N'SQL Server', N'AdventureWorks2017 Dev', N'adsgofast-onpre', N'SQLAuth', N'sqladfir', N'adsgofast-onpre-sqladfir-password', N'https://adsgofastkeyvault.vault.azure.net/', N' +INSERT [dbo].[SourceAndTargetSystems] ([SystemId], [SystemName], [SystemType], [SystemDescription], [SystemServer], [SystemAuthType], [SystemUserName], [SystemSecretName], [SystemKeyVaultBaseUrl], [SystemJSON], [ActiveYN], [IntegrationRuntime]) VALUES (6, N'Sample - VM Hosted SQL Server', N'SQL Server', N'Sample - VM Hosted SQL Server', N'adsgofast-onpre', N'SQLAuth', N'sqladfir', N'adsgofast-onpre-sqladfir-password', N'https://adsgofastkeyvault.vault.azure.net/', N' { "Database" : "AdventureWorks2017" } -', 1) +', 1, NULL) GO -INSERT [dbo].[SourceAndTargetSystems] ([SystemId], [SystemName], [SystemType], [SystemDescription], [SystemServer], [SystemAuthType], [SystemUserName], [SystemSecretName], [SystemKeyVaultBaseUrl], [SystemJSON], [ActiveYN]) VALUES (7, N'datalakelanding-adsgofastdatalakeaccelst', N'Azure Blob', N'datalakelanding Dev', N'https://adsgofastdatalakeaccelst.blob.core.windows.net', N'MSI', NULL, NULL, N'https://adsgofastkeyvault.vault.azure.net/', N' +INSERT [dbo].[SourceAndTargetSystems] ([SystemId], [SystemName], [SystemType], [SystemDescription], [SystemServer], [SystemAuthType], [SystemUserName], [SystemSecretName], [SystemKeyVaultBaseUrl], [SystemJSON], [ActiveYN], [IntegrationRuntime]) VALUES (7, N'Sample - DataLake Landing (Blob)', N'Azure Blob', N'Sample - DataLake Landing (Blob)', N'https://adsgofastdatalakeaccelst.blob.core.windows.net', N'MSI', NULL, NULL, N'https://adsgofastkeyvault.vault.azure.net/', N' { "Container" : "datalakelanding" } -', 1) +', 1, NULL) GO -INSERT [dbo].[SourceAndTargetSystems] ([SystemId], [SystemName], [SystemType], [SystemDescription], [SystemServer], [SystemAuthType], [SystemUserName], [SystemSecretName], [SystemKeyVaultBaseUrl], [SystemJSON], [ActiveYN]) VALUES (8, N'datalakelanding-adsgofastdatalakeadls', N'ADLS', N'datalakelanding Dev ADLS Gen2', N'https://adsgofastdatalakeadls.dfs.core.windows.net', N'MSI', NULL, NULL, N'https://adsgofastkeyvault.vault.azure.net/', N' +INSERT [dbo].[SourceAndTargetSystems] ([SystemId], [SystemName], [SystemType], [SystemDescription], [SystemServer], [SystemAuthType], [SystemUserName], [SystemSecretName], [SystemKeyVaultBaseUrl], [SystemJSON], [ActiveYN], [IntegrationRuntime]) VALUES (8, N'Sample - DataLake Landing (ADLS)', N'ADLS', N'Sample - DataLake Landing (ADLS)', N'https://adsgofastdatalakeadls.dfs.core.windows.net', N'MSI', NULL, NULL, N'https://adsgofastkeyvault.vault.azure.net/', N' { "Container" : "datalakelanding" } -', 1) +', 1, NULL) GO -INSERT [dbo].[SourceAndTargetSystems] ([SystemId], [SystemName], [SystemType], [SystemDescription], [SystemServer], [SystemAuthType], [SystemUserName], [SystemSecretName], [SystemKeyVaultBaseUrl], [SystemJSON], [ActiveYN]) VALUES (9, N'adsgofasttransientstg', N'Azure Blob', N'adsgofasttransientstg Transient In Dev', N'https://adsgofasttransientstg.blob.core.windows.net', N'MSI', NULL, NULL, N'https://adsgofastkeyvault.vault.azure.net/', N' +INSERT [dbo].[SourceAndTargetSystems] ([SystemId], [SystemName], [SystemType], [SystemDescription], [SystemServer], [SystemAuthType], [SystemUserName], [SystemSecretName], [SystemKeyVaultBaseUrl], [SystemJSON], [ActiveYN], [IntegrationRuntime]) VALUES (9, N'Sample - Transient-In (Blob)', N'Azure Blob', N'Sample - Transient-In (Blob)', N'https://adsgofasttransientstg.blob.core.windows.net', N'MSI', NULL, NULL, N'https://adsgofastkeyvault.vault.azure.net/', N' { "Container" : "transientin" } -', 1) +', 1, NULL) GO -INSERT [dbo].[SourceAndTargetSystems] ([SystemId], [SystemName], [SystemType], [SystemDescription], [SystemServer], [SystemAuthType], [SystemUserName], [SystemSecretName], [SystemKeyVaultBaseUrl], [SystemJSON], [ActiveYN]) VALUES (10, N'SendGrid Email For PipQi Upload', N'SendGrid', N'SendGrid Dev', N'ADSGoFastSendGrid', N'Key', NULL, NULL, N'https://adsgofastkeyvault.vault.azure.net/', N' { +INSERT [dbo].[SourceAndTargetSystems] ([SystemId], [SystemName], [SystemType], [SystemDescription], [SystemServer], [SystemAuthType], [SystemUserName], [SystemSecretName], [SystemKeyVaultBaseUrl], [SystemJSON], [ActiveYN], [IntegrationRuntime]) VALUES (10, N'Sample - Sendgrid Email Account', N'SendGrid', N'Sample - Sendgrid Email Account', N'ADSGoFastSendGrid', N'Key', NULL, NULL, N'https://adsgofastkeyvault.vault.azure.net/', N' { "SenderEmail" : "noreply@cleverchiro.com", "SenderDescription" : "ADS Go Fast (No Reply)", "Subject" : "ADS GO Fast SAS Uri for File Upload", "PlainTextContent" : "Hello, Email!", "HtmlContent" : "Hi ,

Use the link below to upload files:


" - }', 1) + }', 1, NULL) GO -INSERT [dbo].[SourceAndTargetSystems] ([SystemId], [SystemName], [SystemType], [SystemDescription], [SystemServer], [SystemAuthType], [SystemUserName], [SystemSecretName], [SystemKeyVaultBaseUrl], [SystemJSON], [ActiveYN]) VALUES (11, N'Task Master Meta Data Database', N'Azure SQL', N'Task Master Meta Data Database', N'adsgofastdatakakeaccelsqlsvr.database.windows.net', N'MSI', NULL, NULL, N'https://adsgofastkeyvault.vault.azure.net/', N' +INSERT [dbo].[SourceAndTargetSystems] ([SystemId], [SystemName], [SystemType], [SystemDescription], [SystemServer], [SystemAuthType], [SystemUserName], [SystemSecretName], [SystemKeyVaultBaseUrl], [SystemJSON], [ActiveYN], [IntegrationRuntime]) VALUES (11, N'Sample - AdsGoFast Database', N'Azure SQL', N'Sample - AdsGoFast Database', N'adsgofastdatakakeaccelsqlsvr.database.windows.net', N'MSI', NULL, NULL, N'https://adsgofastkeyvault.vault.azure.net/', N' { "Database" : "AdsGoFast" } -', 1) +', 1, NULL) GO -INSERT [dbo].[SourceAndTargetSystems] ([SystemId], [SystemName], [SystemType], [SystemDescription], [SystemServer], [SystemAuthType], [SystemUserName], [SystemSecretName], [SystemKeyVaultBaseUrl], [SystemJSON], [ActiveYN]) VALUES (15, N'Email For PipQi Upload Confirmation', N'SendGrid', N'SendGrid Dev', N'ADSGoFastSendGrid', N'Key', NULL, NULL, N'https://adsgofastkeyvault.vault.azure.net/', N' { +INSERT [dbo].[SourceAndTargetSystems] ([SystemId], [SystemName], [SystemType], [SystemDescription], [SystemServer], [SystemAuthType], [SystemUserName], [SystemSecretName], [SystemKeyVaultBaseUrl], [SystemJSON], [ActiveYN], [IntegrationRuntime]) VALUES (12, N'Sample - Sendgrid Email Account2', N'SendGrid', N'Sample - Sendgrid Email Account2', N'ADSGoFastSendGrid', N'Key', NULL, NULL, N'https://adsgofastkeyvault.vault.azure.net/', N' { "SenderEmail" : "noreply@cleverchiro.com", "SenderDescription" : "ADS Go Fast (No Reply)", "Subject" : "ADS GO Fast SAS Uri for File Upload", "PlainTextContent" : "Hello, Email!", "HtmlContent" : "Hi ,

Use the link below to upload files:


" - }', 1) + }', 1, NULL) GO -INSERT [dbo].[SourceAndTargetSystems] ([SystemId], [SystemName], [SystemType], [SystemDescription], [SystemServer], [SystemAuthType], [SystemUserName], [SystemSecretName], [SystemKeyVaultBaseUrl], [SystemJSON], [ActiveYN]) VALUES (16, N'SH IR - Integration Runtime VM', N'AzureVM', N'SH IR - Integration Runtime VM', N'adsgofastadfir', N'MSI', NULL, NULL, N'https://adsgofastkeyvault.vault.azure.net/', N' { "SubscriptionUid" : "035a1364-f00d-48e2-b582-4fe125905ee3", "VMname" : "adsgofastadfir", "ResourceGroup":"ADSGOFASTDATALAKEACCEL" }', 1) +INSERT [dbo].[SourceAndTargetSystems] ([SystemId], [SystemName], [SystemType], [SystemDescription], [SystemServer], [SystemAuthType], [SystemUserName], [SystemSecretName], [SystemKeyVaultBaseUrl], [SystemJSON], [ActiveYN], [IntegrationRuntime]) VALUES (13, N'Sample - VM Target', N'AzureVM', N'Sample - VM Target', N'adsgofastadfir', N'MSI', NULL, NULL, N'https://adsgofastkeyvault.vault.azure.net/', N' { "SubscriptionUid" : "035a1364-f00d-48e2-b582-4fe125905ee3", "VMname" : "adsgofastadfir", "ResourceGroup":"ADSGOFASTDATALAKEACCEL" }', 1, NULL) GO SET IDENTITY_INSERT [dbo].[SourceAndTargetSystems] OFF GO -INSERT [dbo].[SourceAndTargetSystems_JsonSchema] ([SystemType], [JsonSchema]) VALUES (N'ADLS', N'{ - "$schema": "http://json-schema.org/draft-04/schema#", - "type": "object", - "properties": { - "Container": { - "type": "string" - } - }, - "required": [ - "Container" - ]}') -GO -INSERT [dbo].[SourceAndTargetSystems_JsonSchema] ([SystemType], [JsonSchema]) VALUES (N'Azure Blob', N'{ - "$schema": "http://json-schema.org/draft-04/schema#", - "type": "object", - "properties": { - "Container": { - "type": "string" - } - }, - "required": [ - "Container" - ]}') -GO -INSERT [dbo].[SourceAndTargetSystems_JsonSchema] ([SystemType], [JsonSchema]) VALUES (N'Azure SQL', N'{ - "$schema": "http://json-schema.org/draft-04/schema#", - "type": "object", - "properties": { - "Database": { - "type": "string" - } - }, - "required": [ - "Database" - ] -}') -GO -INSERT [dbo].[SourceAndTargetSystems_JsonSchema] ([SystemType], [JsonSchema]) VALUES (N'AzureVM', N'{ "$schema": "http://json-schema.org/draft-04/schema#", "type": "object", "properties": { "SubscriptionUid": { "type": "string" }, "VMname": { "type": "string" }, "ResourceGroup": { "type": "string" } }, "required": [ "SubscriptionUid", "VMname", "ResourceGroup" ] }') -GO -INSERT [dbo].[SourceAndTargetSystems_JsonSchema] ([SystemType], [JsonSchema]) VALUES (N'SendGrid', N'{ - "$schema": "http://json-schema.org/draft-04/schema#", - "type": "object", - "properties": { - "SenderEmail": { - "type": "string" - }, - "SenderDescription": { - "type": "string" - } - }, - "required": [ - "SenderEmail", - "SenderDescription" - ] -}') -GO -INSERT [dbo].[SourceAndTargetSystems_JsonSchema] ([SystemType], [JsonSchema]) VALUES (N'SQL Server', N'{ - "$schema": "http://json-schema.org/draft-04/schema#", - "type": "object", - "properties": { - "Database": { - "type": "string" - } - }, - "required": [ - "Database" - ] -}') -GO \ No newline at end of file diff --git a/solution/Database/ADSGoFastDbUp/AdsGoFastDbUp/0-1-0-beta/A-Journaled/010-TaskTypeMapping_Update.sql b/solution/Database/ADSGoFastDbUp/AdsGoFastDbUp/0-1-0-beta/A-Journaled/010-TaskTypeMapping_Update.sql new file mode 100644 index 00000000..40e2761e --- /dev/null +++ b/solution/Database/ADSGoFastDbUp/AdsGoFastDbUp/0-1-0-beta/A-Journaled/010-TaskTypeMapping_Update.sql @@ -0,0 +1,20 @@ + +Update +[dbo].[TaskTypeMapping] +Set MappingName = replace(replace(replace(MappingName, '-', '_'), 'OnP_SH_IR', 'IRB'), 'SH_IR', 'IRA') + + +Update +[dbo].[TaskTypeMapping] +Set TaskDataFactoryIR = replace(replace(replace(TaskDataFactoryIR, '-', '_'), 'OnP SH IR', 'IRB'), 'SH IR', 'IRA') + +Update +[dbo].[TaskMaster] +Set TaskDataFactoryIR = replace(replace(replace(TaskDataFactoryIR, '-', '_'), 'OnP SH IR', 'IRB'), 'SH IR', 'IRA') + + +Update [dbo].[TaskTypeMapping] +set SourceType = replace(replace(replace(SourceType,'csv', 'Csv'), 'json','Json'), 'JSON','Json') + +Update [dbo].[TaskTypeMapping] +set TargetType = replace(replace(replace(TargetType,'csv', 'Csv'), 'json','Json'), 'JSON','Json') \ No newline at end of file diff --git a/solution/Deployment/.devcontainer/devcontainer.json b/solution/Deployment/.devcontainer/devcontainer.json index 85572a9f..fe510047 100644 --- a/solution/Deployment/.devcontainer/devcontainer.json +++ b/solution/Deployment/.devcontainer/devcontainer.json @@ -6,8 +6,8 @@ // Set *default* container specific settings.json values on container create. "settings": { - "terminal.integrated.defaultProfile.linux": "pwsh", - "terminal.integrated.cwd": "/home/vscode" + "terminal.integrated.defaultProfile.linux": "pwsh" + //"terminal.integrated.cwd": "/home/vscode" }, // Add the IDs of extensions you want installed when the container is created. @@ -19,11 +19,11 @@ // "forwardPorts": [], // Use 'postCreateCommand' to run commands after the container is created. - "postCreateCommand": "sudo apt-get update && sudo apt-get install -y wget apt-transport-https software-properties-common && wget -q https://packages.microsoft.com/config/ubuntu/16.04/packages-microsoft-prod.deb && sudo dpkg -i packages-microsoft-prod.deb && sudo apt-get update && sudo apt-get install -y powershell && rm ./packages-microsoft-prod.deb && sudo apt install -y dotnet-sdk-3.1 && printf '$var = (Get-ChildItem \"../../workspaces\")[0].Name\n rsync -avzh ~/../../workspaces/$var ~/ ' > ~/synccodebase.ps1 && chmod +x ~/synccodebase.ps1", + "postCreateCommand": "sudo apt-get update && sudo apt-get install -y wget apt-transport-https software-properties-common && wget -q https://packages.microsoft.com/config/ubuntu/16.04/packages-microsoft-prod.deb && sudo dpkg -i packages-microsoft-prod.deb && sudo apt-get update && sudo apt-get install -y powershell && rm ./packages-microsoft-prod.deb && sudo apt install -y dotnet-sdk-3.1 && curl -fsSL https://deb.nodesource.com/setup_17.x | sudo -E bash - && apt-get install -y nodejs", // Uncomment when using a ptrace-based debugger like C++, Go, and Rust // "runArgs": [ "--cap-add=SYS_PTRACE", "--security-opt", "seccomp=unconfined" ], // Comment out connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root. - "remoteUser": "vscode" + //"remoteUser": "vscode" } \ No newline at end of file diff --git a/solution/Deployment/arm/AppService_Func.bicep b/solution/Deployment/arm/AppService_Func.bicep new file mode 100644 index 00000000..c72f2506 --- /dev/null +++ b/solution/Deployment/arm/AppService_Func.bicep @@ -0,0 +1,28 @@ +@description('Location for all resources.') +param location string = resourceGroup().location + +@description('') +param asp_name string = 'test' + +resource asp_name_resource 'Microsoft.Web/serverfarms@2020-06-01' = { + name: asp_name + location: location + kind: 'functionapp' + properties: { + perSiteScaling: false + maximumElasticWorkerCount: 1 + isSpot: false + reserved: false + isXenon: false + hyperV: false + targetWorkerCount: 0 + targetWorkerSizeId: 0 + } + sku: { + name: 'Y1' + tier: 'Dynamic' + size: 'Y1' + family: 'Y' + capacity: 0 + } +} \ No newline at end of file diff --git a/solution/Deployment/arm/AppService_Web.bicep b/solution/Deployment/arm/AppService_Web.bicep new file mode 100644 index 00000000..e3db98c0 --- /dev/null +++ b/solution/Deployment/arm/AppService_Web.bicep @@ -0,0 +1,27 @@ +@description('Location for all resources.') +param location string = resourceGroup().location + +@description('') +param asp_name string = 'test' + +resource asp_name_resource 'Microsoft.Web/serverfarms@2018-02-01' = { + name: asp_name + location: location + properties: { + perSiteScaling: false + maximumElasticWorkerCount: 1 + isSpot: false + reserved: false + isXenon: false + hyperV: false + targetWorkerCount: 0 + targetWorkerSizeId: 0 + } + sku: { + name: 'S1' + tier: 'Standard' + size: 'S1' + family: 'S' + capacity: 1 + } +} \ No newline at end of file diff --git a/solution/Deployment/arm/ApplicationInsights.bicep b/solution/Deployment/arm/ApplicationInsights.bicep new file mode 100644 index 00000000..7f7667de --- /dev/null +++ b/solution/Deployment/arm/ApplicationInsights.bicep @@ -0,0 +1,288 @@ +param appinsights_name string = 'appinsights-adsgofastyckrmqteklajm' +param location string = 'australiaeast' + +resource appinsights_name_resource 'microsoft.insights/components@2020-02-02-preview' = { + name: appinsights_name + location: location + tags: {} + kind: 'web' + properties: { + Application_Type: 'web' + IngestionMode: 'ApplicationInsights' + publicNetworkAccessForIngestion: 'Enabled' + publicNetworkAccessForQuery: 'Enabled' + } +} + +resource appinsights_name_degradationindependencyduration 'microsoft.insights/components/ProactiveDetectionConfigs@2018-05-01-preview' = { + parent: appinsights_name_resource + name: 'degradationindependencyduration' + location: location + properties: { + RuleDefinitions: { + Name: 'degradationindependencyduration' + DisplayName: 'Degradation in dependency duration' + Description: 'Smart Detection rules notify you of performance anomaly issues.' + HelpUrl: 'https://docs.microsoft.com/en-us/azure/application-insights/app-insights-proactive-performance-diagnostics' + IsHidden: false + IsEnabledByDefault: true + IsInPreview: false + SupportsEmailNotifications: true + } + Enabled: true + SendEmailsToSubscriptionOwners: true + CustomEmails: [] + } +} + +resource appinsights_name_degradationinserverresponsetime 'microsoft.insights/components/ProactiveDetectionConfigs@2018-05-01-preview' = { + parent: appinsights_name_resource + name: 'degradationinserverresponsetime' + location: location + properties: { + RuleDefinitions: { + Name: 'degradationinserverresponsetime' + DisplayName: 'Degradation in server response time' + Description: 'Smart Detection rules notify you of performance anomaly issues.' + HelpUrl: 'https://docs.microsoft.com/en-us/azure/application-insights/app-insights-proactive-performance-diagnostics' + IsHidden: false + IsEnabledByDefault: true + IsInPreview: false + SupportsEmailNotifications: true + } + Enabled: true + SendEmailsToSubscriptionOwners: true + CustomEmails: [] + } +} + +resource appinsights_name_digestMailConfiguration 'microsoft.insights/components/ProactiveDetectionConfigs@2018-05-01-preview' = { + parent: appinsights_name_resource + name: 'digestMailConfiguration' + location: location + properties: { + RuleDefinitions: { + Name: 'digestMailConfiguration' + DisplayName: 'Digest Mail Configuration' + Description: 'This rule describes the digest mail preferences' + HelpUrl: 'www.homail.com' + IsHidden: true + IsEnabledByDefault: true + IsInPreview: false + SupportsEmailNotifications: true + } + Enabled: true + SendEmailsToSubscriptionOwners: true + CustomEmails: [] + } +} + +resource appinsights_name_extension_billingdatavolumedailyspikeextension 'microsoft.insights/components/ProactiveDetectionConfigs@2018-05-01-preview' = { + parent: appinsights_name_resource + name: 'extension_billingdatavolumedailyspikeextension' + location: location + properties: { + RuleDefinitions: { + Name: 'extension_billingdatavolumedailyspikeextension' + DisplayName: 'Abnormal rise in daily data volume (preview)' + Description: 'This detection rule automatically analyzes the billing data generated by your application, and can warn you about an unusual increase in your application\'s billing costs' + HelpUrl: 'https://github.com/Microsoft/ApplicationInsights-Home/tree/master/SmartDetection/billing-data-volume-daily-spike.md' + IsHidden: false + IsEnabledByDefault: true + IsInPreview: true + SupportsEmailNotifications: false + } + Enabled: true + SendEmailsToSubscriptionOwners: true + CustomEmails: [] + } +} + +resource appinsights_name_extension_canaryextension 'microsoft.insights/components/ProactiveDetectionConfigs@2018-05-01-preview' = { + parent: appinsights_name_resource + name: 'extension_canaryextension' + location: location + properties: { + RuleDefinitions: { + Name: 'extension_canaryextension' + DisplayName: 'Canary extension' + Description: 'Canary extension' + HelpUrl: 'https://github.com/Microsoft/ApplicationInsights-Home/blob/master/SmartDetection/' + IsHidden: true + IsEnabledByDefault: true + IsInPreview: true + SupportsEmailNotifications: false + } + Enabled: true + SendEmailsToSubscriptionOwners: true + CustomEmails: [] + } +} + +resource appinsights_name_extension_exceptionchangeextension 'microsoft.insights/components/ProactiveDetectionConfigs@2018-05-01-preview' = { + parent: appinsights_name_resource + name: 'extension_exceptionchangeextension' + location: location + properties: { + RuleDefinitions: { + Name: 'extension_exceptionchangeextension' + DisplayName: 'Abnormal rise in exception volume (preview)' + Description: 'This detection rule automatically analyzes the exceptions thrown in your application, and can warn you about unusual patterns in your exception telemetry.' + HelpUrl: 'https://github.com/Microsoft/ApplicationInsights-Home/blob/master/SmartDetection/abnormal-rise-in-exception-volume.md' + IsHidden: false + IsEnabledByDefault: true + IsInPreview: true + SupportsEmailNotifications: false + } + Enabled: true + SendEmailsToSubscriptionOwners: true + CustomEmails: [] + } +} + +resource appinsights_name_extension_memoryleakextension 'microsoft.insights/components/ProactiveDetectionConfigs@2018-05-01-preview' = { + parent: appinsights_name_resource + name: 'extension_memoryleakextension' + location: location + properties: { + RuleDefinitions: { + Name: 'extension_memoryleakextension' + DisplayName: 'Potential memory leak detected (preview)' + Description: 'This detection rule automatically analyzes the memory consumption of each process in your application, and can warn you about potential memory leaks or increased memory consumption.' + HelpUrl: 'https://github.com/Microsoft/ApplicationInsights-Home/tree/master/SmartDetection/memory-leak.md' + IsHidden: false + IsEnabledByDefault: true + IsInPreview: true + SupportsEmailNotifications: false + } + Enabled: true + SendEmailsToSubscriptionOwners: true + CustomEmails: [] + } +} + +resource appinsights_name_extension_securityextensionspackage 'microsoft.insights/components/ProactiveDetectionConfigs@2018-05-01-preview' = { + parent: appinsights_name_resource + name: 'extension_securityextensionspackage' + location: location + properties: { + RuleDefinitions: { + Name: 'extension_securityextensionspackage' + DisplayName: 'Potential security issue detected (preview)' + Description: 'This detection rule automatically analyzes the telemetry generated by your application and detects potential security issues.' + HelpUrl: 'https://github.com/Microsoft/ApplicationInsights-Home/blob/master/SmartDetection/application-security-detection-pack.md' + IsHidden: false + IsEnabledByDefault: true + IsInPreview: true + SupportsEmailNotifications: false + } + Enabled: true + SendEmailsToSubscriptionOwners: true + CustomEmails: [] + } +} + +resource appinsights_name_extension_traceseveritydetector 'microsoft.insights/components/ProactiveDetectionConfigs@2018-05-01-preview' = { + parent: appinsights_name_resource + name: 'extension_traceseveritydetector' + location: location + properties: { + RuleDefinitions: { + Name: 'extension_traceseveritydetector' + DisplayName: 'Degradation in trace severity ratio (preview)' + Description: 'This detection rule automatically analyzes the trace logs emitted from your application, and can warn you about unusual patterns in the severity of your trace telemetry.' + HelpUrl: 'https://github.com/Microsoft/ApplicationInsights-Home/blob/master/SmartDetection/degradation-in-trace-severity-ratio.md' + IsHidden: false + IsEnabledByDefault: true + IsInPreview: true + SupportsEmailNotifications: false + } + Enabled: true + SendEmailsToSubscriptionOwners: true + CustomEmails: [] + } +} + +resource appinsights_name_longdependencyduration 'microsoft.insights/components/ProactiveDetectionConfigs@2018-05-01-preview' = { + parent: appinsights_name_resource + name: 'longdependencyduration' + location: location + properties: { + RuleDefinitions: { + Name: 'longdependencyduration' + DisplayName: 'Long dependency duration' + Description: 'Smart Detection rules notify you of performance anomaly issues.' + HelpUrl: 'https://docs.microsoft.com/en-us/azure/application-insights/app-insights-proactive-performance-diagnostics' + IsHidden: false + IsEnabledByDefault: true + IsInPreview: false + SupportsEmailNotifications: true + } + Enabled: true + SendEmailsToSubscriptionOwners: true + CustomEmails: [] + } +} + +resource appinsights_name_migrationToAlertRulesCompleted 'microsoft.insights/components/ProactiveDetectionConfigs@2018-05-01-preview' = { + parent: appinsights_name_resource + name: 'migrationToAlertRulesCompleted' + location: location + properties: { + RuleDefinitions: { + Name: 'migrationToAlertRulesCompleted' + DisplayName: 'Migration To Alert Rules Completed' + Description: 'A configuration that controls the migration state of Smart Detection to Smart Alerts' + HelpUrl: 'https://docs.microsoft.com/en-us/azure/application-insights/app-insights-proactive-performance-diagnostics' + IsHidden: true + IsEnabledByDefault: false + IsInPreview: true + SupportsEmailNotifications: false + } + Enabled: false + SendEmailsToSubscriptionOwners: true + CustomEmails: [] + } +} + +resource appinsights_name_slowpageloadtime 'microsoft.insights/components/ProactiveDetectionConfigs@2018-05-01-preview' = { + parent: appinsights_name_resource + name: 'slowpageloadtime' + location: location + properties: { + RuleDefinitions: { + Name: 'slowpageloadtime' + DisplayName: 'Slow page load time' + Description: 'Smart Detection rules notify you of performance anomaly issues.' + HelpUrl: 'https://docs.microsoft.com/en-us/azure/application-insights/app-insights-proactive-performance-diagnostics' + IsHidden: false + IsEnabledByDefault: true + IsInPreview: false + SupportsEmailNotifications: true + } + Enabled: true + SendEmailsToSubscriptionOwners: true + CustomEmails: [] + } +} + +resource appinsights_name_slowserverresponsetime 'microsoft.insights/components/ProactiveDetectionConfigs@2018-05-01-preview' = { + parent: appinsights_name_resource + name: 'slowserverresponsetime' + location: location + properties: { + RuleDefinitions: { + Name: 'slowserverresponsetime' + DisplayName: 'Slow server response time' + Description: 'Smart Detection rules notify you of performance anomaly issues.' + HelpUrl: 'https://docs.microsoft.com/en-us/azure/application-insights/app-insights-proactive-performance-diagnostics' + IsHidden: false + IsEnabledByDefault: true + IsInPreview: false + SupportsEmailNotifications: true + } + Enabled: true + SendEmailsToSubscriptionOwners: true + CustomEmails: [] + } +} \ No newline at end of file diff --git a/solution/Deployment/arm/AzureSQLServer.json b/solution/Deployment/arm/AzureSQLServer.json index be1a2df9..dfb72c8e 100644 --- a/solution/Deployment/arm/AzureSQLServer.json +++ b/solution/Deployment/arm/AzureSQLServer.json @@ -1,168 +1,172 @@ { - "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", - "contentVersion": "1.0.0.0", - "parameters": { - "location": { - "type": "string", - "defaultValue": "[resourceGroup().location]", - "metadata": { - "description": "Location for all resources." - } - }, - "sql-server-name": { - "type": "string", - "defaultValue": "[concat('adsgofast-srv-', uniqueString(resourceGroup().id))]", - "metadata": { - "description": "Azure SQL Server Name (Logical Server)." - } - }, - "sql-admin-login": { - "type": "string", - "defaultValue": "adsgofastadmin", - "metadata": { - "description": "The administrator username of the SQL logical server" - } - }, - "sql-admin-password": { - "type": "securestring", - "metadata": { - "description": "The administrator password of the SQL logical server." - } - }, - "sample-db-name": { - "defaultValue": "AdventureWorksLT", - "type": "String" - }, - "ads-go-fast-db-name": { - "defaultValue": "adsgofast", - "type": "String" - }, - "staging-db-name": { - "defaultValue": "staging", - "type": "String" - }, - "vnet-name": { - "type": "string", - "defaultValue": "adsgofast-vnet", - "metadata": { - "description": "Name of Azure Bastion resource" - } - } + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "metadata": { + "_generator": { + "name": "bicep", + "version": "0.4.613.9944", + "templateHash": "6917878293454276384" + } + }, + "parameters": { + "location": { + "type": "string", + "defaultValue": "[resourceGroup().location]", + "metadata": { + "description": "Location for all resources." + } }, - "variables": { - "data-subnet-name": "Data", - "sample-database-name": "[concat(parameters('sql-server-name'),'/',parameters('sample-db-name'))]", - "ads-go-fast-database-name": "[concat(parameters('sql-server-name'),'/',parameters('ads-go-fast-db-name'))]", - "staging-database-name": "[concat(parameters('sql-server-name'),'/',parameters('staging-db-name'))]", - "vnet-data-subnet-resource-id": "[resourceId('Microsoft.Network/virtualNetworks/subnets', parameters('vnet-name'), variables('data-subnet-name'))]" + "sql_server_name": { + "type": "string", + "defaultValue": "[format('adsgofast-srv-{0}', uniqueString(resourceGroup().id))]", + "metadata": { + "description": "Azure SQL Server Name (Logical Server)." + } }, - "resources": [ - { - "type": "Microsoft.Sql/servers", - "apiVersion": "2019-06-01-preview", - "name": "[parameters('sql-server-name')]", - "location": "[parameters('location')]", - "tags": { - "displayName": "[parameters('sql-server-name')]" - }, - "kind": "v12.0", - "properties": { - "administratorLogin": "[parameters('sql-admin-login')]", - "administratorLoginPassword": "[parameters('sql-admin-password')]", - "version": "12.0", - "publicNetworkAccess": "Enabled" - }, - "resources": [ - { - "type": "virtualNetworkRules", - "name": "[variables('data-subnet-name')]", - "apiVersion": "2015-05-01-preview", - "dependsOn": [ - "[resourceId('Microsoft.Sql/servers',parameters('sql-server-name'))]" - ], - "properties": { - "virtualNetworkSubnetId": "[variables('vnet-data-subnet-resource-id')]", - "ignoreMissingVnetServiceEndpoint": false - } - } - ] - }, - { - "type": "Microsoft.Sql/servers/databases", - "apiVersion": "2019-06-01-preview", - "name": "[variables('sample-database-name')]", - "location": "[parameters('location')]", - "tags": { - "displayName": "[variables('sample-database-name')]" - }, - "dependsOn": [ - "[resourceId('Microsoft.Sql/servers', parameters('sql-server-name'))]" - ], - "sku": { - "name": "Standard", - "tier": "Standard", - "capacity": 50 - }, - "properties": { - "collation": "SQL_Latin1_General_CP1_CI_AS", - "maxSizeBytes": 32212254720, - "catalogCollation": "SQL_Latin1_General_CP1_CI_AS", - "zoneRedundant": false, - "readScale": "Disabled", - "storageAccountType": "GRS", - "sampleName": "AdventureWorksLT" - } - }, - { - "type": "Microsoft.Sql/servers/databases", - "apiVersion": "2019-06-01-preview", - "name": "[variables('ads-go-fast-database-name')]", - "location": "[parameters('location')]", - "tags": { - "displayName": "[variables('ads-go-fast-database-name')]" - }, - "dependsOn": [ - "[resourceId('Microsoft.Sql/servers', parameters('sql-server-name'))]" - ], - "sku": { - "name": "Standard", - "tier": "Standard", - "capacity": 50 - }, - "properties": { - "collation": "SQL_Latin1_General_CP1_CI_AS", - "maxSizeBytes": 32212254720, - "catalogCollation": "SQL_Latin1_General_CP1_CI_AS", - "zoneRedundant": false, - "readScale": "Disabled", - "storageAccountType": "GRS" - } - }, - { - "type": "Microsoft.Sql/servers/databases", - "apiVersion": "2019-06-01-preview", - "name": "[variables('staging-database-name')]", - "location": "[parameters('location')]", - "tags": { - "displayName": "[variables('staging-database-name')]" - }, - "dependsOn": [ - "[resourceId('Microsoft.Sql/servers', parameters('sql-server-name'))]" - ], - "sku": { - "name": "Standard", - "tier": "Standard", - "capacity": 100 - }, - "properties": { - "collation": "SQL_Latin1_General_CP1_CI_AS", - "maxSizeBytes": 32212254720, - "catalogCollation": "SQL_Latin1_General_CP1_CI_AS", - "zoneRedundant": false, - "readScale": "Disabled", - "storageAccountType": "GRS" - } - } - ], - "outputs":{} -} + "sql_admin_login": { + "type": "string", + "defaultValue": "adsgofastadmin", + "metadata": { + "description": "The administrator username of the SQL logical server" + } + }, + "sql_admin_password": { + "type": "secureString", + "metadata": { + "description": "The administrator password of the SQL logical server." + } + }, + "sample_db_name": { + "type": "string", + "defaultValue": "AdventureWorksLT" + }, + "ads_go_fast_db_name": { + "type": "string", + "defaultValue": "adsgofast" + }, + "staging_db_name": { + "type": "string", + "defaultValue": "staging" + }, + "vnet_name": { + "type": "string", + "defaultValue": "adsgofast-vnet", + "metadata": { + "description": "Name of Azure Bastion resource" + } + } + }, + "functions": [], + "variables": { + "data_subnet_name": "Data", + "sample_database_name_var": "[format('{0}/{1}', parameters('sql_server_name'), parameters('sample_db_name'))]", + "ads_go_fast_database_name_var": "[format('{0}/{1}', parameters('sql_server_name'), parameters('ads_go_fast_db_name'))]", + "staging_database_name_var": "[format('{0}/{1}', parameters('sql_server_name'), parameters('staging_db_name'))]", + "vnet_data_subnet_resource_id": "[resourceId('Microsoft.Network/virtualNetworks/subnets', parameters('vnet_name'), variables('data_subnet_name'))]" + }, + "resources": [ + { + "type": "Microsoft.Sql/servers", + "apiVersion": "2019-06-01-preview", + "name": "[parameters('sql_server_name')]", + "location": "[parameters('location')]", + "tags": { + "displayName": "[parameters('sql_server_name')]" + }, + "properties": { + "administratorLogin": "[parameters('sql_admin_login')]", + "administratorLoginPassword": "[parameters('sql_admin_password')]", + "version": "12.0", + "publicNetworkAccess": "Enabled" + } + }, + { + "type": "Microsoft.Sql/servers/virtualNetworkRules", + "apiVersion": "2015-05-01-preview", + "name": "[format('{0}/{1}', parameters('sql_server_name'), variables('data_subnet_name'))]", + "properties": { + "virtualNetworkSubnetId": "[variables('vnet_data_subnet_resource_id')]", + "ignoreMissingVnetServiceEndpoint": false + }, + "dependsOn": [ + "[resourceId('Microsoft.Sql/servers', parameters('sql_server_name'))]" + ] + }, + { + "type": "Microsoft.Sql/servers/databases", + "apiVersion": "2019-06-01-preview", + "name": "[variables('sample_database_name_var')]", + "location": "[parameters('location')]", + "tags": { + "displayName": "[variables('sample_database_name_var')]" + }, + "sku": { + "name": "Standard", + "tier": "Standard", + "capacity": 50 + }, + "properties": { + "collation": "SQL_Latin1_General_CP1_CI_AS", + "maxSizeBytes": 32212254720, + "catalogCollation": "SQL_Latin1_General_CP1_CI_AS", + "zoneRedundant": false, + "readScale": "Disabled", + "storageAccountType": "GRS", + "sampleName": "AdventureWorksLT" + }, + "dependsOn": [ + "[resourceId('Microsoft.Sql/servers', parameters('sql_server_name'))]" + ] + }, + { + "type": "Microsoft.Sql/servers/databases", + "apiVersion": "2019-06-01-preview", + "name": "[variables('ads_go_fast_database_name_var')]", + "location": "[parameters('location')]", + "tags": { + "displayName": "[variables('ads_go_fast_database_name_var')]" + }, + "sku": { + "name": "Standard", + "tier": "Standard", + "capacity": 50 + }, + "properties": { + "collation": "SQL_Latin1_General_CP1_CI_AS", + "maxSizeBytes": 32212254720, + "catalogCollation": "SQL_Latin1_General_CP1_CI_AS", + "zoneRedundant": false, + "readScale": "Disabled", + "storageAccountType": "GRS" + }, + "dependsOn": [ + "[resourceId('Microsoft.Sql/servers', parameters('sql_server_name'))]" + ] + }, + { + "type": "Microsoft.Sql/servers/databases", + "apiVersion": "2019-06-01-preview", + "name": "[variables('staging_database_name_var')]", + "location": "[parameters('location')]", + "tags": { + "displayName": "[variables('staging_database_name_var')]" + }, + "sku": { + "name": "Standard", + "tier": "Standard", + "capacity": 100 + }, + "properties": { + "collation": "SQL_Latin1_General_CP1_CI_AS", + "maxSizeBytes": 32212254720, + "catalogCollation": "SQL_Latin1_General_CP1_CI_AS", + "zoneRedundant": false, + "readScale": "Disabled", + "storageAccountType": "GRS" + }, + "dependsOn": [ + "[resourceId('Microsoft.Sql/servers', parameters('sql_server_name'))]" + ] + } + ] +} \ No newline at end of file diff --git a/solution/Deployment/arm/AzureSqlServer.bicep b/solution/Deployment/arm/AzureSqlServer.bicep new file mode 100644 index 00000000..0d745cbf --- /dev/null +++ b/solution/Deployment/arm/AzureSqlServer.bicep @@ -0,0 +1,120 @@ +@description('Location for all resources.') +param location string = resourceGroup().location + +@description('Azure SQL Server Name (Logical Server).') +param sql_server_name string = 'adsgofast-srv-${uniqueString(resourceGroup().id)}' + +@description('The administrator username of the SQL logical server') +param sql_admin_login string = 'adsgofastadmin' + +@description('The administrator password of the SQL logical server.') +@secure() +param sql_admin_password string +param sample_db_name string = 'AdventureWorksLT' +param ads_go_fast_db_name string = 'adsgofast' +param staging_db_name string = 'staging' + +@description('Name of Azure Bastion resource') +param vnet_name string = 'adsgofast-vnet' + +var data_subnet_name = 'Data' +var sample_database_name_var = '${sql_server_name}/${sample_db_name}' +var ads_go_fast_database_name_var = '${sql_server_name}/${ads_go_fast_db_name}' +var staging_database_name_var = '${sql_server_name}/${staging_db_name}' +var vnet_data_subnet_resource_id = resourceId('Microsoft.Network/virtualNetworks/subnets', vnet_name, data_subnet_name) + +resource sql_server_name_resource 'Microsoft.Sql/servers@2019-06-01-preview' = { + name: sql_server_name + location: location + tags: { + displayName: sql_server_name + } + properties: { + administratorLogin: sql_admin_login + administratorLoginPassword: sql_admin_password + version: '12.0' + publicNetworkAccess: 'Enabled' + } +} + +resource sql_server_name_data_subnet_name 'Microsoft.Sql/servers/virtualNetworkRules@2015-05-01-preview' = { + parent: sql_server_name_resource + name: '${data_subnet_name}' + properties: { + virtualNetworkSubnetId: vnet_data_subnet_resource_id + ignoreMissingVnetServiceEndpoint: false + } +} + +resource sample_database_name 'Microsoft.Sql/servers/databases@2019-06-01-preview' = { + name: sample_database_name_var + location: location + tags: { + displayName: sample_database_name_var + } + sku: { + name: 'Standard' + tier: 'Standard' + capacity: 50 + } + properties: { + collation: 'SQL_Latin1_General_CP1_CI_AS' + maxSizeBytes: 32212254720 + catalogCollation: 'SQL_Latin1_General_CP1_CI_AS' + zoneRedundant: false + readScale: 'Disabled' + storageAccountType: 'GRS' + sampleName: 'AdventureWorksLT' + } + dependsOn: [ + sql_server_name_resource + ] +} + +resource ads_go_fast_database_name 'Microsoft.Sql/servers/databases@2019-06-01-preview' = { + name: ads_go_fast_database_name_var + location: location + tags: { + displayName: ads_go_fast_database_name_var + } + sku: { + name: 'Standard' + tier: 'Standard' + capacity: 50 + } + properties: { + collation: 'SQL_Latin1_General_CP1_CI_AS' + maxSizeBytes: 32212254720 + catalogCollation: 'SQL_Latin1_General_CP1_CI_AS' + zoneRedundant: false + readScale: 'Disabled' + storageAccountType: 'GRS' + } + dependsOn: [ + sql_server_name_resource + ] +} + +resource staging_database_name 'Microsoft.Sql/servers/databases@2019-06-01-preview' = { + name: staging_database_name_var + location: location + tags: { + displayName: staging_database_name_var + } + sku: { + name: 'Standard' + tier: 'Standard' + capacity: 100 + } + properties: { + collation: 'SQL_Latin1_General_CP1_CI_AS' + maxSizeBytes: 32212254720 + catalogCollation: 'SQL_Latin1_General_CP1_CI_AS' + zoneRedundant: false + readScale: 'Disabled' + storageAccountType: 'GRS' + } + dependsOn: [ + sql_server_name_resource + ] +} diff --git a/solution/Deployment/arm/DataFactory.bicep b/solution/Deployment/arm/DataFactory.bicep new file mode 100644 index 00000000..966a275e --- /dev/null +++ b/solution/Deployment/arm/DataFactory.bicep @@ -0,0 +1,14 @@ +@description('Location for all resources.') +param location string = resourceGroup().location + +@description('') +param adf_name string = 'test' + +resource adf_name_resource 'Microsoft.DataFactory/factories@2018-06-01' = { + name: adf_name + location: location + identity: { + type: 'SystemAssigned' + } + properties: {} +} \ No newline at end of file diff --git a/solution/Deployment/arm/FunctionApp.bicep b/solution/Deployment/arm/FunctionApp.bicep new file mode 100644 index 00000000..20aefce7 --- /dev/null +++ b/solution/Deployment/arm/FunctionApp.bicep @@ -0,0 +1,48 @@ +@description('Location for all resources.') +param location string = resourceGroup().location + +@description('The name of you Web Site.') +param azure_function_site_name string = 'FuncApp-${uniqueString(resourceGroup().id)}' + +@description('The name of Azure Application Insights.') +param app_insights_name string = 'appinsights-adsgofast' + +@description('The name of storage account used for logging') +param storage_log_account_name string = 'adsgofastlog' + +@description('A key to the storage account') +param storage_log_account_key string = '' + +@description('') +param appservice_name string = '' + +resource azure_function_site_name_resource 'Microsoft.Web/sites@2020-06-01' = { + name: azure_function_site_name + kind: 'functionapp' + location: location + properties: { + name: azure_function_site_name + siteConfig: { + appSettings: [ + { + name: 'FUNCTIONS_WORKER_RUNTIME' + value: 'dotnet' + } + { + name: 'FUNCTIONS_EXTENSION_VERSION' + value: '~3' + } + { + name: 'AzureWebJobsStorage' + value: 'DefaultEndpointsProtocol=https;AccountName=${storage_log_account_name};AccountKey=${storage_log_account_key}' + } + { + name: 'APPINSIGHTS_INSTRUMENTATIONKEY' + value: reference(resourceId('microsoft.insights/components/', app_insights_name), '2015-05-01').InstrumentationKey + } + ] + } + serverFarmId: resourceId('Microsoft.Web/serverfarms', appservice_name) + clientAffinityEnabled: false + } +} \ No newline at end of file diff --git a/solution/Deployment/arm/KeyVault.bicep b/solution/Deployment/arm/KeyVault.bicep new file mode 100644 index 00000000..a1fe1975 --- /dev/null +++ b/solution/Deployment/arm/KeyVault.bicep @@ -0,0 +1,28 @@ +@description('Location for all resources.') +param location string = '' + +@description('kv') +param keyvault_name string = 'kv' + +@description('kv') +param tenant_id string = 'kv' + +resource keyvault_name_resource 'Microsoft.KeyVault/vaults@2018-02-14' = { + name: keyvault_name + location: location + properties: { + enabledForDeployment: true + enabledForDiskEncryption: true + enabledForTemplateDeployment: true + tenantId: tenant_id + accessPolicies: [] + sku: { + name: 'standard' + family: 'A' + } + networkAcls: { + defaultAction: 'Allow' + bypass: 'AzureServices' + } + } +} \ No newline at end of file diff --git a/solution/Deployment/arm/LogAnalytics.bicep b/solution/Deployment/arm/LogAnalytics.bicep new file mode 100644 index 00000000..242057cc --- /dev/null +++ b/solution/Deployment/arm/LogAnalytics.bicep @@ -0,0 +1,36 @@ +@description('Location for all resources.') +param location string = resourceGroup().location +param workspaces_adsgofastloganalytics_name string = 'adsloganalytics' + +@description('Pricing tier: PerGB2018 or legacy tiers (Free, Standalone, PerNode, Standard or Premium) which are not available to all customers.') +@allowed([ + 'pergb2018' + 'Free' + 'Standalone' + 'PerNode' + 'Standard' + 'Premium' +]) +param log_analytics_sku string = 'pergb2018' + +@description('Number of days to retain data.') +param log_analytics_retentionInDays int = 30 + +@description('true to use resource or workspace permissions. false to require workspace permissions.') +param log_analytics_resourcePermissions bool = false + +resource workspaces_adsgofastloganalytics_name_resource 'microsoft.operationalinsights/workspaces@2020-08-01' = { + name: workspaces_adsgofastloganalytics_name + location: location + properties: { + sku: { + name: log_analytics_sku + } + retentionInDays: log_analytics_retentionInDays + features: { + searchVersion: 1 + legacy: 0 + enableLogAccessUsingOnlyResourcePermissions: log_analytics_resourcePermissions + } + } +} \ No newline at end of file diff --git a/solution/Deployment/arm/Networking.bicep b/solution/Deployment/arm/Networking.bicep new file mode 100644 index 00000000..ad82604a --- /dev/null +++ b/solution/Deployment/arm/Networking.bicep @@ -0,0 +1,107 @@ +@description('Location for all resources.') +param location string = resourceGroup().location + +@description('Name of Azure Bastion resource') +param vnet_name string = 'adsgofast-vnet' + +@description('Bastion subnet IP prefix MUST be within vnet IP prefix address space') +param vnet_address_prefix string = '10.1.0.0/16' + +@description('Bastion subnet IP prefix MUST be within vnet IP prefix address space') +param bastion_subnet_ip_prefix string = '10.1.1.0/27' + +@description('Bastion subnet IP prefix MUST be within vnet IP prefix address space') +param data_subnet_ip_prefix string = '10.1.2.0/27' + +@description('Name of Azure Bastion resource') +param bastion_host_name string = 'azure-bastion-ads-go-fast' + +var bastion_subnet_name = 'AzureBastionSubnet' +var data_subnet_name = 'Data' +var public_ip_address_name_bastion_var = '${bastion_host_name}-pip' + +resource public_ip_address_name_bastion 'Microsoft.Network/publicIpAddresses@2019-02-01' = { + name: public_ip_address_name_bastion_var + location: location + sku: { + name: 'Standard' + } + properties: { + publicIPAllocationMethod: 'Static' + } +} + +resource vnet_name_resource 'Microsoft.Network/virtualNetworks@2019-02-01' = { + name: vnet_name + location: location + properties: { + addressSpace: { + addressPrefixes: [ + vnet_address_prefix + ] + } + subnets: [ + { + name: bastion_subnet_name + properties: { + addressPrefix: bastion_subnet_ip_prefix + } + } + { + name: data_subnet_name + properties: { + addressPrefix: data_subnet_ip_prefix + } + } + ] + } +} + +resource vnet_name_bastion_subnet_name 'Microsoft.Network/virtualNetworks/subnets@2019-02-01' = { + parent: vnet_name_resource + name: '${bastion_subnet_name}' + location: location + properties: { + addressPrefix: bastion_subnet_ip_prefix + } + dependsOn: [ + vnet_name_data_subnet_name + ] +} + +resource vnet_name_data_subnet_name 'Microsoft.Network/virtualNetworks/subnets@2019-02-01' = { + parent: vnet_name_resource + name: '${data_subnet_name}' + location: location + properties: { + addressPrefix: data_subnet_ip_prefix + serviceEndpoints: [ + { + service: 'Microsoft.Sql' + } + ] + } +} + +resource bastion_host_name_resource 'Microsoft.Network/bastionHosts@2019-04-01' = { + name: bastion_host_name + location: location + properties: { + ipConfigurations: [ + { + name: 'IpConf' + properties: { + subnet: { + id: vnet_name_bastion_subnet_name.id + } + publicIPAddress: { + id: public_ip_address_name_bastion.id + } + } + } + ] + } + dependsOn: [ + vnet_name_resource + ] +} \ No newline at end of file diff --git a/solution/Deployment/arm/Storage_ADLS.bicep b/solution/Deployment/arm/Storage_ADLS.bicep new file mode 100644 index 00000000..f4819ba8 --- /dev/null +++ b/solution/Deployment/arm/Storage_ADLS.bicep @@ -0,0 +1,53 @@ +@description('Location for all resources.') +param location string = '' + +@description('') +param storage_account_name string = 'adsgfadls' + +@description('datalakelanding') +param storage_landing_container_name string = 'datalakelanding' + +@description('datalakeraw') +param storage_raw_container_name string = 'datalakeraw' + +@description('') +param storage_account_sku string = 'Standard_GRS' + +resource storage_account_name_resource 'Microsoft.Storage/storageAccounts@2019-04-01' = { + location: location + name: storage_account_name + kind: 'StorageV2' + sku: { + name: storage_account_sku + } + properties: { + encryption: { + keySource: 'Microsoft.Storage' + services: { + blob: { + enabled: true + } + file: { + enabled: true + } + } + } + isHnsEnabled: true + supportsHttpsTrafficOnly: true + accessTier: 'Hot' + } +} + +resource storage_account_name_default_storage_landing_container_name 'Microsoft.Storage/storageAccounts/blobServices/containers@2019-06-01' = { + name: '${storage_account_name}/default/${storage_landing_container_name}' + dependsOn: [ + storage_account_name_resource + ] +} + +resource storage_account_name_default_storage_raw_container_name 'Microsoft.Storage/storageAccounts/blobServices/containers@2019-06-01' = { + name: '${storage_account_name}/default/${storage_raw_container_name}' + dependsOn: [ + storage_account_name_resource + ] +} \ No newline at end of file diff --git a/solution/Deployment/arm/Storage_Blob.bicep b/solution/Deployment/arm/Storage_Blob.bicep new file mode 100644 index 00000000..78c3f191 --- /dev/null +++ b/solution/Deployment/arm/Storage_Blob.bicep @@ -0,0 +1,53 @@ +@description('Location for all resources.') +param location string = '' + +@description('') +param storage_account_name string = 'adsgfadls' + +@description('') +param storage_landing_container_name string = 'datalakelanding' + +@description('') +param storage_raw_container_name string = 'datalakeraw' + +@description('') +param storage_account_sku string = 'Standard_GRS' + +resource storage_account_name_resource 'Microsoft.Storage/storageAccounts@2019-04-01' = { + location: location + name: storage_account_name + kind: 'StorageV2' + sku: { + name: storage_account_sku + } + properties: { + encryption: { + keySource: 'Microsoft.Storage' + services: { + blob: { + enabled: true + } + file: { + enabled: true + } + } + } + isHnsEnabled: false + supportsHttpsTrafficOnly: true + accessTier: 'Hot' + } +} + +resource storage_account_name_default_storage_landing_container_name 'Microsoft.Storage/storageAccounts/blobServices/containers@2019-06-01' = { + name: '${storage_account_name}/default/${storage_landing_container_name}' + dependsOn: [ + storage_account_name_resource + ] +} + +resource storage_account_name_default_storage_raw_container_name 'Microsoft.Storage/storageAccounts/blobServices/containers@2019-06-01' = { + name: '${storage_account_name}/default/${storage_raw_container_name}' + dependsOn: [ + storage_account_name_resource + ] +} \ No newline at end of file diff --git a/solution/Deployment/arm/Storage_Logging.bicep b/solution/Deployment/arm/Storage_Logging.bicep new file mode 100644 index 00000000..e00121cb --- /dev/null +++ b/solution/Deployment/arm/Storage_Logging.bicep @@ -0,0 +1,21 @@ +@description('Location for all resources.') +param location string = '' + +@description('The name of the Log Store account to create.') +param storage_log_account_name string = 'logstg' + +resource storage_log_account_name_resource 'Microsoft.Storage/storageAccounts@2019-06-01' = { + name: storage_log_account_name + location: location + kind: 'StorageV2' + sku: { + name: 'Standard_LRS' + tier: 'Standard' + } + properties: { + accessTier: 'Hot' + } +} + +output stringSubcriptionId string = subscription().id +output stringLogStorageAccount string = storage_log_account_name \ No newline at end of file diff --git a/solution/Deployment/arm/VirtualMachine.bicep b/solution/Deployment/arm/VirtualMachine.bicep new file mode 100644 index 00000000..89eff9ec --- /dev/null +++ b/solution/Deployment/arm/VirtualMachine.bicep @@ -0,0 +1,254 @@ +@description('Location for all resources.') +param location string = resourceGroup().location + +@description('The size of the VM') +param adf_ir_vm_size string = 'Standard_D4s_v3' + +@description('Username for the Virtual Machine.') +param adf_ir_vm_admin_username string = 'adsgofastadmin' + +@description('Password for the Virtual Machine. The password must be at least 12 characters long and have lower case, upper characters, digit and a special character (Regex match)') +@secure() +param adf_ir_vm_admin_password string + +@description('Defines the type of storage account to use for the data lake store') +@allowed([ + 'Standard_LRS' + 'Standard_ZRS' + 'Standard_GRS' + 'Standard_RAGRS' +]) +param os_disk_type_adfir_vm string = 'Standard_LRS' +param adf_ir_onp_vm_name string + +@description('Windows Server and SQL Offer') +@allowed([ + 'sql2019-ws2019' + 'sql2017-ws2019' + 'SQL2017-WS2016' + 'SQL2016SP1-WS2016' + 'SQL2016SP2-WS2016' + 'SQL2014SP3-WS2012R2' + 'SQL2014SP2-WS2012R2' +]) +param imageOffer string = 'sql2019-ws2019' + +@description('SQL Server Sku') +@allowed([ + 'Standard' + 'Enterprise' + 'SQLDEV' + 'Web' + 'Express' +]) +param sqlSku string = 'Standard' + +@description('Amount of data disks (1TB each) for SQL Data files') +@minValue(1) +@maxValue(8) +param sqlDataDisksCount int = 1 + +@description('Amount of data disks (1TB each) for SQL Log files') +@minValue(1) +@maxValue(8) +param sqlLogDisksCount int = 1 + +@description('SQL Server Workload Type') +@allowed([ + 'General' + 'OLTP' + 'DW' +]) +param storageWorkloadType string = 'General' + +@description('Path for SQL Data files. Please choose drive letter from F to Z, and other drives from A to E are reserved for system') +param dataPath string = 'F:\\SQLData' + +@description('Path for SQL Log files. Please choose drive letter from F to Z and different than the one used for SQL data. Drive letter from A to E are reserved for system') +param logPath string = 'G:\\SQLLog' + +@description('Name of Azure Bastion resource') +param vnet_name string = 'adsgofast-vnet' + +var adf_ir_vm_name_var = take('IR-Az-${uniqueString(resourceGroup().id)}-VM', 15) +var adf_ir_az_network_interface_name_var = '${adf_ir_vm_name_var}NetInt' +var adf_ir_onp_network_interface_name_var = '${adf_ir_onp_vm_name}NetInt' +var dataDisks = { + createOption: 'empty' + caching: 'ReadOnly' + writeAcceleratorEnabled: false + storageAccountType: 'Premium_LRS' + diskSizeGB: 1023 +} +var diskConfigurationType = 'NEW' +var dataDisksLuns = array(range(0, sqlDataDisksCount)) +var logDisksLuns = array(range(sqlDataDisksCount, sqlLogDisksCount)) +var tempDbPath = 'D:\\SQLTemp' +var data_subnet_name = 'Data' + +resource adf_ir_az_network_interface_name 'Microsoft.Network/networkInterfaces@2019-09-01' = { + name: adf_ir_az_network_interface_name_var + location: location + tags: { + displayName: adf_ir_az_network_interface_name_var + } + properties: { + ipConfigurations: [ + { + name: 'ipConfig1' + properties: { + privateIPAllocationMethod: 'Dynamic' + subnet: { + id: resourceId('Microsoft.Network/virtualNetworks/subnets', vnet_name, data_subnet_name) + } + } + } + ] + } + dependsOn: [] +} + +resource adf_ir_onp_network_interface_name 'Microsoft.Network/networkInterfaces@2019-09-01' = { + name: adf_ir_onp_network_interface_name_var + location: location + tags: { + displayName: adf_ir_onp_network_interface_name_var + } + properties: { + ipConfigurations: [ + { + name: 'ipConfig1' + properties: { + privateIPAllocationMethod: 'Dynamic' + subnet: { + id: resourceId('Microsoft.Network/virtualNetworks/subnets', vnet_name, data_subnet_name) + } + } + } + ] + } + dependsOn: [] +} + +resource adf_ir_vm_name 'Microsoft.Compute/virtualMachines@2019-07-01' = { + name: adf_ir_vm_name_var + location: location + tags: { + displayName: adf_ir_vm_name_var + } + properties: { + hardwareProfile: { + vmSize: adf_ir_vm_size + } + osProfile: { + computerName: adf_ir_vm_name_var + adminUsername: adf_ir_vm_admin_username + adminPassword: adf_ir_vm_admin_password + } + storageProfile: { + imageReference: { + publisher: 'MicrosoftWindowsServer' + offer: 'WindowsServer' + sku: '2019-Datacenter' + version: 'latest' + } + osDisk: { + name: '${adf_ir_vm_name_var}OsDisk' + caching: 'ReadWrite' + createOption: 'FromImage' + managedDisk: { + storageAccountType: os_disk_type_adfir_vm + } + diskSizeGB: 128 + } + } + networkProfile: { + networkInterfaces: [ + { + id: adf_ir_az_network_interface_name.id + } + ] + } + } +} + +resource adf_ir_onp_vm_name_resource 'Microsoft.Compute/virtualMachines@2019-07-01' = { + name: adf_ir_onp_vm_name + location: location + tags: { + displayName: adf_ir_onp_vm_name + } + properties: { + hardwareProfile: { + vmSize: adf_ir_vm_size + } + osProfile: { + computerName: adf_ir_onp_vm_name + adminUsername: adf_ir_vm_admin_username + adminPassword: adf_ir_vm_admin_password + windowsConfiguration: { + enableAutomaticUpdates: true + provisionVMAgent: true + } + } + storageProfile: { + imageReference: { + publisher: 'MicrosoftSQLServer' + offer: imageOffer + sku: sqlSku + version: 'latest' + } + osDisk: { + name: '${adf_ir_onp_vm_name}OsDisk' + caching: 'ReadWrite' + createOption: 'FromImage' + managedDisk: { + storageAccountType: os_disk_type_adfir_vm + } + diskSizeGB: 128 + } + dataDisks: [for j in range(0, (sqlDataDisksCount + sqlLogDisksCount)): { + lun: j + createOption: dataDisks.createOption + caching: ((j >= sqlDataDisksCount) ? 'None' : dataDisks.caching) + writeAcceleratorEnabled: dataDisks.writeAcceleratorEnabled + diskSizeGB: dataDisks.diskSizeGB + managedDisk: { + storageAccountType: dataDisks.storageAccountType + } + }] + } + networkProfile: { + networkInterfaces: [ + { + id: adf_ir_onp_network_interface_name.id + } + ] + } + } +} + +resource Microsoft_SqlVirtualMachine_SqlVirtualMachines_adf_ir_onp_vm_name 'Microsoft.SqlVirtualMachine/SqlVirtualMachines@2017-03-01-preview' = { + name: adf_ir_onp_vm_name + location: location + properties: { + virtualMachineResourceId: adf_ir_onp_vm_name_resource.id + sqlManagement: 'Full' + sqlServerLicenseType: 'PAYG' + storageConfigurationSettings: { + diskConfigurationType: diskConfigurationType + storageWorkloadType: storageWorkloadType + sqlDataSettings: { + luns: dataDisksLuns + defaultFilePath: dataPath + } + sqlLogSettings: { + luns: logDisksLuns + defaultFilePath: logPath + } + sqlTempDbSettings: { + defaultFilePath: tempDbPath + } + } + } +} diff --git a/solution/Deployment/arm/WebApp.bicep b/solution/Deployment/arm/WebApp.bicep new file mode 100644 index 00000000..63aad64f --- /dev/null +++ b/solution/Deployment/arm/WebApp.bicep @@ -0,0 +1,42 @@ +@description('Location for all resources.') +param location string = resourceGroup().location + +@description('Resource Group.') +param resource_group_name string = '' + +@description('The name of Web Application.') +param sites_AdsGoFastWebApp_name string = 'adsgofastWebApp' + +@description('') +param appservice_name string = '' + +resource sites_AdsGoFastWebApp_name_resource 'Microsoft.Web/sites@2018-11-01' = { + name: sites_AdsGoFastWebApp_name + location: location + tags: {} + properties: { + name: sites_AdsGoFastWebApp_name + siteConfig: { + appSettings: [ + { + name: 'XDT_MicrosoftApplicationInsights_Mode' + value: 'default' + } + { + name: 'ANCM_ADDITIONAL_ERROR_PAGE_LINK' + value: 'https://${sites_AdsGoFastWebApp_name}.scm.azurewebsites.net/detectors?type=tools&name=eventviewer' + } + ] + metadata: [ + { + name: 'CURRENT_STACK' + value: 'dotnetcore' + } + ] + phpVersion: 'OFF' + alwaysOn: true + } + serverFarmId: resourceId('Microsoft.Web/serverfarms', appservice_name) + clientAffinityEnabled: true + } +} \ No newline at end of file diff --git a/solution/Deployment/arm/WebApp.json b/solution/Deployment/arm/WebApp.json index f11a4d05..6e0785c8 100644 --- a/solution/Deployment/arm/WebApp.json +++ b/solution/Deployment/arm/WebApp.json @@ -1,68 +1,76 @@ { - "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", - "contentVersion": "1.0.0.0", - "parameters": { - "location": { - "type": "string", - "defaultValue": "[resourceGroup().location]", - "metadata": { - "description": "Location for all resources." - } - }, - "resource-group-name": { - "type": "string", - "defaultValue": "", - "metadata": { - "description": "Resource Group." - } - }, - "sites_AdsGoFastWebApp_name": { - "type": "string", - "defaultValue": "adsgofastWebApp", - "metadata": { - "description": "The name of Web Application." - } - }, - "appservice-name": { - "type": "string", - "defaultValue": "", - "metadata": { - "description": "" - } - } + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "metadata": { + "_generator": { + "name": "bicep", + "version": "0.4.613.9944", + "templateHash": "4770209531422889309" + } + }, + "parameters": { + "location": { + "type": "string", + "defaultValue": "[resourceGroup().location]", + "metadata": { + "description": "Location for all resources." + } }, - "resources": [ - { - "apiVersion": "2018-11-01", - "name": "[parameters('sites_AdsGoFastWebApp_name')]", - "type": "Microsoft.Web/sites", - "location": "[parameters('location')]", - "tags": {}, - "properties": { - "name": "[parameters('sites_AdsGoFastWebApp_name')]", - "siteConfig": { - "appSettings": [ - { - "name": "XDT_MicrosoftApplicationInsights_Mode", - "value": "default" - }, - { - "name": "ANCM_ADDITIONAL_ERROR_PAGE_LINK", - "value": "[concat('https://',parameters('sites_AdsGoFastWebApp_name'),'.scm.azurewebsites.net/detectors?type=tools&name=eventviewer')]" - } - ], - "metadata": [ - { - "name": "CURRENT_STACK", - "value": "dotnetcore" - } - ], - "phpVersion": "OFF", - "alwaysOn": true - }, - "serverFarmId": "[resourceId('Microsoft.Web/serverfarms', parameters('appservice-name'))]", - "clientAffinityEnabled": true + "resource_group_name": { + "type": "string", + "defaultValue": "", + "metadata": { + "description": "Resource Group." + } + }, + "sites_AdsGoFastWebApp_name": { + "type": "string", + "defaultValue": "adsgofastWebApp", + "metadata": { + "description": "The name of Web Application." + } + }, + "appservice_name": { + "type": "string", + "defaultValue": "", + "metadata": { + "description": "" + } + } + }, + "functions": [], + "resources": [ + { + "type": "Microsoft.Web/sites", + "apiVersion": "2018-11-01", + "name": "[parameters('sites_AdsGoFastWebApp_name')]", + "location": "[parameters('location')]", + "tags": {}, + "properties": { + "name": "[parameters('sites_AdsGoFastWebApp_name')]", + "siteConfig": { + "appSettings": [ + { + "name": "XDT_MicrosoftApplicationInsights_Mode", + "value": "default" + }, + { + "name": "ANCM_ADDITIONAL_ERROR_PAGE_LINK", + "value": "[format('https://{0}.scm.azurewebsites.net/detectors?type=tools&name=eventviewer', parameters('sites_AdsGoFastWebApp_name'))]" } - } - ] + ], + "metadata": [ + { + "name": "CURRENT_STACK", + "value": "dotnetcore" + } + ], + "phpVersion": "OFF", + "alwaysOn": true + }, + "serverFarmId": "[resourceId('Microsoft.Web/serverfarms', parameters('appservice_name'))]", + "clientAffinityEnabled": true + } + } + ] } \ No newline at end of file diff --git a/solution/Deployment/environments/EditSettings.html b/solution/Deployment/environments/EditSettings.html new file mode 100644 index 00000000..115a1718 --- /dev/null +++ b/solution/Deployment/environments/EditSettings.html @@ -0,0 +1,67 @@ + + + +Page Title + + + + + + + +

Azure Data Services Go Fast Settings Editor

+ +
+ + + + diff --git a/solution/Deployment/environments/Node/package-lock.json b/solution/Deployment/environments/Node/package-lock.json new file mode 100644 index 00000000..efafaecb --- /dev/null +++ b/solution/Deployment/environments/Node/package-lock.json @@ -0,0 +1,872 @@ +{ + "name": "ads_gofast_configuration_app", + "version": "1.0.0", + "lockfileVersion": 2, + "requires": true, + "packages": { + "": { + "name": "ads_gofast_configuration_app", + "version": "1.0.0", + "dependencies": { + "express": "^4.16.1", + "fs": "0.0.2", + "node-static": "0.7.11", + "path": "0.12.7" + } + }, + "node_modules/accepts": { + "version": "1.3.7", + "license": "MIT", + "dependencies": { + "mime-types": "~2.1.24", + "negotiator": "0.6.2" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/array-flatten": { + "version": "1.1.1", + "license": "MIT" + }, + "node_modules/body-parser": { + "version": "1.19.0", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.19.0.tgz", + "integrity": "sha512-dhEPs72UPbDnAQJ9ZKMNTP6ptJaionhP5cBb541nXPlW60Jepo9RV/a4fX4XWW9CuFNK22krhrj1+rgzifNCsw==", + "dependencies": { + "bytes": "3.1.0", + "content-type": "~1.0.4", + "debug": "2.6.9", + "depd": "~1.1.2", + "http-errors": "1.7.2", + "iconv-lite": "0.4.24", + "on-finished": "~2.3.0", + "qs": "6.7.0", + "raw-body": "2.4.0", + "type-is": "~1.6.17" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/bytes": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.0.tgz", + "integrity": "sha512-zauLjrfCG+xvoyaqLoV8bLVXXNGC4JqlxFCutSDWA6fJrTo2ZuvLYTqZ7aHBLZSMOopbzwv8f+wZcVzfVTI2Dg==", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/colors": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/colors/-/colors-1.4.0.tgz", + "integrity": "sha512-a+UqTh4kgZg/SlGvfbzDHpgRu7AAQOmmqRHJnxhRZICKFUT91brVhNNt58CMWU9PsBbv3PDCZUHbVxuDiH2mtA==", + "engines": { + "node": ">=0.1.90" + } + }, + "node_modules/content-disposition": { + "version": "0.5.3", + "license": "MIT", + "dependencies": { + "safe-buffer": "5.1.2" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/content-type": { + "version": "1.0.4", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/cookie": { + "version": "0.4.0", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/cookie-signature": { + "version": "1.0.6", + "license": "MIT" + }, + "node_modules/debug": { + "version": "2.6.9", + "license": "MIT", + "dependencies": { + "ms": "2.0.0" + } + }, + "node_modules/depd": { + "version": "1.1.2", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/destroy": { + "version": "1.0.4", + "license": "MIT" + }, + "node_modules/ee-first": { + "version": "1.1.1", + "license": "MIT" + }, + "node_modules/encodeurl": { + "version": "1.0.2", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/escape-html": { + "version": "1.0.3", + "license": "MIT" + }, + "node_modules/etag": { + "version": "1.8.1", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/express": { + "version": "4.17.1", + "resolved": "https://registry.npmjs.org/express/-/express-4.17.1.tgz", + "integrity": "sha512-mHJ9O79RqluphRrcw2X/GTh3k9tVv8YcoyY4Kkh4WDMUYKRZUq0h1o0w2rrrxBqM7VoeUVqgb27xlEMXTnYt4g==", + "dependencies": { + "accepts": "~1.3.7", + "array-flatten": "1.1.1", + "body-parser": "1.19.0", + "content-disposition": "0.5.3", + "content-type": "~1.0.4", + "cookie": "0.4.0", + "cookie-signature": "1.0.6", + "debug": "2.6.9", + "depd": "~1.1.2", + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "etag": "~1.8.1", + "finalhandler": "~1.1.2", + "fresh": "0.5.2", + "merge-descriptors": "1.0.1", + "methods": "~1.1.2", + "on-finished": "~2.3.0", + "parseurl": "~1.3.3", + "path-to-regexp": "0.1.7", + "proxy-addr": "~2.0.5", + "qs": "6.7.0", + "range-parser": "~1.2.1", + "safe-buffer": "5.1.2", + "send": "0.17.1", + "serve-static": "1.14.1", + "setprototypeof": "1.1.1", + "statuses": "~1.5.0", + "type-is": "~1.6.18", + "utils-merge": "1.0.1", + "vary": "~1.1.2" + }, + "engines": { + "node": ">= 0.10.0" + } + }, + "node_modules/finalhandler": { + "version": "1.1.2", + "license": "MIT", + "dependencies": { + "debug": "2.6.9", + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "on-finished": "~2.3.0", + "parseurl": "~1.3.3", + "statuses": "~1.5.0", + "unpipe": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/forwarded": { + "version": "0.2.0", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/fresh": { + "version": "0.5.2", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/fs": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/fs/-/fs-0.0.2.tgz", + "integrity": "sha1-4fJE7zkzwbKmS9R5kTYGDQ9ZFPg=" + }, + "node_modules/http-errors": { + "version": "1.7.2", + "license": "MIT", + "dependencies": { + "depd": "~1.1.2", + "inherits": "2.0.3", + "setprototypeof": "1.1.1", + "statuses": ">= 1.5.0 < 2", + "toidentifier": "1.0.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/iconv-lite": { + "version": "0.4.24", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", + "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/inherits": { + "version": "2.0.3", + "license": "ISC" + }, + "node_modules/ipaddr.js": { + "version": "1.9.1", + "license": "MIT", + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/media-typer": { + "version": "0.3.0", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/merge-descriptors": { + "version": "1.0.1", + "license": "MIT" + }, + "node_modules/methods": { + "version": "1.1.2", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime": { + "version": "1.6.0", + "license": "MIT", + "bin": { + "mime": "cli.js" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/mime-db": { + "version": "1.51.0", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime-types": { + "version": "2.1.34", + "license": "MIT", + "dependencies": { + "mime-db": "1.51.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/minimist": { + "version": "0.0.10", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.10.tgz", + "integrity": "sha1-3j+YVD2/lggr5IrRoMfNqDYwHc8=" + }, + "node_modules/ms": { + "version": "2.0.0", + "license": "MIT" + }, + "node_modules/negotiator": { + "version": "0.6.2", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/node-static": { + "version": "0.7.11", + "resolved": "https://registry.npmjs.org/node-static/-/node-static-0.7.11.tgz", + "integrity": "sha512-zfWC/gICcqb74D9ndyvxZWaI1jzcoHmf4UTHWQchBNuNMxdBLJMDiUgZ1tjGLEIe/BMhj2DxKD8HOuc2062pDQ==", + "dependencies": { + "colors": ">=0.6.0", + "mime": "^1.2.9", + "optimist": ">=0.3.4" + }, + "bin": { + "static": "bin/cli.js" + }, + "engines": { + "node": ">= 0.4.1" + } + }, + "node_modules/on-finished": { + "version": "2.3.0", + "license": "MIT", + "dependencies": { + "ee-first": "1.1.1" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/optimist": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/optimist/-/optimist-0.6.1.tgz", + "integrity": "sha1-2j6nRob6IaGaERwybpDrFaAZZoY=", + "dependencies": { + "minimist": "~0.0.1", + "wordwrap": "~0.0.2" + } + }, + "node_modules/parseurl": { + "version": "1.3.3", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/path": { + "version": "0.12.7", + "resolved": "https://registry.npmjs.org/path/-/path-0.12.7.tgz", + "integrity": "sha1-1NwqUGxM4hl+tIHr/NWzbAFAsQ8=", + "dependencies": { + "process": "^0.11.1", + "util": "^0.10.3" + } + }, + "node_modules/path-to-regexp": { + "version": "0.1.7", + "license": "MIT" + }, + "node_modules/process": { + "version": "0.11.10", + "resolved": "https://registry.npmjs.org/process/-/process-0.11.10.tgz", + "integrity": "sha1-czIwDoQBYb2j5podHZGn1LwW8YI=", + "engines": { + "node": ">= 0.6.0" + } + }, + "node_modules/proxy-addr": { + "version": "2.0.7", + "license": "MIT", + "dependencies": { + "forwarded": "0.2.0", + "ipaddr.js": "1.9.1" + }, + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/qs": { + "version": "6.7.0", + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.6" + } + }, + "node_modules/range-parser": { + "version": "1.2.1", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/raw-body": { + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.4.0.tgz", + "integrity": "sha512-4Oz8DUIwdvoa5qMJelxipzi/iJIi40O5cGV1wNYp5hvZP8ZN0T+jiNkL0QepXs+EsQ9XJ8ipEDoiH70ySUJP3Q==", + "dependencies": { + "bytes": "3.1.0", + "http-errors": "1.7.2", + "iconv-lite": "0.4.24", + "unpipe": "1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/safe-buffer": { + "version": "5.1.2", + "license": "MIT" + }, + "node_modules/safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" + }, + "node_modules/send": { + "version": "0.17.1", + "license": "MIT", + "dependencies": { + "debug": "2.6.9", + "depd": "~1.1.2", + "destroy": "~1.0.4", + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "etag": "~1.8.1", + "fresh": "0.5.2", + "http-errors": "~1.7.2", + "mime": "1.6.0", + "ms": "2.1.1", + "on-finished": "~2.3.0", + "range-parser": "~1.2.1", + "statuses": "~1.5.0" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/send/node_modules/ms": { + "version": "2.1.1", + "license": "MIT" + }, + "node_modules/serve-static": { + "version": "1.14.1", + "license": "MIT", + "dependencies": { + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "parseurl": "~1.3.3", + "send": "0.17.1" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/setprototypeof": { + "version": "1.1.1", + "license": "ISC" + }, + "node_modules/statuses": { + "version": "1.5.0", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/toidentifier": { + "version": "1.0.0", + "license": "MIT", + "engines": { + "node": ">=0.6" + } + }, + "node_modules/type-is": { + "version": "1.6.18", + "license": "MIT", + "dependencies": { + "media-typer": "0.3.0", + "mime-types": "~2.1.24" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/unpipe": { + "version": "1.0.0", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/util": { + "version": "0.10.4", + "resolved": "https://registry.npmjs.org/util/-/util-0.10.4.tgz", + "integrity": "sha512-0Pm9hTQ3se5ll1XihRic3FDIku70C+iHUdT/W926rSgHV5QgXsYbKZN8MSC3tJtSkhuROzvsQjAaFENRXr+19A==", + "dependencies": { + "inherits": "2.0.3" + } + }, + "node_modules/utils-merge": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", + "integrity": "sha1-n5VxD1CiZ5R7LMwSR0HBAoQn5xM=", + "engines": { + "node": ">= 0.4.0" + } + }, + "node_modules/vary": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", + "integrity": "sha1-IpnwLG3tMNSllhsLn3RSShj2NPw=", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/wordwrap": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/wordwrap/-/wordwrap-0.0.3.tgz", + "integrity": "sha1-o9XabNXAvAAI03I0u68b7WMFkQc=", + "engines": { + "node": ">=0.4.0" + } + } + }, + "dependencies": { + "accepts": { + "version": "1.3.7", + "requires": { + "mime-types": "~2.1.24", + "negotiator": "0.6.2" + } + }, + "array-flatten": { + "version": "1.1.1" + }, + "body-parser": { + "version": "1.19.0", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.19.0.tgz", + "integrity": "sha512-dhEPs72UPbDnAQJ9ZKMNTP6ptJaionhP5cBb541nXPlW60Jepo9RV/a4fX4XWW9CuFNK22krhrj1+rgzifNCsw==", + "requires": { + "bytes": "3.1.0", + "content-type": "~1.0.4", + "debug": "2.6.9", + "depd": "~1.1.2", + "http-errors": "1.7.2", + "iconv-lite": "0.4.24", + "on-finished": "~2.3.0", + "qs": "6.7.0", + "raw-body": "2.4.0", + "type-is": "~1.6.17" + } + }, + "bytes": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.0.tgz", + "integrity": "sha512-zauLjrfCG+xvoyaqLoV8bLVXXNGC4JqlxFCutSDWA6fJrTo2ZuvLYTqZ7aHBLZSMOopbzwv8f+wZcVzfVTI2Dg==" + }, + "colors": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/colors/-/colors-1.4.0.tgz", + "integrity": "sha512-a+UqTh4kgZg/SlGvfbzDHpgRu7AAQOmmqRHJnxhRZICKFUT91brVhNNt58CMWU9PsBbv3PDCZUHbVxuDiH2mtA==" + }, + "content-disposition": { + "version": "0.5.3", + "requires": { + "safe-buffer": "5.1.2" + } + }, + "content-type": { + "version": "1.0.4" + }, + "cookie": { + "version": "0.4.0" + }, + "cookie-signature": { + "version": "1.0.6" + }, + "debug": { + "version": "2.6.9", + "requires": { + "ms": "2.0.0" + } + }, + "depd": { + "version": "1.1.2" + }, + "destroy": { + "version": "1.0.4" + }, + "ee-first": { + "version": "1.1.1" + }, + "encodeurl": { + "version": "1.0.2" + }, + "escape-html": { + "version": "1.0.3" + }, + "etag": { + "version": "1.8.1" + }, + "express": { + "version": "4.17.1", + "resolved": "https://registry.npmjs.org/express/-/express-4.17.1.tgz", + "integrity": "sha512-mHJ9O79RqluphRrcw2X/GTh3k9tVv8YcoyY4Kkh4WDMUYKRZUq0h1o0w2rrrxBqM7VoeUVqgb27xlEMXTnYt4g==", + "requires": { + "accepts": "~1.3.7", + "array-flatten": "1.1.1", + "body-parser": "1.19.0", + "content-disposition": "0.5.3", + "content-type": "~1.0.4", + "cookie": "0.4.0", + "cookie-signature": "1.0.6", + "debug": "2.6.9", + "depd": "~1.1.2", + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "etag": "~1.8.1", + "finalhandler": "~1.1.2", + "fresh": "0.5.2", + "merge-descriptors": "1.0.1", + "methods": "~1.1.2", + "on-finished": "~2.3.0", + "parseurl": "~1.3.3", + "path-to-regexp": "0.1.7", + "proxy-addr": "~2.0.5", + "qs": "6.7.0", + "range-parser": "~1.2.1", + "safe-buffer": "5.1.2", + "send": "0.17.1", + "serve-static": "1.14.1", + "setprototypeof": "1.1.1", + "statuses": "~1.5.0", + "type-is": "~1.6.18", + "utils-merge": "1.0.1", + "vary": "~1.1.2" + } + }, + "finalhandler": { + "version": "1.1.2", + "requires": { + "debug": "2.6.9", + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "on-finished": "~2.3.0", + "parseurl": "~1.3.3", + "statuses": "~1.5.0", + "unpipe": "~1.0.0" + } + }, + "forwarded": { + "version": "0.2.0" + }, + "fresh": { + "version": "0.5.2" + }, + "fs": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/fs/-/fs-0.0.2.tgz", + "integrity": "sha1-4fJE7zkzwbKmS9R5kTYGDQ9ZFPg=" + }, + "http-errors": { + "version": "1.7.2", + "requires": { + "depd": "~1.1.2", + "inherits": "2.0.3", + "setprototypeof": "1.1.1", + "statuses": ">= 1.5.0 < 2", + "toidentifier": "1.0.0" + } + }, + "iconv-lite": { + "version": "0.4.24", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", + "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", + "requires": { + "safer-buffer": ">= 2.1.2 < 3" + } + }, + "inherits": { + "version": "2.0.3" + }, + "ipaddr.js": { + "version": "1.9.1" + }, + "media-typer": { + "version": "0.3.0" + }, + "merge-descriptors": { + "version": "1.0.1" + }, + "methods": { + "version": "1.1.2" + }, + "mime": { + "version": "1.6.0" + }, + "mime-db": { + "version": "1.51.0" + }, + "mime-types": { + "version": "2.1.34", + "requires": { + "mime-db": "1.51.0" + } + }, + "minimist": { + "version": "0.0.10", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.10.tgz", + "integrity": "sha1-3j+YVD2/lggr5IrRoMfNqDYwHc8=" + }, + "ms": { + "version": "2.0.0" + }, + "negotiator": { + "version": "0.6.2" + }, + "node-static": { + "version": "0.7.11", + "resolved": "https://registry.npmjs.org/node-static/-/node-static-0.7.11.tgz", + "integrity": "sha512-zfWC/gICcqb74D9ndyvxZWaI1jzcoHmf4UTHWQchBNuNMxdBLJMDiUgZ1tjGLEIe/BMhj2DxKD8HOuc2062pDQ==", + "requires": { + "colors": ">=0.6.0", + "mime": "^1.2.9", + "optimist": ">=0.3.4" + } + }, + "on-finished": { + "version": "2.3.0", + "requires": { + "ee-first": "1.1.1" + } + }, + "optimist": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/optimist/-/optimist-0.6.1.tgz", + "integrity": "sha1-2j6nRob6IaGaERwybpDrFaAZZoY=", + "requires": { + "minimist": "~0.0.1", + "wordwrap": "~0.0.2" + } + }, + "parseurl": { + "version": "1.3.3" + }, + "path": { + "version": "0.12.7", + "resolved": "https://registry.npmjs.org/path/-/path-0.12.7.tgz", + "integrity": "sha1-1NwqUGxM4hl+tIHr/NWzbAFAsQ8=", + "requires": { + "process": "^0.11.1", + "util": "^0.10.3" + } + }, + "path-to-regexp": { + "version": "0.1.7" + }, + "process": { + "version": "0.11.10", + "resolved": "https://registry.npmjs.org/process/-/process-0.11.10.tgz", + "integrity": "sha1-czIwDoQBYb2j5podHZGn1LwW8YI=" + }, + "proxy-addr": { + "version": "2.0.7", + "requires": { + "forwarded": "0.2.0", + "ipaddr.js": "1.9.1" + } + }, + "qs": { + "version": "6.7.0" + }, + "range-parser": { + "version": "1.2.1" + }, + "raw-body": { + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.4.0.tgz", + "integrity": "sha512-4Oz8DUIwdvoa5qMJelxipzi/iJIi40O5cGV1wNYp5hvZP8ZN0T+jiNkL0QepXs+EsQ9XJ8ipEDoiH70ySUJP3Q==", + "requires": { + "bytes": "3.1.0", + "http-errors": "1.7.2", + "iconv-lite": "0.4.24", + "unpipe": "1.0.0" + } + }, + "safe-buffer": { + "version": "5.1.2" + }, + "safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" + }, + "send": { + "version": "0.17.1", + "requires": { + "debug": "2.6.9", + "depd": "~1.1.2", + "destroy": "~1.0.4", + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "etag": "~1.8.1", + "fresh": "0.5.2", + "http-errors": "~1.7.2", + "mime": "1.6.0", + "ms": "2.1.1", + "on-finished": "~2.3.0", + "range-parser": "~1.2.1", + "statuses": "~1.5.0" + }, + "dependencies": { + "ms": { + "version": "2.1.1" + } + } + }, + "serve-static": { + "version": "1.14.1", + "requires": { + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "parseurl": "~1.3.3", + "send": "0.17.1" + } + }, + "setprototypeof": { + "version": "1.1.1" + }, + "statuses": { + "version": "1.5.0" + }, + "toidentifier": { + "version": "1.0.0" + }, + "type-is": { + "version": "1.6.18", + "requires": { + "media-typer": "0.3.0", + "mime-types": "~2.1.24" + } + }, + "unpipe": { + "version": "1.0.0" + }, + "util": { + "version": "0.10.4", + "resolved": "https://registry.npmjs.org/util/-/util-0.10.4.tgz", + "integrity": "sha512-0Pm9hTQ3se5ll1XihRic3FDIku70C+iHUdT/W926rSgHV5QgXsYbKZN8MSC3tJtSkhuROzvsQjAaFENRXr+19A==", + "requires": { + "inherits": "2.0.3" + } + }, + "utils-merge": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", + "integrity": "sha1-n5VxD1CiZ5R7LMwSR0HBAoQn5xM=" + }, + "vary": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", + "integrity": "sha1-IpnwLG3tMNSllhsLn3RSShj2NPw=" + }, + "wordwrap": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/wordwrap/-/wordwrap-0.0.3.tgz", + "integrity": "sha1-o9XabNXAvAAI03I0u68b7WMFkQc=" + } + } +} diff --git a/solution/Deployment/environments/Node/package.json b/solution/Deployment/environments/Node/package.json new file mode 100644 index 00000000..4ee23d2b --- /dev/null +++ b/solution/Deployment/environments/Node/package.json @@ -0,0 +1,16 @@ +{ + "name": "ads_gofast_configuration_app", + "version": "1.0.0", + "description": "ads_gofast_configuration_app", + "author": "john.rampono@microsoft.com", + "main": "server.js", + "scripts": { + "start": "node server.js" + }, + "dependencies": { + "express": "^4.16.1", + "node-static": "0.7.11", + "fs": "0.0.2", + "path": "0.12.7" + } + } \ No newline at end of file diff --git a/solution/Deployment/environments/Node/server.js b/solution/Deployment/environments/Node/server.js new file mode 100644 index 00000000..eabee95e --- /dev/null +++ b/solution/Deployment/environments/Node/server.js @@ -0,0 +1,37 @@ +var static = require('node-static'); +var http = require('http'); +// Import the path module +const path = require('path'); +var fs = require('fs'); + +var directory = path.resolve(__dirname + "/../"); +var file = new(static.Server)(directory, { cache: 1 }); + +http.createServer(function (req, res) { + + if (req.method === "GET") + { + file.serve(req, res); + } + else + { + var body = ''; + filePath = directory + '/development.json'; + req.on('data', function(data) { + body += data; + body = decodeURI(body); + body = JSON.parse(body); + body = JSON.stringify(body, null, 2); + //body = body.replace("\n", "\r\n"); + }); + + req.on('end', function (){ + fs.writeFile(filePath, body, function() { + res.end(); + }); + }); + } +}).listen(8080); + + + diff --git a/solution/Deployment/environments/development.json b/solution/Deployment/environments/development.json index 3eeb29b6..ebd23061 100644 --- a/solution/Deployment/environments/development.json +++ b/solution/Deployment/environments/development.json @@ -1,15 +1,26 @@ { - "$schema": "./environment.schema.json", "AdsOpts": { "CI": { "Enable": true, "BuildFunctionApp": true, "BuildWebApp": true, - "BuildAdsGoFastDatabase": true + "BuildAdsGoFastDatabase": true, + "BuildDataFactory": true }, "CD": { "EnableDeploy": true, "EnableConfigure": true, + "ResourceGroup": { + "Enable": true, + "Id": "/subscriptions/035a1364-f00d-48e2-b582-4fe125905ee3/resourceGroups/AdsTest", + "Subscription": "Jorampon Internal Consumption", + "Domain": "microsoft.com", + "TenantId": "72f988bf-86f1-41af-91ab-2d7cd011db47", + "Location": "australiaeast", + "Name": "AdsTest", + "AADUser": "jorampon@microsoft.com", + "Hash": "DummyValueToBeReplacedAtRuntime" + }, "ServicePrincipals": { "DeploymentSP": { "Enable": true, @@ -20,26 +31,15 @@ "Enable": true, "Name": "AdsGFWebAuthSP", "ApplyNamePostFix": true, - "ClientId": "d733e6a7-01e6-4af6-a6b8-99b029ed7237" + "ClientId": "6112fad2-f8f2-4313-987c-690703898480" }, "FunctionAppAuthenticationSP": { "Enable": true, "Name": "AdsGFFuncAppAuthSP", "ApplyNamePostFix": true, - "ClientId": "cb3e089a-6d2d-4b3c-949f-353437b78e23" + "ClientId": "e7bf45f2-019c-4f6e-b908-58b71278bbe1" } }, - "ResourceGroup": { - "Id": "/subscriptions/035a1364-f00d-48e2-b582-4fe125905ee3/resourceGroups/AdsTest", - "Enable": true, - "Subscription": "Jorampon Internal Consumption", - "Domain": "microsoft.com", - "TenantId": "72f988bf-86f1-41af-91ab-2d7cd011db47", - "Location": "australiaeast", - "Name": "AdsTest", - "AADUser": "jorampon@microsoft.com", - "Hash": "DummyValueToBeReplacedAtRuntime" - }, "FolderPaths": { "ArmTemplates": "./arm", "Environments": "./../environments", @@ -82,13 +82,13 @@ }, "AzureSQLServer": { "Enable": true, - "Name": "adsgfazsql", + "Name": "adsgfsvr", "ApplyNamePostFix": true, "AdminUser": "AdsAdmin", "AdminPassword": "EPuiX2K0^T4t", "AdsGoFastDB": { "Enable": true, - "Name": "AdsGf", + "Name": "AdsGF", "UpdateSourceAndTargetSystems": true, "UpdateDataFactory": true, "ApplyNamePostFix": false @@ -116,19 +116,19 @@ "ApplyNamePostFix": true, "AzVnetIr": { "Enable": true, - "Name": "SelfHostedIntegrationRuntime-Azure-VNET", + "Name": "IRA", "Type": "ManagedVnet" }, "OnPremVnetIr": { "Enable": false, - "Name": "SelfHostedIntegrationRuntime-OnPem-Net", + "Name": "IRB", "Type": "SelfHosted", - "IrInstallConfig":{ - "LocalDrive":"C:", - "LocalVMFolder":"ADFInstaller", - "IrDownloadURL":"https://download.microsoft.com/download/E/4/7/E4771905-1079-445B-8BF9-8A1A075D8A10/IntegrationRuntime_5.9.7900.1.msi", - "JDKDownloadURL":"https://github.com/adoptium/temurin11-binaries/releases/download/jdk-11.0.12%2B7/OpenJDK11U-jdk_x64_windows_hotspot_11.0.12_7.msi", - "JDKInstallFolder":"c:\\Program Files\\Eclipse Foundation\\" + "IrInstallConfig": { + "LocalDrive": "C:", + "LocalVMFolder": "ADFInstaller", + "IrDownloadURL": "https://download.microsoft.com/download/E/4/7/E4771905-1079-445B-8BF9-8A1A075D8A10/IntegrationRuntime_5.9.7900.1.msi", + "JDKDownloadURL": "https://github.com/adoptium/temurin11-binaries/releases/download/jdk-11.0.12%2B7/OpenJDK11U-jdk_x64_windows_hotspot_11.0.12_7.msi", + "JDKInstallFolder": "c:\\Program Files\\Eclipse Foundation\\" } } }, @@ -149,7 +149,7 @@ "ApplyNamePostFix": true }, "Vnet": { - "Enable": true, + "Enable": false, "Name": "AdsGoFastVnet", "ApplyNamePostFix": true, "vNetAddressRange": "10.3.0.0/16", @@ -157,14 +157,10 @@ "DataSubnetAddressRange": "10.3.2.0/27", "WebAppSubnetAddressRange": "10.3.3.0/27", "FuncAppSubnetAddressRange": "10.3.4.0/27", - "BastionSubnetName":"AzureBastionSubnet", - "DataSubnetName":"Data", - "WebAppSubnetName":"WebApp", - "FuncAppSubnetName":"FuncApp" - }, - "Bastion": { - "Name": "adsgfbastion", - "ApplyNamePostFix": true + "BastionSubnetName": "AzureBastionSubnet", + "DataSubnetName": "Data", + "WebAppSubnetName": "WebApp", + "FuncAppSubnetName": "FuncApp" }, "Storage": { "Logging": { @@ -178,19 +174,24 @@ "ApplyNamePostFix": true, "Enable": true }, - "ADLSTransient": { - "Name": "adlstran", - "ApplyNamePostFix": true, - "Enable": false - }, "Blob": { "Name": "blob", "ApplyNamePostFix": true, "Enable": true, "ResourceId": "" + }, + "ADLSTransient": { + "Name": "adlstran", + "ApplyNamePostFix": true, + "Enable": false } + }, + "Bastion": { + "Name": "adsgfbastion", + "ApplyNamePostFix": true, + "Enable": true } } } } -} +} \ No newline at end of file diff --git a/solution/Deployment/environments/development_bak.json b/solution/Deployment/environments/development_bak.json new file mode 100644 index 00000000..ff4b4653 --- /dev/null +++ b/solution/Deployment/environments/development_bak.json @@ -0,0 +1,196 @@ +{ + "AdsOpts": { + "CI": { + "Enable": true, + "BuildFunctionApp": true, + "BuildWebApp": true, + "BuildAdsGoFastDatabase": true, + "BuildDataFactory": true + }, + "CD": { + "EnableDeploy": true, + "EnableConfigure": true, + "ServicePrincipals": { + "DeploymentSP": { + "Enable": true, + "Name": "AdsGoFastDeployer", + "ApplyNamePostFix": false + }, + "WebAppAuthenticationSP": { + "Enable": true, + "Name": "AdsGFWebAuthSP", + "ApplyNamePostFix": true, + "ClientId": "6112fad2-f8f2-4313-987c-690703898480" + }, + "FunctionAppAuthenticationSP": { + "Enable": true, + "Name": "AdsGFFuncAppAuthSP", + "ApplyNamePostFix": true, + "ClientId": "e7bf45f2-019c-4f6e-b908-58b71278bbe1" + } + }, + "ResourceGroup": { + "Id": "/subscriptions/035a1364-f00d-48e2-b582-4fe125905ee3/resourceGroups/AdsTest", + "Enable": true, + "Subscription": "Jorampon Internal Consumption", + "Domain": "microsoft.com", + "TenantId": "72f988bf-86f1-41af-91ab-2d7cd011db47", + "Location": "australiaeast", + "Name": "AdsTest", + "AADUser": "jorampon@microsoft.com", + "Hash": "DummyValueToBeReplacedAtRuntime" + }, + "FolderPaths": { + "ArmTemplates": "./arm", + "Environments": "./../environments", + "PublishZip": "./../bin/publish/zipped", + "PublishUnZip": "./../bin/publish/unzipped" + }, + "AzureLoginOptions": { + "UseInteractiveAzCliLogin": true + }, + "EnviroInstalls": { + "PerformLocalInstalls": true, + "PerformLocalInstallsAzCli": false, + "PerformLocalInstallsAzCliAddToPath": true + }, + "ArmOptions": { + "PerformDeployment": false, + "PerformDeploymentStorageLogging": false, + "PerformDeploymentAppService": false + }, + "Services": { + "UseARMDefaults": false, + "AppInsights": { + "Enable": true, + "Name": "adsgfappin", + "ApplyNamePostFix": true + }, + "AppPlans": { + "WebApp": { + "Enable": true, + "Name": "adsgfappplanweb", + "ApplyNamePostFix": true, + "ResourceGroup": null + }, + "FunctionApp": { + "Enable": true, + "Name": "adsgfappplanfnc", + "ApplyNamePostFix": true, + "ResourceGroup": null + } + }, + "AzureSQLServer": { + "Enable": true, + "Name": "adsgfsvr", + "ApplyNamePostFix": true, + "AdminUser": "AdsAdmin", + "AdminPassword": "EPuiX2K0^T4t", + "AdsGoFastDB": { + "Enable": true, + "Name": "AdsGF", + "UpdateSourceAndTargetSystems": true, + "UpdateDataFactory": true, + "ApplyNamePostFix": false + }, + "StagingDB": { + "Enable": true, + "Name": "AdsGfStaging", + "ApplyNamePostFix": false + }, + "SampleDB": { + "Enable": true, + "Name": "AdsGfSample", + "ApplyNamePostFix": false + } + }, + "CoreFunctionApp": { + "Enable": true, + "Name": "adsgofastfunc", + "ApplyNamePostFix": true, + "PrincipalId": "" + }, + "DataFactory": { + "Enable": true, + "Name": "adsgfadf", + "ApplyNamePostFix": true, + "AzVnetIr": { + "Enable": true, + "Name": "IRA", + "Type": "ManagedVnet" + }, + "OnPremVnetIr": { + "Enable": false, + "Name": "IRB", + "Type": "SelfHosted", + "IrInstallConfig": { + "LocalDrive": "C:", + "LocalVMFolder": "ADFInstaller", + "IrDownloadURL": "https://download.microsoft.com/download/E/4/7/E4771905-1079-445B-8BF9-8A1A075D8A10/IntegrationRuntime_5.9.7900.1.msi", + "JDKDownloadURL": "https://github.com/adoptium/temurin11-binaries/releases/download/jdk-11.0.12%2B7/OpenJDK11U-jdk_x64_windows_hotspot_11.0.12_7.msi", + "JDKInstallFolder": "c:\\Program Files\\Eclipse Foundation\\" + } + } + }, + "WebSite": { + "Enable": true, + "Name": "adsgofastweb", + "ApplyNamePostFix": true, + "PrincipalId": "" + }, + "LogAnalytics": { + "Enable": true, + "Name": "adsgofastloganalytics", + "ApplyNamePostFix": true + }, + "KeyVault": { + "Enable": true, + "Name": "adsgfkv", + "ApplyNamePostFix": true + }, + "Vnet": { + "Enable": false, + "Name": "AdsGoFastVnet", + "ApplyNamePostFix": true, + "vNetAddressRange": "10.3.0.0/16", + "BastionSubnetAddressRange": "10.3.1.0/27", + "DataSubnetAddressRange": "10.3.2.0/27", + "WebAppSubnetAddressRange": "10.3.3.0/27", + "FuncAppSubnetAddressRange": "10.3.4.0/27", + "BastionSubnetName": "AzureBastionSubnet", + "DataSubnetName": "Data", + "WebAppSubnetName": "WebApp", + "FuncAppSubnetName": "FuncApp" + }, + "Bastion": { + "Name": "adsgfbastion", + "ApplyNamePostFix": true + }, + "Storage": { + "Logging": { + "Name": "logstg", + "ApplyNamePostFix": true, + "Enable": true, + "Dummy": "" + }, + "ADLS": { + "Name": "adls", + "ApplyNamePostFix": true, + "Enable": true + }, + "ADLSTransient": { + "Name": "adlstran", + "ApplyNamePostFix": true, + "Enable": false + }, + "Blob": { + "Name": "blob", + "ApplyNamePostFix": true, + "Enable": true, + "ResourceId": "" + } + } + } + } + } +} \ No newline at end of file diff --git a/solution/Deployment/environments/development_test.json b/solution/Deployment/environments/development_test.json new file mode 100644 index 00000000..1a7fdc7c --- /dev/null +++ b/solution/Deployment/environments/development_test.json @@ -0,0 +1,175 @@ +{ + "$schema": "./environment.schema.json", + "AdsOpts": { + "CI": { + "Enable": true, + "BuildFunctionApp": true, + "BuildWebApp": true, + "BuildAdsGoFastDatabase": true + }, + "CD": { + "EnableDeploy": true, + "EnableConfigure": true, + "ServicePrincipals": { + "DeploymentSP": { + "Enable": true, + "Name": "AdsGoFastDeployer", + "ApplyNamePostFix": false + }, + "WebAppAuthenticationSP": { + "Enable": true, + "Name": "AdsGFWebAuthSP", + "ApplyNamePostFix": true, + "ClientId": "f5aa088b-ea94-4070-8ecb-e89ccdb1758f" + }, + "FunctionAppAuthenticationSP": { + "Enable": true, + "Name": "AdsGFFuncAppAuthSP", + "ApplyNamePostFix": true, + "ClientId": "92321a9b-1261-4352-9be7-40c635a5d928" + } + }, + "ResourceGroup": { + "Id": "/subscriptions/2ba4a248-b4cd-4c44-8936-8e598057fffc/resourceGroups/AdsTMsdn", + "Enable": true, + "Subscription": "MS Employee - MSDN - Jorampon", + "Domain": "msdemoauxx2.online", + "TenantId": "e26ed924-1847-4ea1-b43e-9ba7c65d3538", + "Location": "australiaeast", + "Name": "AdsTMsdn", + "AADUser": "jrampono@msdemoauxx2.online", + "Hash": "DummyValueToBeReplacedAtRuntime" + }, + "FolderPaths": { + "ArmTemplates": "./arm", + "Environments": "./../environments", + "PublishZip": "./../bin/publish/zipped", + "PublishUnZip": "./../bin/publish/unzipped" + }, + "AzureLoginOptions": { + "UseInteractiveAzCliLogin": true + }, + "EnviroInstalls": { + "PerformLocalInstalls": true, + "PerformLocalInstallsAzCli": false, + "PerformLocalInstallsAzCliAddToPath": true + }, + "ArmOptions": { + "PerformDeployment": false, + "PerformDeploymentStorageLogging": false, + "PerformDeploymentAppService": false + }, + "Services": { + "UseARMDefaults": false, + "AppInsights": { + "Enable": true, + "Name": "adsgfappin", + "ApplyNamePostFix": true + }, + "AppPlans": { + "WebApp": { + "Enable": true, + "Name": "adsgfappplanweb", + "ApplyNamePostFix": true, + "ResourceGroup": null + }, + "FunctionApp": { + "Enable": true, + "Name": "adsgfappplanfnc", + "ApplyNamePostFix": true, + "ResourceGroup": null + } + }, + "AzureSQLServer": { + "Enable": true, + "Name": "adsgfazsql", + "ApplyNamePostFix": true, + "AdminUser": "AdsAdmin", + "AdminPassword": "EPuiX2K0^T4t", + "AdsGoFastDB": { + "Enable": true, + "Name": "AdsGf", + "UpdateSourceAndTargetSystems": true, + "UpdateDataFactory": true, + "ApplyNamePostFix": false + }, + "StagingDB": { + "Enable": true, + "Name": "AdsGfStaging", + "ApplyNamePostFix": false + }, + "SampleDB": { + "Enable": true, + "Name": "AdsGfSample", + "ApplyNamePostFix": false + } + }, + "CoreFunctionApp": { + "Enable": true, + "Name": "adsgofastfunc", + "ApplyNamePostFix": true, + "PrincipalId": "" + }, + "DataFactory": { + "Enable": true, + "Name": "adsgfadf", + "ApplyNamePostFix": true, + "AzVnetIr": { + "Enable": true, + "Name": "SelfHostedIntegrationRuntime-Azure-VNET", + "Type": "ManagedVnet" + }, + "OnPremVnetIr": { + "Enable": false, + "Name": "SelfHostedIntegrationRuntime-OnPem-Net", + "Type": "ManagedVnet" + } + }, + "WebSite": { + "Enable": true, + "Name": "adsgofastweb", + "ApplyNamePostFix": true, + "PrincipalId": "" + }, + "LogAnalytics": { + "Enable": true, + "Name": "adsgofastloganalytics", + "ApplyNamePostFix": true + }, + "KeyVault": { + "Enable": true, + "Name": "adsgfkv", + "ApplyNamePostFix": true + }, + "Vnet": { + "Enable": true, + "Name": "AdsGoFastVnet" + }, + "Storage": { + "Logging": { + "Name": "logstg", + "ApplyNamePostFix": true, + "Enable": true, + "Dummy": "" + }, + "ADLS": { + "Name": "adls", + "ApplyNamePostFix": true, + "Enable": true + }, + "ADLSTransient": { + "Name": "adlstran", + "ApplyNamePostFix": true, + "Enable": false + }, + "Blob": { + "Name": "blob", + "ApplyNamePostFix": true, + "Enable": true, + "ResourceId": "" + } + } + } + } + } +} diff --git a/solution/Deployment/environments/environment.schema.json b/solution/Deployment/environments/environment.schema.json index 51126198..6b68c9dd 100644 --- a/solution/Deployment/environments/environment.schema.json +++ b/solution/Deployment/environments/environment.schema.json @@ -1,2007 +1,2061 @@ { - "$schema": "http://json-schema.org/draft-07/schema", - "$id": "http://adsgofast.com/environment.schema.json", - "type": "object", - "title": "The root schema", - "description": "The root schema comprises the entire JSON document.", - "default": {}, - "required": [ - "AdsOpts" - ], - "properties": { - "AdsOpts": { - "$id": "#/properties/AdsOpts", - "type": "object", - "title": "The AdsOpts schema", - "description": "AdsOpts is the Primary configuration element for CICD in this project.", - "default": {}, - "required": [ - "CI", - "CD" - ], - "properties": { - "CI": { - "$id": "#/properties/AdsOpts/properties/CI", - "type": "object", - "title": "The CI schema", - "description": "//CI (Continuous Integration - Set this section to true if you want the source binaries to be built out. In ordinary operation these should be true. Switches provided for ease of debugging only)", - "default": { - "Enable": true, - "BuildFunctionApp": true, - "BuildWebApp": true, - "BuildAdsGoFastDatabase": true - }, - "required": [ - "Enable", - "BuildFunctionApp", - "BuildWebApp", - "BuildAdsGoFastDatabase" - ], - "properties": { - "Enable": { - "$id": "#/properties/AdsOpts/properties/CI/properties/Enable", - "type": "boolean", - "title": "Enable", - "description": "Set this section to true if you want the source binaries to be built out. In ordinary operation these should be true. ", - "default": false, - "examples": [ - true, - false - ] - }, - "BuildFunctionApp": { - "$id": "#/properties/AdsOpts/properties/CI/properties/BuildFunctionApp", - "type": "boolean", - "title": "The BuildFunctionApp schema", - "description": "Set this section to true if you want the source binaries to be built out. In ordinary operation these should be true.", - "default": true, - "examples": [ - true, - false - ] - }, - "BuildWebApp": { - "$id": "#/properties/AdsOpts/properties/CI/properties/BuildWebApp", - "type": "boolean", - "title": "The BuildWebApp schema", - "description": "Set this section to true if you want the source binaries to be built out. In ordinary operation these should be true.", - "default": true, - "examples": [ - true, - false - ] - }, - "BuildAdsGoFastDatabase": { - "$id": "#/properties/AdsOpts/properties/CI/properties/BuildAdsGoFastDatabase", - "type": "boolean", - "title": "The BuildAdsGoFastDatabase schema", - "description": "Set this section to true if you want the source binaries to be built out. In ordinary operation these should be true.", - "default": true, - "examples": [ - true, - false - ] - } - }, - "additionalProperties": false + "$schema": "http://json-schema.org/draft-07/schema", + "$id": "http://adsgofast.com/environment.schema.json", + "type": "object", + "title": "Environment Configuration for the ADS Go Fast Framework", + "description": "Environment Configuration for the ADS Go Fast Framework.", + "default": {}, + "required": [ + "AdsOpts" + ], + "properties": { + "AdsOpts": { + "$id": "#/properties/AdsOpts", + "type": "object", + "title": "Ads Go Fast Settings", + "description": "Primary, top-level configuration element for CICD in this project.", + "default": {}, + "required": [ + "CI", + "CD" + ], + "properties": { + "CI": { + "$id": "#/properties/AdsOpts/properties/CI", + "type": "object", + "title": "Continuous Integration Settings", + "description": "//CI (Continuous Integration - Set this section to true if you want the source binaries to be built out. In ordinary operation these should be true. Switches provided for ease of debugging only)", + "default": { + "Enable": true, + "BuildFunctionApp": true, + "BuildWebApp": true, + "BuildAdsGoFastDatabase": true, + "BuildDataFactory": true + }, + "required": [ + "Enable", + "BuildFunctionApp", + "BuildWebApp", + "BuildAdsGoFastDatabase", + "BuildDataFactory" + ], + "properties": { + "Enable": { + "$id": "#/properties/AdsOpts/properties/CI/properties/Enable", + "type": "boolean", + "format": "checkbox", + "title": "Enable All", + "default": true, + "enum": [ + true, + false + ], + "options": { + "infoText": "Set this section to true if you want the source binaries to be built out. In ordinary operation these should be true. " + } + }, + "BuildFunctionApp": { + "$id": "#/properties/AdsOpts/properties/CI/properties/BuildFunctionApp", + "type": "boolean", + "format": "checkbox", + "title": "BuildFunctionApp", + "options": { + "infoText": "Set this section to true if you want the source binaries for the Function Application to be built out. In ordinary operation these should be true." + }, + "default": true, + "enum": [ + true, + false + ] + }, + "BuildWebApp": { + "$id": "#/properties/AdsOpts/properties/CI/properties/BuildWebApp", + "type": "boolean", + "format": "checkbox", + "title": "BuildWebApp", + "options": { + "infoText": "Set this section to true if you want the source binaries for the Web Appilication to be built out. In ordinary operation these should be true." + }, + "default": true, + "enum": [ + true, + false + ] + }, + "BuildAdsGoFastDatabase": { + "$id": "#/properties/AdsOpts/properties/CI/properties/BuildAdsGoFastDatabase", + "type": "boolean", + "format": "checkbox", + "title": "BuildAdsGoFastDatabase", + "options": { + "infoText": "Set this section to true if you want the source binaries for the AdsGoFastDatabase to be built out. In ordinary operation these should be true." + }, + "default": true, + "enum": [ + true, + false + ] + }, + "BuildDataFactory": { + "$id": "#/properties/AdsOpts/properties/CI/properties/BuildDataFactory", + "type": "boolean", + "format": "checkbox", + "title": "DataFactory", + "options": { + "infoText": "Set this section to true if you want the source binaries for the AdsGoFastDatabase to be built out. In ordinary operation these should be true." + }, + "default": true, + "enum": [ + true, + false + ] + } + }, + "additionalProperties": false + }, + "CD": { + "$id": "#/properties/AdsOpts/properties/CD", + "type": "object", + "title": "Continuous Deployment Settings", + "description": "This configuration element controls continous deployment specifics", + "default": {}, + "required": [ + "EnableDeploy", + "EnableConfigure", + "ServicePrincipals", + "ResourceGroup", + "FolderPaths", + "AzureLoginOptions", + "EnviroInstalls", + "ArmOptions", + "Services" + ], + "properties": { + "EnableDeploy": { + "$id": "#/properties/AdsOpts/properties/CD/properties/EnableDeploy", + "type": "boolean", + "format": "checkbox", + "title": "Enable/Disable Service Deployment", + "options": { + "infoText": "Set to false if you want no continous deployment operations to occur." + }, + "default": true, + "enum": [ + true, + false + ] + }, + "EnableConfigure": { + "$id": "#/properties/AdsOpts/properties/CD/properties/EnableConfigure", + "type": "boolean", + "format": "checkbox", + "title": "Enable/Disable Post Deployment Configuration", + "options": { + "infoText": "Set to false if you don't want post deployment configuration activities to occur." + }, + "default": true, + "enum": [ + true, + false + ] + }, + "ResourceGroup": { + "$id": "#/properties/AdsOpts/properties/CD/properties/ResourceGroup", + "type": "object", + "title": "Primary Resource Group Settings", + "options": { + "infoText": "" + }, + "default": {}, + "required": [ + "Id", + "Enable", + "Subscription", + "Domain", + "TenantId", + "Location", + "Name", + "AADUser", + "Hash" + ], + "properties": { + "Enable": { + "$id": "#/properties/AdsOpts/properties/CD/properties/ResourceGroup/properties/Enable", + "type": "boolean", + "format": "checkbox", + "title": "Enable Resource Group Deployment", + "options": { + "infoText": "Enable or Disable the Deployment of the Resource Group" + }, + "default": true, + "enum": [ + true, + false + ] }, - "CD": { - "$id": "#/properties/AdsOpts/properties/CD", - "type": "object", - "title": "The CD schema", - "description": "This configuration element controls continous deployment specifics", - "default": {}, - "required": [ + "Id": { + "$id": "#/properties/AdsOpts/properties/CD/properties/ResourceGroup/properties/Id", + "type": "string", + "title": "Resource Group ID", + "options": { + "infoText": "", + "inputAttributes": { + "placeholder": "eg. /subscriptions/92f988bf-86f1-41af-91ab-2d7cd011db48/resourceGroups/AdsTest" + } + }, + "default": "", + "examples": [ + "/subscriptions/92f988bf-86f1-41af-91ab-2d7cd011db48/resourceGroups/AdsTest" + ] + }, + "Subscription": { + "$id": "#/properties/AdsOpts/properties/CD/properties/ResourceGroup/properties/Subscription", + "type": "string", + "title": "Subscription Name", + "options": { + "infoText": "name of the subscription which will host the deployment.", + "inputAttributes": { + "placeholder": "eg. Ads Go Fast Demo Subscription" + } + }, + "default": "", + "examples": [ + "Ads Go Fast Demo Subscription" + ] + }, + "Domain": { + "$id": "#/properties/AdsOpts/properties/CD/properties/ResourceGroup/properties/Domain", + "type": "string", + "title": "Domain ", + "options": { + "infoText": "Azure Active Directory Domain Name to use for the deployment.", + "inputAttributes": { + "placeholder": "eg. adventureworks.com" + } + }, + "default": "", + "examples": [ + "adventureworks.com" + ] + }, + "TenantId": { + "$id": "#/properties/AdsOpts/properties/CD/properties/ResourceGroup/properties/TenantId", + "type": "string", + "title": "TenantId", + "options": { + "infoText": "tenant Azure Subscription Tenant Id to be used for deployment", + "inputAttributes": { + "placeholder": "eg. 82f988bf-86f1-41af-91ab-2d7cd011db48" + } + }, + "default": "", + "examples": [ + "82f988bf-86f1-41af-91ab-2d7cd011db48" + ] + }, + "Location": { + "$id": "#/properties/AdsOpts/properties/CD/properties/ResourceGroup/properties/Location", + "type": "string", + "title": "Azure Region", + "options": { + "infoText": "Azure Region to deploy into.", + "inputAttributes": { + "placeholder": "eg. australiaeast" + } + }, + "default": "australiaeast", + "enum": [ + "australiaeast", + "australiasoutheast" + ] + }, + "Name": { + "$id": "#/properties/AdsOpts/properties/CD/properties/ResourceGroup/properties/Name", + "type": "string", + "title": "Resource Group Name", + "options": { + "infoText": "Name of the Resource Group to use for deployment." + }, + "default": "AdsTest", + "examples": [ + "AdsTest" + ] + }, + "AADUser": { + "$id": "#/properties/AdsOpts/properties/CD/properties/ResourceGroup/properties/AADUser", + "type": "string", + "title": "AADUser", + "options": { + "infoText": "", + "inputAttributes": { + "placeholder": "eg. admin@advetureworks.com" + } + }, + "default": "", + "examples": [ + "jorampon@microsoft.com" + ] + }, + "Hash": { + "$id": "#/properties/AdsOpts/properties/CD/properties/ResourceGroup/properties/Hash", + "type": "string", + "title": "Hash", + "description": "This is the used by the deployment scripts to store a hash value that is optionally postfixed to resource names to ensure uniqueness. Leave this item bank.", + "default": "", + "options": { + "inputAttributes": { + "placeholder": "LEAVE THIS BLANK" + } + }, + "examples": [ + "DummyValueToBeReplacedAtRuntime" + ] + } + }, + "additionalProperties": true + }, + "ServicePrincipals": { + "$id": "#/properties/AdsOpts/properties/CD/properties/ServicePrincipals", + "type": "object", + "title": "Service Principal Options", + "description": "Controls the specifics of the Service Principals and Azure App Registrations used.", + "default": { + "DeploymentSP": { + "Enable": true, + "Name": "AdsGFDeploySP", + "ApplyNamePostFix": true + }, + "WebAppAuthenticationSP": { + "Enable": true, + "Name": "AdsGFWebAuthSP", + "ApplyNamePostFix": true, + "ClientId": "#######" + }, + "FunctionAppAuthenticationSP": { + "Enable": true, + "Name": "AdsGFFuncAppAuthSP", + "ApplyNamePostFix": true, + "ClientId": "########" + } + }, + "options": { + "collapsed": true + }, + "properties": { + "DeploymentSP": { + "$id": "#/properties/AdsOpts/properties/CD/properties/ServicePrincipals/properties/DeploymentSP", + "type": "object", + "title": "DeploymentSP", + "description": "Set enable to true if you want the script to create the Deployment SP for you othwerise you need to provide the SP details in Secrets.json or if depoloying via GitHub Actions add in github secrets - https://github.com/Azure/actions-workflow-samples/blob/master/assets/create-secrets-for-GitHub-workflows.md#set-secret-with-azure-credentials", + "default": { + "Enable": true, + "Name": "AdsGFDeploySP", + "ApplyNamePostFix": true + }, + "required": [ + "Enable", + "Name", + "ApplyNamePostFix" + ], + "properties": { + "Enable": { + "$id": "#/properties/AdsOpts/properties/CD/properties/ServicePrincipals/properties/DeploymentSP/properties/Enable", + "type": "boolean", + "format": "checkbox", + "title": "Enable Deployment", + "description": "Set enable to true if you want the script to create the Deployment SP for you othwerise you need to provide the SP details in Secrets.json or if depoloying via GitHub Actions add in github secrets - https://github.com/Azure/actions-workflow-samples/blob/master/assets/create-secrets-for-GitHub-workflows.md#set-secret-with-azure-credentials", + "default": true, + "examples": [ + true, + false + ] + }, + "Name": { + "$id": "#/properties/AdsOpts/properties/CD/properties/ServicePrincipals/properties/DeploymentSP/properties/Name", + "type": "string", + "title": "Name", + "description": "Name of the Service Principal", + "default": "", + "examples": [ + "AdsGFDeploySP" + ] + }, + "ApplyNamePostFix": { + "$id": "#/properties/AdsOpts/properties/CD/properties/ServicePrincipals/properties/DeploymentSP/properties/ApplyNamePostFix", + "type": "boolean", + "format": "checkbox", + "title": "Apply Postfix to Resource Name", + "description": "If true then the script will add a hash based on the resource group name as a postfix to the objects name", + "default": true, + "examples": [ + true, + false + ] + } + }, + "additionalProperties": true + }, + "WebAppAuthenticationSP": { + "$id": "#/properties/AdsOpts/properties/CD/properties/ServicePrincipals/properties/WebAppAuthenticationSP", + "type": "object", + "title": "WebAppAuthenticationSP", + "description": "", + "default": { + "Enable": true, + "Name": "AdsGFWebAuthSP", + "ApplyNamePostFix": true, + "ClientId": "###################" + }, + "required": [ + "Enable", + "Name", + "ApplyNamePostFix", + "ClientId" + ], + "properties": { + "Enable": { + "$id": "#/properties/AdsOpts/properties/CD/properties/ServicePrincipals/properties/WebAppAuthenticationSP/properties/Enable", + "type": "boolean", + "format": "checkbox", + "title": "Enable Deployment", + "description": "Set to true if you want the script to create the App Registration Used by the Web App to validate AAD authentication requests.", + "default": true, + "examples": [ + true, + false + ] + }, + "Name": { + "$id": "#/properties/AdsOpts/properties/CD/properties/ServicePrincipals/properties/WebAppAuthenticationSP/properties/Name", + "type": "string", + "title": "Name", + "description": "Name of the App Registration", + "default": "AdsGFWebAuthSP", + "examples": [ + "AdsGFWebAuthSP" + ] + }, + "ApplyNamePostFix": { + "$id": "#/properties/AdsOpts/properties/CD/properties/ServicePrincipals/properties/WebAppAuthenticationSP/properties/ApplyNamePostFix", + "type": "boolean", + "format": "checkbox", + "title": "Apply Postfix to Resource Name", + "description": "If true then the script will add a hash based on the resource group name as a postfix to the objects name", + "default": true, + "examples": [ + true, + false + ] + }, + "ClientId": { + "$id": "#/properties/AdsOpts/properties/CD/properties/ServicePrincipals/properties/WebAppAuthenticationSP/properties/ClientId", + "type": "string", + "title": "ClientId", + "description": "client id of the App Registration. If you have enable set to true then ignore this as your script will autopopulate once it creates the App Reg", + "default": "", + "examples": [ + "904a7841-c19d-4c38-bb7e-ccd78e49a55b" + ] + } + }, + "additionalProperties": false + }, + "FunctionAppAuthenticationSP": { + "$id": "#/properties/AdsOpts/properties/CD/properties/ServicePrincipals/properties/FunctionAppAuthenticationSP", + "type": "object", + "title": "FunctionAppAuthenticationSP", + "description": "", + "default": {}, + "examples": [ + { + "Enable": true, + "Name": "AdsGFFuncAppAuthSP", + "ApplyNamePostFix": true, + "ClientId": "835b082c-6a63-4fcc-b5d8-005dba169819" + } + ], + "required": [ + "Enable", + "Name", + "ApplyNamePostFix", + "ClientId" + ], + "properties": { + "Enable": { + "$id": "#/properties/AdsOpts/properties/CD/properties/ServicePrincipals/properties/FunctionAppAuthenticationSP/properties/Enable", + "type": "boolean", + "format": "checkbox", + "title": "Enable Deployment", + "description": "", + "default": false, + "examples": [ + true + ] + }, + "Name": { + "$id": "#/properties/AdsOpts/properties/CD/properties/ServicePrincipals/properties/FunctionAppAuthenticationSP/properties/Name", + "type": "string", + "title": "Name", + "description": "", + "default": "", + "examples": [ + "AdsGFFuncAppAuthSP" + ] + }, + "ApplyNamePostFix": { + "$id": "#/properties/AdsOpts/properties/CD/properties/ServicePrincipals/properties/FunctionAppAuthenticationSP/properties/ApplyNamePostFix", + "type": "boolean", + "format": "checkbox", + "title": "Apply Postfix to Resource Name", + "description": "", + "default": false, + "examples": [ + true + ] + }, + "ClientId": { + "$id": "#/properties/AdsOpts/properties/CD/properties/ServicePrincipals/properties/FunctionAppAuthenticationSP/properties/ClientId", + "type": "string", + "title": "ClientId", + "description": "", + "default": "", + "examples": [ + "835b082c-6a63-4fcc-b5d8-005dba169819" + ] + } + }, + "additionalProperties": true + } + }, + "additionalProperties": true + }, + "FolderPaths": { + "$id": "#/properties/AdsOpts/properties/CD/properties/FolderPaths", + "type": "object", + "title": "Folder Path Options", + "description": "Controls the various folder addresses that the deployment scripts use.", + "default": {}, + "options": { + "collapsed": true + }, + "examples": [ + { + "ArmTemplates": "./arm", + "Environments": "./../environments", + "PublishZip": "./../bin/publish/zipped", + "PublishUnZip": "./../bin/publish/unzipped" + } + ], + "required": [ + "ArmTemplates", + "Environments", + "PublishZip", + "PublishUnZip" + ], + "properties": { + "ArmTemplates": { + "$id": "#/properties/AdsOpts/properties/CD/properties/FolderPaths/properties/ArmTemplates", + "type": "string", + "title": "ArmTemplates", + "description": "", + "default": "", + "examples": [ + "./arm" + ] + }, + "Environments": { + "$id": "#/properties/AdsOpts/properties/CD/properties/FolderPaths/properties/Environments", + "type": "string", + "title": "Environments", + "description": "", + "default": "", + "examples": [ + "./../environments" + ] + }, + "PublishZip": { + "$id": "#/properties/AdsOpts/properties/CD/properties/FolderPaths/properties/PublishZip", + "type": "string", + "title": "PublishZip", + "description": "", + "default": "", + "examples": [ + "./../bin/publish/zipped" + ] + }, + "PublishUnZip": { + "$id": "#/properties/AdsOpts/properties/CD/properties/FolderPaths/properties/PublishUnZip", + "type": "string", + "title": "PublishUnZip", + "description": "", + "default": "", + "examples": [ + "./../bin/publish/unzipped" + ] + } + }, + "additionalProperties": true + }, + "AzureLoginOptions": { + "$id": "#/properties/AdsOpts/properties/CD/properties/AzureLoginOptions", + "type": "object", + "title": "AzureLoginOptions", + "description": "Azure Login Options.", + "default": {}, + "options": { + "collapsed": true + }, + "examples": [ + { + "UseInteractiveAzCliLogin": true + } + ], + "required": [ + "UseInteractiveAzCliLogin" + ], + "properties": { + "UseInteractiveAzCliLogin": { + "$id": "#/properties/AdsOpts/properties/CD/properties/AzureLoginOptions/properties/UseInteractiveAzCliLogin", + "type": "boolean", + "format": "checkbox", + "title": "UseInteractiveAzCliLogin", + "description": "", + "default": false, + "examples": [ + true + ] + } + }, + "additionalProperties": true + }, + "EnviroInstalls": { + "$id": "#/properties/AdsOpts/properties/CD/properties/EnviroInstalls", + "type": "object", + "title": "Control Local Environment Installs", + "description": "", + "default": {}, + "options": { + "collapsed": true + }, + "examples": [ + { + "PerformLocalInstalls": true, + "PerformLocalInstallsAzCli": false, + "PerformLocalInstallsAzCliAddToPath": true + } + ], + "required": [ + "PerformLocalInstalls", + "PerformLocalInstallsAzCli", + "PerformLocalInstallsAzCliAddToPath" + ], + "properties": { + "PerformLocalInstalls": { + "$id": "#/properties/AdsOpts/properties/CD/properties/EnviroInstalls/properties/PerformLocalInstalls", + "type": "boolean", + "format": "checkbox", + "title": "PerformLocalInstalls", + "description": "", + "default": false, + "examples": [ + true + ] + }, + "PerformLocalInstallsAzCli": { + "$id": "#/properties/AdsOpts/properties/CD/properties/EnviroInstalls/properties/PerformLocalInstallsAzCli", + "type": "boolean", + "format": "checkbox", + "title": "PerformLocalInstallsAzCli", + "description": "", + "default": false, + "examples": [ + false + ] + }, + "PerformLocalInstallsAzCliAddToPath": { + "$id": "#/properties/AdsOpts/properties/CD/properties/EnviroInstalls/properties/PerformLocalInstallsAzCliAddToPath", + "type": "boolean", + "format": "checkbox", + "title": "PerformLocalInstallsAzCliAddToPath", + "description": "", + "default": false, + "examples": [ + true + ] + } + }, + "additionalProperties": true + }, + "ArmOptions": { + "$id": "#/properties/AdsOpts/properties/CD/properties/ArmOptions", + "type": "object", + "title": "Arm Deployment Options", + "description": "", + "default": {}, + "options": { + "collapsed": true + }, + "examples": [ + { + "PerformDeployment": false, + "PerformDeploymentStorageLogging": false, + "PerformDeploymentAppService": false + } + ], + "required": [ + "PerformDeployment", + "PerformDeploymentStorageLogging", + "PerformDeploymentAppService" + ], + "properties": { + "PerformDeployment": { + "$id": "#/properties/AdsOpts/properties/CD/properties/ArmOptions/properties/PerformDeployment", + "type": "boolean", + "format": "checkbox", + "title": "PerformDeployment", + "description": "", + "default": false, + "examples": [ + false + ] + }, + "PerformDeploymentStorageLogging": { + "$id": "#/properties/AdsOpts/properties/CD/properties/ArmOptions/properties/PerformDeploymentStorageLogging", + "type": "boolean", + "format": "checkbox", + "title": "PerformDeploymentStorageLogging", + "description": "", + "default": false, + "examples": [ + false + ] + }, + "PerformDeploymentAppService": { + "$id": "#/properties/AdsOpts/properties/CD/properties/ArmOptions/properties/PerformDeploymentAppService", + "type": "boolean", + "format": "checkbox", + "title": "PerformDeploymentAppService", + "description": "", + "default": false, + "examples": [ + false + ] + } + }, + "additionalProperties": true + }, + "Services": { + "$id": "#/properties/AdsOpts/properties/CD/properties/Services", + "type": "object", + "title": "Azure Service Options", + "description": "This configuration element controls the specifics of the Azure services deployed.", + "default": {}, + "options": { + "collapsed": true + }, + "required": [ + "UseARMDefaults", + "AppInsights", + "AppPlans", + "AzureSQLServer", + "CoreFunctionApp", + "DataFactory", + "WebSite", + "LogAnalytics", + "KeyVault", + "Vnet", + "Storage" + ], + "properties": { + "UseARMDefaults": { + "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/UseARMDefaults", + "type": "boolean", + "format": "checkbox", + "title": "UseARMDefaults", + "description": "", + "default": false, + "examples": [ + false + ] + }, + "AppInsights": { + "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/AppInsights", + "type": "object", + "title": "AppInsights", + "description": "", + "default": {}, + "examples": [ + { + "Enable": true, + "Name": "adsgfappin", + "ApplyNamePostFix": true + } + ], + "required": [ + "Enable", + "Name", + "ApplyNamePostFix" + ], + "properties": { + "Enable": { + "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/AppInsights/properties/Enable", + "type": "boolean", + "format": "checkbox", + "title": "Enable Deployment", + "description": "", + "default": false, + "examples": [ + true + ] + }, + "Name": { + "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/AppInsights/properties/Name", + "type": "string", + "title": "Name", + "description": "", + "default": "", + "examples": [ + "adsgfappin" + ] + }, + "ApplyNamePostFix": { + "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/AppInsights/properties/ApplyNamePostFix", + "type": "boolean", + "format": "checkbox", + "title": "Apply Postfix to Resource Name", + "description": "", + "default": false, + "examples": [ + true + ] + } + }, + "additionalProperties": true + }, + "AppPlans": { + "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/AppPlans", + "type": "object", + "title": "AppPlans", + "description": "", + "default": {}, + "examples": [ + { + "WebApp": { + "Enable": true, + "Name": "adsgfappplanweb", + "ApplyNamePostFix": true, + "ResourceGroup": null + }, + "FunctionApp": { + "Enable": true, + "Name": "adsgfappplanfnc", + "ApplyNamePostFix": true, + "ResourceGroup": null + } + } + ], + "required": [ + "WebApp", + "FunctionApp" + ], + "properties": { + "WebApp": { + "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/AppPlans/properties/WebApp", + "type": "object", + "title": "WebApp", + "description": "", + "default": {}, + "examples": [ + { + "Enable": true, + "Name": "adsgfappplanweb", + "ApplyNamePostFix": true, + "ResourceGroup": null + } + ], + "required": [ "Enable", - "ServicePrincipals", - "ResourceGroup", - "FolderPaths", - "AzureLoginOptions", - "EnviroInstalls", - "ArmOptions", - "Services" - ], - "properties": { + "Name", + "ApplyNamePostFix", + "ResourceGroup" + ], + "properties": { "Enable": { - "$id": "#/properties/AdsOpts/properties/CD/properties/Enable", - "type": "boolean", - "title": "The Enable schema", - "description": "Set to false if you want no continous deployment operations to occur.", - "default": true, - "examples": [ - true, - false - ] + "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/AppPlans/properties/WebApp/properties/Enable", + "type": "boolean", + "format": "checkbox", + "title": "Enable Deployment", + "description": "", + "default": false, + "examples": [ + true + ] }, - "ServicePrincipals": { - "$id": "#/properties/AdsOpts/properties/CD/properties/ServicePrincipals", - "type": "object", - "title": "The ServicePrincipals schema", - "description": "An explanation about the purpose of this instance.", - "default": { - "DeploymentSP": { - "Enable": true, - "Name": "AdsGFDeploySP", - "ApplyNamePostFix": true - }, - "WebAppAuthenticationSP": { - "Enable": true, - "Name": "AdsGFWebAuthSP", - "ApplyNamePostFix": true, - "ClientId": "#######" - }, - "FunctionAppAuthenticationSP": { - "Enable": true, - "Name": "AdsGFFuncAppAuthSP", - "ApplyNamePostFix": true, - "ClientId": "########" - } - }, - "properties": { - "DeploymentSP": { - "$id": "#/properties/AdsOpts/properties/CD/properties/ServicePrincipals/properties/DeploymentSP", - "type": "object", - "title": "The DeploymentSP schema", - "description": "Set enable to true if you want the script to create the Deployment SP for you othwerise you need to provide the SP details in Secrets.json or if depoloying via GitHub Actions add in github secrets - https://github.com/Azure/actions-workflow-samples/blob/master/assets/create-secrets-for-GitHub-workflows.md#set-secret-with-azure-credentials", - "default": { - "Enable": true, - "Name": "AdsGFDeploySP", - "ApplyNamePostFix": true - }, - "required": [ - "Enable", - "Name", - "ApplyNamePostFix" - ], - "properties": { - "Enable": { - "$id": "#/properties/AdsOpts/properties/CD/properties/ServicePrincipals/properties/DeploymentSP/properties/Enable", - "type": "boolean", - "title": "The Enable schema", - "description": "Set enable to true if you want the script to create the Deployment SP for you othwerise you need to provide the SP details in Secrets.json or if depoloying via GitHub Actions add in github secrets - https://github.com/Azure/actions-workflow-samples/blob/master/assets/create-secrets-for-GitHub-workflows.md#set-secret-with-azure-credentials", - "default": true, - "examples": [ - true, - false - ] - }, - "Name": { - "$id": "#/properties/AdsOpts/properties/CD/properties/ServicePrincipals/properties/DeploymentSP/properties/Name", - "type": "string", - "title": "The Name schema", - "description": "Name of the Service Principal", - "default": "", - "examples": [ - "AdsGFDeploySP" - ] - }, - "ApplyNamePostFix": { - "$id": "#/properties/AdsOpts/properties/CD/properties/ServicePrincipals/properties/DeploymentSP/properties/ApplyNamePostFix", - "type": "boolean", - "title": "The ApplyNamePostFix schema", - "description": "If true then the script will add a hash based on the resource group name as a postfix to the objects name", - "default": true, - "examples": [ - true, - false - ] - } - }, - "additionalProperties": true - }, - "WebAppAuthenticationSP": { - "$id": "#/properties/AdsOpts/properties/CD/properties/ServicePrincipals/properties/WebAppAuthenticationSP", - "type": "object", - "title": "The WebAppAuthenticationSP schema", - "description": "An explanation about the purpose of this instance.", - "default": { - "Enable": true, - "Name": "AdsGFWebAuthSP", - "ApplyNamePostFix": true, - "ClientId": "###################" - }, - "required": [ - "Enable", - "Name", - "ApplyNamePostFix", - "ClientId" - ], - "properties": { - "Enable": { - "$id": "#/properties/AdsOpts/properties/CD/properties/ServicePrincipals/properties/WebAppAuthenticationSP/properties/Enable", - "type": "boolean", - "title": "The Enable schema", - "description": "Set to true if you want the script to create the App Registration Used by the Web App to validate AAD authentication requests.", - "default": true, - "examples": [ - true, - false - ] - }, - "Name": { - "$id": "#/properties/AdsOpts/properties/CD/properties/ServicePrincipals/properties/WebAppAuthenticationSP/properties/Name", - "type": "string", - "title": "The Name schema", - "description": "Name of the App Registration", - "default": "AdsGFWebAuthSP", - "examples": [ - "AdsGFWebAuthSP" - ] - }, - "ApplyNamePostFix": { - "$id": "#/properties/AdsOpts/properties/CD/properties/ServicePrincipals/properties/WebAppAuthenticationSP/properties/ApplyNamePostFix", - "type": "boolean", - "title": "The ApplyNamePostFix schema", - "description": "If true then the script will add a hash based on the resource group name as a postfix to the objects name", - "default": true, - "examples": [ - true, - false - ] - }, - "ClientId": { - "$id": "#/properties/AdsOpts/properties/CD/properties/ServicePrincipals/properties/WebAppAuthenticationSP/properties/ClientId", - "type": "string", - "title": "The ClientId schema", - "description": "The client id of the App Registration. If you have enable set to true then ignore this as your script will autopopulate once it creates the App Reg", - "default": "", - "examples": [ - "904a7841-c19d-4c38-bb7e-ccd78e49a55b" - ] - } - }, - "additionalProperties": false - }, - "FunctionAppAuthenticationSP": { - "$id": "#/properties/AdsOpts/properties/CD/properties/ServicePrincipals/properties/FunctionAppAuthenticationSP", - "type": "object", - "title": "The FunctionAppAuthenticationSP schema", - "description": "An explanation about the purpose of this instance.", - "default": {}, - "examples": [ - { - "Enable": true, - "Name": "AdsGFFuncAppAuthSP", - "ApplyNamePostFix": true, - "ClientId": "835b082c-6a63-4fcc-b5d8-005dba169819" - } - ], - "required": [ - "Enable", - "Name", - "ApplyNamePostFix", - "ClientId" - ], - "properties": { - "Enable": { - "$id": "#/properties/AdsOpts/properties/CD/properties/ServicePrincipals/properties/FunctionAppAuthenticationSP/properties/Enable", - "type": "boolean", - "title": "The Enable schema", - "description": "An explanation about the purpose of this instance.", - "default": false, - "examples": [ - true - ] - }, - "Name": { - "$id": "#/properties/AdsOpts/properties/CD/properties/ServicePrincipals/properties/FunctionAppAuthenticationSP/properties/Name", - "type": "string", - "title": "The Name schema", - "description": "An explanation about the purpose of this instance.", - "default": "", - "examples": [ - "AdsGFFuncAppAuthSP" - ] - }, - "ApplyNamePostFix": { - "$id": "#/properties/AdsOpts/properties/CD/properties/ServicePrincipals/properties/FunctionAppAuthenticationSP/properties/ApplyNamePostFix", - "type": "boolean", - "title": "The ApplyNamePostFix schema", - "description": "An explanation about the purpose of this instance.", - "default": false, - "examples": [ - true - ] - }, - "ClientId": { - "$id": "#/properties/AdsOpts/properties/CD/properties/ServicePrincipals/properties/FunctionAppAuthenticationSP/properties/ClientId", - "type": "string", - "title": "The ClientId schema", - "description": "An explanation about the purpose of this instance.", - "default": "", - "examples": [ - "835b082c-6a63-4fcc-b5d8-005dba169819" - ] - } - }, - "additionalProperties": true - } - }, - "additionalProperties": true + "Name": { + "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/AppPlans/properties/WebApp/properties/Name", + "type": "string", + "title": "Name", + "description": "", + "default": "", + "examples": [ + "adsgfappplanweb" + ] + }, + "ApplyNamePostFix": { + "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/AppPlans/properties/WebApp/properties/ApplyNamePostFix", + "type": "boolean", + "format": "checkbox", + "title": "Apply Postfix to Resource Name", + "description": "", + "default": false, + "examples": [ + true + ] }, "ResourceGroup": { - "$id": "#/properties/AdsOpts/properties/CD/properties/ResourceGroup", - "type": "object", - "title": "The ResourceGroup schema", - "description": "An explanation about the purpose of this instance.", - "default": {}, - "examples": [ - { - "Id": "/subscriptions/035a1364-f00d-48e2-b582-4fe125905ee3/resourceGroups/AdsTest", - "Enable": true, - "Subscription": "Jorampon Internal Consumption", - "Domain": "microsoft.com", - "TenantId": "72f988bf-86f1-41af-91ab-2d7cd011db47", - "Location": "australiaeast", - "Name": "AdsTest", - "AADUser": "jorampon@microsoft.com", - "ServicePrincipal_Deployment": "AdsGoFastDeploymentSP", - "ServicePrincipalNameAFDownstream": "AdsGoFastDeployTestSP", - "ServicePrincipalNameWebAuth": "adsgofastwebappauth", - "ServicePrincipalNameAFAuth": "adsgofastwebappauth", - "Hash": "DummyValueToBeReplacedAtRuntime" - } - ], - "required": [ - "Id", - "Enable", - "Subscription", - "Domain", - "TenantId", - "Location", - "Name", - "AADUser", - "ServicePrincipal_Deployment", - "ServicePrincipalNameAFDownstream", - "ServicePrincipalNameWebAuth", - "ServicePrincipalNameAFAuth", - "Hash" - ], - "properties": { - "Id": { - "$id": "#/properties/AdsOpts/properties/CD/properties/ResourceGroup/properties/Id", - "type": "string", - "title": "The Id schema", - "description": "An explanation about the purpose of this instance.", - "default": "", - "examples": [ - "/subscriptions/035a1364-f00d-48e2-b582-4fe125905ee3/resourceGroups/AdsTest" - ] - }, - "Enable": { - "$id": "#/properties/AdsOpts/properties/CD/properties/ResourceGroup/properties/Enable", - "type": "boolean", - "title": "The Enable schema", - "description": "An explanation about the purpose of this instance.", - "default": false, - "examples": [ - true - ] - }, - "Subscription": { - "$id": "#/properties/AdsOpts/properties/CD/properties/ResourceGroup/properties/Subscription", - "type": "string", - "title": "The Subscription schema", - "description": "An explanation about the purpose of this instance.", - "default": "", - "examples": [ - "Jorampon Internal Consumption" - ] - }, - "Domain": { - "$id": "#/properties/AdsOpts/properties/CD/properties/ResourceGroup/properties/Domain", - "type": "string", - "title": "The Domain schema", - "description": "An explanation about the purpose of this instance.", - "default": "", - "examples": [ - "microsoft.com" - ] - }, - "TenantId": { - "$id": "#/properties/AdsOpts/properties/CD/properties/ResourceGroup/properties/TenantId", - "type": "string", - "title": "The TenantId schema", - "description": "An explanation about the purpose of this instance.", - "default": "", - "examples": [ - "72f988bf-86f1-41af-91ab-2d7cd011db47" - ] - }, - "Location": { - "$id": "#/properties/AdsOpts/properties/CD/properties/ResourceGroup/properties/Location", - "type": "string", - "title": "The Location schema", - "description": "An explanation about the purpose of this instance.", - "default": "", - "examples": [ - "australiaeast" - ] - }, - "Name": { - "$id": "#/properties/AdsOpts/properties/CD/properties/ResourceGroup/properties/Name", - "type": "string", - "title": "The Name schema", - "description": "An explanation about the purpose of this instance.", - "default": "", - "examples": [ - "AdsTest" - ] - }, - "AADUser": { - "$id": "#/properties/AdsOpts/properties/CD/properties/ResourceGroup/properties/AADUser", - "type": "string", - "title": "The AADUser schema", - "description": "An explanation about the purpose of this instance.", - "default": "", - "examples": [ - "jorampon@microsoft.com" - ] - }, - "ServicePrincipal_Deployment": { - "$id": "#/properties/AdsOpts/properties/CD/properties/ResourceGroup/properties/ServicePrincipal_Deployment", - "type": "string", - "title": "The ServicePrincipal_Deployment schema", - "description": "An explanation about the purpose of this instance.", - "default": "", - "examples": [ - "AdsGoFastDeploymentSP" - ] - }, - "ServicePrincipalNameAFDownstream": { - "$id": "#/properties/AdsOpts/properties/CD/properties/ResourceGroup/properties/ServicePrincipalNameAFDownstream", - "type": "string", - "title": "The ServicePrincipalNameAFDownstream schema", - "description": "An explanation about the purpose of this instance.", - "default": "", - "examples": [ - "AdsGoFastDeployTestSP" - ] - }, - "ServicePrincipalNameWebAuth": { - "$id": "#/properties/AdsOpts/properties/CD/properties/ResourceGroup/properties/ServicePrincipalNameWebAuth", - "type": "string", - "title": "The ServicePrincipalNameWebAuth schema", - "description": "An explanation about the purpose of this instance.", - "default": "", - "examples": [ - "adsgofastwebappauth" - ] - }, - "ServicePrincipalNameAFAuth": { - "$id": "#/properties/AdsOpts/properties/CD/properties/ResourceGroup/properties/ServicePrincipalNameAFAuth", - "type": "string", - "title": "The ServicePrincipalNameAFAuth schema", - "description": "An explanation about the purpose of this instance.", - "default": "", - "examples": [ - "adsgofastwebappauth" - ] - }, - "Hash": { - "$id": "#/properties/AdsOpts/properties/CD/properties/ResourceGroup/properties/Hash", - "type": "string", - "title": "The Hash schema", - "description": "An explanation about the purpose of this instance.", - "default": "", - "examples": [ - "DummyValueToBeReplacedAtRuntime" - ] - } - }, - "additionalProperties": true - }, - "FolderPaths": { - "$id": "#/properties/AdsOpts/properties/CD/properties/FolderPaths", - "type": "object", - "title": "The FolderPaths schema", - "description": "An explanation about the purpose of this instance.", - "default": {}, - "examples": [ - { - "ArmTemplates": "./arm", - "Environments": "./../environments", - "PublishZip": "./../bin/publish/zipped", - "PublishUnZip": "./../bin/publish/unzipped" - } - ], - "required": [ - "ArmTemplates", - "Environments", - "PublishZip", - "PublishUnZip" - ], - "properties": { - "ArmTemplates": { - "$id": "#/properties/AdsOpts/properties/CD/properties/FolderPaths/properties/ArmTemplates", - "type": "string", - "title": "The ArmTemplates schema", - "description": "An explanation about the purpose of this instance.", - "default": "", - "examples": [ - "./arm" - ] - }, - "Environments": { - "$id": "#/properties/AdsOpts/properties/CD/properties/FolderPaths/properties/Environments", - "type": "string", - "title": "The Environments schema", - "description": "An explanation about the purpose of this instance.", - "default": "", - "examples": [ - "./../environments" - ] - }, - "PublishZip": { - "$id": "#/properties/AdsOpts/properties/CD/properties/FolderPaths/properties/PublishZip", - "type": "string", - "title": "The PublishZip schema", - "description": "An explanation about the purpose of this instance.", - "default": "", - "examples": [ - "./../bin/publish/zipped" - ] - }, - "PublishUnZip": { - "$id": "#/properties/AdsOpts/properties/CD/properties/FolderPaths/properties/PublishUnZip", - "type": "string", - "title": "The PublishUnZip schema", - "description": "An explanation about the purpose of this instance.", - "default": "", - "examples": [ - "./../bin/publish/unzipped" - ] - } - }, - "additionalProperties": true - }, - "AzureLoginOptions": { - "$id": "#/properties/AdsOpts/properties/CD/properties/AzureLoginOptions", - "type": "object", - "title": "The AzureLoginOptions schema", - "description": "An explanation about the purpose of this instance.", - "default": {}, - "examples": [ - { - "UseInteractiveAzCliLogin": true - } - ], - "required": [ - "UseInteractiveAzCliLogin" - ], - "properties": { - "UseInteractiveAzCliLogin": { - "$id": "#/properties/AdsOpts/properties/CD/properties/AzureLoginOptions/properties/UseInteractiveAzCliLogin", - "type": "boolean", - "title": "The UseInteractiveAzCliLogin schema", - "description": "An explanation about the purpose of this instance.", - "default": false, - "examples": [ - true - ] - } - }, - "additionalProperties": true - }, - "EnviroInstalls": { - "$id": "#/properties/AdsOpts/properties/CD/properties/EnviroInstalls", - "type": "object", - "title": "The EnviroInstalls schema", - "description": "An explanation about the purpose of this instance.", - "default": {}, - "examples": [ - { - "PerformLocalInstalls": true, - "PerformLocalInstallsAzCli": false, - "PerformLocalInstallsAzCliAddToPath": true - } - ], - "required": [ - "PerformLocalInstalls", - "PerformLocalInstallsAzCli", - "PerformLocalInstallsAzCliAddToPath" - ], - "properties": { - "PerformLocalInstalls": { - "$id": "#/properties/AdsOpts/properties/CD/properties/EnviroInstalls/properties/PerformLocalInstalls", - "type": "boolean", - "title": "The PerformLocalInstalls schema", - "description": "An explanation about the purpose of this instance.", - "default": false, - "examples": [ - true - ] - }, - "PerformLocalInstallsAzCli": { - "$id": "#/properties/AdsOpts/properties/CD/properties/EnviroInstalls/properties/PerformLocalInstallsAzCli", - "type": "boolean", - "title": "The PerformLocalInstallsAzCli schema", - "description": "An explanation about the purpose of this instance.", - "default": false, - "examples": [ - false - ] - }, - "PerformLocalInstallsAzCliAddToPath": { - "$id": "#/properties/AdsOpts/properties/CD/properties/EnviroInstalls/properties/PerformLocalInstallsAzCliAddToPath", - "type": "boolean", - "title": "The PerformLocalInstallsAzCliAddToPath schema", - "description": "An explanation about the purpose of this instance.", - "default": false, - "examples": [ - true - ] - } - }, - "additionalProperties": true - }, - "ArmOptions": { - "$id": "#/properties/AdsOpts/properties/CD/properties/ArmOptions", - "type": "object", - "title": "The ArmOptions schema", - "description": "An explanation about the purpose of this instance.", - "default": {}, - "examples": [ - { - "PerformDeployment": false, - "PerformDeploymentStorageLogging": false, - "PerformDeploymentAppService": false - } - ], - "required": [ - "PerformDeployment", - "PerformDeploymentStorageLogging", - "PerformDeploymentAppService" - ], - "properties": { - "PerformDeployment": { - "$id": "#/properties/AdsOpts/properties/CD/properties/ArmOptions/properties/PerformDeployment", - "type": "boolean", - "title": "The PerformDeployment schema", - "description": "An explanation about the purpose of this instance.", - "default": false, - "examples": [ - false - ] - }, - "PerformDeploymentStorageLogging": { - "$id": "#/properties/AdsOpts/properties/CD/properties/ArmOptions/properties/PerformDeploymentStorageLogging", - "type": "boolean", - "title": "The PerformDeploymentStorageLogging schema", - "description": "An explanation about the purpose of this instance.", - "default": false, - "examples": [ - false - ] - }, - "PerformDeploymentAppService": { - "$id": "#/properties/AdsOpts/properties/CD/properties/ArmOptions/properties/PerformDeploymentAppService", - "type": "boolean", - "title": "The PerformDeploymentAppService schema", - "description": "An explanation about the purpose of this instance.", - "default": false, - "examples": [ - false - ] - } - }, - "additionalProperties": true - }, - "Services": { - "$id": "#/properties/AdsOpts/properties/CD/properties/Services", - "type": "object", - "title": "The Services schema", - "description": "An explanation about the purpose of this instance.", - "default": {}, - "examples": [ - { - "UseARMDefaults": false, - "AppInsights": { - "Enable": true, - "Name": "adsgfappin", - "ApplyNamePostFix": true - }, - "AppPlans": { - "WebApp": { - "Enable": true, - "Name": "adsgfappplanweb", - "ApplyNamePostFix": true, - "ResourceGroup": null - }, - "FunctionApp": { - "Enable": true, - "Name": "adsgfappplanfnc", - "ApplyNamePostFix": true, - "ResourceGroup": null - } - }, - "AzureSQLServer": { - "Enable": true, - "Name": "adsgfazsql", - "ApplyNamePostFix": true, - "AdminUser": "AdsAdmin", - "AdminPassword": "EPuiX2K0^T4t", - "AdsGoFastDB": { - "Enable": true, - "Name": "AdsGf", - "UpdateSourceAndTargetSystems": true, - "UpdateDataFactory": true, - "ApplyNamePostFix": false - }, - "StagingDB": { - "Enable": true, - "Name": "AdsGfStaging", - "ApplyNamePostFix": false - }, - "SampleDB": { - "Enable": true, - "Name": "AdsGfSample", - "ApplyNamePostFix": false - } - }, - "CoreFunctionApp": { - "Enable": true, - "Name": "adsgofastfunc", - "ApplyNamePostFix": true, - "PrincipalId": "" - }, - "DataFactory": { - "Enable": true, - "Name": "adsgfadf", - "ApplyNamePostFix": true, - "AzVnetIr": { - "Enable": true, - "Name": "SelfHostedIntegrationRuntime-Azure-VNET", - "Type": "ManagedVnet" - }, - "OnPremVnetIr": { - "Enable": false, - "Name": "SelfHostedIntegrationRuntime-OnPem-Net", - "Type": "SelfHosted", - "IrInstallConfig":{ - "LocalDrive":"C:", - "LocalVMFolder":"ADFInstaller", - "IrDownloadURL":"https://download.microsoft.com/download/E/4/7/E4771905-1079-445B-8BF9-8A1A075D8A10/IntegrationRuntime_5.9.7900.1.msi", - "JDKDownloadURL":"https://github.com/adoptium/temurin11-binaries/releases/download/jdk-11.0.12%2B7/OpenJDK11U-jdk_x64_windows_hotspot_11.0.12_7.msi", - "JDKInstallFolder":"c:\\Program Files\\Eclipse Foundation\\" - } - } - }, - "WebSite": { - "Enable": true, - "Name": "adsgofastweb", - "ApplyNamePostFix": true, - "PrincipalId": "" - }, - "LogAnalytics": { - "Enable": true, - "Name": "adsgofastloganalytics", - "ApplyNamePostFix": false - }, - "KeyVault": { - "Enable": true, - "Name": "adsgfkv", - "ApplyNamePostFix": true - }, - "Vnet": { - "Enable": true, - "Name": "AdsGoFastVnet", - "ApplyNamePostFix": true, - "vNetAddressRange": "10.3.0.0/16", - "BastionSubnetAddressRange": "10.3.1.0/27", - "DataSubnetAddressRange": "10.3.2.0/27", - "WebAppSubnetAddressRange": "10.3.3.0/27", - "FuncAppSubnetAddressRange": "10.3.4.0/27", - "BastionSubnetName":"AzureBastionSubnet", - "DataSubnetName":"Data", - "WebAppSubnetName":"WebApp", - "FuncAppSubnetName":"FuncApp" - }, - "Bastion": { - "Name": "adsgfbastion", - "ApplyNamePostFix": true - }, - "Storage": { - "Logging": { - "Name": "logstg", - "ApplyNamePostFix": true, - "Enable": true, - "Dummy": "" - }, - "ADLS": { - "Name": "adls", - "ApplyNamePostFix": true, - "Enable": true - }, - "ADLSTransient": { - "Name": "adlstran", - "ApplyNamePostFix": true, - "Enable": false - }, - "Blob": { - "Name": "blob", - "ApplyNamePostFix": true, - "Enable": true, - "ResourceId": "" - } - } - } - ], - "required": [ - "UseARMDefaults", - "AppInsights", - "AppPlans", - "AzureSQLServer", - "CoreFunctionApp", - "DataFactory", - "WebSite", - "LogAnalytics", - "KeyVault", - "Vnet", - "Storage" - ], - "properties": { - "UseARMDefaults": { - "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/UseARMDefaults", - "type": "boolean", - "title": "The UseARMDefaults schema", - "description": "An explanation about the purpose of this instance.", - "default": false, - "examples": [ - false - ] - }, - "AppInsights": { - "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/AppInsights", - "type": "object", - "title": "The AppInsights schema", - "description": "An explanation about the purpose of this instance.", - "default": {}, - "examples": [ - { - "Enable": true, - "Name": "adsgfappin", - "ApplyNamePostFix": true - } - ], - "required": [ - "Enable", - "Name", - "ApplyNamePostFix" - ], - "properties": { - "Enable": { - "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/AppInsights/properties/Enable", - "type": "boolean", - "title": "The Enable schema", - "description": "An explanation about the purpose of this instance.", - "default": false, - "examples": [ - true - ] - }, - "Name": { - "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/AppInsights/properties/Name", - "type": "string", - "title": "The Name schema", - "description": "An explanation about the purpose of this instance.", - "default": "", - "examples": [ - "adsgfappin" - ] - }, - "ApplyNamePostFix": { - "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/AppInsights/properties/ApplyNamePostFix", - "type": "boolean", - "title": "The ApplyNamePostFix schema", - "description": "An explanation about the purpose of this instance.", - "default": false, - "examples": [ - true - ] - } - }, - "additionalProperties": true - }, - "AppPlans": { - "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/AppPlans", - "type": "object", - "title": "The AppPlans schema", - "description": "An explanation about the purpose of this instance.", - "default": {}, - "examples": [ - { - "WebApp": { - "Enable": true, - "Name": "adsgfappplanweb", - "ApplyNamePostFix": true, - "ResourceGroup": null - }, - "FunctionApp": { - "Enable": true, - "Name": "adsgfappplanfnc", - "ApplyNamePostFix": true, - "ResourceGroup": null - } - } - ], - "required": [ - "WebApp", - "FunctionApp" - ], - "properties": { - "WebApp": { - "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/AppPlans/properties/WebApp", - "type": "object", - "title": "The WebApp schema", - "description": "An explanation about the purpose of this instance.", - "default": {}, - "examples": [ - { - "Enable": true, - "Name": "adsgfappplanweb", - "ApplyNamePostFix": true, - "ResourceGroup": null - } - ], - "required": [ - "Enable", - "Name", - "ApplyNamePostFix", - "ResourceGroup" - ], - "properties": { - "Enable": { - "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/AppPlans/properties/WebApp/properties/Enable", - "type": "boolean", - "title": "The Enable schema", - "description": "An explanation about the purpose of this instance.", - "default": false, - "examples": [ - true - ] - }, - "Name": { - "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/AppPlans/properties/WebApp/properties/Name", - "type": "string", - "title": "The Name schema", - "description": "An explanation about the purpose of this instance.", - "default": "", - "examples": [ - "adsgfappplanweb" - ] - }, - "ApplyNamePostFix": { - "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/AppPlans/properties/WebApp/properties/ApplyNamePostFix", - "type": "boolean", - "title": "The ApplyNamePostFix schema", - "description": "An explanation about the purpose of this instance.", - "default": false, - "examples": [ - true - ] - }, - "ResourceGroup": { - "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/AppPlans/properties/WebApp/properties/ResourceGroup", - "type": "null", - "title": "The ResourceGroup schema", - "description": "An explanation about the purpose of this instance.", - "default": null, - "examples": [ - null - ] - } - }, - "additionalProperties": true - }, - "FunctionApp": { - "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/AppPlans/properties/FunctionApp", - "type": "object", - "title": "The FunctionApp schema", - "description": "An explanation about the purpose of this instance.", - "default": {}, - "examples": [ - { - "Enable": true, - "Name": "adsgfappplanfnc", - "ApplyNamePostFix": true, - "ResourceGroup": null - } - ], - "required": [ - "Enable", - "Name", - "ApplyNamePostFix", - "ResourceGroup" - ], - "properties": { - "Enable": { - "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/AppPlans/properties/FunctionApp/properties/Enable", - "type": "boolean", - "title": "The Enable schema", - "description": "An explanation about the purpose of this instance.", - "default": false, - "examples": [ - true - ] - }, - "Name": { - "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/AppPlans/properties/FunctionApp/properties/Name", - "type": "string", - "title": "The Name schema", - "description": "An explanation about the purpose of this instance.", - "default": "", - "examples": [ - "adsgfappplanfnc" - ] - }, - "ApplyNamePostFix": { - "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/AppPlans/properties/FunctionApp/properties/ApplyNamePostFix", - "type": "boolean", - "title": "The ApplyNamePostFix schema", - "description": "An explanation about the purpose of this instance.", - "default": false, - "examples": [ - true - ] - }, - "ResourceGroup": { - "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/AppPlans/properties/FunctionApp/properties/ResourceGroup", - "type": "null", - "title": "The ResourceGroup schema", - "description": "An explanation about the purpose of this instance.", - "default": null, - "examples": [ - null - ] - } - }, - "additionalProperties": true - } - }, - "additionalProperties": true - }, - "AzureSQLServer": { - "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/AzureSQLServer", - "type": "object", - "title": "The AzureSQLServer schema", - "description": "An explanation about the purpose of this instance.", - "default": {}, - "examples": [ - { - "Enable": true, - "Name": "adsgfazsql", - "ApplyNamePostFix": true, - "AdminUser": "AdsAdmin", - "AdminPassword": "EPuiX2K0^T4t", - "AdsGoFastDB": { - "Enable": true, - "Name": "AdsGf", - "UpdateSourceAndTargetSystems": true, - "UpdateDataFactory": true, - "ApplyNamePostFix": false - }, - "StagingDB": { - "Enable": true, - "Name": "AdsGfStaging", - "ApplyNamePostFix": false - }, - "SampleDB": { - "Enable": true, - "Name": "AdsGfSample", - "ApplyNamePostFix": false - } - } - ], - "required": [ - "Enable", - "Name", - "ApplyNamePostFix", - "AdminUser", - "AdminPassword", - "AdsGoFastDB", - "StagingDB", - "SampleDB" - ], - "properties": { - "Enable": { - "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/AzureSQLServer/properties/Enable", - "type": "boolean", - "title": "The Enable schema", - "description": "An explanation about the purpose of this instance.", - "default": false, - "examples": [ - true - ] - }, - "Name": { - "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/AzureSQLServer/properties/Name", - "type": "string", - "title": "The Name schema", - "description": "An explanation about the purpose of this instance.", - "default": "", - "examples": [ - "adsgfazsql" - ] - }, - "ApplyNamePostFix": { - "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/AzureSQLServer/properties/ApplyNamePostFix", - "type": "boolean", - "title": "The ApplyNamePostFix schema", - "description": "An explanation about the purpose of this instance.", - "default": false, - "examples": [ - true - ] - }, - "AdminUser": { - "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/AzureSQLServer/properties/AdminUser", - "type": "string", - "title": "The AdminUser schema", - "description": "An explanation about the purpose of this instance.", - "default": "", - "examples": [ - "AdsAdmin" - ] - }, - "AdminPassword": { - "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/AzureSQLServer/properties/AdminPassword", - "type": "string", - "title": "The AdminPassword schema", - "description": "An explanation about the purpose of this instance.", - "default": "", - "examples": [ - "EPuiX2K0^T4t" - ] - }, - "AdsGoFastDB": { - "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/AzureSQLServer/properties/AdsGoFastDB", - "type": "object", - "title": "The AdsGoFastDB schema", - "description": "An explanation about the purpose of this instance.", - "default": {}, - "examples": [ - { - "Enable": true, - "Name": "AdsGf", - "UpdateSourceAndTargetSystems": true, - "UpdateDataFactory": true, - "ApplyNamePostFix": false - } - ], - "required": [ - "Enable", - "Name", - "UpdateSourceAndTargetSystems", - "UpdateDataFactory", - "ApplyNamePostFix" - ], - "properties": { - "Enable": { - "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/AzureSQLServer/properties/AdsGoFastDB/properties/Enable", - "type": "boolean", - "title": "The Enable schema", - "description": "An explanation about the purpose of this instance.", - "default": false, - "examples": [ - true - ] - }, - "Name": { - "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/AzureSQLServer/properties/AdsGoFastDB/properties/Name", - "type": "string", - "title": "The Name schema", - "description": "An explanation about the purpose of this instance.", - "default": "", - "examples": [ - "AdsGf" - ] - }, - "UpdateSourceAndTargetSystems": { - "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/AzureSQLServer/properties/AdsGoFastDB/properties/UpdateSourceAndTargetSystems", - "type": "boolean", - "title": "The UpdateSourceAndTargetSystems schema", - "description": "An explanation about the purpose of this instance.", - "default": false, - "examples": [ - true - ] - }, - "UpdateDataFactory": { - "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/AzureSQLServer/properties/AdsGoFastDB/properties/UpdateDataFactory", - "type": "boolean", - "title": "The UpdateDataFactory schema", - "description": "An explanation about the purpose of this instance.", - "default": false, - "examples": [ - true - ] - }, - "ApplyNamePostFix": { - "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/AzureSQLServer/properties/AdsGoFastDB/properties/ApplyNamePostFix", - "type": "boolean", - "title": "The ApplyNamePostFix schema", - "description": "An explanation about the purpose of this instance.", - "default": false, - "examples": [ - false - ] - } - }, - "additionalProperties": true - }, - "StagingDB": { - "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/AzureSQLServer/properties/StagingDB", - "type": "object", - "title": "The StagingDB schema", - "description": "An explanation about the purpose of this instance.", - "default": {}, - "examples": [ - { - "Enable": true, - "Name": "AdsGfStaging", - "ApplyNamePostFix": false - } - ], - "required": [ - "Enable", - "Name", - "ApplyNamePostFix" - ], - "properties": { - "Enable": { - "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/AzureSQLServer/properties/StagingDB/properties/Enable", - "type": "boolean", - "title": "The Enable schema", - "description": "An explanation about the purpose of this instance.", - "default": false, - "examples": [ - true - ] - }, - "Name": { - "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/AzureSQLServer/properties/StagingDB/properties/Name", - "type": "string", - "title": "The Name schema", - "description": "An explanation about the purpose of this instance.", - "default": "", - "examples": [ - "AdsGfStaging" - ] - }, - "ApplyNamePostFix": { - "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/AzureSQLServer/properties/StagingDB/properties/ApplyNamePostFix", - "type": "boolean", - "title": "The ApplyNamePostFix schema", - "description": "An explanation about the purpose of this instance.", - "default": false, - "examples": [ - false - ] - } - }, - "additionalProperties": true - }, - "SampleDB": { - "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/AzureSQLServer/properties/SampleDB", - "type": "object", - "title": "The SampleDB schema", - "description": "An explanation about the purpose of this instance.", - "default": {}, - "examples": [ - { - "Enable": true, - "Name": "AdsGfSample", - "ApplyNamePostFix": false - } - ], - "required": [ - "Enable", - "Name", - "ApplyNamePostFix" - ], - "properties": { - "Enable": { - "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/AzureSQLServer/properties/SampleDB/properties/Enable", - "type": "boolean", - "title": "The Enable schema", - "description": "An explanation about the purpose of this instance.", - "default": false, - "examples": [ - true - ] - }, - "Name": { - "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/AzureSQLServer/properties/SampleDB/properties/Name", - "type": "string", - "title": "The Name schema", - "description": "An explanation about the purpose of this instance.", - "default": "", - "examples": [ - "AdsGfSample" - ] - }, - "ApplyNamePostFix": { - "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/AzureSQLServer/properties/SampleDB/properties/ApplyNamePostFix", - "type": "boolean", - "title": "The ApplyNamePostFix schema", - "description": "An explanation about the purpose of this instance.", - "default": false, - "examples": [ - false - ] - } - }, - "additionalProperties": true - } - }, - "additionalProperties": true - }, - "CoreFunctionApp": { - "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/CoreFunctionApp", - "type": "object", - "title": "The CoreFunctionApp schema", - "description": "An explanation about the purpose of this instance.", - "default": {}, - "examples": [ - { - "Enable": true, - "Name": "adsgofastfunc", - "ApplyNamePostFix": true, - "PrincipalId": "" - } - ], - "required": [ - "Enable", - "Name", - "ApplyNamePostFix", - "PrincipalId" - ], - "properties": { - "Enable": { - "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/CoreFunctionApp/properties/Enable", - "type": "boolean", - "title": "The Enable schema", - "description": "An explanation about the purpose of this instance.", - "default": false, - "examples": [ - true - ] - }, - "Name": { - "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/CoreFunctionApp/properties/Name", - "type": "string", - "title": "The Name schema", - "description": "An explanation about the purpose of this instance.", - "default": "", - "examples": [ - "adsgofastfunc" - ] - }, - "ApplyNamePostFix": { - "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/CoreFunctionApp/properties/ApplyNamePostFix", - "type": "boolean", - "title": "The ApplyNamePostFix schema", - "description": "An explanation about the purpose of this instance.", - "default": false, - "examples": [ - true - ] - }, - "PrincipalId": { - "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/CoreFunctionApp/properties/PrincipalId", - "type": "string", - "title": "The PrincipalId schema", - "description": "An explanation about the purpose of this instance.", - "default": "", - "examples": [ - "" - ] - } - }, - "additionalProperties": true - }, - "DataFactory": { - "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/DataFactory", - "type": "object", - "title": "The DataFactory schema", - "description": "An explanation about the purpose of this instance.", - "default": {}, - "examples": [ - { - "Enable": true, - "Name": "adsgfadf", - "ApplyNamePostFix": true, - "AzVnetIr": { - "Enable": true, - "Name": "SelfHostedIntegrationRuntime-Azure-VNET", - "Type": "ManagedVnet" - }, - "OnPremVnetIr": { - "Enable": false, - "Name": "SelfHostedIntegrationRuntime-OnPem-Net", - "Type": "ManagedVnet" - } - } - ], - "required": [ - "Enable", - "Name", - "ApplyNamePostFix", - "AzVnetIr", - "OnPremVnetIr" - ], - "properties": { - "Enable": { - "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/DataFactory/properties/Enable", - "type": "boolean", - "title": "The Enable schema", - "description": "An explanation about the purpose of this instance.", - "default": false, - "examples": [ - true - ] - }, - "Name": { - "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/DataFactory/properties/Name", - "type": "string", - "title": "The Name schema", - "description": "An explanation about the purpose of this instance.", - "default": "", - "examples": [ - "adsgfadf" - ] - }, - "ApplyNamePostFix": { - "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/DataFactory/properties/ApplyNamePostFix", - "type": "boolean", - "title": "The ApplyNamePostFix schema", - "description": "An explanation about the purpose of this instance.", - "default": false, - "examples": [ - true - ] - }, - "AzVnetIr": { - "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/DataFactory/properties/AzVnetIr", - "type": "object", - "title": "The AzVnetIr schema", - "description": "An explanation about the purpose of this instance.", - "default": {}, - "examples": [ - { - "Enable": true, - "Name": "SelfHostedIntegrationRuntime-Azure-VNET", - "Type": "ManagedVnet" - } - ], - "required": [ - "Enable", - "Name", - "Type" - ], - "properties": { - "Enable": { - "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/DataFactory/properties/AzVnetIr/properties/Enable", - "type": "boolean", - "title": "The Enable schema", - "description": "An explanation about the purpose of this instance.", - "default": false, - "examples": [ - true - ] - }, - "Name": { - "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/DataFactory/properties/AzVnetIr/properties/Name", - "type": "string", - "title": "The Name schema", - "description": "An explanation about the purpose of this instance.", - "default": "", - "examples": [ - "SelfHostedIntegrationRuntime-Azure-VNET" - ] - }, - "Type": { - "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/DataFactory/properties/AzVnetIr/properties/Type", - "type": "string", - "title": "The Type schema", - "description": "An explanation about the purpose of this instance.", - "default": "", - "examples": [ - "ManagedVnet" - ] - } - }, - "additionalProperties": true - }, - "OnPremVnetIr": { - "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/DataFactory/properties/OnPremVnetIr", - "type": "object", - "title": "The OnPremVnetIr schema", - "description": "An explanation about the purpose of this instance.", - "default": {}, - "examples": [ - { - "Enable": false, - "Name": "SelfHostedIntegrationRuntime-OnPem-Net", - "Type": "ManagedVnet" - } - ], - "required": [ - "Enable", - "Name", - "Type" - ], - "properties": { - "Enable": { - "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/DataFactory/properties/OnPremVnetIr/properties/Enable", - "type": "boolean", - "title": "The Enable schema", - "description": "An explanation about the purpose of this instance.", - "default": false, - "examples": [ - false - ] - }, - "Name": { - "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/DataFactory/properties/OnPremVnetIr/properties/Name", - "type": "string", - "title": "The Name schema", - "description": "An explanation about the purpose of this instance.", - "default": "", - "examples": [ - "SelfHostedIntegrationRuntime-OnPem-Net" - ] - }, - "Type": { - "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/DataFactory/properties/OnPremVnetIr/properties/Type", - "type": "string", - "title": "The Type schema", - "description": "An explanation about the purpose of this instance.", - "default": "", - "examples": [ - "ManagedVnet" - ] - } - }, - "additionalProperties": true - } - }, - "additionalProperties": true - }, - "WebSite": { - "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/WebSite", - "type": "object", - "title": "The WebSite schema", - "description": "An explanation about the purpose of this instance.", - "default": {}, - "examples": [ - { - "Enable": true, - "Name": "adsgofastweb", - "ApplyNamePostFix": true, - "PrincipalId": "" - } - ], - "required": [ - "Enable", - "Name", - "ApplyNamePostFix", - "PrincipalId" - ], - "properties": { - "Enable": { - "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/WebSite/properties/Enable", - "type": "boolean", - "title": "The Enable schema", - "description": "An explanation about the purpose of this instance.", - "default": false, - "examples": [ - true - ] - }, - "Name": { - "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/WebSite/properties/Name", - "type": "string", - "title": "The Name schema", - "description": "An explanation about the purpose of this instance.", - "default": "", - "examples": [ - "adsgofastweb" - ] - }, - "ApplyNamePostFix": { - "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/WebSite/properties/ApplyNamePostFix", - "type": "boolean", - "title": "The ApplyNamePostFix schema", - "description": "An explanation about the purpose of this instance.", - "default": false, - "examples": [ - true - ] - }, - "PrincipalId": { - "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/WebSite/properties/PrincipalId", - "type": "string", - "title": "The PrincipalId schema", - "description": "An explanation about the purpose of this instance.", - "default": "", - "examples": [ - "" - ] - } - }, - "additionalProperties": true - }, - "LogAnalytics": { - "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/LogAnalytics", - "type": "object", - "title": "The LogAnalytics schema", - "description": "An explanation about the purpose of this instance.", - "default": {}, - "examples": [ - { - "Enable": true, - "Name": "adsgofastloganalytics", - "ApplyNamePostFix": false - } - ], - "required": [ - "Enable", - "Name", - "ApplyNamePostFix" - ], - "properties": { - "Enable": { - "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/LogAnalytics/properties/Enable", - "type": "boolean", - "title": "The Enable schema", - "description": "An explanation about the purpose of this instance.", - "default": false, - "examples": [ - true - ] - }, - "Name": { - "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/LogAnalytics/properties/Name", - "type": "string", - "title": "The Name schema", - "description": "An explanation about the purpose of this instance.", - "default": "", - "examples": [ - "adsgofastloganalytics" - ] - }, - "ApplyNamePostFix": { - "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/LogAnalytics/properties/ApplyNamePostFix", - "type": "boolean", - "title": "The ApplyNamePostFix schema", - "description": "An explanation about the purpose of this instance.", - "default": false, - "examples": [ - false - ] - } - }, - "additionalProperties": true - }, - "KeyVault": { - "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/KeyVault", - "type": "object", - "title": "The KeyVault schema", - "description": "An explanation about the purpose of this instance.", - "default": {}, - "examples": [ - { - "Enable": true, - "Name": "adsgfkv", - "ApplyNamePostFix": true - } - ], - "required": [ - "Enable", - "Name", - "ApplyNamePostFix" - ], - "properties": { - "Enable": { - "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/KeyVault/properties/Enable", - "type": "boolean", - "title": "The Enable schema", - "description": "An explanation about the purpose of this instance.", - "default": false, - "examples": [ - true - ] - }, - "Name": { - "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/KeyVault/properties/Name", - "type": "string", - "title": "The Name schema", - "description": "An explanation about the purpose of this instance.", - "default": "", - "examples": [ - "adsgfkv" - ] - }, - "ApplyNamePostFix": { - "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/KeyVault/properties/ApplyNamePostFix", - "type": "boolean", - "title": "The ApplyNamePostFix schema", - "description": "An explanation about the purpose of this instance.", - "default": false, - "examples": [ - true - ] - } - }, - "additionalProperties": true - }, - "Vnet": { - "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/Vnet", - "type": "object", - "title": "The Vnet schema", - "description": "An explanation about the purpose of this instance.", - "default": {}, - "examples": [ - { - "Enable": true, - "Name": "AdsGoFastVnet" - } - ], - "required": [ - "Enable", - "Name" - ], - "properties": { - "Enable": { - "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/Vnet/properties/Enable", - "type": "boolean", - "title": "The Enable schema", - "description": "An explanation about the purpose of this instance.", - "default": false, - "examples": [ - true - ] - }, - "Name": { - "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/Vnet/properties/Name", - "type": "string", - "title": "The Name schema", - "description": "An explanation about the purpose of this instance.", - "default": "", - "examples": [ - "AdsGoFastVnet" - ] - } - }, - "additionalProperties": true - }, - "Storage": { - "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/Storage", - "type": "object", - "title": "The Storage schema", - "description": "An explanation about the purpose of this instance.", - "default": {}, - "examples": [ - { - "Logging": { - "Name": "logstg", - "ApplyNamePostFix": true, - "Enable": true, - "Dummy": "" - }, - "ADLS": { - "Name": "adls", - "ApplyNamePostFix": true, - "Enable": true - }, - "Blob": { - "Name": "blob", - "ApplyNamePostFix": true, - "Enable": true, - "ResourceId": "" - } - } - ], - "required": [ - "Logging", - "ADLS", - "Blob" - ], - "properties": { - "Logging": { - "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/Storage/properties/Logging", - "type": "object", - "title": "The Logging schema", - "description": "An explanation about the purpose of this instance.", - "default": {}, - "examples": [ - { - "Name": "logstg", - "ApplyNamePostFix": true, - "Enable": true, - "Dummy": "" - } - ], - "required": [ - "Name", - "ApplyNamePostFix", - "Enable", - "Dummy" - ], - "properties": { - "Name": { - "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/Storage/properties/Logging/properties/Name", - "type": "string", - "title": "The Name schema", - "description": "An explanation about the purpose of this instance.", - "default": "", - "examples": [ - "logstg" - ] - }, - "ApplyNamePostFix": { - "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/Storage/properties/Logging/properties/ApplyNamePostFix", - "type": "boolean", - "title": "The ApplyNamePostFix schema", - "description": "An explanation about the purpose of this instance.", - "default": false, - "examples": [ - true - ] - }, - "Enable": { - "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/Storage/properties/Logging/properties/Enable", - "type": "boolean", - "title": "The Enable schema", - "description": "An explanation about the purpose of this instance.", - "default": false, - "examples": [ - true - ] - }, - "Dummy": { - "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/Storage/properties/Logging/properties/Dummy", - "type": "string", - "title": "The Dummy schema", - "description": "An explanation about the purpose of this instance.", - "default": "", - "examples": [ - "" - ] - } - }, - "additionalProperties": true - }, - "ADLS": { - "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/Storage/properties/ADLS", - "type": "object", - "title": "The ADLS schema", - "description": "An explanation about the purpose of this instance.", - "default": {}, - "examples": [ - { - "Name": "adls", - "ApplyNamePostFix": true, - "Enable": true - } - ], - "required": [ - "Name", - "ApplyNamePostFix", - "Enable" - ], - "properties": { - "Name": { - "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/Storage/properties/ADLS/properties/Name", - "type": "string", - "title": "The Name schema", - "description": "An explanation about the purpose of this instance.", - "default": "", - "examples": [ - "adls" - ] - }, - "ApplyNamePostFix": { - "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/Storage/properties/ADLS/properties/ApplyNamePostFix", - "type": "boolean", - "title": "The ApplyNamePostFix schema", - "description": "An explanation about the purpose of this instance.", - "default": false, - "examples": [ - true - ] - }, - "Enable": { - "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/Storage/properties/ADLS/properties/Enable", - "type": "boolean", - "title": "The Enable schema", - "description": "An explanation about the purpose of this instance.", - "default": false, - "examples": [ - true - ] - } - }, - "additionalProperties": true - }, - "Blob": { - "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/Storage/properties/Blob", - "type": "object", - "title": "The Blob schema", - "description": "An explanation about the purpose of this instance.", - "default": {}, - "examples": [ - { - "Name": "blob", - "ApplyNamePostFix": true, - "Enable": true, - "ResourceId": "" - } - ], - "required": [ - "Name", - "ApplyNamePostFix", - "Enable", - "ResourceId" - ], - "properties": { - "Name": { - "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/Storage/properties/Blob/properties/Name", - "type": "string", - "title": "The Name schema", - "description": "An explanation about the purpose of this instance.", - "default": "", - "examples": [ - "blob" - ] - }, - "ApplyNamePostFix": { - "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/Storage/properties/Blob/properties/ApplyNamePostFix", - "type": "boolean", - "title": "The ApplyNamePostFix schema", - "description": "An explanation about the purpose of this instance.", - "default": false, - "examples": [ - true - ] - }, - "Enable": { - "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/Storage/properties/Blob/properties/Enable", - "type": "boolean", - "title": "The Enable schema", - "description": "An explanation about the purpose of this instance.", - "default": false, - "examples": [ - true - ] - }, - "ResourceId": { - "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/Storage/properties/Blob/properties/ResourceId", - "type": "string", - "title": "The ResourceId schema", - "description": "An explanation about the purpose of this instance.", - "default": "", - "examples": [ - "" - ] - } - }, - "additionalProperties": true - } - }, - "additionalProperties": true - } - }, - "additionalProperties": true + "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/AppPlans/properties/WebApp/properties/ResourceGroup", + "type": "null", + "title": "ResourceGroup", + "description": "", + "default": null, + "examples": [ + null + ] } + }, + "additionalProperties": true }, - "additionalProperties": true - } + "FunctionApp": { + "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/AppPlans/properties/FunctionApp", + "type": "object", + "title": "FunctionApp", + "description": "", + "default": {}, + "examples": [ + { + "Enable": true, + "Name": "adsgfappplanfnc", + "ApplyNamePostFix": true, + "ResourceGroup": null + } + ], + "required": [ + "Enable", + "Name", + "ApplyNamePostFix", + "ResourceGroup" + ], + "properties": { + "Enable": { + "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/AppPlans/properties/FunctionApp/properties/Enable", + "type": "boolean", + "format": "checkbox", + "title": "Enable Deployment", + "description": "", + "default": false, + "examples": [ + true + ] + }, + "Name": { + "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/AppPlans/properties/FunctionApp/properties/Name", + "type": "string", + "title": "Name", + "description": "", + "default": "", + "examples": [ + "adsgfappplanfnc" + ] + }, + "ApplyNamePostFix": { + "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/AppPlans/properties/FunctionApp/properties/ApplyNamePostFix", + "type": "boolean", + "format": "checkbox", + "title": "Apply Postfix to Resource Name", + "description": "", + "default": false, + "examples": [ + true + ] + }, + "ResourceGroup": { + "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/AppPlans/properties/FunctionApp/properties/ResourceGroup", + "type": "null", + "title": "ResourceGroup", + "description": "", + "default": null, + "examples": [ + null + ] + } + }, + "additionalProperties": true + } + }, + "additionalProperties": true + }, + "AzureSQLServer": { + "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/AzureSQLServer", + "type": "object", + "title": "AzureSQLServer", + "description": "", + "default": {}, + "examples": [ + { + "Enable": true, + "Name": "adsgfazsql", + "ApplyNamePostFix": true, + "AdminUser": "AdsAdmin", + "AdminPassword": "EPuiX2K0^T4t", + "AdsGoFastDB": { + "Enable": true, + "Name": "AdsGf", + "UpdateSourceAndTargetSystems": true, + "UpdateDataFactory": true, + "ApplyNamePostFix": false + }, + "StagingDB": { + "Enable": true, + "Name": "AdsGfStaging", + "ApplyNamePostFix": false + }, + "SampleDB": { + "Enable": true, + "Name": "AdsGfSample", + "ApplyNamePostFix": false + } + } + ], + "required": [ + "Enable", + "Name", + "ApplyNamePostFix", + "AdminUser", + "AdminPassword", + "AdsGoFastDB", + "StagingDB", + "SampleDB" + ], + "properties": { + "Enable": { + "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/AzureSQLServer/properties/Enable", + "type": "boolean", + "format": "checkbox", + "title": "Enable Deployment", + "description": "", + "default": false, + "examples": [ + true + ] + }, + "Name": { + "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/AzureSQLServer/properties/Name", + "type": "string", + "title": "Name", + "description": "", + "default": "", + "examples": [ + "adsgfazsql" + ] + }, + "ApplyNamePostFix": { + "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/AzureSQLServer/properties/ApplyNamePostFix", + "type": "boolean", + "format": "checkbox", + "title": "Apply Postfix to Resource Name", + "description": "", + "default": false, + "examples": [ + true + ] + }, + "AdminUser": { + "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/AzureSQLServer/properties/AdminUser", + "type": "string", + "title": "AdminUser", + "description": "", + "default": "", + "examples": [ + "AdsAdmin" + ] + }, + "AdminPassword": { + "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/AzureSQLServer/properties/AdminPassword", + "type": "string", + "title": "AdminPassword", + "description": "", + "default": "", + "examples": [ + "EPuiX2K0^T4t" + ] + }, + "AdsGoFastDB": { + "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/AzureSQLServer/properties/AdsGoFastDB", + "type": "object", + "title": "AdsGoFastDB", + "description": "", + "default": {}, + "examples": [ + { + "Enable": true, + "Name": "AdsGf", + "UpdateSourceAndTargetSystems": true, + "UpdateDataFactory": true, + "ApplyNamePostFix": false + } + ], + "required": [ + "Enable", + "Name", + "UpdateSourceAndTargetSystems", + "UpdateDataFactory", + "ApplyNamePostFix" + ], + "properties": { + "Enable": { + "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/AzureSQLServer/properties/AdsGoFastDB/properties/Enable", + "type": "boolean", + "format": "checkbox", + "title": "Enable Deployment", + "description": "", + "default": false, + "examples": [ + true + ] + }, + "Name": { + "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/AzureSQLServer/properties/AdsGoFastDB/properties/Name", + "type": "string", + "title": "Name", + "description": "", + "default": "", + "examples": [ + "AdsGf" + ] + }, + "UpdateSourceAndTargetSystems": { + "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/AzureSQLServer/properties/AdsGoFastDB/properties/UpdateSourceAndTargetSystems", + "type": "boolean", + "format": "checkbox", + "title": "UpdateSourceAndTargetSystems", + "description": "", + "default": false, + "examples": [ + true + ] + }, + "UpdateDataFactory": { + "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/AzureSQLServer/properties/AdsGoFastDB/properties/UpdateDataFactory", + "type": "boolean", + "format": "checkbox", + "title": "UpdateDataFactory", + "description": "", + "default": false, + "examples": [ + true + ] + }, + "ApplyNamePostFix": { + "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/AzureSQLServer/properties/AdsGoFastDB/properties/ApplyNamePostFix", + "type": "boolean", + "format": "checkbox", + "title": "Apply Postfix to Resource Name", + "description": "", + "default": false, + "examples": [ + false + ] + } + }, + "additionalProperties": true + }, + "StagingDB": { + "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/AzureSQLServer/properties/StagingDB", + "type": "object", + "title": "StagingDB", + "description": "", + "default": {}, + "examples": [ + { + "Enable": true, + "Name": "AdsGfStaging", + "ApplyNamePostFix": false + } + ], + "required": [ + "Enable", + "Name", + "ApplyNamePostFix" + ], + "properties": { + "Enable": { + "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/AzureSQLServer/properties/StagingDB/properties/Enable", + "type": "boolean", + "format": "checkbox", + "title": "Enable Deployment", + "description": "", + "default": false, + "examples": [ + true + ] + }, + "Name": { + "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/AzureSQLServer/properties/StagingDB/properties/Name", + "type": "string", + "title": "Name", + "description": "", + "default": "", + "examples": [ + "AdsGfStaging" + ] + }, + "ApplyNamePostFix": { + "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/AzureSQLServer/properties/StagingDB/properties/ApplyNamePostFix", + "type": "boolean", + "format": "checkbox", + "title": "Apply Postfix to Resource Name", + "description": "", + "default": false, + "examples": [ + false + ] + } + }, + "additionalProperties": true + }, + "SampleDB": { + "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/AzureSQLServer/properties/SampleDB", + "type": "object", + "title": "SampleDB", + "description": "", + "default": {}, + "examples": [ + { + "Enable": true, + "Name": "AdsGfSample", + "ApplyNamePostFix": false + } + ], + "required": [ + "Enable", + "Name", + "ApplyNamePostFix" + ], + "properties": { + "Enable": { + "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/AzureSQLServer/properties/SampleDB/properties/Enable", + "type": "boolean", + "format": "checkbox", + "title": "Enable Deployment", + "description": "", + "default": false, + "examples": [ + true + ] + }, + "Name": { + "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/AzureSQLServer/properties/SampleDB/properties/Name", + "type": "string", + "title": "Name", + "description": "", + "default": "", + "examples": [ + "AdsGfSample" + ] + }, + "ApplyNamePostFix": { + "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/AzureSQLServer/properties/SampleDB/properties/ApplyNamePostFix", + "type": "boolean", + "format": "checkbox", + "title": "Apply Postfix to Resource Name", + "description": "", + "default": false, + "examples": [ + false + ] + } + }, + "additionalProperties": true + } + }, + "additionalProperties": true + }, + "CoreFunctionApp": { + "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/CoreFunctionApp", + "type": "object", + "title": "CoreFunctionApp", + "description": "", + "default": {}, + "examples": [ + { + "Enable": true, + "Name": "adsgofastfunc", + "ApplyNamePostFix": true, + "PrincipalId": "" + } + ], + "required": [ + "Enable", + "Name", + "ApplyNamePostFix", + "PrincipalId" + ], + "properties": { + "Enable": { + "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/CoreFunctionApp/properties/Enable", + "type": "boolean", + "format": "checkbox", + "title": "Enable Deployment", + "description": "", + "default": false, + "examples": [ + true + ] + }, + "Name": { + "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/CoreFunctionApp/properties/Name", + "type": "string", + "title": "Name", + "description": "", + "default": "", + "examples": [ + "adsgofastfunc" + ] + }, + "ApplyNamePostFix": { + "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/CoreFunctionApp/properties/ApplyNamePostFix", + "type": "boolean", + "format": "checkbox", + "title": "Apply Postfix to Resource Name", + "description": "", + "default": false, + "examples": [ + true + ] + }, + "PrincipalId": { + "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/CoreFunctionApp/properties/PrincipalId", + "type": "string", + "title": "PrincipalId", + "description": "", + "default": "", + "examples": [ + "" + ] + } + }, + "additionalProperties": true + }, + "DataFactory": { + "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/DataFactory", + "type": "object", + "title": "DataFactory", + "description": "", + "default": {}, + "examples": [ + { + "Enable": true, + "Name": "adsgfadf", + "ApplyNamePostFix": true, + "AzVnetIr": { + "Enable": true, + "Name": "SelfHostedIntegrationRuntime-Azure-VNET", + "Type": "ManagedVnet" + }, + "OnPremVnetIr": { + "Enable": false, + "Name": "SelfHostedIntegrationRuntime-OnPem-Net", + "Type": "ManagedVnet" + } + } + ], + "required": [ + "Enable", + "Name", + "ApplyNamePostFix", + "AzVnetIr", + "OnPremVnetIr" + ], + "properties": { + "Enable": { + "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/DataFactory/properties/Enable", + "type": "boolean", + "format": "checkbox", + "title": "Enable Deployment", + "description": "", + "default": false, + "examples": [ + true + ] + }, + "Name": { + "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/DataFactory/properties/Name", + "type": "string", + "title": "Name", + "description": "", + "default": "", + "examples": [ + "adsgfadf" + ] + }, + "ApplyNamePostFix": { + "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/DataFactory/properties/ApplyNamePostFix", + "type": "boolean", + "format": "checkbox", + "title": "Apply Postfix to Resource Name", + "description": "", + "default": false, + "examples": [ + true + ] + }, + "AzVnetIr": { + "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/DataFactory/properties/AzVnetIr", + "type": "object", + "title": "AzVnetIr", + "description": "", + "default": {}, + "examples": [ + { + "Enable": true, + "Name": "SelfHostedIntegrationRuntime-Azure-VNET", + "Type": "ManagedVnet" + } + ], + "required": [ + "Enable", + "Name", + "Type" + ], + "properties": { + "Enable": { + "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/DataFactory/properties/AzVnetIr/properties/Enable", + "type": "boolean", + "format": "checkbox", + "title": "Enable Deployment", + "description": "", + "default": false, + "examples": [ + true + ] + }, + "Name": { + "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/DataFactory/properties/AzVnetIr/properties/Name", + "type": "string", + "title": "Name", + "description": "", + "default": "", + "examples": [ + "SelfHostedIntegrationRuntime-Azure-VNET" + ] + }, + "Type": { + "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/DataFactory/properties/AzVnetIr/properties/Type", + "type": "string", + "title": "Type", + "description": "", + "default": "", + "examples": [ + "ManagedVnet" + ] + } + }, + "additionalProperties": true + }, + "OnPremVnetIr": { + "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/DataFactory/properties/OnPremVnetIr", + "type": "object", + "title": "OnPremVnetIr", + "description": "", + "default": {}, + "examples": [ + { + "Enable": false, + "Name": "SelfHostedIntegrationRuntime-OnPem-Net", + "Type": "ManagedVnet" + } + ], + "required": [ + "Enable", + "Name", + "Type" + ], + "properties": { + "Enable": { + "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/DataFactory/properties/OnPremVnetIr/properties/Enable", + "type": "boolean", + "format": "checkbox", + "title": "Enable Deployment", + "description": "", + "default": false, + "examples": [ + false + ] + }, + "Name": { + "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/DataFactory/properties/OnPremVnetIr/properties/Name", + "type": "string", + "title": "Name", + "description": "", + "default": "", + "examples": [ + "SelfHostedIntegrationRuntime-OnPem-Net" + ] + }, + "Type": { + "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/DataFactory/properties/OnPremVnetIr/properties/Type", + "type": "string", + "title": "Type", + "description": "", + "default": "", + "examples": [ + "ManagedVnet" + ] + } + }, + "additionalProperties": true + } + }, + "additionalProperties": true + }, + "WebSite": { + "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/WebSite", + "type": "object", + "title": "WebSite", + "description": "", + "default": {}, + "examples": [ + { + "Enable": true, + "Name": "adsgofastweb", + "ApplyNamePostFix": true, + "PrincipalId": "" + } + ], + "required": [ + "Enable", + "Name", + "ApplyNamePostFix", + "PrincipalId" + ], + "properties": { + "Enable": { + "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/WebSite/properties/Enable", + "type": "boolean", + "format": "checkbox", + "title": "Enable Deployment", + "description": "", + "default": false, + "examples": [ + true + ] + }, + "Name": { + "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/WebSite/properties/Name", + "type": "string", + "title": "Name", + "description": "", + "default": "", + "examples": [ + "adsgofastweb" + ] + }, + "ApplyNamePostFix": { + "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/WebSite/properties/ApplyNamePostFix", + "type": "boolean", + "format": "checkbox", + "title": "Apply Postfix to Resource Name", + "description": "", + "default": false, + "examples": [ + true + ] + }, + "PrincipalId": { + "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/WebSite/properties/PrincipalId", + "type": "string", + "title": "PrincipalId", + "description": "", + "default": "", + "examples": [ + "" + ] + } + }, + "additionalProperties": true + }, + "LogAnalytics": { + "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/LogAnalytics", + "type": "object", + "title": "LogAnalytics", + "description": "", + "default": {}, + "examples": [ + { + "Enable": true, + "Name": "adsgofastloganalytics", + "ApplyNamePostFix": false + } + ], + "required": [ + "Enable", + "Name", + "ApplyNamePostFix" + ], + "properties": { + "Enable": { + "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/LogAnalytics/properties/Enable", + "type": "boolean", + "format": "checkbox", + "title": "Enable Deployment", + "description": "", + "default": false, + "examples": [ + true + ] + }, + "Name": { + "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/LogAnalytics/properties/Name", + "type": "string", + "title": "Name", + "description": "", + "default": "", + "examples": [ + "adsgofastloganalytics" + ] + }, + "ApplyNamePostFix": { + "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/LogAnalytics/properties/ApplyNamePostFix", + "type": "boolean", + "format": "checkbox", + "title": "Apply Postfix to Resource Name", + "description": "", + "default": false, + "examples": [ + false + ] + } + }, + "additionalProperties": true + }, + "KeyVault": { + "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/KeyVault", + "type": "object", + "title": "KeyVault", + "description": "", + "default": {}, + "examples": [ + { + "Enable": true, + "Name": "adsgfkv", + "ApplyNamePostFix": true + } + ], + "required": [ + "Enable", + "Name", + "ApplyNamePostFix" + ], + "properties": { + "Enable": { + "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/KeyVault/properties/Enable", + "type": "boolean", + "format": "checkbox", + "title": "Enable Deployment", + "description": "", + "default": false, + "examples": [ + true + ] + }, + "Name": { + "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/KeyVault/properties/Name", + "type": "string", + "title": "Name", + "description": "", + "default": "", + "examples": [ + "adsgfkv" + ] + }, + "ApplyNamePostFix": { + "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/KeyVault/properties/ApplyNamePostFix", + "type": "boolean", + "format": "checkbox", + "title": "Apply Postfix to Resource Name", + "description": "", + "default": false, + "examples": [ + true + ] + } + }, + "additionalProperties": true + }, + "Vnet": { + "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/Vnet", + "type": "object", + "title": "Vnet", + "description": "", + "default": {}, + "examples": [ + { + "Enable": true, + "Name": "AdsGoFastVnet" + } + ], + "required": [ + "Enable", + "Name" + ], + "properties": { + "Enable": { + "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/Vnet/properties/Enable", + "type": "boolean", + "format": "checkbox", + "title": "Enable Deployment", + "description": "", + "default": false, + "examples": [ + true + ] + }, + "Name": { + "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/Vnet/properties/Name", + "type": "string", + "title": "Name", + "description": "", + "default": "", + "examples": [ + "AdsGoFastVnet" + ] + } + }, + "additionalProperties": true + }, + "Storage": { + "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/Storage", + "type": "object", + "title": "Storage", + "description": "", + "default": {}, + "examples": [ + { + "Logging": { + "Name": "logstg", + "ApplyNamePostFix": true, + "Enable": true, + "Dummy": "" + }, + "ADLS": { + "Name": "adls", + "ApplyNamePostFix": true, + "Enable": true + }, + "Blob": { + "Name": "blob", + "ApplyNamePostFix": true, + "Enable": true, + "ResourceId": "" + } + } + ], + "required": [ + "Logging", + "ADLS", + "Blob" + ], + "properties": { + "Logging": { + "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/Storage/properties/Logging", + "type": "object", + "title": "Logging", + "description": "", + "default": {}, + "examples": [ + { + "Name": "logstg", + "ApplyNamePostFix": true, + "Enable": true, + "Dummy": "" + } + ], + "required": [ + "Name", + "ApplyNamePostFix", + "Enable", + "Dummy" + ], + "properties": { + "Name": { + "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/Storage/properties/Logging/properties/Name", + "type": "string", + "title": "Name", + "description": "", + "default": "", + "examples": [ + "logstg" + ] + }, + "ApplyNamePostFix": { + "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/Storage/properties/Logging/properties/ApplyNamePostFix", + "type": "boolean", + "format": "checkbox", + "title": "Apply Postfix to Resource Name", + "description": "", + "default": false, + "examples": [ + true + ] + }, + "Enable": { + "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/Storage/properties/Logging/properties/Enable", + "type": "boolean", + "format": "checkbox", + "title": "Enable Deployment", + "description": "", + "default": false, + "examples": [ + true + ] + }, + "Dummy": { + "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/Storage/properties/Logging/properties/Dummy", + "type": "string", + "title": "Dummy", + "description": "", + "default": "", + "examples": [ + "" + ] + } + }, + "additionalProperties": true + }, + "ADLS": { + "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/Storage/properties/ADLS", + "type": "object", + "title": "ADLS", + "description": "", + "default": {}, + "examples": [ + { + "Name": "adls", + "ApplyNamePostFix": true, + "Enable": true + } + ], + "required": [ + "Name", + "ApplyNamePostFix", + "Enable" + ], + "properties": { + "Name": { + "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/Storage/properties/ADLS/properties/Name", + "type": "string", + "title": "Name", + "description": "", + "default": "", + "examples": [ + "adls" + ] + }, + "ApplyNamePostFix": { + "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/Storage/properties/ADLS/properties/ApplyNamePostFix", + "type": "boolean", + "format": "checkbox", + "title": "Apply Postfix to Resource Name", + "description": "", + "default": false, + "examples": [ + true + ] + }, + "Enable": { + "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/Storage/properties/ADLS/properties/Enable", + "type": "boolean", + "format": "checkbox", + "title": "Enable Deployment", + "description": "", + "default": false, + "examples": [ + true + ] + } + }, + "additionalProperties": true + }, + "Blob": { + "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/Storage/properties/Blob", + "type": "object", + "title": "Blob", + "description": "", + "default": {}, + "examples": [ + { + "Name": "blob", + "ApplyNamePostFix": true, + "Enable": true, + "ResourceId": "" + } + ], + "required": [ + "Name", + "ApplyNamePostFix", + "Enable", + "ResourceId" + ], + "properties": { + "Name": { + "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/Storage/properties/Blob/properties/Name", + "type": "string", + "title": "Name", + "description": "", + "default": "", + "examples": [ + "blob" + ] + }, + "ApplyNamePostFix": { + "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/Storage/properties/Blob/properties/ApplyNamePostFix", + "type": "boolean", + "format": "checkbox", + "title": "Apply Postfix to Resource Name", + "description": "", + "default": false, + "examples": [ + true + ] + }, + "Enable": { + "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/Storage/properties/Blob/properties/Enable", + "type": "boolean", + "format": "checkbox", + "title": "Enable Deployment", + "description": "", + "default": false, + "examples": [ + true + ] + }, + "ResourceId": { + "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/Storage/properties/Blob/properties/ResourceId", + "type": "string", + "title": "ResourceId", + "description": "", + "default": "", + "examples": [ + "" + ] + } + }, + "additionalProperties": true + }, + "ADLSTransient": { + "type": "object", + "title": "ADLS Transient", + "description": "", + "default": {}, + "required": [ + "Name", + "ApplyNamePostFix", + "Enable" + ], + "properties": { + "Name": { + "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/Storage/properties/ADLSTransient/properties/Name", + "type": "string", + "title": "Name", + "description": "", + "default": "", + "examples": [ + "adls" + ] + }, + "ApplyNamePostFix": { + "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/Storage/properties/ADLSTransient/properties/ApplyNamePostFix", + "type": "boolean", + "format": "checkbox", + "title": "Apply Postfix to Resource Name", + "description": "", + "default": false, + "examples": [ + true + ] + }, + "Enable": { + "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/Storage/properties/ADLSTransient/properties/Enable", + "type": "boolean", + "format": "checkbox", + "title": "Enable Deployment", + "description": "", + "default": false, + "examples": [ + true + ] + } + }, + "additionalProperties": true + } + }, + "additionalProperties": true + }, + "Bastion": { + "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/Bastion", + "type": "object", + "title": "Bastion Settings", + "description": "", + "default": {}, + "required": [ + "Name", + "ApplyNamePostFix", + "Enable" + ], + "properties": { + "Name": { + "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/Bastion/properties/Name", + "type": "string", + "title": "Bastion Name", + "description": "", + "default": "", + "examples": [ + "logstg" + ] + }, + "ApplyNamePostFix": { + "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/Bastion/properties/ApplyNamePostFix", + "type": "boolean", + "format": "checkbox", + "title": "ApplyNamePostFix", + "description": "", + "default": true, + "enum": [ + true + ] + }, + "Enable": { + "$id": "#/properties/AdsOpts/properties/CD/properties/Services/properties/Bastion/properties/Enable", + "type": "boolean", + "format": "checkbox", + "title": "Enable deployment of Bastion", + "description": "", + "default": true, + "enum": [ + true + ] + } + } + }, + "additionalProperties": true + } }, "additionalProperties": true - } + } + }, + "additionalProperties": false + } }, - "additionalProperties": true + "additionalProperties": false + } } \ No newline at end of file diff --git a/solution/Deployment/workflows/CD_0a_CreateServicePrincipals_AAD_Elevated.ps1 b/solution/Deployment/workflows/CD_0a_CreateServicePrincipals_AAD_Elevated.ps1 index 5d89ed49..f50280a8 100644 --- a/solution/Deployment/workflows/CD_0a_CreateServicePrincipals_AAD_Elevated.ps1 +++ b/solution/Deployment/workflows/CD_0a_CreateServicePrincipals_AAD_Elevated.ps1 @@ -9,18 +9,18 @@ Invoke-Expression -Command ".\Steps\CD_DeployResourceGroup.ps1" ######################################################################### if($env:AdsOpts_CD_ServicePrincipals_DeploymentSP_Enable -eq "True") { - Write-Host "Creating Deployment Service Principal" -ForegroundColor Yellow + Write-Debug "Creating Deployment Service Principal" $subid = ((az account show -s $env:AdsOpts_CD_ResourceGroup_Subscription) | ConvertFrom-Json).id $spcheck = az ad sp list --filter "displayname eq '$env:AdsOpts_CD_ServicePrincipals_DeploymentSP_Name'" | ConvertFrom-Json if ($null -eq $spcheck) { - Write-Host "Deployment Principal does not exist so creating now." -ForegroundColor Yellow + Write-Debug "Deployment Principal does not exist so creating now." $SP = az ad sp create-for-rbac --name $env:AdsOpts_CD_ServicePrincipals_DeploymentSP_Name --role contributor --scopes /subscriptions/$subid/resourceGroups/$env:AdsOpts_CD_ResourceGroup_Name } else { - Write-Host "Deployment Prinicpal Already Exists So Just Adding Contributor Role on Resource Group" -ForegroundColor Yellow - az role assignment create --assignee $spcheck[0].objectId --role "Contributor" --scope /subscriptions/$subid/resourceGroups/$env:AdsOpts_CD_ResourceGroup_Name + Write-Debug "Deployment Prinicpal Already Exists So Just Adding Contributor Role on Resource Group" + $SP = az role assignment create --assignee $spcheck[0].objectId --role "Contributor" --scope /subscriptions/$subid/resourceGroups/$env:AdsOpts_CD_ResourceGroup_Name } } @@ -30,7 +30,7 @@ $envsettings = Get-Content $environmentfile | ConvertFrom-Json if($env:AdsOpts_CD_ServicePrincipals_WebAppAuthenticationSP_Enable -eq "True") { - Write-Host "Creating WebAppAuthentication Service Principal" -ForegroundColor Yellow + Write-Debug "Creating WebAppAuthentication Service Principal" $roleid = [guid]::NewGuid() $roles = '[{\"allowedMemberTypes\": [\"Application\"],\"description\": \"Administrator\",\"displayName\": \"Administrator\",\"id\": \"@Id\",\"isEnabled\": true,\"lang\": null,\"origin\": \"Users\\Groups\",\"value\": \"Administrator\"}]' @@ -46,7 +46,7 @@ if($env:AdsOpts_CD_ServicePrincipals_WebAppAuthenticationSP_Enable -eq "True") if($env:AdsOpts_CD_ServicePrincipals_FunctionAppAuthenticationSP_Enable -eq "True") { - Write-Host "Creating FunctionAppAuthentication Service Principal" -ForegroundColor Yellow + Write-Debug "Creating FunctionAppAuthentication Service Principal" $roleid = [guid]::NewGuid() $roles = '[{\"allowedMemberTypes\": [\"Application\"],\"description\": \"Used to applications to call the ADS Go Fast functions\",\"displayName\": \"FunctionAPICaller\",\"id\": \"@Id\",\"isEnabled\": true,\"lang\": null,\"origin\": \"Application\",\"value\": \"FunctionAPICaller\"}]' @@ -74,5 +74,5 @@ $envsettings | ConvertTo-Json -Depth 10 | set-content $environmentfile #Check Status of Errors -Write-Host "Script Complete Please Check below for Errors:" -ForegroundColor Yellow +Write-Host "Script Complete Please Check below for Errors:" Write-Host $error \ No newline at end of file diff --git a/solution/Deployment/workflows/CD_1a_DeployServices.ps1 b/solution/Deployment/workflows/CD_1a_DeployServices.ps1 index 86e82294..003a0548 100644 --- a/solution/Deployment/workflows/CD_1a_DeployServices.ps1 +++ b/solution/Deployment/workflows/CD_1a_DeployServices.ps1 @@ -4,7 +4,7 @@ ######################################################Write-Host ([Environment]::GetEnvironmentVariable("AdsOpts_CI_Enable")) if (([Environment]::GetEnvironmentVariable("AdsOpts_CD_EnableDeploy")) -eq "True") { - Write-Host "Starting CD.." + Write-Debug "Starting CD.." Invoke-Expression -Command ".\Steps\CD_DeployKeyVault.ps1" @@ -33,12 +33,12 @@ if (([Environment]::GetEnvironmentVariable("AdsOpts_CD_EnableDeploy")) -eq "True Invoke-Expression -Command ".\Steps\CD_DeployADF.ps1" - Write-Host "Finishing CD.." + Write-Debug "Finishing CD.." } else { - Write-Host "CD_1a_DeployServices.ps1 skipped as flag in environment file is set to false" -ForegroundColor Yellow + Write-Warning "CD_1a_DeployServices.ps1 skipped as flag in environment file is set to false" } #Invoke-Expression -Command ".\Cleanup_RemoveAll.ps1" \ No newline at end of file diff --git a/solution/Deployment/workflows/CD_2a_CreateMSIs_AAD_Elevated.ps1 b/solution/Deployment/workflows/CD_2a_CreateMSIs_AAD_Elevated.ps1 index 704cd59b..42ea81ce 100644 --- a/solution/Deployment/workflows/CD_2a_CreateMSIs_AAD_Elevated.ps1 +++ b/solution/Deployment/workflows/CD_2a_CreateMSIs_AAD_Elevated.ps1 @@ -38,7 +38,7 @@ az extension add --name datafactory $dfpid = ((az datafactory show --factory-name $env:AdsOpts_CD_Services_DataFactory_Name --resource-group $env:AdsOpts_CD_ResourceGroup_Name) | ConvertFrom-Json).identity.principalId $dfoid = ((az ad sp show --id $dfpid) | ConvertFrom-Json).objectId #Allow ADF to Read Key Vault -az keyvault set-policy --name $env:AdsOpts_CD_Services_KeyVault_Name --certificate-permissions get list --key-permissions get list --object-id $dfoid --resource-group $env:AdsOpts_CD_ResourceGroup_Name --secret-permissions get list --storage-permissions get --subscription $env:AdsOpts_CD_ResourceGroup_Subscription +$result = az keyvault set-policy --name $env:AdsOpts_CD_Services_KeyVault_Name --certificate-permissions get list --key-permissions get list --object-id $dfoid --resource-group $env:AdsOpts_CD_ResourceGroup_Name --secret-permissions get list --storage-permissions get --subscription $env:AdsOpts_CD_ResourceGroup_Subscription @@ -47,7 +47,7 @@ az keyvault set-policy --name $env:AdsOpts_CD_Services_KeyVault_Name --certifica #Give MSIs Required AD Privileges #Assign SQL Admin $cu = az ad signed-in-user show | ConvertFrom-Json -az sql server ad-admin create --display-name $cu.DisplayName --object-id $cu.ObjectId --resource-group $env:AdsOpts_CD_ResourceGroup_Name --server $env:AdsOpts_CD_Services_AzureSQLServer_Name --subscription $env:AdsOpts_CD_ResourceGroup_Subscription +$result = az sql server ad-admin create --display-name $cu.DisplayName --object-id $cu.ObjectId --resource-group $env:AdsOpts_CD_ResourceGroup_Name --server $env:AdsOpts_CD_Services_AzureSQLServer_Name --subscription $env:AdsOpts_CD_ResourceGroup_Subscription #az login --service-principal --username $env:secrets_AZURE_CREDENTIALS_clientId --password $env:secrets_AZURE_CREDENTIALS_clientSecret --tenant $env:secrets_AZURE_CREDENTIALS_tenantId @@ -62,7 +62,7 @@ if($null -eq $SqlInstalled) #Add Ip to SQL Firewall write-host "Creating SQL Server Firewall Rules" $myIp = (Invoke-WebRequest ifconfig.me/ip).Content -az sql server firewall-rule create -g $env:AdsOpts_CD_ResourceGroup_Name -s $env:AdsOpts_CD_Services_AzureSQLServer_Name -n "MySetupIP" --start-ip-address $myIp --end-ip-address $myIp +$result = az sql server firewall-rule create -g $env:AdsOpts_CD_ResourceGroup_Name -s $env:AdsOpts_CD_Services_AzureSQLServer_Name -n "MySetupIP" --start-ip-address $myIp --end-ip-address $myIp #May Need to add a wait here to allow MSI creation to have propogated completely @@ -112,6 +112,7 @@ $sqlcommand = " CREATE USER [$env:AdsOpts_CD_Services_DataFactory_Name] FROM EXTERNAL PROVIDER; ALTER ROLE db_datareader ADD MEMBER [$env:AdsOpts_CD_Services_DataFactory_Name]; ALTER ROLE db_datawriter ADD MEMBER [$env:AdsOpts_CD_Services_DataFactory_Name]; + ALTER ROLE db_ddladmin ADD MEMBER [$env:AdsOpts_CD_Services_DataFactory_Name]; GRANT EXECUTE ON SCHEMA::[dbo] TO [$env:AdsOpts_CD_Services_DataFactory_Name]; GO " @@ -121,14 +122,31 @@ $token=$(az account get-access-token --resource=https://database.windows.net --q Invoke-Sqlcmd -ServerInstance "$env:AdsOpts_CD_Services_AzureSQLServer_Name.database.windows.net,1433" -Database $env:AdsOpts_CD_Services_AzureSQLServer_SampleDB_Name -AccessToken $token -query $sqlcommand +#STAGING DB +$sqlcommand = " + DROP USER IF EXISTS [$env:AdsOpts_CD_Services_DataFactory_Name] + CREATE USER [$env:AdsOpts_CD_Services_DataFactory_Name] FROM EXTERNAL PROVIDER; + ALTER ROLE db_datareader ADD MEMBER [$env:AdsOpts_CD_Services_DataFactory_Name]; + ALTER ROLE db_datawriter ADD MEMBER [$env:AdsOpts_CD_Services_DataFactory_Name]; + ALTER ROLE db_ddladmin ADD MEMBER [$env:AdsOpts_CD_Services_DataFactory_Name]; + GRANT EXECUTE ON SCHEMA::[dbo] TO [$env:AdsOpts_CD_Services_DataFactory_Name]; + GO +" + +write-host "Granting MSI Privileges on STAGING DB" +$token=$(az account get-access-token --resource=https://database.windows.net --query accessToken --output tsv) +Invoke-Sqlcmd -ServerInstance "$env:AdsOpts_CD_Services_AzureSQLServer_Name.database.windows.net,1433" -Database $env:AdsOpts_CD_Services_AzureSQLServer_StagingDB_Name -AccessToken $token -query $sqlcommand + + #Next Add MSIs Permissions #Function App MSI Access to App Role to allow chained function calls +write-host "Granting Function App MSI Access to App Role to allow chained function calls" $authapp = az ad app show --id "api://$env:AdsOpts_CD_ServicePrincipals_FunctionAppAuthenticationSP_Name" | ConvertFrom-Json $callingappid = ((az functionapp identity show --name $env:AdsOpts_CD_Services_CoreFunctionApp_Name --resource-group $env:AdsOpts_CD_ResourceGroup_Name) | ConvertFrom-Json).principalId $authappid = $authapp.appId $permissionid = $authapp.oauth2Permissions.id -$authappobjectid = (az ad sp show --id $authapp.appId | ConvertFrom-Json).objectId +$authappobjectid = (az ad sp show --id $authappid | ConvertFrom-Json).objectId $body = '{"principalId": "@principalid","resourceId":"@resourceId","appRoleId": "@appRoleId"}' | ConvertFrom-Json $body.resourceId = $authappobjectid @@ -136,10 +154,11 @@ $body.appRoleId = ($authapp.appRoles | Where-Object {$_.value -eq "FunctionAPIC $body.principalId = $callingappid $body = ($body | ConvertTo-Json -compress | Out-String).Replace('"','\"') -az rest --method post --uri "https://graph.microsoft.com/v1.0/servicePrincipals/$authappobjectid/appRoleAssignedTo" --headers '{\"Content-Type\":\"application/json\"}' --body $body +$result = az rest --method post --uri "https://graph.microsoft.com/v1.0/servicePrincipals/$authappobjectid/appRoleAssignedTo" --headers '{\"Content-Type\":\"application/json\"}' --body $body #Web App +write-host "Adding Admin Role To WebApp" $authapp = az ad app show --id "api://$env:AdsOpts_CD_ServicePrincipals_WebAppAuthenticationSP_Name" | ConvertFrom-Json $callinguser = $cu.objectId $authappid = $authapp.appId @@ -153,7 +172,7 @@ $body.appRoleId = ($authapp.appRoles | Where-Object {$_.value -eq "Administrato $body.principalId = $callinguser $body = ($body | ConvertTo-Json -compress | Out-String).Replace('"','\"') -az rest --method post --uri "https://graph.microsoft.com/v1.0/servicePrincipals/$authappobjectid/appRoleAssignedTo" --headers '{\"Content-Type\":\"application/json\"}' --body $body +$result = az rest --method post --uri "https://graph.microsoft.com/v1.0/servicePrincipals/$authappobjectid/appRoleAssignedTo" --headers '{\"Content-Type\":\"application/json\"}' --body $body Invoke-Expression -Command ".\Steps\CD_GrantRBAC.ps1" diff --git a/solution/Deployment/workflows/CD_2b_ConfigureServices.ps1 b/solution/Deployment/workflows/CD_2b_ConfigureServices.ps1 index 8c79cdbf..95ceb354 100644 --- a/solution/Deployment/workflows/CD_2b_ConfigureServices.ps1 +++ b/solution/Deployment/workflows/CD_2b_ConfigureServices.ps1 @@ -25,6 +25,8 @@ if (([Environment]::GetEnvironmentVariable("AdsOpts_CD_EnableConfigure")) -eq "T Invoke-Expression -Command ".\Steps\CD_ConfigureVnet.ps1" + Invoke-Expression -Command ".\Steps\CD_ConfigureAzureSqlServer_UpdateTaskTypeMappingJson.ps1" + Write-Host "Finishing CD.." } diff --git a/solution/Deployment/workflows/LocalDevOnly_EnvironmentSetUp.ps1 b/solution/Deployment/workflows/LocalDevOnly_EnvironmentSetUp.ps1 index 3d9119f1..658ffcfb 100644 --- a/solution/Deployment/workflows/LocalDevOnly_EnvironmentSetUp.ps1 +++ b/solution/Deployment/workflows/LocalDevOnly_EnvironmentSetUp.ps1 @@ -1,15 +1,16 @@ #az login #az account set -s "jorampon internal consumption" - +#$DebugPreference = "Continue" +#$DebugPreference = "SilentlyContinue" [Environment]::SetEnvironmentVariable("ENVIRONMENT_NAME", "development") -if (Test-Path -PathType Container -Path "../bin/"){New-Item -ItemType Directory -Force -Path "../bin"} -New-Item -Path "../bin/" -Name "GitEnv.txt" -type "file" -value "" -force +if (Test-Path -PathType Container -Path "../bin/"){$newitem = New-Item -ItemType Directory -Force -Path "../bin"} +$newitem = New-Item -Path "../bin/" -Name "GitEnv.txt" -type "file" -value "" -force . .\Steps\PushEnvFileIntoVariables.ps1 ParseEnvFile("$env:ENVIRONMENT_NAME") -Invoke-Expression -Command ".\Steps\CD_SetResourceGroupHash.ps1" +Invoke-Expression -Command ".\Steps\CD_SetResourceGroupHash.ps1" #Load Secrets into Environment Variables -ParseSecretsFile ($SecretFile) +ParseSecretsFile ($SecretFile) diff --git a/solution/Deployment/workflows/Steps/CD_ConfigureADF.ps1 b/solution/Deployment/workflows/Steps/CD_ConfigureADF.ps1 index 6374b28a..99fc944d 100644 --- a/solution/Deployment/workflows/Steps/CD_ConfigureADF.ps1 +++ b/solution/Deployment/workflows/Steps/CD_ConfigureADF.ps1 @@ -10,7 +10,7 @@ az monitor diagnostic-settings create --name ADF-Diagnostics --export-to-resourc $subid = (az account show -s $env:AdsOpts_CD_ResourceGroup_Subscription | ConvertFrom-Json).id $uri = "https://management.azure.com/subscriptions/$subid/resourceGroups/$env:AdsOpts_CD_ResourceGroup_Name/providers/Microsoft.DataFactory/factories/$env:AdsOpts_CD_Services_DataFactory_Name/managedVirtualNetworks/default" + '?api-version=2018-06-01' -az rest --method put --uri $uri --headers '{\"Content-Type\":\"application/json\"}' --body '{\"properties\": {}}' +$rest = az rest --method put --uri $uri --headers '{\"Content-Type\":\"application/json\"}' --body '{\"properties\": {}}' #VNET $body = ' @@ -37,7 +37,7 @@ $body = ' $body = ($body | ConvertTo-Json -compress -Depth 10 | Out-String).Replace('"','\"') $uri = "https://management.azure.com/$env:AdsOpts_CD_ResourceGroup_Id/providers/Microsoft.DataFactory/factories/$env:AdsOpts_CD_Services_DataFactory_Name/integrationRuntimes/$env:AdsOpts_CD_Services_DataFactory_AzVnetIR_Name" + '?api-version=2018-06-01' Write-Host "Creating IR: $env:AdsOpts_CD_Services_DataFactory_AzVnetIR_Name" -az rest --method put --uri $uri --headers '{\"Content-Type\":\"application/json\"}' --body $body --uri-parameters 'api-version=2018-06-01' +$rest = az rest --method put --uri $uri --headers '{\"Content-Type\":\"application/json\"}' --body $body --uri-parameters 'api-version=2018-06-01' #On Prem - Note we are using a managed VNET IR to mimic on prem # $body = ' @@ -74,9 +74,12 @@ if (([Environment]::GetEnvironmentVariable("AdsOpts_CD_Services_DataFactory_OnPr $body = ($body | ConvertTo-Json -compress -Depth 10 | Out-String).Replace('"','\"') $uri = "https://management.azure.com/$env:AdsOpts_CD_ResourceGroup_Id/providers/Microsoft.DataFactory/factories/$env:AdsOpts_CD_Services_DataFactory_Name/integrationRuntimes/$env:AdsOpts_CD_Services_DataFactory_OnPremVnetIr_Name" + '?api-version=2018-06-01' Write-Host "Creating IR: $env:AdsOpts_CD_Services_DataFactory_OnPremVnetIr_Name" - az rest --method put --uri $uri --headers '{\"Content-Type\":\"application/json\"}' --body $body --uri-parameters 'api-version=2018-06-01' + $rest = az rest --method put --uri $uri --headers '{\"Content-Type\":\"application/json\"}' --body $body --uri-parameters 'api-version=2018-06-01' } +$IRA_PostFix = "_" + $env:AdsOpts_CD_Services_DataFactory_AzVnetIR_Name +$IRB_PostFix = "_" + $env:AdsOpts_CD_Services_DataFactory_OnPremVnetIr_Name + $dfbase = "$env:AdsOpts_CD_FolderPaths_PublishUnZip/datafactory" #Data Factory - LinkedServices @@ -108,9 +111,9 @@ Foreach-Object { Copy-Item -Path $fileName -Destination "FileForUpload.json" if ( - (($env:AdsOpts_CD_Services_DataFactory_OnPremVnetIr_Enable -eq "True") -and ($lsName.Contains("_OnPrem_Net") -eq $true)) -or - (($env:AdsOpts_CD_Services_DataFactory_AzVnetIR_Enable -eq "True") -and ($lsName.Contains("_SH_IR") -eq $true)) -or - (($lsName.Contains("_SH_IR") -eq $false) -and ($lsName.Contains("_OnPrem_Net") -eq $false)) + (($env:AdsOpts_CD_Services_DataFactory_OnPremVnetIr_Enable -eq "True") -and ($lsName.Contains($IRB_PostFix) -eq $true)) -or + (($env:AdsOpts_CD_Services_DataFactory_AzVnetIR_Enable -eq "True") -and ($lsName.Contains($IRA_PostFix) -eq $true)) -or + (($lsName.Contains($IRA_PostFix) -eq $false) -and ($lsName.Contains($IRB_PostFix) -eq $false)) ) { write-host ("LinkedService:" + $lsName) -ForegroundColor Yellow -BackgroundColor DarkGreen @@ -118,12 +121,12 @@ Foreach-Object { $body = ($jsonobject | ConvertTo-Json -compress -Depth 10 | Out-String).Replace('"','\"') $uri = "https://management.azure.com/$env:AdsOpts_CD_ResourceGroup_Id/providers/Microsoft.DataFactory/factories/$env:AdsOpts_CD_Services_DataFactory_Name/linkedservices/$name" write-host $uri - az rest --method put --uri $uri --headers '{\"Content-Type\":\"application/json\"}' --body "@FileForUpload.json" --uri-parameters 'api-version=2018-06-01' + $rest = az rest --method put --uri $uri --headers '{\"Content-Type\":\"application/json\"}' --body "@FileForUpload.json" --uri-parameters 'api-version=2018-06-01' } else { - Write-Host "Skipped: $lsName" -ForegroundColor Black -BackgroundColor Yellow + Write-Warning "Skipped: $lsName" -ForegroundColor Black -BackgroundColor Yellow } } @@ -144,20 +147,20 @@ Foreach-Object { Copy-Item -Path $fileName -Destination "FileForUpload.json" if ( - (($env:AdsOpts_CD_Services_DataFactory_OnPremVnetIr_Enable -eq "True") -and ($lsName.Contains("_OnPrem_SH_IR") -eq $true)) -or - (($env:AdsOpts_CD_Services_DataFactory_AzVnetIR_Enable -eq "True") -and ($lsName.Contains("_SH_IR") -eq $true) -and ($lsName.Contains("_OnPrem_SH_IR") -eq $false)) -or - (($lsName.Contains("_SH_IR") -eq $false) -and ($lsName.Contains("_OnPrem_SH_IR") -eq $false)) + (($env:AdsOpts_CD_Services_DataFactory_OnPremVnetIr_Enable -eq "True") -and ($lsName.Contains($IRB_PostFix) -eq $true)) -or + (($env:AdsOpts_CD_Services_DataFactory_AzVnetIR_Enable -eq "True") -and ($lsName.Contains($IRA_PostFix) -eq $true) -and ($lsName.Contains($IRB_PostFix) -eq $false)) -or + (($lsName.Contains($IRA_PostFix) -eq $false) -and ($lsName.Contains($IRB_PostFix) -eq $false)) ) { write-host ("Dataset: " + $fileName) -ForegroundColor Yellow -BackgroundColor DarkGreen #Set-AzDataFactoryV2Dataset -DataFactoryName $env:AdsOpts_CD_Services_DataFactory_Name -ResourceGroupName $env:AdsOpts_CD_ResourceGroup_Name -Name $lsName -DefinitionFile $fileName -Force $body = ($jsonobject | ConvertTo-Json -compress -Depth 10 | Out-String).Replace('"','\"') $uri = "https://management.azure.com/$env:AdsOpts_CD_ResourceGroup_Id/providers/Microsoft.DataFactory/factories/$env:AdsOpts_CD_Services_DataFactory_Name/datasets/$name" - az rest --method put --uri $uri --headers '{\"Content-Type\":\"application/json\"}' --body "@FileForUpload.json" --uri-parameters 'api-version=2018-06-01' + $rest = az rest --method put --uri $uri --headers '{\"Content-Type\":\"application/json\"}' --body "@FileForUpload.json" --uri-parameters 'api-version=2018-06-01' } else { - Write-Host "Skipped: $lsName" -ForegroundColor Black -BackgroundColor Yellow + Write-Warning "Skipped: $lsName" -ForegroundColor Black -BackgroundColor Yellow } } @@ -166,8 +169,11 @@ Foreach-Object { $CurrentPath = (Get-Location).Path Set-Location "..\bin\publish\unzipped\datafactory\pipeline" + #Data Factory - Pipelines -Get-ChildItem "./" -Recurse -Include "AZ-Function*.json", "OnP-SQL-Watermark-OnP-SH-IR.json", "AZ-SQL-Watermark-SH-IR.json" | +Write-Host "Starting Pipelines" +Write-Host "Uploading Level 0 Dependencies" +Get-ChildItem "./" -Recurse -Include "AZ_Function*.json", "OnP_SQL_Watermark_*.json", "AZ_SQL_Watermark_*.json" | Foreach-Object { $lsName = $_.BaseName $fileName = $_.FullName @@ -183,24 +189,25 @@ Foreach-Object { Copy-Item -Path $fileName -Destination "FileForUpload.json" if ( - (($env:AdsOpts_CD_Services_DataFactory_OnPremVnetIr_Enable -eq "True") -and ($lsName.Contains('-OnP-SH-IR') -eq $true)) -or - (($env:AdsOpts_CD_Services_DataFactory_AzVnetIR_Enable -eq "True") -and ($lsName.Contains('-SH-IR') -eq $true) -and ($lsName.Contains('-OnP-SH-IR') -eq $false)) -or - (($lsName.Contains('SH-IR') -eq $false) -and ($lsName.Contains('-OnP-SH-IR') -eq $false)) + (($env:AdsOpts_CD_Services_DataFactory_OnPremVnetIr_Enable -eq "True") -and ($lsName.Contains($IRB_PostFix) -eq $true)) -or + (($env:AdsOpts_CD_Services_DataFactory_AzVnetIR_Enable -eq "True") -and ($lsName.Contains($IRA_PostFix) -eq $true) -and ($lsName.Contains($IRB_PostFix) -eq $false)) -or + (($lsName.Contains($IRA_PostFix) -eq $false) -and ($lsName.Contains($IRB_PostFix) -eq $false)) ) { write-host $fileName -ForegroundColor Yellow -BackgroundColor DarkGreen #Set-AzDataFactoryV2Pipeline -DataFactoryName $env:AdsOpts_CD_Services_DataFactory_Name -ResourceGroupName $env:AdsOpts_CD_ResourceGroup_Name -Name $lsName -DefinitionFile $fileName -force #$body = ($jsonobject | ConvertTo-Json -compress -Depth 100 | Out-String).Replace('"','\"') $uri = "https://management.azure.com/$env:AdsOpts_CD_ResourceGroup_Id/providers/Microsoft.DataFactory/factories/$env:AdsOpts_CD_Services_DataFactory_Name/pipelines/$name" - az rest --method put --uri $uri --headers '{\"Content-Type\":\"application/json\"}' --body "@FileForUpload.json" --uri-parameters 'api-version=2018-06-01' + $rest = az rest --method put --uri $uri --headers '{\"Content-Type\":\"application/json\"}' --body "@FileForUpload.json" --uri-parameters 'api-version=2018-06-01' } else { - Write-Host "Skipped: $lsName" -ForegroundColor Black -BackgroundColor Yellow + Write-Warning "Skipped: $lsName" -ForegroundColor Black -BackgroundColor Yellow } } -Get-ChildItem "./" -Recurse -Include "az-sql-full-load-sh-ir.json", "sh-sql-full-load-sh-ir.json", "onp-sql-full-load-onp-sh-ir.json", "sh-sql-watermark-sh-ir.json" | +Write-Host "Uploading Level 1 Dependencies" +Get-ChildItem "./" -Recurse -Include "az_sql_full_load_*.json", "sh_sql_full_load_*.json", "onp_sql_full_load_*.json", "sh_sql_watermark_*.json" | Foreach-Object { $lsName = $_.BaseName $fileName = $_.FullName @@ -216,23 +223,23 @@ Foreach-Object { Copy-Item -Path $fileName -Destination "FileForUpload.json" if ( - (($env:AdsOpts_CD_Services_DataFactory_OnPremVnetIr_Enable -eq "True") -and ($lsName.Contains('-OnP-SH-IR') -eq $true)) -or - (($env:AdsOpts_CD_Services_DataFactory_AzVnetIR_Enable -eq "True") -and ($lsName.Contains('-SH-IR') -eq $true) -and ($lsName.Contains('-OnP-SH-IR') -eq $false)) -or - (($lsName.Contains('SH-IR') -eq $false) -and ($lsName.Contains('-OnP-SH-IR') -eq $false)) + (($env:AdsOpts_CD_Services_DataFactory_OnPremVnetIr_Enable -eq "True") -and ($lsName.Contains($IRB_PostFix) -eq $true)) -or + (($env:AdsOpts_CD_Services_DataFactory_AzVnetIR_Enable -eq "True") -and ($lsName.Contains($IRA_PostFix) -eq $true) -and ($lsName.Contains($IRB_PostFix) -eq $false)) -or + (($lsName.Contains($IRA_PostFix) -eq $false) -and ($lsName.Contains($IRB_PostFix) -eq $false)) ) { write-host $fileName -ForegroundColor Yellow -BackgroundColor DarkGreen #Set-AzDataFactoryV2Pipeline -DataFactoryName $env:AdsOpts_CD_Services_DataFactory_Name -ResourceGroupName $env:AdsOpts_CD_ResourceGroup_Name -Name $lsName -DefinitionFile $fileName -Force $body = ($jsonobject | ConvertTo-Json -compress -Depth 100 | Out-String).Replace('"','\"') $uri = "https://management.azure.com/$env:AdsOpts_CD_ResourceGroup_Id/providers/Microsoft.DataFactory/factories/$env:AdsOpts_CD_Services_DataFactory_Name/pipelines/$name" - az rest --method put --uri $uri --headers '{\"Content-Type\":\"application/json\"}' --body "@FileForUpload.json" --uri-parameters 'api-version=2018-06-01' + $rest = az rest --method put --uri $uri --headers '{\"Content-Type\":\"application/json\"}' --body "@FileForUpload.json" --uri-parameters 'api-version=2018-06-01' } else { - Write-Host "Skipped: $lsName" -ForegroundColor Black -BackgroundColor Yellow + Write-Warning "Skipped: $lsName" -ForegroundColor Black -BackgroundColor Yellow } } - +Write-Host "Uploading Level 3 Dependencies - Chunks" Get-ChildItem "./" -Filter *chunk*.json | Foreach-Object { $lsName = $_.BaseName @@ -249,25 +256,25 @@ Foreach-Object { Copy-Item -Path $fileName -Destination "FileForUpload.json" if ( - (($env:AdsOpts_CD_Services_DataFactory_OnPremVnetIr_Enable -eq "True") -and ($lsName.Contains('-OnP-SH-IR') -eq $true)) -or - (($env:AdsOpts_CD_Services_DataFactory_AzVnetIR_Enable -eq "True") -and ($lsName.Contains('-SH-IR') -eq $true) -and ($lsName.Contains('-OnP-SH-IR') -eq $false)) -or - (($lsName.Contains('SH-IR') -eq $false) -and ($lsName.Contains('-OnP-SH-IR') -eq $false)) + (($env:AdsOpts_CD_Services_DataFactory_OnPremVnetIr_Enable -eq "True") -and ($lsName.Contains($IRB_PostFix) -eq $true)) -or + (($env:AdsOpts_CD_Services_DataFactory_AzVnetIR_Enable -eq "True") -and ($lsName.Contains($IRA_PostFix) -eq $true) -and ($lsName.Contains($IRB_PostFix) -eq $false)) -or + (($lsName.Contains($IRA_PostFix) -eq $false) -and ($lsName.Contains($IRB_PostFix) -eq $false)) ) { write-host $fileName -ForegroundColor Yellow -BackgroundColor DarkGreen #Set-AzDataFactoryV2Pipeline -DataFactoryName $env:AdsOpts_CD_Services_DataFactory_Name -ResourceGroupName $env:AdsOpts_CD_ResourceGroup_Name -Name $lsName -DefinitionFile $fileName -Force $body = ($jsonobject | ConvertTo-Json -compress -Depth 100 | Out-String).Replace('"','\"') $uri = "https://management.azure.com/$env:AdsOpts_CD_ResourceGroup_Id/providers/Microsoft.DataFactory/factories/$env:AdsOpts_CD_Services_DataFactory_Name/pipelines/$name" - az rest --method put --uri $uri --headers '{\"Content-Type\":\"application/json\"}' --body "@FileForUpload.json" --uri-parameters 'api-version=2018-06-01' + $rest = az rest --method put --uri $uri --headers '{\"Content-Type\":\"application/json\"}' --body "@FileForUpload.json" --uri-parameters 'api-version=2018-06-01' } else { - Write-Host "Skipped: $lsName" -ForegroundColor Black -BackgroundColor Yellow + Write-Warning "Skipped: $lsName" -ForegroundColor Black -BackgroundColor Yellow } } - -Get-ChildItem "./" -Exclude "FileForUpload.json", "Master.json","AZ-Function*.json", "OnP-SQL-Watermark-OnP-SH-IR.json", "AZ-SQL-Watermark-SH-IR.json", "*chunk*.json", "az-sql-full-load-sh-ir.json", "sh-sql-full-load-sh-ir.json", "onp-sql-full-load-onp-sh-ir.json", "sh-sql-watermark-sh-ir.json" | +Write-Host "Uploading Level 4 Dependencies" +Get-ChildItem "./" -Exclude "FileForUpload.json", "Master*.json","AZ_Function*.json", "OnP_SQL_Watermark_*.json", "AZ_SQL_Watermark_*.json", "*chunk*.json", "az_sql_full_load*.json", "sh_sql_full_load*.json", "onp_sql_full_load*.json", "sh_sql_watermark*.json" | Foreach-Object { $lsName = $_.BaseName $fileName = $_.FullName @@ -283,24 +290,25 @@ Foreach-Object { Copy-Item -Path $fileName -Destination "FileForUpload.json" if ( - (($env:AdsOpts_CD_Services_DataFactory_OnPremVnetIr_Enable -eq "True") -and ($lsName.Contains('-OnP-SH-IR') -eq $true)) -or - (($env:AdsOpts_CD_Services_DataFactory_AzVnetIR_Enable -eq "True") -and ($lsName.Contains('-SH-IR') -eq $true) -and ($lsName.Contains('-OnP-SH-IR') -eq $false)) -or - (($lsName.Contains('SH-IR') -eq $false) -and ($lsName.Contains('-OnP-SH-IR') -eq $false)) + (($env:AdsOpts_CD_Services_DataFactory_OnPremVnetIr_Enable -eq "True") -and ($lsName.Contains($IRB_PostFix) -eq $true)) -or + (($env:AdsOpts_CD_Services_DataFactory_AzVnetIR_Enable -eq "True") -and ($lsName.Contains($IRA_PostFix) -eq $true) -and ($lsName.Contains($IRB_PostFix) -eq $false)) -or + (($lsName.Contains($IRA_PostFix) -eq $false) -and ($lsName.Contains($IRB_PostFix) -eq $false)) ) { write-host $fileName -ForegroundColor Yellow -BackgroundColor DarkGreen #Set-AzDataFactoryV2Pipeline -DataFactoryName $env:AdsOpts_CD_Services_DataFactory_Name -ResourceGroupName $env:AdsOpts_CD_ResourceGroup_Name -Name $lsName -DefinitionFile $fileName -Force $body = ($jsonobject | ConvertTo-Json -compress -Depth 100 | Out-String).Replace('"','\"') $uri = "https://management.azure.com/$env:AdsOpts_CD_ResourceGroup_Id/providers/Microsoft.DataFactory/factories/$env:AdsOpts_CD_Services_DataFactory_Name/pipelines/$name" - az rest --method put --uri $uri --headers '{\"Content-Type\":\"application/json\"}' --body "@FileForUpload.json" --uri-parameters 'api-version=2018-06-01' + $rest = az rest --method put --uri $uri --headers '{\"Content-Type\":\"application/json\"}' --body "@FileForUpload.json" --uri-parameters 'api-version=2018-06-01' } else { - Write-Host "Skipped: $lsName" -ForegroundColor Black -BackgroundColor Yellow + Write-Warning "Skipped: $lsName" -ForegroundColor Black -BackgroundColor Yellow } } -Get-ChildItem "./" -Filter Master.json | +Write-Host "Processing Master" +Get-ChildItem "./" -Filter Master*.json | Foreach-Object { #filter out any undelpoyed pipelines $jsonobject = $_ | Get-Content | ConvertFrom-Json @@ -309,9 +317,9 @@ Foreach-Object { $casesfiltered = @() foreach ($item in $cases) { if ( - (($env:AdsOpts_CD_Services_DataFactory_OnPremVnetIr_Enable -eq "True") -and ($item.Value.Contains('-OnP-SH-IR') -eq $true)) -or - (($env:AdsOpts_CD_Services_DataFactory_AzVnetIR_Enable -eq "True") -and ($item.Value.Contains('-SH-IR') -eq $true) -and ($item.Value.Contains('-OnP-SH-IR') -eq $false)) -or - (($item.Value.Contains('SH-IR') -eq $false) -and ($item.Value.Contains('-OnP-SH-IR') -eq $false)) + (($env:AdsOpts_CD_Services_DataFactory_OnPremVnetIr_Enable -eq "True") -and ($item.Value.Contains($IRB_PostFix) -eq $true)) -or + (($env:AdsOpts_CD_Services_DataFactory_AzVnetIR_Enable -eq "True") -and ($item.Value.Contains($IRA_PostFix) -eq $true) -and ($item.Value.Contains($IRB_PostFix) -eq $false)) -or + (($item.Value.Contains($IRA_PostFix) -eq $false) -and ($item.Value.Contains($IRB_PostFix) -eq $false)) ) { #Exclude pipelines that require Self Hosted IRs (eg. Excel ) TODO - make this optional @@ -322,7 +330,7 @@ Foreach-Object { } else { - Write-Host "Skipped: $item.Value" -ForegroundColor Black -BackgroundColor Yellow + Write-Warning "Skipped: $item.Value" -ForegroundColor Black -BackgroundColor Yellow } } $jsonobject.properties.activities[0].typeProperties.cases = $casesfiltered @@ -340,7 +348,8 @@ Foreach-Object { #Set-AzDataFactoryV2Pipeline -DataFactoryName $env:AdsOpts_CD_Services_DataFactory_Name -ResourceGroupName $env:AdsOpts_CD_ResourceGroup_Name -Name $lsName -DefinitionFile $fileName -Force $body = ($jsonobject | ConvertTo-Json -compress -Depth 100 | Out-String).Replace('"','\"') $uri = "https://management.azure.com/$env:AdsOpts_CD_ResourceGroup_Id/providers/Microsoft.DataFactory/factories/$env:AdsOpts_CD_Services_DataFactory_Name/pipelines/$name" - az rest --method put --uri $uri --headers '{\"Content-Type\":\"application/json\"}' --body "@FileForUpload.json" --uri-parameters 'api-version=2018-06-01' + Write-Host "Uploading Master" + $rest = az rest --method put --uri $uri --headers '{\"Content-Type\":\"application/json\"}' --body "@FileForUpload.json" --uri-parameters 'api-version=2018-06-01' } diff --git a/solution/Deployment/workflows/Steps/CD_ConfigureAzureSQLServer.ps1 b/solution/Deployment/workflows/Steps/CD_ConfigureAzureSQLServer.ps1 index 83c9c8d4..4a213e29 100644 --- a/solution/Deployment/workflows/Steps/CD_ConfigureAzureSQLServer.ps1 +++ b/solution/Deployment/workflows/Steps/CD_ConfigureAzureSQLServer.ps1 @@ -22,7 +22,7 @@ function GeneratePassword { } -Write-Host "Configuring Azure SQL Server" +Write-Debug "Configuring Azure SQL Server" #Install Sql Server Module Install-Module -Name SqlServer -Force @@ -73,7 +73,7 @@ if($env:AdsOpts_CD_Services_AzureSQLServer_AdsGoFastDB_Enable -eq "True") LogAnalyticsWorkspaceId = '$LogAnalyticsId' where id = 1" - Write-Host "Updating DataFactory in ADS Go Fast DB Config - DataFactory" + Write-Debug "Updating DataFactory in ADS Go Fast DB Config - DataFactory" Invoke-Sqlcmd -ServerInstance "$targetserver,1433" -Database $env:AdsOpts_CD_Services_AzureSQLServer_AdsGoFastDB_Name -AccessToken "$token" -Query $sql } if($env:AdsOpts_CD_Services_AzureSQLServer_AdsGoFastDB_UpdateSourceAndTargetSystems -eq "True") @@ -95,7 +95,7 @@ if($env:AdsOpts_CD_Services_AzureSQLServer_AdsGoFastDB_Enable -eq "True") Set SystemServer = '$env:AdsOpts_CD_Services_AzureSQLServer_Name.database.windows.net', SystemKeyVaultBaseUrl = 'https://$env:AdsOpts_CD_Services_KeyVault_Name.vault.azure.net/', - SystemJSON = '{ ""Database"" : ""$env:AdsOpts_CD_Services_AzureSQLServer_Staging_Name"" }' + SystemJSON = '{ ""Database"" : ""$env:AdsOpts_CD_Services_AzureSQLServer_StagingDB_Name"" }' Where SystemId = '2' GO @@ -111,7 +111,7 @@ if($env:AdsOpts_CD_Services_AzureSQLServer_AdsGoFastDB_Enable -eq "True") GO " - Write-Host "Updating DataFactory in ADS Go Fast DB Config - SourceAndTargetSystems - Azure SQL Servers" + Write-Debug "Updating DataFactory in ADS Go Fast DB Config - SourceAndTargetSystems - Azure SQL Servers" Invoke-Sqlcmd -ServerInstance "$targetserver,1433" -Database $env:AdsOpts_CD_Services_AzureSQLServer_AdsGoFastDB_Name -AccessToken "$token" -Query $sql $sql = @@ -167,7 +167,7 @@ if($env:AdsOpts_CD_Services_AzureSQLServer_AdsGoFastDB_Enable -eq "True") GO " - Write-Host "Updating DataFactory in ADS Go Fast DB Config - SourceAndTargetSystems - Storage Accounts" + Write-Debug "Updating DataFactory in ADS Go Fast DB Config - SourceAndTargetSystems - Storage Accounts" Invoke-Sqlcmd -ServerInstance "$targetserver,1433" -Database $env:AdsOpts_CD_Services_AzureSQLServer_AdsGoFastDB_Name -AccessToken "$token" -Query $sql } @@ -181,5 +181,5 @@ if($env:AdsOpts_CD_Services_AzureSQLServer_AdsGoFastDB_Enable -eq "True") } else { - Write-Host "Skipped Configuring Azure SQL Server" + Write-Warning "Skipped Configuring Azure SQL Server" } \ No newline at end of file diff --git a/solution/Deployment/workflows/Steps/CD_ConfigureAzureSqlServer_UpdateTaskTypeMappingJson.ps1 b/solution/Deployment/workflows/Steps/CD_ConfigureAzureSqlServer_UpdateTaskTypeMappingJson.ps1 index a87fcc82..e10e0636 100644 --- a/solution/Deployment/workflows/Steps/CD_ConfigureAzureSqlServer_UpdateTaskTypeMappingJson.ps1 +++ b/solution/Deployment/workflows/Steps/CD_ConfigureAzureSqlServer_UpdateTaskTypeMappingJson.ps1 @@ -21,5 +21,11 @@ Foreach-Object { } +#Loop through all Existing Task Master JSON Entries in DB +#$tm = Invoke-Sqlcmd -ServerInstance "$targetserver,1433" -Database $env:AdsOpts_CD_Services_AzureSQLServer_AdsGoFastDB_Name -AccessToken "$token" -Query "Select * from dbo.TaskMaster" + +#$ttm = Invoke-Sqlcmd -ServerInstance "$targetserver,1433" -Database $env:AdsOpts_CD_Services_AzureSQLServer_AdsGoFastDB_Name -AccessToken "$token" -Query "Select * from dbo.TaskTypeMapping" + + diff --git a/solution/Deployment/workflows/Steps/CD_ConfigureFunctionApp.ps1 b/solution/Deployment/workflows/Steps/CD_ConfigureFunctionApp.ps1 index fc049d5f..2c02648b 100644 --- a/solution/Deployment/workflows/Steps/CD_ConfigureFunctionApp.ps1 +++ b/solution/Deployment/workflows/Steps/CD_ConfigureFunctionApp.ps1 @@ -1,4 +1,4 @@ -Write-Host "Configuring Function App" +Write-Debug "Configuring Function App" $SourceFile = $env:AdsOpts_CD_FolderPaths_PublishZip + "/functionapp/Publish.zip" if($env:AdsOpts_CD_Services_CoreFunctionApp_Enable -eq "True") @@ -41,10 +41,10 @@ if($env:AdsOpts_CD_Services_CoreFunctionApp_Enable -eq "True") Set-Location $CurrentPath # Deploy CoreFunctionApp App - az functionapp deployment source config-zip --resource-group $env:AdsOpts_CD_ResourceGroup_Name --name $env:AdsOpts_CD_Services_CoreFunctionApp_Name --src $SourceFile + $result = az functionapp deployment source config-zip --resource-group $env:AdsOpts_CD_ResourceGroup_Name --name $env:AdsOpts_CD_Services_CoreFunctionApp_Name --src $SourceFile } else { - Write-Host "Skipped Configuring Function App" + Write-Warning "Skipped Configuring Function App" } diff --git a/solution/Deployment/workflows/Steps/CD_ConfigureKeyVault.ps1 b/solution/Deployment/workflows/Steps/CD_ConfigureKeyVault.ps1 index 1381e60c..1857c68c 100644 --- a/solution/Deployment/workflows/Steps/CD_ConfigureKeyVault.ps1 +++ b/solution/Deployment/workflows/Steps/CD_ConfigureKeyVault.ps1 @@ -1,4 +1,4 @@ -Write-Host "Configuring Key Vault" +Write-Debug"Configuring Key Vault" if($env:AdsOpts_CD_Services_KeyVault_Enable -eq "True") { @@ -12,7 +12,7 @@ if($env:AdsOpts_CD_Services_KeyVault_Enable -eq "True") $functionkey = (az functionapp keys list -g $env:AdsOpts_CD_ResourceGroup_Name -n $env:AdsOpts_CD_Services_CoreFunctionApp_Name | ConvertFrom-Json).functionKeys.default - Write-Host "Enabling Access to KeyVault and Adding Secrets" + Write-Debug"Enabling Access to KeyVault and Adding Secrets" #Set KeyVault Policy to allow logged in user to add key az keyvault set-policy --name $env:AdsOpts_CD_Services_KeyVault_Name --certificate-permissions backup create delete deleteissuers get getissuers import list listissuers managecontacts manageissuers purge recover restore setissuers update --key-permissions backup create decrypt delete encrypt get import list purge recover restore sign unwrapKey update verify wrapKey --object-id $AADUserId --resource-group $env:AdsOpts_CD_ResourceGroup_Name --secret-permissions backup delete get list purge recover restore set --storage-permissions backup delete deletesas get getsas list listsas purge recover regeneratekey restore set setsas update --subscription $env:AdsOpts_CD_ResourceGroup_Subscription #Set KeyVault Policy to allow MSI for ADF to Retrieve Key Vault Key @@ -24,5 +24,5 @@ if($env:AdsOpts_CD_Services_KeyVault_Enable -eq "True") } else { - Write-Host "Skipped Configuring Key Vault" + Write-Warning "Skipped Configuring Key Vault" } diff --git a/solution/Deployment/workflows/Steps/CD_ConfigureSampleData.ps1 b/solution/Deployment/workflows/Steps/CD_ConfigureSampleData.ps1 new file mode 100644 index 00000000..bdff6184 --- /dev/null +++ b/solution/Deployment/workflows/Steps/CD_ConfigureSampleData.ps1 @@ -0,0 +1,10 @@ + +$pathbase = "./../../SampleFiles/" +$files = @("yellow_tripdata_2017-03.xlsx","yellow_tripdata_2017-03.csv") + +$files | ForEach-Object -Parallel { + + az storage blob upload --file $using:pathbase/$_ --container-name "datalakeraw" --name samples/$_ --account-name $env:AdsOpts_CD_Services_Storage_ADLS_Name + az storage blob upload --file $using:pathbase/$_ --container-name "datalakeraw" --name samples/$_ --account-name $env:AdsOpts_CD_Services_Storage_Blob_Name +} + diff --git a/solution/Deployment/workflows/Steps/CD_ConfigureVnet.ps1 b/solution/Deployment/workflows/Steps/CD_ConfigureVnet.ps1 index 2574a5bd..5d6e6fbd 100644 --- a/solution/Deployment/workflows/Steps/CD_ConfigureVnet.ps1 +++ b/solution/Deployment/workflows/Steps/CD_ConfigureVnet.ps1 @@ -3,16 +3,16 @@ if($env:AdsOpts_CD_Services_Vnet_Enable -eq "True") ############################################################################################################## #Firewall and Virtual Network Rules for Services ############################################################################################################## - Write-Host "Configuring VNet rules for provisioned services" + Write-Debug"Configuring VNet rules for provisioned services" #Enable Service Endpoints on the subnet. Required to be done before adding network rules. az network vnet subnet update --resource-group $env:AdsOpts_CD_ResourceGroup_Name --vnet-name $env:AdsOpts_CD_Services_Vnet_Name --name $env:AdsOpts_CD_Services_Vnet_DataSubnetName --service-endpoints Microsoft.Sql Microsoft.Storage Microsoft.KeyVault Microsoft.Web - Write-Host "Configured Microsoft.Storage Service Endpoint to subnet $env:AdsOpts_CD_Services_Vnet_DataSubnetName" + Write-Debug"Configured Microsoft.Storage Service Endpoint to subnet $env:AdsOpts_CD_Services_Vnet_DataSubnetName" #adls az storage account network-rule add --resource-group $env:AdsOpts_CD_ResourceGroup_Name --account-name $env:AdsOpts_CD_Services_Storage_ADLS_Name --vnet-name $env:AdsOpts_CD_Services_Vnet_Name --subnet $env:AdsOpts_CD_Services_Vnet_DataSubnetName az storage account update --resource-group $env:AdsOpts_CD_ResourceGroup_Name --name $env:AdsOpts_CD_Services_Storage_ADLS_Name --default-action Deny #Default action to apply when no rule matches i.e. allow access from selected network/PE only. - Write-Host "Completed network rule configuration for storage $env:AdsOpts_CD_Services_Storage_ADLS_Name" + Write-Debug"Completed network rule configuration for storage $env:AdsOpts_CD_Services_Storage_ADLS_Name" #adlstran #Note: Skipping as it will be outside for importing data into it. @@ -21,23 +21,23 @@ if($env:AdsOpts_CD_Services_Vnet_Enable -eq "True") #Note: Commented below to allow App Insights access to Storage account. # az storage account network-rule add --resource-group $env:AdsOpts_CD_ResourceGroup_Name --account-name $env:AdsOpts_CD_Services_Storage_Logging_Name --vnet-name $env:AdsOpts_CD_Services_Vnet_Name --subnet $env:AdsOpts_CD_Services_Vnet_DataSubnetName # az storage account update --resource-group $env:AdsOpts_CD_ResourceGroup_Name --name $env:AdsOpts_CD_Services_Storage_Logging_Name --default-action Deny #Default action to apply when no rule matches i.e. allow access from selected network/PE only. - # Write-Host "Completed network rule configuration for storage $env:AdsOpts_CD_Services_Storage_Logging_Name" + # Write-Debug"Completed network rule configuration for storage $env:AdsOpts_CD_Services_Storage_Logging_Name" #Blob az storage account network-rule add --resource-group $env:AdsOpts_CD_ResourceGroup_Name --account-name $env:AdsOpts_CD_Services_Storage_Blob_Name --vnet-name $env:AdsOpts_CD_Services_Vnet_Name --subnet $env:AdsOpts_CD_Services_Vnet_DataSubnetName az storage account update --resource-group $env:AdsOpts_CD_ResourceGroup_Name --name $env:AdsOpts_CD_Services_Storage_Blob_Name --default-action Deny #Default action to apply when no rule matches i.e. allow access from selected network/PE only. - Write-Host "Completed network rule configuration for storage $env:AdsOpts_CD_Services_Storage_Blob_Name" + Write-Debug"Completed network rule configuration for storage $env:AdsOpts_CD_Services_Storage_Blob_Name" #Azure Sql #Note: Commenting below as will use the PE for SQL, so vNet rule is not required. # az sql server vnet-rule create --server $env:AdsOpts_CD_Services_AzureSQLServer_Name --name $env:AdsOpts_CD_Services_Vnet_Name --resource-group $env:AdsOpts_CD_ResourceGroup_Name --vnet-name $env:AdsOpts_CD_Services_Vnet_Name --subnet $env:AdsOpts_CD_Services_Vnet_DataSubnetName - # Write-Host "Completed network rule configuration for Azure SQL Server $env:AdsOpts_CD_Services_AzureSQLServer_Name" + # Write-Debug"Completed network rule configuration for Azure SQL Server $env:AdsOpts_CD_Services_AzureSQLServer_Name" #Key Vault #Note: Commenting below as will use the PE for AKV, so vNet rule is not required. # az keyvault network-rule add --name $env:AdsOpts_CD_Services_KeyVault_Name --resource-group $env:AdsOpts_CD_ResourceGroup_Name --vnet-name $env:AdsOpts_CD_Services_Vnet_Name --subnet $env:AdsOpts_CD_Services_Vnet_DataSubnetName # az keyvault update --name $env:AdsOpts_CD_Services_KeyVault_Name --resource-group $env:AdsOpts_CD_ResourceGroup_Name --default-action Deny #Default action to apply when no rule matches i.e. allow access from selected network/PE only. - # Write-Host "Completed network rule configuration for Azure Key Vault $env:AdsOpts_CD_Services_KeyVault_Name" + # Write-Debug"Completed network rule configuration for Azure Key Vault $env:AdsOpts_CD_Services_KeyVault_Name" #Azure Function App az appservice plan update --name $env:AdsOpts_CD_Services_AppPlans_FunctionApp_Name --resource-group $env:AdsOpts_CD_ResourceGroup_Name --sku P1V2 #Upgrade SKU to 'P1V2' to support PE and vNet Integration. @@ -47,18 +47,18 @@ if($env:AdsOpts_CD_Services_Vnet_Enable -eq "True") az functionapp config access-restriction add --priority 300 --rule-name "Allow WebApp Subnet" --resource-group $env:AdsOpts_CD_ResourceGroup_Name --name $env:AdsOpts_CD_Services_CoreFunctionApp_Name --vnet-name $env:AdsOpts_CD_Services_Vnet_Name --subnet $env:AdsOpts_CD_Services_Vnet_WebAppSubnetName --description "Allow vNet" --action Allow #Configure Access Restrictions. az functionapp config access-restriction add --priority 400 --rule-name "Allow Azure Portal" --resource-group $env:AdsOpts_CD_ResourceGroup_Name --name $env:AdsOpts_CD_Services_CoreFunctionApp_Name --service-tag AzurePortal --description "Allow vNet" --action Allow #Configure Access Restrictions. az functionapp config access-restriction add --priority 500 --rule-name "Allow ADF" --resource-group $env:AdsOpts_CD_ResourceGroup_Name --name $env:AdsOpts_CD_Services_CoreFunctionApp_Name --service-tag DataFactory --description "Allow vNet" --action Allow #Configure Access Restrictions. - Write-Host "Completed network rule configuration for Azure Function App $env:AdsOpts_CD_Services_CoreFunctionApp_Name" + Write-Debug"Completed network rule configuration for Azure Function App $env:AdsOpts_CD_Services_CoreFunctionApp_Name" #Azure Web App az appservice plan update --name $env:AdsOpts_CD_Services_AppPlans_WebApp_Name --resource-group $env:AdsOpts_CD_ResourceGroup_Name --sku P1V2 #Upgrade SKU to 'P1V2' to support PE. az webapp vnet-integration add --resource-group $env:AdsOpts_CD_ResourceGroup_Name --name $env:AdsOpts_CD_Services_Website_Name --vnet $env:AdsOpts_CD_Services_Vnet_Name --subnet $env:AdsOpts_CD_Services_Vnet_WebAppSubnetName #Note: Commented below to keep Web App accessible over internet. For customers who have VPN/Express Route/use Bastion -> Uncomment below. #az functionapp config access-restriction add --priority 100 --resource-group $env:AdsOpts_CD_ResourceGroup_Name --name $env:AdsOpts_CD_Services_Website_Name --vnet-name $env:AdsOpts_CD_Services_Vnet_Name --subnet $env:AdsOpts_CD_Services_Vnet_WebAppSubnetName --description "Allow vNet" --action Allow #Configure Access Restrictions. - Write-Host "Completed network rule configuration for Azure Web App $env:AdsOpts_CD_Services_Website_Name" + Write-Debug"Completed network rule configuration for Azure Web App $env:AdsOpts_CD_Services_Website_Name" } else { - Write-Host "Skipped Configuration of Vnet rules for provisioned services" + Write-Warning "Skipped Configuration of Vnet rules for provisioned services" } if($env:AdsOpts_CD_Services_Vnet_Enable -eq "True") @@ -66,7 +66,7 @@ if($env:AdsOpts_CD_Services_Vnet_Enable -eq "True") ############################################################################################################## #Private Endpoints for Services (ADF Managed and vNet Managed) ############################################################################################################## - Write-Host "Configuring Private Endpoints for provisioned services" + Write-Debug"Configuring Private Endpoints for provisioned services" ########## ADF Managed PE @@ -78,42 +78,42 @@ if($env:AdsOpts_CD_Services_Vnet_Enable -eq "True") $managedPrivateEndpointName = "ADF-Managed-PE-"+$env:AdsOpts_CD_Services_Storage_ADLS_Name $privateEndpointResourceId = "$env:AdsOpts_CD_ResourceGroup_Id/providers/Microsoft.DataFactory/factories/$env:AdsOpts_CD_Services_DataFactory_Name/managedVirtualNetworks/default/managedprivateendpoints/${managedPrivateEndpointName}" $privateLinkResourceId = az storage account show --name $env:AdsOpts_CD_Services_Storage_ADLS_Name --resource-group $env:AdsOpts_CD_ResourceGroup_Name --query '[id][0]' --output tsv - Write-Host "Creating $managedPrivateEndpointName" + Write-Debug"Creating $managedPrivateEndpointName" New-AzResource -Force -ApiVersion "${apiVersion}" -ResourceId "${privateEndpointResourceId}" -Properties @{ privateLinkResourceId = "${privateLinkResourceId}" groupId = "dfs" } - Write-Host "Created $managedPrivateEndpointName" + Write-Debug"Created $managedPrivateEndpointName" #Blob (ADF Managed PE) $managedPrivateEndpointName = "ADF-Managed-PE-"+$env:AdsOpts_CD_Services_Storage_Blob_Name $privateEndpointResourceId = "$env:AdsOpts_CD_ResourceGroup_Id/providers/Microsoft.DataFactory/factories/$env:AdsOpts_CD_Services_DataFactory_Name/managedVirtualNetworks/default/managedprivateendpoints/${managedPrivateEndpointName}" $privateLinkResourceId = az storage account show --name $env:AdsOpts_CD_Services_Storage_Blob_Name --resource-group $env:AdsOpts_CD_ResourceGroup_Name --query '[id][0]' --output tsv - Write-Host "Creating $managedPrivateEndpointName" + Write-Debug"Creating $managedPrivateEndpointName" New-AzResource -Force -ApiVersion "${apiVersion}" -ResourceId "${privateEndpointResourceId}" -Properties @{ privateLinkResourceId = "${privateLinkResourceId}" groupId = "blob" } - Write-Host "Created $managedPrivateEndpointName" + Write-Debug"Created $managedPrivateEndpointName" #Note: Commented below to allow On-Prem SHIR to communicate with AKV (Non-VPN,Non-ExpressRoute, Non-Peering scenario) # #Key Vault (ADF Managed PE) # $managedPrivateEndpointName = "ADF-Managed-PE-"+$env:AdsOpts_CD_Services_KeyVault_Name # $privateEndpointResourceId = "$env:AdsOpts_CD_ResourceGroup_Id/providers/Microsoft.DataFactory/factories/$env:AdsOpts_CD_Services_DataFactory_Name/managedVirtualNetworks/default/managedprivateendpoints/${managedPrivateEndpointName}" # $privateLinkResourceId = az keyvault show --name $env:AdsOpts_CD_Services_KeyVault_Name --resource-group $env:AdsOpts_CD_ResourceGroup_Name --query '[id][0]' --output tsv - # Write-Host "Creating $managedPrivateEndpointName" + # Write-Debug"Creating $managedPrivateEndpointName" # New-AzResource -Force -ApiVersion "${apiVersion}" -ResourceId "${privateEndpointResourceId}" -Properties @{ # privateLinkResourceId = "${privateLinkResourceId}" # groupId = "vault" # } - # Write-Host "Created $managedPrivateEndpointName" + # Write-Debug"Created $managedPrivateEndpointName" ########## PEs #Note: Not using to allow SHIR to communicate with AKV (Non-VPN,Non-ExpressRoute, Peering scenario) #Key Vault (vNet Managed PE) - Write-Host "Creating PE for Azure Key Vault" + Write-Debug"Creating PE for Azure Key Vault" $PE_Name = "PE-AKV" $Private_DNS_Zone_Name = "privatelink.vaultcore.azure.net" $id= az keyvault list --resource-group $env:AdsOpts_CD_ResourceGroup_Name --query '[].[id]' --output tsv @@ -146,11 +146,11 @@ if($env:AdsOpts_CD_Services_Vnet_Enable -eq "True") --private-dns-zone $Private_DNS_Zone_Name ` --zone-name "akv" - Write-Host "Completed PE creation for Azure Key Vault" + Write-Debug"Completed PE creation for Azure Key Vault" #Azure Sql (vNet Managed) - Write-Host "Creating PE for Azure SQL Server" + Write-Debug"Creating PE for Azure SQL Server" $PE_Name = "PE-SQL" $Private_DNS_Zone_Name = "privatelink.database.windows.net" $id= az sql server list --resource-group $env:AdsOpts_CD_ResourceGroup_Name --query '[].[id]' --output tsv @@ -183,21 +183,21 @@ if($env:AdsOpts_CD_Services_Vnet_Enable -eq "True") --private-dns-zone $Private_DNS_Zone_Name ` --zone-name "sql" - Write-Host "Completed PE creation for Azure SQL Server" + Write-Debug"Completed PE creation for Azure SQL Server" ############################## #Post PE Addition Task az sql server update --name $env:AdsOpts_CD_Services_AzureSQLServer_Name --resource-group $env:AdsOpts_CD_ResourceGroup_Name --enable-public-network false #Disable Public Access (Requires PE to be enabled first). - Write-Host "Disabled Public Access to Azure SQL" + Write-Debug"Disabled Public Access to Azure SQL" #Enable soft-delete on AKV az keyvault update --name $env:AdsOpts_CD_Services_KeyVault_Name --enable-soft-delete true - Write-Host "Soft-delete enabled on AKV" + Write-Debug"Soft-delete enabled on AKV" ############################## } else { - Write-Host "Skipped Configuration of Private Endpoints for provisioned services" + Write-Warning "Skipped Configuration of Private Endpoints for provisioned services" } \ No newline at end of file diff --git a/solution/Deployment/workflows/Steps/CD_ConfigureWebApp.ps1 b/solution/Deployment/workflows/Steps/CD_ConfigureWebApp.ps1 index 39dc9dbd..1cf34829 100644 --- a/solution/Deployment/workflows/Steps/CD_ConfigureWebApp.ps1 +++ b/solution/Deployment/workflows/Steps/CD_ConfigureWebApp.ps1 @@ -1,4 +1,4 @@ -Write-Host "Configuring Web App" +Write-Debug"Configuring Web App" $SourceFile = $env:AdsOpts_CD_FolderPaths_PublishZip + "/webapplication/Publish.zip" if($env:AdsOpts_CD_Services_WebSite_Enable -eq "True") @@ -43,5 +43,5 @@ if($env:AdsOpts_CD_Services_WebSite_Enable -eq "True") } else { - Write-Host "Skipped Configuring Web App" + Write-Warning "Skipped Configuring Web App" } \ No newline at end of file diff --git a/solution/Deployment/workflows/Steps/CD_DeployADF.ps1 b/solution/Deployment/workflows/Steps/CD_DeployADF.ps1 index 0bd9fb84..525cb189 100644 --- a/solution/Deployment/workflows/Steps/CD_DeployADF.ps1 +++ b/solution/Deployment/workflows/Steps/CD_DeployADF.ps1 @@ -1,10 +1,10 @@ if ($env:AdsOpts_CD_Services_DataFactory_Enable -eq "True") { - Write-Host "Creating Data Factory" + Write-Debug"Creating Data Factory" az deployment group create -g $env:AdsOpts_CD_ResourceGroup_Name --template-file ./../arm/DataFactory.json --parameters location=$env:AdsOpts_CD_ResourceGroup_Location adf-name=$env:AdsOpts_CD_Services_DataFactory_Name } else { - Write-Host "Skipped Creation of Data Factory" + Write-Warning "Skipped Creation of Data Factory" } diff --git a/solution/Deployment/workflows/Steps/CD_DeployADFOnPremSHIR.ps1 b/solution/Deployment/workflows/Steps/CD_DeployADFOnPremSHIR.ps1 index 885336ea..37cc000c 100644 --- a/solution/Deployment/workflows/Steps/CD_DeployADFOnPremSHIR.ps1 +++ b/solution/Deployment/workflows/Steps/CD_DeployADFOnPremSHIR.ps1 @@ -12,16 +12,16 @@ $ADFJDKDownloadURL = $env:AdsOpts_CD_Services_DataFactory_OnPremVnetIr_IrInstall $ADFJDKLocalFileName = $ADFJDKDownloadURL.Split("/")[$ADFJDKDownloadURL.Split("/").Length-1] #Get the .msi filename. $ADFJDKInstallerLocalFileLocation = $ADFLocalDrive + '\' + $ADFLocalVMFolder + '\' + $ADFJDKLocalFileName #Local Path of downloaded installer. -Write-Host "Creating directory to download the SHIR installable." +Write-Debug"Creating directory to download the SHIR installable." New-Item -Path $ADFLocalDrive -Name $ADFLocalVMFolder -ItemType Directory -Force #'-Force' Ok if directory already exists. -Write-Host "Downloading the SHIR installable at $ADFIRInstallerLocalFileLocation." +Write-Debug"Downloading the SHIR installable at $ADFIRInstallerLocalFileLocation." Invoke-WebRequest -Uri $ADFIRDownloadURL -OutFile $ADFIRInstallerLocalFileLocation #Download SHIR installable. -Write-Host "Downloading the OpenJDK for SHIR at $ADFJDKInstallerLocalFileLocation." +Write-Debug"Downloading the OpenJDK for SHIR at $ADFJDKInstallerLocalFileLocation." Invoke-WebRequest -Uri $ADFJDKDownloadURL -OutFile $ADFJDKInstallerLocalFileLocation #Download OpenJDK. -Write-Host "Installing OpenJDK." +Write-Debug"Installing OpenJDK." #msiexec /i $ADFJDKInstallerLocalFileLocation ADDLOCAL=FeatureMain,FeatureEnvironment,FeatureJarFileRunWith,FeatureJavaHome /quiet #Ensure command prompt is run as administrator @@ -34,17 +34,17 @@ $MSIInstallArguments = @( "/qb!" "/norestart" ) -Write-Host $MSIInstallArguments +Write-Debug$MSIInstallArguments Start-Process "msiexec.exe" -ArgumentList $MSIInstallArguments -Wait -NoNewWindow -Write-Host "Installing SHIR." +Write-Debug"Installing SHIR." # Data Factory - VM AZ IR - Install IR # $irKey1 = az datafactory integration-runtime list-auth-key --factory-name $DataFactoryName --name "SelfHostedIntegrationRuntime-Azure-VNET" --resource-group $ResourceGroupName --query authKey1 --out tsv # az vm run-command invoke --command-id RunPowerShellScript --name $VMAzIR -g $ResourceGroupName --scripts "$ADFIRInstallerLocalFileLocation -path $ADFIRLocalFileLocation -authKey '$irKey1'" # $irKey1 = az datafactory integration-runtime list-auth-key --factory-name $env:AdsOpts_CD_Services_DataFactory_Name --name $env:AdsOpts_CD_Services_DataFactory_OnPremVnetIr_Name --resource-group $env:AdsOpts_CD_ResourceGroup_Name --query authKey1 --out tsv -Write-Host "irKey1 retrieved." +Write-Debug"irKey1 retrieved." # #Ensure command prompt is run as administrator # $MSIInstallArguments = @( @@ -54,7 +54,7 @@ Write-Host "irKey1 retrieved." # "/qb!" # "/norestart" # ) -# Write-Host $MSIInstallArguments +# Write-Debug$MSIInstallArguments # Start-Process "msiexec.exe" -ArgumentList $MSIInstallArguments -Wait -NoNewWindow . .\Steps\InstallGatewayFunctions.ps1 -path "$ADFIRInstallerLocalFileLocation" -authKey "$irKey1" \ No newline at end of file diff --git a/solution/Deployment/workflows/Steps/CD_DeployAppInsights.ps1 b/solution/Deployment/workflows/Steps/CD_DeployAppInsights.ps1 index 80f1f462..884bfa9b 100644 --- a/solution/Deployment/workflows/Steps/CD_DeployAppInsights.ps1 +++ b/solution/Deployment/workflows/Steps/CD_DeployAppInsights.ps1 @@ -1,12 +1,12 @@ if ($env:AdsOpts_CD_Services_AppInsights_Enable -eq "True") { - Write-Host "Creating App Insights" + Write-Debug"Creating App Insights" $storageaccountkey = (az storage account keys list -g $env:AdsOpts_CD_ResourceGroup_Name -n $env:AdsOpts_CD_Services_Storage_Logging_Name | ConvertFrom-Json)[0].value az deployment group create -g $env:AdsOpts_CD_ResourceGroup_Name --template-file ./../arm/ApplicationInsights.json --parameters location=$env:AdsOpts_CD_ResourceGroup_Location appinsights_name=$env:AdsOpts_CD_Services_AppInsights_Name } else { - Write-Host "Skipped Creation of App Insights" + Write-Warning "Skipped Creation of App Insights" } diff --git a/solution/Deployment/workflows/Steps/CD_DeployAppService.ps1 b/solution/Deployment/workflows/Steps/CD_DeployAppService.ps1 index 02bcf436..146f0c99 100644 --- a/solution/Deployment/workflows/Steps/CD_DeployAppService.ps1 +++ b/solution/Deployment/workflows/Steps/CD_DeployAppService.ps1 @@ -1,7 +1,7 @@ if ($env:AdsOpts_CD_Services_AppPlans_WebApp_Enable -eq "True") { - Write-Host "Creating App Service for Web App" + Write-Debug"Creating App Service for Web App" #App Service (Includes both functions and web) $storageaccountkey = (az storage account keys list -g $env:AdsOpts_CD_ResourceGroup_Name -n $env:AdsOpts_CD_Services_Storage_Logging_Name | ConvertFrom-Json)[0].value @@ -9,13 +9,13 @@ if ($env:AdsOpts_CD_Services_AppPlans_WebApp_Enable -eq "True") } else { - Write-Host "Skipped Creation of App Service for Web App" + Write-Warning "Skipped Creation of App Service for Web App" } if ($env:AdsOpts_CD_Services_AppPlans_FunctionApp_Enable -eq "True") { - Write-Host "Creating App Service for Function App" + Write-Debug"Creating App Service for Function App" #App Service (Includes both functions and web) $storageaccountkey = (az storage account keys list -g $env:AdsOpts_CD_ResourceGroup_Name -n $env:AdsOpts_CD_Services_Storage_Logging_Name | ConvertFrom-Json)[0].value @@ -23,5 +23,5 @@ if ($env:AdsOpts_CD_Services_AppPlans_FunctionApp_Enable -eq "True") } else { - Write-Host "Skipped Creation of App Service For Func App" + Write-Warning "Skipped Creation of App Service For Func App" } \ No newline at end of file diff --git a/solution/Deployment/workflows/Steps/CD_DeployAzureSqlServer.ps1 b/solution/Deployment/workflows/Steps/CD_DeployAzureSqlServer.ps1 index db90aca9..df1421f9 100644 --- a/solution/Deployment/workflows/Steps/CD_DeployAzureSqlServer.ps1 +++ b/solution/Deployment/workflows/Steps/CD_DeployAzureSqlServer.ps1 @@ -1,19 +1,19 @@ -Write-Host "Creating Azure SQL Server" -Write-Host $env:AdsOpts_CD_Services_AzureSQLServer_Name -Write-Host $env:AdsOpts_CD_Services_AzureSQLServer_SampleDB_Name -Write-Host $env:AdsOpts_CD_Services_AzureSQLServer_AdsGoFastDB_Name -Write-Host $env:AdsOpts_CD_Services_AzureSQLServer_AdminUser -Write-Host $env:AdsOpts_CD_Services_AzureSQLServer_StagingDB_Name +Write-Debug"Creating Azure SQL Server" +Write-Debug$env:AdsOpts_CD_Services_AzureSQLServer_Name +Write-Debug$env:AdsOpts_CD_Services_AzureSQLServer_SampleDB_Name +Write-Debug$env:AdsOpts_CD_Services_AzureSQLServer_AdsGoFastDB_Name +Write-Debug$env:AdsOpts_CD_Services_AzureSQLServer_AdminUser +Write-Debug$env:AdsOpts_CD_Services_AzureSQLServer_StagingDB_Name if($env:AdsOpts_CD_Services_AzureSQLServer_Enable -eq "True") { - Write-Host "Creating Azure SQL Server" + Write-Debug"Creating Azure SQL Server" #StorageAccount For Logging - az deployment group create -g $env:AdsOpts_CD_ResourceGroup_Name --template-file ./../arm/AzureSQLServer.json --parameters location=$env:AdsOpts_CD_ResourceGroup_Location sql-server-name=$env:AdsOpts_CD_Services_AzureSQLServer_Name sql-admin-login=$env:AdsOpts_CD_Services_AzureSQLServer_AdminUser sql-admin-password=$env:AdsOpts_CD_Services_AzureSQLServer_AdminPassword sample-db-name=$env:AdsOpts_CD_Services_AzureSQLServer_SampleDB_Name ads-go-fast-db-name=$env:AdsOpts_CD_Services_AzureSQLServer_AdsGoFastDB_Name staging-db-name=$env:AdsOpts_CD_Services_AzureSQLServer_StagingDB_Name vnet-name=$env:AdsOpts_CD_Services_Vnet_Name + az deployment group create -g $env:AdsOpts_CD_ResourceGroup_Name --template-file ./../arm/AzureSQLServer.json --parameters location=$env:AdsOpts_CD_ResourceGroup_Location sql_server_name=$env:AdsOpts_CD_Services_AzureSQLServer_Name sql_admin_login=$env:AdsOpts_CD_Services_AzureSQLServer_AdminUser sql_admin_password=$env:AdsOpts_CD_Services_AzureSQLServer_AdminPassword sample_db_name=$env:AdsOpts_CD_Services_AzureSQLServer_SampleDB_Name ads_go_fast_db_name=$env:AdsOpts_CD_Services_AzureSQLServer_AdsGoFastDB_Name staging_db_name=$env:AdsOpts_CD_Services_AzureSQLServer_StagingDB_Name vnet_name=$env:AdsOpts_CD_Services_Vnet_Name #Make sure password is correct az sql server update -n $env:AdsOpts_CD_Services_AzureSQLServer_Name -g $env:AdsOpts_CD_ResourceGroup_Name -p ($env:AdsOpts_CD_Services_AzureSQLServer_AdminPassword | Out-String) } else { - Write-Host "Skipped Creation of Azure SQL Server" + Write-Warning "Skipped Creation of Azure SQL Server" } \ No newline at end of file diff --git a/solution/Deployment/workflows/Steps/CD_DeployFunctionApp.ps1 b/solution/Deployment/workflows/Steps/CD_DeployFunctionApp.ps1 index df007699..509d142e 100644 --- a/solution/Deployment/workflows/Steps/CD_DeployFunctionApp.ps1 +++ b/solution/Deployment/workflows/Steps/CD_DeployFunctionApp.ps1 @@ -1,5 +1,5 @@ -Write-Host "Creating Function App" +Write-Debug"Creating Function App" if ($env:AdsOpts_CD_Services_CoreFunctionApp_Enable -eq "True") { if ($env:AdsOpts_CD_Services_AppPlans_FunctionApp_ResourceGroup -eq $null) @@ -22,6 +22,6 @@ if ($env:AdsOpts_CD_Services_CoreFunctionApp_Enable -eq "True") } else { - Write-Host "Skipped Creation of Function App" + Write-Warning "Skipped Creation of Function App" } diff --git a/solution/Deployment/workflows/Steps/CD_DeployKeyVault.ps1 b/solution/Deployment/workflows/Steps/CD_DeployKeyVault.ps1 index 14218858..9f429ded 100644 --- a/solution/Deployment/workflows/Steps/CD_DeployKeyVault.ps1 +++ b/solution/Deployment/workflows/Steps/CD_DeployKeyVault.ps1 @@ -1,10 +1,10 @@ if ($env:AdsOpts_CD_Services_KeyVault_Enable -eq "True") { - Write-Host "Creating Key Vault" + Write-Debug"Creating Key Vault" az deployment group create -g $env:AdsOpts_CD_ResourceGroup_Name --template-file ./../arm/KeyVault.json --parameters location=$env:AdsOpts_CD_ResourceGroup_Location keyvault-name=$env:AdsOpts_CD_Services_KeyVault_Name tenant-id=$env:AdsOpts_CD_ResourceGroup_TenantId } else { - Write-Host "Skipped Creation of Key Vault" + Write-Warning "Skipped Creation of Key Vault" } diff --git a/solution/Deployment/workflows/Steps/CD_DeployLogAnalytics.ps1 b/solution/Deployment/workflows/Steps/CD_DeployLogAnalytics.ps1 index d984acd7..394b1052 100644 --- a/solution/Deployment/workflows/Steps/CD_DeployLogAnalytics.ps1 +++ b/solution/Deployment/workflows/Steps/CD_DeployLogAnalytics.ps1 @@ -1,10 +1,10 @@ if ($env:AdsOpts_CD_Services_DataFactory_Enable -eq "True") { - Write-Host "Creating Log Analyticss" + Write-Debug"Creating Log Analyticss" az deployment group create -g $env:AdsOpts_CD_ResourceGroup_Name --template-file ./../arm/LogAnalytics.json --parameters location=$env:AdsOpts_CD_ResourceGroup_Location workspaces_adsgofastloganalytics_name=$env:AdsOpts_CD_Services_LogAnalytics_Name } else { - Write-Host "Skipped Creation of Log Analytics" + Write-Warning "Skipped Creation of Log Analytics" } diff --git a/solution/Deployment/workflows/Steps/CD_DeployResourceGroup.ps1 b/solution/Deployment/workflows/Steps/CD_DeployResourceGroup.ps1 index 8992ffa5..917ecace 100644 --- a/solution/Deployment/workflows/Steps/CD_DeployResourceGroup.ps1 +++ b/solution/Deployment/workflows/Steps/CD_DeployResourceGroup.ps1 @@ -1,7 +1,7 @@ if($env:AdsOpts_CD_ResourceGroup_Enable -eq "True") { - Write-Host "Creating Resource Group" - az group create --name $env:AdsOpts_CD_ResourceGroup_Name --location $env:AdsOpts_CD_ResourceGroup_Location + Write-Debug "Creating Resource Group" + $groupcreate = az group create --name $env:AdsOpts_CD_ResourceGroup_Name --location $env:AdsOpts_CD_ResourceGroup_Location } #Get ResourceGroup Object ID diff --git a/solution/Deployment/workflows/Steps/CD_DeployStorageADLS.ps1 b/solution/Deployment/workflows/Steps/CD_DeployStorageADLS.ps1 index 21d629f2..7319fdbb 100644 --- a/solution/Deployment/workflows/Steps/CD_DeployStorageADLS.ps1 +++ b/solution/Deployment/workflows/Steps/CD_DeployStorageADLS.ps1 @@ -1,23 +1,23 @@ -Write-Host "Creating Storage Account (ADLS) For Data Lake" +Write-Debug"Creating Storage Account (ADLS) For Data Lake" if($env:AdsOpts_CD_Services_Storage_ADLS_Enable -eq "True") { #StorageAccount For Logging az deployment group create -g $env:AdsOpts_CD_ResourceGroup_Name --template-file ./../arm/Storage_ADLS.json --parameters location=$env:AdsOpts_CD_ResourceGroup_Location storage-account-name=$env:AdsOpts_CD_Services_Storage_ADLS_Name - Write-Host "Creating Storage Account (ADLS) For Data Lake" + Write-Debug"Creating Storage Account (ADLS) For Data Lake" } else { - Write-Host "Skipped Creation of Storage (ADLS) For Data Lake" + Write-Warning "Skipped Creation of Storage (ADLS) For Data Lake" } #Transient Storage Account if($env:AdsOpts_CD_Services_Storage_ADLSTransient_Enable -eq "True") { - Write-Host "Creating Transient Storage Account (ADLS)" + Write-Debug"Creating Transient Storage Account (ADLS)" az deployment group create -g $env:AdsOpts_CD_ResourceGroup_Name --template-file ./../arm/Storage_ADLS.json --parameters location=$env:AdsOpts_CD_ResourceGroup_Location storage-account-name=$env:AdsOpts_CD_Services_Storage_ADLSTransient_Name storage-raw-container-name=transient } else { - Write-Host "Skipped Creation of Transient Storage (ADLS)" + Write-Warning "Skipped Creation of Transient Storage (ADLS)" } \ No newline at end of file diff --git a/solution/Deployment/workflows/Steps/CD_DeployStorageBlob.ps1 b/solution/Deployment/workflows/Steps/CD_DeployStorageBlob.ps1 index f2b0895d..0aae0caa 100644 --- a/solution/Deployment/workflows/Steps/CD_DeployStorageBlob.ps1 +++ b/solution/Deployment/workflows/Steps/CD_DeployStorageBlob.ps1 @@ -1,11 +1,11 @@ -Write-Host "Creating Storage Account (Blob) For Data Lake" +Write-Debug"Creating Storage Account (Blob) For Data Lake" if($env:AdsOpts_CD_Services_Storage_Blob_Enable -eq "True") { #StorageAccount For Logging az deployment group create -g $env:AdsOpts_CD_ResourceGroup_Name --template-file ./../arm/Storage_Blob.json --parameters location=$env:AdsOpts_CD_ResourceGroup_Location storage-account-name=$env:AdsOpts_CD_Services_Storage_Blob_Name - Write-Host "Creating Storage Account (Blob) For Data Lake" + Write-Debug"Creating Storage Account (Blob) For Data Lake" } else { - Write-Host "Skipped Creation of Storage (Blob) For Data Lake" + Write-Warning "Skipped Creation of Storage (Blob) For Data Lake" } \ No newline at end of file diff --git a/solution/Deployment/workflows/Steps/CD_DeployStorageForLogging.ps1 b/solution/Deployment/workflows/Steps/CD_DeployStorageForLogging.ps1 index f7179dc3..600ed29e 100644 --- a/solution/Deployment/workflows/Steps/CD_DeployStorageForLogging.ps1 +++ b/solution/Deployment/workflows/Steps/CD_DeployStorageForLogging.ps1 @@ -1,12 +1,12 @@ -Write-Host "Creating Storage Account For Logging" -Write-Host $env:AdsOpts_CD_Services_Storage_Logging_Name +Write-Debug"Creating Storage Account For Logging" +Write-Debug$env:AdsOpts_CD_Services_Storage_Logging_Name if($env:AdsOpts_CD_Services_Storage_Logging_Enable -eq "True") { #StorageAccount For Logging az deployment group create -g $env:AdsOpts_CD_ResourceGroup_Name --template-file ./../arm/Storage_Logging.json --parameters location=$env:AdsOpts_CD_ResourceGroup_Location storage-log-account-name=$env:AdsOpts_CD_Services_Storage_Logging_Name - Write-Host "Creating Storage Account For Logging" + Write-Debug"Creating Storage Account For Logging" } else { - Write-Host "Skipped Creation of Storage Account For Logging" + Write-Warning "Skipped Creation of Storage Account For Logging" } \ No newline at end of file diff --git a/solution/Deployment/workflows/Steps/CD_DeployVnet.ps1 b/solution/Deployment/workflows/Steps/CD_DeployVnet.ps1 index e17ca220..330c1450 100644 --- a/solution/Deployment/workflows/Steps/CD_DeployVnet.ps1 +++ b/solution/Deployment/workflows/Steps/CD_DeployVnet.ps1 @@ -2,8 +2,8 @@ if($env:AdsOpts_CD_Services_Vnet_Enable -eq "True") { - Write-Host "Creating Vnet + Subnets (Bastion, Data, WebApp)" - Write-Host $env:AdsOpts_CD_Services_Vnet_Name + Write-Debug"Creating Vnet + Subnets (Bastion, Data, WebApp)" + Write-Debug$env:AdsOpts_CD_Services_Vnet_Name #vNet az deployment group create -g $env:AdsOpts_CD_ResourceGroup_Name --template-file ./../arm/Networking.json --parameters location=$env:AdsOpts_CD_ResourceGroup_Location vnet-name=$env:AdsOpts_CD_Services_Vnet_Name vnet-address-prefix=$env:AdsOpts_CD_Services_Vnet_vNetAddressRange ` @@ -17,9 +17,9 @@ if($env:AdsOpts_CD_Services_Vnet_Enable -eq "True") webapp-subnet-name=$env:AdsOpts_CD_Services_Vnet_WebAppSubnetName ` funcapp-subnet-name=$env:AdsOpts_CD_Services_Vnet_FuncAppSubnetName - Write-Host "Creating Vnet" + Write-Debug"Creating Vnet" } else { - Write-Host "Skipped Creation of Vnet" + Write-Warning "Skipped Creation of Vnet" } \ No newline at end of file diff --git a/solution/Deployment/workflows/Steps/CD_DeployWebSite.ps1 b/solution/Deployment/workflows/Steps/CD_DeployWebSite.ps1 index 509d03f0..eec85573 100644 --- a/solution/Deployment/workflows/Steps/CD_DeployWebSite.ps1 +++ b/solution/Deployment/workflows/Steps/CD_DeployWebSite.ps1 @@ -13,10 +13,10 @@ if ($env:AdsOpts_CD_Services_WebSite_Enable -eq "True") $sn = $env:AdsOpts_CD_Services_AppPlans_WebApp_Name - Write-Host "Deploying Wesite to $sn in resource group $rg" - az deployment group create -g $rg --template-file ./../arm/WebApp.json --parameters resource-group-name=$rg sites_AdsGoFastWebApp_name=$env:AdsOpts_CD_Services_WebSite_Name appservice-name=$sn} + Write-Debug"Deploying Wesite to $sn in resource group $rg" + az deployment group create -g $rg --template-file ./../arm/WebApp.json --parameters resource_group_name=$rg sites_AdsGoFastWebApp_name=$env:AdsOpts_CD_Services_WebSite_Name appservice_name=$sn} else { - Write-Host "Skipped Creation of Web Site" + Write-Warning "Skipped Creation of Web Site" } diff --git a/solution/Deployment/workflows/Steps/CD_GrantRBAC.ps1 b/solution/Deployment/workflows/Steps/CD_GrantRBAC.ps1 index 0dc9af50..e17e8564 100644 --- a/solution/Deployment/workflows/Steps/CD_GrantRBAC.ps1 +++ b/solution/Deployment/workflows/Steps/CD_GrantRBAC.ps1 @@ -7,49 +7,49 @@ #RBAC Rights # MSI Access from Azure Function to ADF - az role assignment create --assignee $AzureFunctionId --role "Contributor" --scope "$basescope/Microsoft.DataFactory/factories/$env:AdsOpts_CD_Services_DataFactory_Name" + $result = az role assignment create --assignee $AzureFunctionId --role "Contributor" --scope "$basescope/Microsoft.DataFactory/factories/$env:AdsOpts_CD_Services_DataFactory_Name" # MSI Access from Azure Function to ADF Log Analytics - az role assignment create --assignee $AzureFunctionId --role "Contributor" --scope "$basescope/microsoft.operationalinsights/workspaces/$env:AdsOpts_CD_Services_LogAnalytics_Name" + $result = az role assignment create --assignee $AzureFunctionId --role "Contributor" --scope "$basescope/microsoft.operationalinsights/workspaces/$env:AdsOpts_CD_Services_LogAnalytics_Name" # MSI Access from AF to ADLS Gen2 - az role assignment create --assignee $AzureFunctionId --role "Storage Blob Data Contributor" --scope "$basescope/Microsoft.Storage/storageAccounts/$env:AdsOpts_CD_Services_Storage_ADLS_Name" + $result = az role assignment create --assignee $AzureFunctionId --role "Storage Blob Data Contributor" --scope "$basescope/Microsoft.Storage/storageAccounts/$env:AdsOpts_CD_Services_Storage_ADLS_Name" # MSI Access from AF to Blob Gen2 - az role assignment create --assignee $AzureFunctionId --role "Storage Blob Data Contributor" --scope "$basescope/Microsoft.Storage/storageAccounts/$env:AdsOpts_CD_Services_Storage_Blob_Name" + $result = az role assignment create --assignee $AzureFunctionId --role "Storage Blob Data Contributor" --scope "$basescope/Microsoft.Storage/storageAccounts/$env:AdsOpts_CD_Services_Storage_Blob_Name" # MSI Access from ADF to ADLS Gen2 - az role assignment create --assignee $DataFactoryId --role "Storage Blob Data Contributor" --scope "$basescope/Microsoft.Storage/storageAccounts/$env:AdsOpts_CD_Services_Storage_ADLS_Name" + $result = az role assignment create --assignee $DataFactoryId --role "Storage Blob Data Contributor" --scope "$basescope/Microsoft.Storage/storageAccounts/$env:AdsOpts_CD_Services_Storage_ADLS_Name" # MSI Access from ADF to Blob - az role assignment create --assignee $DataFactoryId --role "Storage Blob Data Contributor" --scope "$basescope/Microsoft.Storage/storageAccounts/$env:AdsOpts_CD_Services_Storage_Blob_Name" + $result = az role assignment create --assignee $DataFactoryId --role "Storage Blob Data Contributor" --scope "$basescope/Microsoft.Storage/storageAccounts/$env:AdsOpts_CD_Services_Storage_Blob_Name" # MSI Access from ADF to KeyVault - az keyvault set-policy --name $env:AdsOpts_CD_Services_KeyVault_Name --object-id $DataFactoryId --certificate-permissions get list --key-permissions get list --resource-group $env:AdsOpts_CD_ResourceGroup_Name --secret-permissions get list --subscription $subid + $result = az keyvault set-policy --name $env:AdsOpts_CD_Services_KeyVault_Name --object-id $DataFactoryId --certificate-permissions get list --key-permissions get list --resource-group $env:AdsOpts_CD_ResourceGroup_Name --secret-permissions get list --subscription $subid # MSI Access from WebApp to ADLS Gen2 - az role assignment create --assignee $WebAppID --role "Storage Blob Data Contributor" --scope "$basescope/Microsoft.Storage/storageAccounts/$env:AdsOpts_CD_Services_Storage_ADLS_Name" + $result = az role assignment create --assignee $WebAppID --role "Storage Blob Data Contributor" --scope "$basescope/Microsoft.Storage/storageAccounts/$env:AdsOpts_CD_Services_Storage_ADLS_Name" # MSI Access from WebApp to ADF Log Analytics - az role assignment create --assignee $WebAppID --role "Contributor" --scope "$basescope/microsoft.operationalinsights/workspaces/$env:AdsOpts_CD_Services_LogAnalytics_Name" + $result = az role assignment create --assignee $WebAppID --role "Contributor" --scope "$basescope/microsoft.operationalinsights/workspaces/$env:AdsOpts_CD_Services_LogAnalytics_Name" # MSI Access from WebApp to ADF App Insights - az role assignment create --assignee $WebAppID --role "Contributor" --scope "$basescope/microsoft.insights/components/$env:AdsOpts_CD_Services_AppInsights_Name" + $result = az role assignment create --assignee $WebAppID --role "Contributor" --scope "$basescope/microsoft.insights/components/$env:AdsOpts_CD_Services_AppInsights_Name" # AAD User Access to ADLS Gen2 - to upload sample data files - az role assignment create --assignee $AADUserId --role "Storage Blob Data Contributor" --scope "$basescope/Microsoft.Storage/storageAccounts/$env:AdsOpts_CD_Services_Storage_ADLS_Name" - az role assignment create --assignee $AADUserId --role "Owner" --scope "$basescope/Microsoft.Storage/storageAccounts/$env:AdsOpts_CD_Services_Storage_ADLS_Name" + $result = az role assignment create --assignee $AADUserId --role "Storage Blob Data Contributor" --scope "$basescope/Microsoft.Storage/storageAccounts/$env:AdsOpts_CD_Services_Storage_ADLS_Name" + $result = az role assignment create --assignee $AADUserId --role "Owner" --scope "$basescope/Microsoft.Storage/storageAccounts/$env:AdsOpts_CD_Services_Storage_ADLS_Name" #Transient Storage Account if($env:AdsOpts_CD_Services_Storage_ADLSTransient_Enable -eq "True") { - Write-Host "Granting RBAC on Transient Storage Account (ADLS)" + Write-Debug"Granting RBAC on Transient Storage Account (ADLS)" # MSI Access from AF to ADLS Gen2 - az role assignment create --assignee $AzureFunctionId --role "Storage Blob Data Contributor" --scope "$basescope/Microsoft.Storage/storageAccounts/$env:AdsOpts_CD_Services_Storage_ADLSTransient_Name" + $result = az role assignment create --assignee $AzureFunctionId --role "Storage Blob Data Contributor" --scope "$basescope/Microsoft.Storage/storageAccounts/$env:AdsOpts_CD_Services_Storage_ADLSTransient_Name" # MSI Access from ADF to ADLS Gen2 - az role assignment create --assignee $DataFactoryId --role "Storage Blob Data Contributor" --scope "$basescope/Microsoft.Storage/storageAccounts/$env:AdsOpts_CD_Services_Storage_ADLSTransient_Name" + $result = az role assignment create --assignee $DataFactoryId --role "Storage Blob Data Contributor" --scope "$basescope/Microsoft.Storage/storageAccounts/$env:AdsOpts_CD_Services_Storage_ADLSTransient_Name" # MSI Access from WebApp to ADLS Gen2 - az role assignment create --assignee $WebAppID --role "Storage Blob Data Contributor" --scope "$basescope/Microsoft.Storage/storageAccounts/$env:AdsOpts_CD_Services_Storage_ADLSTransient_Name" + $result = az role assignment create --assignee $WebAppID --role "Storage Blob Data Contributor" --scope "$basescope/Microsoft.Storage/storageAccounts/$env:AdsOpts_CD_Services_Storage_ADLSTransient_Name" } else { - Write-Host "Skipped RBAC on Transient Storage Account (ADLS)" + Write-Warning "Skipped RBAC on Transient Storage Account (ADLS)" } \ No newline at end of file diff --git a/solution/Deployment/workflows/Steps/CD_SetResourceGroupHash.ps1 b/solution/Deployment/workflows/Steps/CD_SetResourceGroupHash.ps1 index 8000d342..8947dee6 100644 --- a/solution/Deployment/workflows/Steps/CD_SetResourceGroupHash.ps1 +++ b/solution/Deployment/workflows/Steps/CD_SetResourceGroupHash.ps1 @@ -6,13 +6,13 @@ function Get-UniqueString ([string]$id, $length=13) -join ($hashArray[1..$length] | ForEach-Object { [char]($_ % 26 + [byte][char]'a') }) } -Write-Host "Creating RG Hash" +Write-Debug "Creating RG Hash" $ResourceGroupHash = Get-UniqueString ($id=$env:AdsOpts_CD_ResourceGroup_Name+$env:AdsOpts_CD_ResourceGroup_TenantId) #Resource Group Name + TenantId to make hash more unique -Write-Host $ResourceGroupHash +Write-Debug $ResourceGroupHash PersistEnvVariable -Name "AdsOpts_CD_ResourceGroup_Hash" -Value $ResourceGroupHash -Write-Host "Created RG Hash" -Write-Host "Setting Service Names" +Write-Debug "Created RG Hash" +Write-Debug "Setting Service Names" SetServiceName -RootElement "AdsOpts_CD_Services_AzureSQLServer" SetServiceName -RootElement "AdsOpts_CD_Services_CoreFunctionApp" SetServiceName -RootElement "AdsOpts_CD_Services_WebSite" diff --git a/solution/Deployment/workflows/Steps/CI_BuildDataFactory.ps1 b/solution/Deployment/workflows/Steps/CI_BuildDataFactory.ps1 index 468a84ff..f321d56b 100644 --- a/solution/Deployment/workflows/Steps/CI_BuildDataFactory.ps1 +++ b/solution/Deployment/workflows/Steps/CI_BuildDataFactory.ps1 @@ -1,18 +1,20 @@ #Move From Workflows to Function App $CurrentPath = (Get-Location).Path -Set-Location "..\..\DataFactory" -if (-Not (Test-Path "..\Deployment\bin\publish\unzipped\datafactory\")) +Set-Location "..\..\DataFactory\ADF\" +if (-Not (Test-Path "..\..\Deployment\bin\publish\unzipped\datafactory\")) { - New-Item -ItemType Directory -Force -Path "..\Deployment\bin\publish\unzipped\datafactory\" + New-Item -ItemType Directory -Force -Path "..\..\Deployment\bin\publish\unzipped\datafactory\" } -Copy-Item -Path ".\*" -Destination "..\Deployment\bin\publish\unzipped\datafactory\" -PassThru -Force -Recurse +Get-ChildItem -Path "..\..\Deployment\bin\publish\unzipped\datafactory\" | Remove-Item -Force -Recurse + +Copy-Item -Path ".\*" -Destination "..\..\Deployment\bin\publish\unzipped\datafactory\" -PassThru -Force -Recurse #Move back to workflows Set-Location $CurrentPath -Set-Location "../bin/publish" -$Path = (Get-Location).Path + "/zipped/datafactory" +Set-Location "..\bin\publish" +$Path = (Get-Location).Path + "\zipped\datafactory" New-Item -ItemType Directory -Force -Path $Path -$Path = $Path + "/Publish.zip" +$Path = $Path + "\Publish.zip" Compress-Archive -Path '.\unzipped\datafactory\*' -DestinationPath $Path -force #Move back to workflows Set-Location $CurrentPath diff --git a/solution/Deployment/workflows/Steps/InstallGatewayFunctions.ps1 b/solution/Deployment/workflows/Steps/InstallGatewayFunctions.ps1 index f3ab6a57..1892fe72 100644 --- a/solution/Deployment/workflows/Steps/InstallGatewayFunctions.ps1 +++ b/solution/Deployment/workflows/Steps/InstallGatewayFunctions.ps1 @@ -8,20 +8,20 @@ function Install-Gateway([string] $gwPath) # uninstall any existing gateway UnInstall-Gateway - Write-Host "Start Gateway installation" + Write-Debug"Start Gateway installation" Start-Process "msiexec.exe" "/i $path /quiet /passive" -Wait Start-Sleep -Seconds 30 - Write-Host "Succeed to install gateway" + Write-Debug"Succeed to install gateway" } function Register-Gateway([string] $key) { - Write-Host "Start to register gateway with key: $key" + Write-Debug"Start to register gateway with key: $key" $cmd = Get-CmdFilePath Start-Process $cmd "-k $key" -Wait - Write-Host "Succeed to register gateway" + Write-Debug"Succeed to register gateway" } @@ -58,11 +58,11 @@ function UnInstall-Gateway() if ($installed -eq $false) { - Write-Host "Microsoft Integration Runtime Preview is not installed." + Write-Debug"Microsoft Integration Runtime Preview is not installed." return } - Write-Host "Microsoft Integration Runtime has been uninstalled from this machine." + Write-Debug"Microsoft Integration Runtime has been uninstalled from this machine." } function Get-CmdFilePath() diff --git a/solution/Deployment/workflows/Steps/PushEnvFileIntoVariables.ps1 b/solution/Deployment/workflows/Steps/PushEnvFileIntoVariables.ps1 index 349e77cb..300cf9a2 100644 --- a/solution/Deployment/workflows/Steps/PushEnvFileIntoVariables.ps1 +++ b/solution/Deployment/workflows/Steps/PushEnvFileIntoVariables.ps1 @@ -12,7 +12,7 @@ function SetServiceName($RootElement) function PersistEnvVariable($Name, $Value) { - Write-Host "Writing $Name to env file" + Write-Debug "Writing $Name to env file" echo "$Name=$Value" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf8 -Append #Also Push Variables to the Session Env Variables for local testing [Environment]::SetEnvironmentVariable($Name, "$Value") @@ -22,14 +22,14 @@ function PersistEnvVariable($Name, $Value) function ParseEnvFragment([object]$Json, [string]$NamePrefix) { - $Json + #$Json foreach($p in ($Json.psobject.properties.where({$_.MemberType -eq "NoteProperty"}))) { $Name = $p.Name - Write-Host "Parsing $($Name)" + Write-Debug "Parsing $($Name)" if($NamePrefix -ne "") { - Write-Host "Prefix is $NamePrefix" + Write-Debug "Prefix is $NamePrefix" $Name = $NamePrefix + "_" + $p.Name } $Value = $p.Value @@ -39,14 +39,14 @@ function ParseEnvFragment([object]$Json, [string]$NamePrefix) PersistEnvVariable -Name $Name -Value $Value ##Push Variables to the GitHub Actions Compatible Store - #Write-Host "Writing $Name to env file" - #Write-Host $p.TypeNameOfValue + #Write-Debug "Writing $Name to env file" + #Write-Debug $p.TypeNameOfValue #echo "$Name=$Value" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf8 -Append ##Also Push Variables to the Session Env Variables for local testing #[Environment]::SetEnvironmentVariable($Name, "$Value") } else { - Write-Host "Further Parsing of $Name required" + Write-Debug "Further Parsing of $Name required" $JsonString = $p.Value #| ConvertTo-Json ParseEnvFragment -Json $JsonString -NamePrefix $Name } @@ -74,6 +74,15 @@ function ParseEnvFile ($EnvFile) } $Json = Get-Content -Path "..\environments\$($EnvFile).json" | ConvertFrom-Json + #$JsonForSchemaValidation = $Json | ConvertTo-Json -Depth 100 + #$schema = Get-Content "..\environments\environment.schema.json" + #$schema = ($schema | ConvertFrom-Json | ConvertTo-Json -depth 100) + #$SchemaValidation = $JsonForSchemaValidation | Test-Json -Schema $schema + #if($SchemaValidation -eq $false) + #{ + # Write-Warning "..\environments\$($EnvFile).json does not adhere to environment.schema.json!" + #} + ParseEnvFragment -Json $Json -NamePrefix "" } diff --git a/solution/FunctionApp/FunctionApp.csproj b/solution/FunctionApp/FunctionApp.csproj index a3035642..dff159c2 100644 --- a/solution/FunctionApp/FunctionApp.csproj +++ b/solution/FunctionApp/FunctionApp.csproj @@ -2,7 +2,7 @@ netcoreapp3.1 v3 - + 61b94ea1-454a-4cfb-b90d-884ec53464f8 @@ -30,7 +30,7 @@ all runtime; build; native; contentfiles; analyzers; buildtransitive - + @@ -81,9 +81,6 @@ PreserveNewest - - Always - PreserveNewest diff --git a/solution/FunctionApp/Startup.cs b/solution/FunctionApp/Startup.cs index a663b2ae..15498fec 100644 --- a/solution/FunctionApp/Startup.cs +++ b/solution/FunctionApp/Startup.cs @@ -50,7 +50,7 @@ namespace AdsGoFast var config = new ConfigurationBuilder() .SetBasePath(actual_root) .AddJsonFile("appsettings.json", optional: false, reloadOnChange: true) - .AddUserSecrets("3956e7aa-4d13-430a-bb5f-a5f8f5a450ee"/*Assembly.GetExecutingAssembly()*/, true) + .AddUserSecrets(Assembly.GetExecutingAssembly(), true) .AddEnvironmentVariables() .Build(); diff --git a/solution/FunctionApp/TaskMetaData/TaskInstance.cs b/solution/FunctionApp/TaskMetaData/TaskInstance.cs index 47b2ab0a..fd3e77b8 100644 --- a/solution/FunctionApp/TaskMetaData/TaskInstance.cs +++ b/solution/FunctionApp/TaskMetaData/TaskInstance.cs @@ -448,7 +448,7 @@ namespace AdsGoFast.TaskMetaData JToken _TableSchema = TaskMasterJSON["Source"]["TableSchema"]; JToken _TableName = TaskMasterJSON["Source"]["TableName"]; - if (TaskMasterJSON["Source"]["ChunkSize"] != null) + if (TaskMasterJSON["Source"]["ChunkSize"] != null && TaskMasterJSON["Source"]["ChunkSize"].ToString() != "0") { if (_IncrementalType.ToString() == "Full" && string.IsNullOrWhiteSpace(TaskMasterJSON["Source"]["ChunkSize"].ToString())) { @@ -588,11 +588,11 @@ namespace AdsGoFast.TaskMetaData if (JSONValue["Source"]["ChunkSize"] != null) { - if (_IncrementalType.ToString() == "Full" && string.IsNullOrWhiteSpace(JSONValue["Source"]["ChunkSize"].ToString())) + if (_IncrementalType.ToString() == "Full" && (string.IsNullOrWhiteSpace(JSONValue["Source"]["ChunkSize"].ToString()) || JSONValue["Source"]["ChunkSize"].ToString() == "0")) { _Type = "Full"; } - else if (_IncrementalType.ToString() == "Full" && !string.IsNullOrWhiteSpace(JSONValue["Source"]["ChunkSize"].ToString())) + else if (_IncrementalType.ToString() == "Full" && !string.IsNullOrWhiteSpace(JSONValue["Source"]["ChunkSize"].ToString()) && JSONValue["Source"]["ChunkSize"].ToString() != "0") { _Type = "Full-Chunk"; } diff --git a/solution/SampleFiles/yellow_tripdata_2017-03.csv b/solution/SampleFiles/yellow_tripdata_2017-03.csv new file mode 100644 index 00000000..6fd185db --- /dev/null +++ b/solution/SampleFiles/yellow_tripdata_2017-03.csv @@ -0,0 +1,61 @@ +VendorID,tpep_pickup_datetime,tpep_dropoff_datetime,passenger_count,trip_distance,RatecodeID,store_and_fwd_flag,PULocationID,DOLocationID,payment_type,fare_amount,extra,mta_tax,tip_amount,tolls_amount,improvement_surcharge,total_amount + +2,2017-03-09 21:30:11,2017-03-09 21:44:20,1,4.06,1,N,148,48,1,14,0.5,0.5,3.06,0,0.3,18.36 +2,2017-03-09 21:47:00,2017-03-09 21:58:01,1,2.73,1,N,48,107,2,11.5,0.5,0.5,0,0,0.3,12.8 +2,2017-03-09 22:01:08,2017-03-09 22:11:16,1,2.27,1,N,79,162,1,10,0.5,0.5,2.82,0,0.3,14.12 +2,2017-03-09 22:16:05,2017-03-10 06:26:11,1,3.86,1,N,237,41,1,12,0.5,0.5,3.99,0,0.3,17.29 +2,2017-03-31 06:31:53,2017-03-31 06:41:48,1,3.45,1,N,41,162,2,12,0.5,0.5,0,0,0.3,13.3 +1,2017-03-01 00:00:00,2017-03-01 00:14:22,1,2.80,1,N,261,79,1,12.5,0.5,0.5,1,0,0.3,14.8 +1,2017-03-01 00:00:00,2017-03-01 00:19:30,1,6.00,1,N,87,142,1,19.5,0.5,0.5,3.5,0,0.3,24.3 +1,2017-03-01 00:00:00,2017-03-01 00:34:27,1,8.70,1,N,142,181,1,30,0.5,0.5,7.8,0,0.3,39.1 +1,2017-03-01 00:00:00,2017-03-01 00:21:31,1,3.70,1,N,68,141,1,16.5,0.5,0.5,1.5,0,0.3,19.3 +2,2017-03-15 00:07:59,2017-03-15 00:38:08,1,4.21,1,N,261,163,1,20.5,0,0.5,4.26,0,0.3,25.56 +2,2017-03-01 00:00:00,2017-03-01 00:07:26,1,2.07,1,N,161,113,1,8.5,0.5,0.5,1.96,0,0.3,11.76 +2,2017-03-01 00:00:00,2017-03-01 00:07:51,2,1.26,1,N,170,161,1,7,0.5,0.5,1.66,0,0.3,9.96 +2,2017-03-01 00:00:00,2017-03-01 00:10:40,1,2.91,1,N,256,65,1,11.5,0.5,0.5,2.56,0,0.3,15.36 +2,2017-03-01 00:00:00,2017-03-01 06:46:27,1,4.70,1,N,132,10,1,15.5,0.3,0.5,0,0,0.3,16.6 +2,2017-03-01 00:00:00,2017-03-01 00:00:00,5,1.86,1,N,132,50,1,19,0,0.5,2.97,0,0.3,22.77 +2,2017-03-01 00:00:00,2017-03-01 00:00:00,5,9.88,1,N,138,25,1,36.5,1,0.5,7.66,0,0.3,45.96 +1,2017-03-01 00:00:01,2017-03-01 00:30:30,3,9.40,1,N,45,116,1,29,0.5,0.5,9.05,0,0.3,39.35 +2,2017-03-01 00:00:01,2017-03-01 00:06:09,1,.83,1,N,100,230,1,6,0.5,0.5,1.82,0,0.3,9.12 +1,2017-03-01 00:00:02,2017-03-01 00:25:33,1,5.70,1,N,170,223,1,21.5,0.5,0.5,4.55,0,0.3,27.35 +2,2017-03-01 00:00:02,2017-03-01 00:10:41,1,2.43,1,N,79,161,1,10.5,0.5,0.5,2.95,0,0.3,14.75 +2,2017-03-01 00:00:02,2017-03-01 00:14:32,2,3.63,1,N,140,24,1,14,0.5,0.5,3.82,0,0.3,19.12 +1,2017-03-01 00:00:03,2017-03-01 00:17:29,1,4.50,1,N,114,181,1,16.5,0.5,0.5,3.55,0,0.3,21.35 +1,2017-03-01 00:00:03,2017-03-01 00:10:12,2,1.20,1,N,211,148,2,8,0.5,0.5,0,0,0.3,9.3 +2,2017-03-01 00:00:03,2017-03-01 00:09:55,1,1.73,1,N,230,143,1,9,0.5,0.5,2.06,0,0.3,12.36 +2,2017-03-01 00:00:03,2017-03-01 00:08:34,2,1.58,1,N,90,144,1,8,0.5,0.5,1.86,0,0.3,11.16 +1,2017-03-01 00:00:04,2017-03-01 00:07:51,1,1.40,1,N,162,186,2,7.5,0.5,0.5,0,0,0.3,8.8 +1,2017-03-01 00:00:05,2017-03-01 00:22:10,4,9.80,1,N,138,163,2,29.5,0.5,0.5,0,5.54,0.3,36.34 +1,2017-03-01 00:00:07,2017-03-01 00:12:35,1,3.70,1,N,162,226,1,13.5,0.5,0.5,2.95,0,0.3,17.75 +1,2017-03-01 00:00:07,2017-03-01 00:27:43,1,10.30,1,N,138,141,1,32.5,0.5,0.5,7.85,5.54,0.3,47.19 +1,2017-03-01 00:00:07,2017-03-01 00:09:41,1,1.40,1,N,114,234,2,8.5,0.5,0.5,0,0,0.3,9.8 +1,2017-03-01 00:00:07,2017-03-01 00:13:00,1,3.50,1,N,125,50,1,13,0.5,0.5,3.55,0,0.3,17.85 +1,2017-03-01 00:00:08,2017-03-01 00:07:26,3,.90,1,N,114,79,2,6.5,0.5,0.5,0,0,0.3,7.8 +1,2017-03-01 00:00:08,2017-03-01 00:02:10,1,.40,1,N,237,141,1,3.5,0.5,0.5,1.2,0,0.3,6 +2,2017-03-01 00:00:08,2017-03-01 00:03:29,1,.66,1,N,161,162,2,4.5,0.5,0.5,0,0,0.3,5.8 +2,2017-03-01 00:00:08,2017-03-01 00:25:29,1,14.90,1,N,132,226,1,41.5,0.5,0.5,7.2,0,0.3,50 +1,2017-03-01 00:00:09,2017-03-01 00:07:15,1,1.40,1,N,163,48,1,7,0.5,0.5,2.05,0,0.3,10.35 +1,2017-03-01 00:00:09,2017-03-01 00:11:12,1,4.00,1,N,263,79,1,12.5,0.5,0.5,2.75,0,0.3,16.55 +2,2017-03-01 00:00:09,2017-03-01 00:00:19,1,.00,5,N,87,87,1,12,0,0,3.08,0,0.3,15.38 +2,2017-03-01 00:00:09,2017-03-01 00:21:59,1,16.53,2,N,132,170,1,52,0,0.5,11.67,5.54,0.3,70.01 +1,2017-03-01 00:00:18,2017-03-01 00:06:31,1,1.20,1,N,234,4,1,6.5,0.5,0.5,1,0,0.3,8.8 +1,2017-03-01 00:00:10,2017-03-01 00:05:58,1,.90,1,N,234,107,1,6,0.5,0.5,0.73,0,0.3,8.03 +1,2017-03-01 00:00:10,2017-03-01 00:02:53,2,1.00,1,N,74,75,1,5,0.5,0.5,1.25,0,0.3,7.55 +2,2017-03-01 00:00:10,2017-03-01 00:06:47,1,1.54,1,N,48,90,1,7.5,0.5,0.5,1.25,0,0.3,10.05 +2,2017-03-01 00:00:10,2017-03-01 00:15:02,1,4.33,1,N,170,24,1,14.5,0.5,0.5,3.16,0,0.3,18.96 +2,2017-03-01 00:00:11,2017-03-01 00:16:34,6,2.79,1,N,114,230,2,13,0.5,0.5,0,0,0.3,14.3 +2,2017-03-01 00:00:12,2017-03-01 00:06:04,1,1.17,1,N,161,186,2,6.5,0.5,0.5,0,0,0.3,7.8 +2,2017-03-01 00:00:12,2017-03-01 00:04:46,1,1.95,1,N,48,239,1,7,0.5,0.5,2.08,0,0.3,10.38 +2,2017-03-01 00:00:13,2017-03-01 00:18:57,2,5.79,1,N,142,223,2,19,0.5,0.5,0,0,0.3,20.3 +2,2017-03-01 00:00:14,2017-03-01 00:24:41,1,6.21,1,N,264,264,1,22.5,0.5,0.5,4.76,0,0.3,28.56 +1,2017-03-01 00:00:15,2017-03-01 00:14:34,1,2.60,1,N,79,186,1,12,0.5,0.5,2.65,0,0.3,15.95 +1,2017-03-01 00:00:15,2017-03-01 00:11:06,2,1.50,1,N,48,229,1,9,0.5,0.5,2.05,0,0.3,12.35 +1,2017-03-01 00:00:15,2017-03-01 00:07:40,1,1.40,1,N,161,186,2,7,0.5,0.5,0,0,0.3,8.3 +1,2017-03-01 00:00:15,2017-03-01 00:18:03,2,3.90,1,N,48,146,1,15.5,0.5,0.5,0,0,0.3,16.8 +1,2017-03-01 00:00:15,2017-03-01 00:08:14,1,2.90,1,N,142,166,2,10,0.5,0.5,0,0,0.3,11.3 +2,2017-03-01 00:00:15,2017-03-01 00:08:56,3,2.27,1,N,142,140,1,9,0.5,0.5,2.06,0,0.3,12.36 +1,2017-03-01 00:00:16,2017-03-01 00:01:47,1,.60,1,N,68,68,1,4,0.5,0.5,1.3,0,0.3,6.6 +2,2017-03-01 00:00:16,2017-03-01 00:27:54,1,17.33,2,N,132,237,1,52,0,0.5,8,0,0.3,60.8 +2,2017-03-01 00:00:16,2017-03-01 00:08:16,1,2.29,1,N,158,261,1,9,0.5,0.5,2,0,0.3,12.3 +2,2017-03-01 00:00:16,2017-03-01 00:19:28,2,14.63,1,N,132,223,1,39.5,0.5,0.5,5.5,0,0.3,46.3 \ No newline at end of file diff --git a/solution/SampleFiles/yellow_tripdata_2017-03.xlsx b/solution/SampleFiles/yellow_tripdata_2017-03.xlsx new file mode 100644 index 0000000000000000000000000000000000000000..b20d5e9787fe99c009df3414545f3607495a612f GIT binary patch literal 16320 zcmeHu1#?_Wl68xjnVFfHSr#)|U`ZA;Gc$uFSq!#hF*90>7Be$5(W)`JO~YJ=^kq$73YtUIBE5I&t8&|4Rzou8$3@#EF`1*_(t6QgAfe zHep%x0A2%$300o{BcA=B*Y*^60ZDqPaAk=Ro}=yv6P8BKczMk=DFLIt_E*q4x;X?* zVlg6Ni!wm*k6P?nNgS^mpHL#Mf@mZDs)W1ueET@0E#IK%Un30S>4d~n6_EHXC(3nk zdOrY9S)hvj8<4)1_bGZlju7PaMh6X2lTO}(jn^CeSKv?&eps<&OJ+scA;j?k)A6nZ zcW4Pk?^_LGUY+I>1lBm`wTE0l{d#`~11SBQk=Cm-lidJ+kptQw9MDJ&98ImAm>7S* z|DSRGFSf{kdGxZxzm0S7es(n;S@eUOh;%E7ny7oVukmhHOcKGuDYr@k42Snwj(d8hk=Q8*R^;?C)%qwgF@z@ z3Mo741M4rT0gBvPKb<=E%aJUPUM`rP(!q|!9Btrtu zjJq|Ho1LSTv7MdO?InL}o?@GzWd9Uv zk!T8ipRU=((8Y1;cKsBR(>Fq7Kr&@tNWJ>jGbLSG;hwa5S@58!g$x7y?G#sv89Num zCkt_=7mZ;=aJoCwiv!Cf`;A%qfdQZsfb^rt^nR$m|o3!dk9aDd_yu zoVrgNK>DQ3hMcu&<@Z6&?WD16*k6p@VZvvs6bzpX@rKySqjeU@{BPZ0i0r8X6bJF1 zGQ{49#3H@NG*)KMdFTkplN7C}GzR6c>70{TClZ5YVaxmELAZX#|*dW$|RF_G}R4yGrKhD{I4Wf(VSAq+lH8 z?Z0G;cA)sHQ!HE48R9m$AmLnbA02kC(~sN)H7jJh zV;*4uj0sk8U@e!8_#+eS)^&HW5Kdf+VwdQE%y)B}sCFrQvq&=LUXg=9+a+k2#IO5I z&q9#T15T%{ig;d*a;v*8Ba2IIO+yHid$Fm_IzeI5T;RORuJ|sZ(R1<_vHK0u4pc|| zRmNQRF_)$wI7z;C$eb!2OL@(=hDF~cOh3Icpc&qJm3r*L`Vm1E?n4kX*{xz6&X8dr z28&54I!y^TaN}U$xUX~A8cl#N(|J5#{U_}X5KK%qV{znw}R~`Sa zhyVhlYd{zwQ9nk49;%~RrX*qWy;;YjZPC}7 z`@jTd7wU07HsE^Cfw%>RcH2!?9El3FPMl=6M7_TK^z;1* zHd@(2T74Gf<2@f_n~eI8%|skyq4Nbcrn0j*S;@*3t-My)1a1+O^RxY7+rj+B;d3|@z*v-upMa>S(*>yhRt;k;T5%dF#6Y~49W37aynwQkvhn0!RSG0f#P zdmW_sL~i*+P&!&(c3S!|e1ui@z)PvIyGS=%bQV1osQt$B0~;k(>)6bEv&M6~Gz|f1 z>tOUV_V9e^>fP&x=}=qpm1k+k;lR6#3x$s`W_7bMQ6h+OP(FV9sHxb=bEP_62;R$N zJsGe*BqQe%fft?}><1DTLG6QNQd~mKJENDld%>jZ?=f?HotHRovcgk{cx~jS3T;fawj&tn^_a`9MqfLE0cfNF^ z=elQlXn&_L@3Jclc0*%NARL&ET&jTp^ah`X33kVxS4B-K z9Idcl>=l96;|&XbFw;rCG`Pr?EQ? zDBc*`gu>JJ?y5nK%d3W)&m4m8>ZvxDt;_ZI(+k6oT(a)0c6b#0U!Nsm ze(hB&|LpNtr`|k#sZ;jl&nALEp`$M$t18A67C!Bi#6mGZp_k`Kc43J=T}aVq3&up* ziW#Um4eKb|#cJ6~-U^6vO5~JSjQu4v@s6eAp7=ml1J<=wsZ$u>smDP|8X}oCHiXGr zDWiuuA!CR0C@^%KAM7en*ERRr+6;2@Qdo}@$qVNH1P$1NzI?_47j~|O=+e5=7oZnX zdc`uvU$VqL=7eY9&s2>J15D_k3$Wjbj!{LVonNW--ZpvCf{<`M0nF+V7n&<&Hu(g| zI!nEm8|&kW#rXJJ{1K6Sru#Lue2o-{`_-qRMR6ZPDg?f{ zFKp^WdN!&EDqr~Iei*k{tIZb0m) zBF0J`kN1*^VA_sd?lQLR>awR@sIXgfq*i{JW6(|KyuBpxQ8y3TKz%QD#WRK?Bts3W zlPmH&{p^Hwb;Kavl#UFf6T5O*eQ#>cRR*}HbA#3U4{kl%@3{Kj&Tc|mA5L#1v|~T5 zBd|Jbzj)-6jLSUnvoS2kz}07y8+HWEsza4GQSu<3bKx3kSnTed7GCIQCanK>4%6wO zvk|O{nw3h*XbdCM=bvDVWxQ)yIE-SQu|aEj04vy!2<4`tL0p<~O|*YRRFHtgubov5 zx@M1IE_HS$$VBf5V!QH6oIlvlyPN3SgJ|U*Dy6)~jFW0{FVx4^KXIv^#xhlpQ(p~1 zmR<3iMBXPSh_tPJ#dTm&fKx;jTHn_J=esYt0{M7{hrmK*5ffGWkB3fb^YbJ~?LGpc z?gCNu=-4ocT-M=N^7yCrjS096xt*C5?mSoV1~XQps7bba6;Y>{`oV;h$A7g`zFM9Y-cF2B}T4@46fk4 zn6Pwpz2**Ib%baLJIem-4!=5N3#_w4pW7Yg+!(7U&Yrelv6T!hwlhP?>(Z0zCLyWRZelL1O578~B;w_f$Zl<}gu5a^+lwBXCLl83Sr@+zu)_ z%A#S526FY#{acBryFle00*b@7kr_r$?@fs@E7;CMV|_r@v8%$*auU5ps%K?@n&7)h z!|^6E9o3vO{ep^(FI=PeB0OebKuDPGTc3=7k^^QN_9}VTVH36s6!_AYB4tOM$O_z| zAfE&IU1V?0YAPdjF~j)n-8o9*Z-jernxH(E8pJ9!tatHXpL34>^H{dRQ;m6h%dqpqjB;(GbzUlNqY)60NBr816!1RuB1bK-&1kBSQ%?0BpF`xLO1Dqg5 zQ$yvIwiPQTBTW#SC(8jMqzcK-edXL5AgTh-54konN6?3!E>trmDMC={xcroE8DVEt zsmr5-BUda~P=N#>Xx}c;OH#o^jyuEx2p(G&0k5?!>A<&q?6zaaohk>3KERDX--e(> zITsP7X`8YEsq2&8UUg5??(@&p#6XkYF&DZvXo{=ZgPZr|IZ`|gd4kK3ciJPfaS4Y( zx9SIHRIm>*fbFBhS2VvUAgD$3S)Ump2AQKGyeiw8D~D} zAGN;o70Sg;j7&0q{vrB-9fGo$e-z8m7$E>GXtv}RBK$0grBc}YXdVp zTmrP%yNuXYmmvoYxWF$VW;1>FOPV=%D2yNJj8;9FJo)}o>GCHLqO-_Kg%jZDPX}-> zDo@M?(6HA#!g9P_K^zETJLZpYwc6PA#g`GE1 zmX^|faQAVsfbwyff7%XV6FHBgyawITC0q#Is>Q_lX5%O04ke@+!$@HAT$Gm#aG{O0 z^T_&4aKXD7Yl}l9!$2sI@fvwIM)RK>#6%g-6;Fp?m0$VxZj2=CH3q-EFJ43Up-I&c zKqD$gVeAffM5ZBt%kS5Ue=N2+^CFFNgsS)q1c z`*UtCEGzsDDp)!AIBceo-fJrh@gh>+m-Ml5*sz2C$mtkTGehYx7hY@<;A)s3JC~6) zfrz9F=i=2<{uJJ)>`Z1Fw&L&OJ?Rq5IK(!;-m8&U+uX>r@D?mpfp8aDV(3f@xLtcu7PQ|4i*aKZDS}Xjd<}9+@4z@IdGKz7)K4Aq3K4qPU&ljy? zq=Z=GG3kN62r5igX2Us=Cn3|sPn5Vw9-P(B4PL*a2uB^>;0G}N~wTgqiL=-4S*UwEjQk1Hx~s25$8_%Gu> zqqRzaH|7}gxy9$6!b&q0HuR@B6wHU`n)i*;O{LDsF2Iv=<#TE>r_|jG~2c0)F#&*tPCbPp3mz{$Xq5gJhts zuzHcKp z%@?4mW!s3L$?;B)M|)!gOuKQ$8r9FtS1lR~B*ao&IxzU4-1ALOZm&5f%PTm&Psr2M zb2lx}(JXGrhI(XeBAN)>f z+Z0>Zu{FcolGG@d9SQ~{#%S|1A3M(qzhHrgh-KGPrY#QAarbGQb&1f36e3q1jWNMm zWSPM!90b7!J$#oKbkBA1-1D7+(Vz>&u0lox)6v#0-D9>7moM|+EC`Q_hM8$eRBLtr z(szo%lTpkOz$A%<(R0Gr2YY5n9AZy3H;XfDM?2s|PZ_HGlA?JV6Ln7e<+xx&hTlEF z4Xw4zwKOi%a7drpIb6&dH7xu`8L8mJZPSS41uAqFFBY`q-~$iu+bE_V0>9SX<=`48?gmAn$) zM_6O#5<#!8iMoZ$S#zgT8*^F|g0T=HkG9n8xN}()#;5i7TRCoS(P-F1A<8yw7D2k) zCBwHF-t-5J(uNB9sk7kVI%-Gnbqd!!8+9PV^UzM}#OpJ24%m{YjoaY89dGrHAue9} zJ4DD69SD4*x|9#it&Lc1^8k7qtpNh?Lh7?88JEx>a$JNa8r6zg$`IYcv9fx22Q0Y~ z$Y6UzW5`e7rtKF323TLkcJj5@EG>Y6I-GbX-euBP`U|)COPT0k}BgQ&Ny^jyHuQMn?j=@~|sM25LbD>Bchz*wj4-f?~S(Zj7-) zk&F4;Zc-^|=sSqAx-f=V%@01e=fqVi)kI%Uhc}Qr7s!2wZK!xU)6U2VFWCyXFOS2W zdA^tkw#0x@Vb@b8<{#Sr*oFTT-4X=&gw4{ky47SY`9^Jqk<>00`TVKGu&6AlL{10Z(-Mi}@)SDuEh z!0ixfE<#fc76k}eO60+Bq&gcyPY2f7D9AjFlSd3vYmWZh;zSLZg6J$6_>TUDb2j>Y zLZ3)Fmm9L&2WunrF+qW$55o?F2OPCr?OBp_>9IgS13H4Xps};Rs z@~hu#Rp@RATM)1Q-UBu!2hYQfe5_Q@Pa#3pGD)Id=qe-A7OHB!hltIpm(SC)Y+(C4 z^7-XtTSDVITIZ(-bKBE8+@5db3e4?ZO)ZJ-f7O*xW3hTtd2}B)$D&3DW?5&MBg(c+-Ek8re|^o}G5B zv=8U7=CJ}ss%engIiHZ{YicS47z93D`4NfXF7FS^T9virez2#DiHau9%XJ7B*L%Dg}f8^S~^M=F`dvR?tc z)(214SkO$aXrLpEhi#6!<$xWwYg4I>ybR5(uXqUIDQe2_lcV?4f&mw1M4QCDxJhgH z)I~B^wOf`KzK9)v>}yQW-=etQx?w70Aw7UaoldP8Xw#9oodw<2uUdAAh&$6fBExFu z9=+Lb#Z$VlOa24jDN*jkF;Oi*+hS$x`KW&k`iH)Lx;_g27Lx8bQRD+OZrZIQjqD0h z69VWj>AJ7{??#o*2l^bgk_)>?<;ZJQ4)xq;d+b1#RGopsO23)0l!Ox?coep{ZEpBR z0Jymva_&k6UnO}0?k=Q)`yz^BSbIQfzjWp#n<2g=;rz8@nRwuvmaF%XRE5tV0x65f z1EKHu6z>7t4Apnu33K@BrvZuAW&<`79SE})0SUa?{O zXp(AdfvkL&Ji}U6#}y$;>aD$~=sw%m^V<1xsy)q9Dh=*u;unJsr7MBV=!G=$=`Vg( z+|=yeiqgr^D5fyBk8t!aAC_}`9Z_tCpa&Mf9>uNmz7WaqEYo84L5C(Zb(wJotW!o0 zNdF|u!5qLR2wF?Sr2D2)j!l;M@G;O>$;*e8gzN+%Mb^*jN1Ng%OM)muH1)b>Ni{`Y z^>$Bc@T@X1|KBSo=EdcIUb?(L`79ji`nMuQ-${!l3x@b;Z7vVobkFh$=mLJLpj&fU0oNUq_9 zE_arTm?>Mp>+Rz3I;>N`@8;mK$DpRUp7w(+6osnl*=(G_`|B&#nnCB=#nC0#;SAN~ z&)&|L$C1>|_e#b+afpdWRKkAAvTo&H?L*XoC8D02gXJB!$R{o!hObn^veF3C z@blG-v<{4L3LYndOH!8b6{1J=@Ng#Lj70-8z1lWkCdfWY*pvzer&Hs zK_{&VBes$K=1EOb!xVxO*)qDparN6wLlD!%6012H$5d!K)BDS0o2t)t4uS5N)DbA_ zg^#%6tFrpsyP0a9S+R)n8yuj!L_1N_H1iM#%!#d?GRHB{+MiQBG0(j9?dg2&Y6lDr z)&x#Fz;V1Saft64@5YdotpfRtLZM;du~q}aG2ZmVvr2WW&yq71GB}aI&?Pn6(UNvz z`hG>!P`$Vf6S{#oK|;joUOp%DW>A~%iwD%g+!N7Td75XhblE4CIn8;fJAIsecz-T2 zoBk+RX*zC|ey!7gdv_~hncn6dFid%Vb&##tR=vwcl6*D-(bp>K<^$d~84bl;ysX?p z)ockePWtebBaMC~nT%oEfl`Qybt!HFUPsJywJS4H?pDp6jWYLBVVDaYM$Sa}v%=jO7OuRq`*5+B`Z^kT$PWxpTj2%N8#&m8+6sJUsULQ*eIV;jkt;VHXdD;pq+ zqJR7x{Bbc|YtmV3Qt2o)Pe;;W7na!u1H*myZbPDdH%w$nB-9K)dH)keQC)glbOr2q zx};|k+mCb*C1WNw;y7p3#TEw-OBaihc8Nj;vS{bp80IPFK&(xAzOgUQ+Wjr*VKc0X z&Bo1mj%-IUNGMhkq*P4-l$5Fyy?nNh^_I|$v}Ej7k7x(jJ@f96&15sWSox|=t~fH; zwhwMbUvRK$!o-t#^^z2ZAz2dAypnqv3>f@U5ZEX2^KIxf{gDNAw!aF~5oI_aJM#^? zvv!kNgsx&MXDbi|Y^qk+zm@c@6O@rJ;{R+Xiiv9n%Db&(pqJxT!@Q(^|JQ`yO z$f&|d*nM={G8{IaZ;PDbNscAsjZ$*WPczB>$q`X8I;luUgsr&vwL9;akw2{j?5-r42~66jLnFsvmUZ-n=?L9VBO3N zIisxd+HDFwk+rqT%XMt@(C~^goDViGlH3B96;wilS))2G!t}?oSr1RcD{Zr8P;LSI zp7ws+MJfkMqIRid)K61*gUOtRQ7cK>G0^L6JX8$tzBJZQtFj9W$+$+H$ZA0_sPXWP3BV<9L}QE~>OSoNDvTOE3f z-VWy|#97bv!@1fvu9hKY^#au3?Ft1GCv&_ZAIo*m5jn$Sw)&!OK7+mQxxIWiv{AIQo&fI*7d% z5;b1W1=u*V*+72Ab4TZ|_90$p9eaP?Tc|BiT&ADx7%)vif7EKMkkgO4F4iJ{v@Cmn zL%s=Z(MVu62)0NV`1HzCaif?-M712;D#$LH73}< z$S_qlBMBIBG*vrhMa04&Nd6XT^VNq8E=jKvrJ`0Ek{oWNXsiVbW09A9h+hWnYP8Ck zA*w>C@+u#GpQgB_eua6eL8G(I4RxVhfJ7TLMM9l^2*V$@&oN#6vZJD76VIqO0dQv{ z{pE9-tIdN+9lfK9g>v-}OcPP&m|C0fa8&2D{~O}j(d|5C!4c7^SK3d44_uT5#nMqK zWE(cjUsi{D=qox|;HFJJQr=jNFn^fZe`=7os{(MWkD%IPwOc4jBoE=7EpAWndEJ>C zB+ioY*r-3ugOu*g++{%t$VAxP|HFq);Kad+pcjs)w8KTb9!!a4^s>U)S9+Osvar}4F7xsJAkhBI-1 z9RwfGMZxS1yz=9K-|VA$IBq7+DfenYl;;LhS}bWe59IaXoykd7>5w?2L)qtX12-rg&Cd#Q*1`Z6}0y5e6Vodhv!%<%~>)NJY3lUw&m?2yFQAxj5vc4)I8 zBtUz{Q|%c?)}!xyt}8$YNP^sNikz-V)UqDy(yMGWWgFaaWe!>v+HfB_gzBDpHYeN& zMqJX9_T?q19!b~wqpH`=N9CV!-E#xZ;a@utBGb;|BU*|f5viFk+VVa>Ho*B|OZc@x z>Oap+t4m>6D-o&}2ILo$BA`p8 zxJdD4mQI+e=xP4DnY*}eBL;sOOqwKx(Uc(k?rhhHN5X>obUL9!biBh7ku97y!BaNBP#+$b zI8hVyL38!2ivz0S^Zjv*yi|UzdxCY1&#}%T)HalDVTXVX_m$Batc7bwTY}NV1IK0( zez)itDFY9{(Kr<~k*uXEf$P@9Mx1QNS{#fVIPwnXOH>C)2{3`-gNsd@v=-D}?=o*3 zukrJkt>;jM;a?mKvFPuN&3qTK#-QQhnt*O@69FT?hKzC7UBl0G7oFa!>kO&v{4ObW ztHhJfE#Ilk`0lj$1?fOfPICF%7rPo_vYb2%5JkPi+QwBWb0kipV-;lsVHh4xXhPem zFPi}(I!zyoe%)tS6Q+WA!H*PHx7HyHRemq)zdX)h*WT40^ zH8SRqgCV?q88M03wUPsoF(cp*Ff{#0HA4L5bF zSH>C3NMRAXEA!Oh#~hn%lh8@-gz@Q;JF@vmdXKh$%K_af4s=IVE5%FTPq6PiZ+c0T8w`#Br*m##Y|F$IORYHa10L<;D`3Arp&pd z)3A6&7f{(i_O(Kw?t?KDunM~^yWsAT&IfxaP%rn@C zX24&cUFr)->zPYuKGj2B6>Ig}H=kC3WO+YGn0GQI+zD4Nu*qq(C}l8sL3@q4T0aO#JaUH^1X0g59gN zpRfc?+sHM44nM=yggvT0|2IV)rPOiC0iE=@&KIobLmS@-XwoCe2Ebi~K zPS2ZFTqk@VnJ(y`8_pHZ+q%SByzKaW?Qp-;^Z9nxw^=2cIDFk{)jA~&J{OJOxWXWd z)+dL5dnc89`>TBEPZrYhkY&IPOb=uO%P7(R$wJJH98FDBogFP~KmSEWoK;2?dYF;h z$S#o*&Sx0HRPx9L>O@6F{gHPq%M;~|QG_mRWuNc4PeFdzA_}^@x|zA2c>6H>Gzy_K+hOh2VugaSA8d@D>^@-vq^tQCM4@GcC)qa7UWO7|WEx zql@l~Xd&2Y3Yh#GcwYL1O$zD_$D5(s1$0aDbw*?tfi%p zeH+H+(cSPYhr5H6`PusoU?D{r!pGAeLDv-)lo|ZUc{VJ?Ng2Up2h_O($R*+%WUvEu z%=J+Maj+>kj6cI)4*2xB1`=mej76~PLMam3n^d%&QYz!8Fs zxyktkQYve!j=jDXTWkj{Zx7)k zvrYDonf_g>`p@daf7Gu2U4!^njXXuNbMR~~((2o6Xa|pQOMLm{qfoLs=ROiw?N!VXK zn~P)FjfSQ8m%2z*@vX2`Vfnr-M6;89AR06+x_9VL9^fwa--`tM;i7D<_F^LZ+J|ma zMAD!SvvmW{^C+}@-_B8YzsH&ML5NiC=ZH8?xI=EC$nbsWy2|hieQ}G5!R`^TSoc4a zUV8C|L#zTH+zI@@)y&z*7)W-R{u=&DHhHU!*e(mhw_zO!L3`Pe7nM&#ehJDC2mlcIPg}hBW!eC?$Mg+*%i6R(H4lPb*;M=jx#HXw%#nNLUD@+=BVMQd@_l4oF zlCU-4w9zE3`^K!*tHM&QR2X@Rv73jkQ}2r<3K{Xq4QyaT7-IcDzelM$2u-TZt*=I^ z-lRG6XbL0hZ<;)|I!QnTlXLU~>=XehjpK8-+P$qfo5bMk2v5ez1fTpmtGtGIcQy=h zW~;N=h_vXRV=td7_UeS^K6PI!&3L_`vsd4Hm{W>%3au}rhKJFOKRYP2y}AlI^kpk1 zBYMV!!XD|@<?T~z}Stx{o4HRRya0qQ${Ji$OAE{28IijbR zu;lR^_^CVZ+7zxMLbIKly`me%!;^ht^W0rXrSf8Y=Fa`Re*>$`q^OxKul_mfCg8g| z_MAoZYL8!@(QY^t!lPo>7*RZ$d4!qRIwi@n(0^2iy2+&nPM|u71HBFkuz1_V?vs+E zoxKy&Cp$;e-}@h67tjCIYy+n*IpMwLps|Bp8J5;OXDE4h1)rJ+fEU7|g=eO(h%?$7chRokb5M9#yu9uC2k3#KQeh zhF~tC4aLsR-^T@kD)m3hzN-}7p5uVOAOYPn^1mvZk-h!@=^9YU{&{640{t&Dvcx^a z6B6h>CqQp?f*=`R$1Sm<XJ!?L($uzEQJLd$0L^ zPtTWr$6ManIlG7IRBRLmel>sdG!Re*ASe6JjmiJDX#X|-hc;!U5C0DE?@g2cCHQ;% z2DFTSX`%d6@XtoFzeR^3|GSIgzuLn%2-+lAH ficj(Ui}*i1wbBO&puzzF@W2-uP-j#5e}DUb@ EditPlus(long? id) { + if (id == null) { return NotFound(); diff --git a/solution/WebApplication/WebApplication/Startup.cs b/solution/WebApplication/WebApplication/Startup.cs index 3f4746f6..339cb642 100644 --- a/solution/WebApplication/WebApplication/Startup.cs +++ b/solution/WebApplication/WebApplication/Startup.cs @@ -95,6 +95,8 @@ namespace WebApplication opt.Filters.Add(new DefaultHelpLinkActionFilter()); }).AddMvcOptions(m => m.ModelMetadataDetailsProviders.Add(new HumanizerMetadataProvider())); + //services.AddServerSideBlazor(); + services.AddRazorPages(); services.AddMicrosoftIdentityWebAppAuthentication(Configuration, "AzureAdAuth"); @@ -141,6 +143,7 @@ namespace WebApplication name: "default", pattern: "{controller=Dashboard}/{action=Index}/{id?}").RequireAuthorization(); endpoints.MapRazorPages(); + // endpoints.MapBlazorHub(); }); } diff --git a/solution/WebApplication/WebApplication/WebApplication.csproj b/solution/WebApplication/WebApplication/WebApplication.csproj index 329fb4fb..c532c0b7 100644 --- a/solution/WebApplication/WebApplication/WebApplication.csproj +++ b/solution/WebApplication/WebApplication/WebApplication.csproj @@ -64,6 +64,7 @@ +