Major changes in this version including:
- Complete refactor of ADF Code Components to improve naming conventions and consistency
- Addition of Node Web App to facilitate configuration editing
On branch development/0.1
Changes to be committed:
modified: README.md
new file: documentation/images/DeploymentForm.png
renamed: solution/DataFactory/dataset/ADLSBinary_AZ_IR.json -> solution/DataFactory/ADF/dataset/ADLSBinary_IRA.json
renamed: solution/DataFactory/dataset/ADLSDelimitedText_SH_IR.json -> solution/DataFactory/ADF/dataset/ADLSDelimitedText_IRA.json
renamed: solution/DataFactory/dataset/ADLSExcel_SH_IR.json -> solution/DataFactory/ADF/dataset/ADLSExcel_IRA.json
renamed: solution/DataFactory/dataset/ADLSJson_SH_IR.json -> solution/DataFactory/ADF/dataset/ADLSJson_IRA.json
renamed: solution/DataFactory/dataset/ADLSParquet_AZ_IR.json -> solution/DataFactory/ADF/dataset/ADLSParquet_IRA.json
renamed: solution/DataFactory/dataset/AzureSqlTable_AZ_IR.json -> solution/DataFactory/ADF/dataset/AzureSqlTable_IRA.json
renamed: solution/DataFactory/dataset/BlobBinary_AZ_IR.json -> solution/DataFactory/ADF/dataset/BlobBinary_IRA.json
renamed: solution/DataFactory/dataset/BlobDelimitedText_SH_IR.json -> solution/DataFactory/ADF/dataset/BlobDelimitedText_IRA.json
renamed: solution/DataFactory/dataset/BlobExcel_SH_IR.json -> solution/DataFactory/ADF/dataset/BlobExcel_IRA.json
renamed: solution/DataFactory/dataset/BlobJson_SH_IR.json -> solution/DataFactory/ADF/dataset/BlobJson_IRA.json
renamed: solution/DataFactory/dataset/BlobParquet_AZ_IR.json -> solution/DataFactory/ADF/dataset/BlobParquet_IRA.json
renamed: solution/DataFactory/dataset/FileBinary_SH_IR.json -> solution/DataFactory/ADF/dataset/FileBinary_IRA.json
renamed: solution/DataFactory/dataset/FileParquet_SH_IR.json -> solution/DataFactory/ADF/dataset/FileParquet_IRA.json
renamed: solution/DataFactory/dataset/SqlServerTable_SH_IR.json -> solution/DataFactory/ADF/dataset/SqlServerTable_IRA.json
renamed: solution/DataFactory/factory/adsgofastdatakakeacceladf.json -> solution/DataFactory/ADF/factory/adsgofastdatakakeacceladf.json
new file: solution/DataFactory/ADF/integrationRuntime/IRA.json
renamed: solution/DataFactory/linkedService/AdsGoFastKeyVault.json -> solution/DataFactory/ADF/linkedService/AdsGoFastKeyVault.json
renamed: solution/DataFactory/linkedService/AzureFunctionAdsGoFastDataLakeAccelFunApp.json -> solution/DataFactory/ADF/linkedService/AzureFunctionAdsGoFastDataLakeAccelFunApp.json
renamed: solution/DataFactory/linkedService/GenericAzureDataLakeStorage_SH_IR.json -> solution/DataFactory/ADF/linkedService/GenericAzureDataLakeStorage_IRA.json
renamed: solution/DataFactory/linkedService/GenericAzureKeyVault.json -> solution/DataFactory/ADF/linkedService/GenericAzureKeyVault_IRA.json
renamed: solution/DataFactory/linkedService/GenericAzureSQL_SH_IR.json -> solution/DataFactory/ADF/linkedService/GenericAzureSQL_IRA.json
renamed: solution/DataFactory/linkedService/GenericBlobStorage_SH_IR.json -> solution/DataFactory/ADF/linkedService/GenericBlobStorage_IRA.json
renamed: solution/DataFactory/linkedService/GenericFileServer_SH_IR.json -> solution/DataFactory/ADF/linkedService/GenericFileServer_IRA.json
renamed: solution/DataFactory/linkedService/GenericMsSqlServer_SH_IR.json -> solution/DataFactory/ADF/linkedService/GenericMsSqlServer_IRA.json
new file: solution/DataFactory/ADF/pipeline/AZ_Function_Generic.json
renamed: solution/DataFactory/pipeline/AZ-SQL-AZ-Storage-Parquet-SH-IR.json -> solution/DataFactory/ADF/pipeline/AZ_SQL_AZ_Storage_Parquet_IRA.json
renamed: solution/DataFactory/pipeline/AZ-SQL-Create-Table-SH-IR.json -> solution/DataFactory/ADF/pipeline/AZ_SQL_Create_Table_IRA.json
renamed: solution/DataFactory/pipeline/AZ-SQL-Full-Load-Chunk-SH-IR.json -> solution/DataFactory/ADF/pipeline/AZ_SQL_Full_Load_Chunk_IRA.json
renamed: solution/DataFactory/pipeline/AZ-SQL-Full-Load-SH-IR.json -> solution/DataFactory/ADF/pipeline/AZ_SQL_Full_Load_IRA.json
renamed: solution/DataFactory/pipeline/AZ-SQL-Post-Copy-SH-IR.json -> solution/DataFactory/ADF/pipeline/AZ_SQL_Post_Copy_IRA.json
renamed: solution/DataFactory/pipeline/AZ-SQL-StoredProcedure-SH-IR.json -> solution/DataFactory/ADF/pipeline/AZ_SQL_StoredProcedure_IRA.json
renamed: solution/DataFactory/pipeline/AZ-SQL-Watermark-Chunk-SH-IR.json -> solution/DataFactory/ADF/pipeline/AZ_SQL_Watermark_Chunk_IRA.json
renamed: solution/DataFactory/pipeline/AZ-SQL-Watermark-SH-IR.json -> solution/DataFactory/ADF/pipeline/AZ_SQL_Watermark_IRA.json
renamed: solution/DataFactory/pipeline/AZ-Storage-Binary-AZ-Storage-Binary-SH-IR.json -> solution/DataFactory/ADF/pipeline/AZ_Storage_Binary_AZ_Storage_Binary_IRA.json
renamed: solution/DataFactory/pipeline/AZ-Storage-CSV-AZ-SQL-SH-IR.json -> solution/DataFactory/ADF/pipeline/AZ_Storage_CSV_AZ_SQL_IRA.json
renamed: solution/DataFactory/pipeline/AZ-Storage-Excel-AZ-SQL-SH-IR.json -> solution/DataFactory/ADF/pipeline/AZ_Storage_Excel_AZ_SQL_IRA.json
renamed: solution/DataFactory/pipeline/AZ-Storage-Excel-AZ-Storage-CSV-SH-IR.json -> solution/DataFactory/ADF/pipeline/AZ_Storage_Excel_AZ_Storage_CSV_IRA.json
renamed: solution/DataFactory/pipeline/AZ-Storage-JSON-AZ-SQL-SH-IR.json -> solution/DataFactory/ADF/pipeline/AZ_Storage_JSON_AZ_SQL_IRA.json
renamed: solution/DataFactory/pipeline/AZ-Storage-Parquet-AZ-SQL-SH-IR.json -> solution/DataFactory/ADF/pipeline/AZ_Storage_Parquet_AZ_SQL_IRA.json
renamed: solution/DataFactory/pipeline/Create-Task-Master-AZ-SQL-SH-IR.json -> solution/DataFactory/ADF/pipeline/Create_Task_Master_AZ_SQL_IRA.json
renamed: solution/DataFactory/pipeline/GEN-File-Binary-AZ-Storage-Binary-SH-IR.json -> solution/DataFactory/ADF/pipeline/GEN_File_Binary_AZ_Storage_Binary_IRA.json
renamed: solution/DataFactory/pipeline/Master.json -> solution/DataFactory/ADF/pipeline/Master_IRA.json
renamed: solution/DataFactory/pipeline/SH-SQL-AZ-Storage-Parquet-SH-IR.json -> solution/DataFactory/ADF/pipeline/SH_SQL_AZ_Storage_Parquet_IRA.json
renamed: solution/DataFactory/pipeline/SH-SQL-Full-Load-Chunk-SH-IR.json -> solution/DataFactory/ADF/pipeline/SH_SQL_Full_Load_Chunk_IRA.json
renamed: solution/DataFactory/pipeline/SH-SQL-Full-Load-SH-IR.json -> solution/DataFactory/ADF/pipeline/SH_SQL_Full_Load_IRA.json
renamed: solution/DataFactory/pipeline/OnP-SQL-GEN-File-Parquet-OnP-SH-IR.json -> solution/DataFactory/ADF/pipeline/SH_SQL_GEN_File_Parquet_IRA.json
renamed: solution/DataFactory/pipeline/SH-SQL-Watermark-Chunk-SH-IR.json -> solution/DataFactory/ADF/pipeline/SH_SQL_Watermark_Chunk_IRA.json
renamed: solution/DataFactory/pipeline/SH-SQL-Watermark-SH-IR.json -> solution/DataFactory/ADF/pipeline/SH_SQL_Watermark_IRA.json
renamed: solution/DataFactory/trigger/Trigger on File End.json -> solution/DataFactory/ADF/trigger/Trigger on File End.json
new file: solution/DataFactory/GenerateADFArtefacts.ps1
new file: solution/DataFactory/IRConfig.json
renamed: solution/DataFactory/dataset/ADLSBinary_SH_IR.json -> solution/DataFactory/Templates/dataset/ADLSBinary_{IR}.json
new file: solution/DataFactory/Templates/dataset/ADLSDelimitedText_{IR}.json
new file: solution/DataFactory/Templates/dataset/ADLSExcel_{IR}.json
new file: solution/DataFactory/Templates/dataset/ADLSJson_{IR}.json
renamed: solution/DataFactory/dataset/ADLSParquet_SH_IR.json -> solution/DataFactory/Templates/dataset/ADLSParquet_{IR}.json
renamed: solution/DataFactory/dataset/AzureSqlTable_SH_IR.json -> solution/DataFactory/Templates/dataset/AzureSqlTable_{IR}.json
renamed: solution/DataFactory/dataset/BlobBinary_SH_IR.json -> solution/DataFactory/Templates/dataset/BlobBinary_{IR}.json
new file: solution/DataFactory/Templates/dataset/BlobDelimitedText_{IR}.json
new file: solution/DataFactory/Templates/dataset/BlobExcel_{IR}.json
new file: solution/DataFactory/Templates/dataset/BlobJson_{IR}.json
renamed: solution/DataFactory/dataset/BlobParquet_SH_IR.json -> solution/DataFactory/Templates/dataset/BlobParquet_{IR}.json
renamed: solution/DataFactory/dataset/FileBinary_OnPrem_SH_IR.json -> solution/DataFactory/Templates/dataset/FileBinary_{IR}.json
renamed: solution/DataFactory/dataset/FileParquet_OnPrem_SH_IR.json -> solution/DataFactory/Templates/dataset/FileParquet_{IR}.json
renamed: solution/DataFactory/dataset/SqlServerTable_OnPrem_SH_IR.json -> solution/DataFactory/Templates/dataset/SqlServerTable_{IR}.json
new file: solution/DataFactory/Templates/factory/adsgofastdatakakeacceladf.json
new file: solution/DataFactory/Templates/integrationRuntime/IRA.json
new file: solution/DataFactory/Templates/integrationRuntime/IRB.json
renamed: solution/DataFactory/linkedService/GenericAzureDataLakeStorage_OnPrem_Net.json -> solution/DataFactory/Templates/linkedService/GenericAzureDataLakeStorage_{IR}.json
new file: solution/DataFactory/Templates/linkedService/GenericAzureKeyVault_{IR}.json
renamed: solution/DataFactory/linkedService/GenericAzureSQL.json -> solution/DataFactory/Templates/linkedService/GenericAzureSQL_{IR}.json
renamed: solution/DataFactory/linkedService/GenericBlobStorage_OnPrem_Net.json -> solution/DataFactory/Templates/linkedService/GenericBlobStorage_{IR}.json
renamed: solution/DataFactory/linkedService/GenericFileServer_OnPrem_Net.json -> solution/DataFactory/Templates/linkedService/GenericFileServer_{IR}.json
renamed: solution/DataFactory/linkedService/GenericMsSqlServer_OnPrem_Net.json -> solution/DataFactory/Templates/linkedService/GenericMsSqlServer_{IR}.json
new file: solution/DataFactory/Templates/linkedService/NamingConventions.md
renamed: solution/DataFactory/pipeline/AZ-Function-Generic.json -> solution/DataFactory/Templates/pipeline/AZ_Function_Generic.json
new file: solution/DataFactory/Templates/pipeline/AZ_SQL_AZ_Storage_Parquet_{IR}.json
new file: solution/DataFactory/Templates/pipeline/AZ_SQL_Create_Table_{IR}.json
renamed: solution/DataFactory/pipeline/OnP-SQL-Full-Load-Chunk-OnP-SH-IR.json -> solution/DataFactory/Templates/pipeline/AZ_SQL_Full_Load_Chunk_{IR}.json
new file: solution/DataFactory/Templates/pipeline/AZ_SQL_Full_Load_{IR}.json
new file: solution/DataFactory/Templates/pipeline/AZ_SQL_Post_Copy_{IR}.json
new file: solution/DataFactory/Templates/pipeline/AZ_SQL_StoredProcedure_{IR}.json
renamed: solution/DataFactory/pipeline/OnP-SQL-Watermark-Chunk-OnP-SH-IR.json -> solution/DataFactory/Templates/pipeline/AZ_SQL_Watermark_Chunk_{IR}.json
new file: solution/DataFactory/Templates/pipeline/AZ_SQL_Watermark_{IR}.json
renamed: solution/DataFactory/pipeline/AZ-Storage-Binary-AZ-Storage-Binary-AZ-IR.json -> solution/DataFactory/Templates/pipeline/AZ_Storage_Binary_AZ_Storage_Binary_{IR}.json
new file: solution/DataFactory/Templates/pipeline/AZ_Storage_CSV_AZ_SQL_{IR}.json
new file: solution/DataFactory/Templates/pipeline/AZ_Storage_Excel_AZ_SQL_{IR}.json
new file: solution/DataFactory/Templates/pipeline/AZ_Storage_Excel_AZ_Storage_CSV_{IR}.json
new file: solution/DataFactory/Templates/pipeline/AZ_Storage_JSON_AZ_SQL_{IR}.json
new file: solution/DataFactory/Templates/pipeline/AZ_Storage_Parquet_AZ_SQL_{IR}.json
renamed: solution/DataFactory/pipeline/Create-Task-Master-AZ-SQL-OnP-SH-IR.json -> solution/DataFactory/Templates/pipeline/Create_Task_Master_AZ_SQL_{IR}.json
renamed: solution/DataFactory/pipeline/GEN-File-Binary-AZ-Storage-Binary-OnP-SH-IR.json -> solution/DataFactory/Templates/pipeline/GEN_File_Binary_AZ_Storage_Binary_{IR}.json
new file: solution/DataFactory/Templates/pipeline/Master_{IR}.json
renamed: solution/DataFactory/pipeline/OnP-SQL-AZ-Storage-Parquet-OnP-SH-IR.json -> solution/DataFactory/Templates/pipeline/SH_SQL_AZ_Storage_Parquet_{IR}.json
new file: solution/DataFactory/Templates/pipeline/SH_SQL_Full_Load_Chunk_{IR}.json
renamed: solution/DataFactory/pipeline/OnP-SQL-Full-Load-OnP-SH-IR.json -> solution/DataFactory/Templates/pipeline/SH_SQL_Full_Load_{IR}.json
new file: solution/DataFactory/Templates/pipeline/SH_SQL_GEN_File_Parquet_{IR}.json
new file: solution/DataFactory/Templates/pipeline/SH_SQL_Watermark_Chunk_{IR}.json
renamed: solution/DataFactory/pipeline/OnP-SQL-Watermark-OnP-SH-IR.json -> solution/DataFactory/Templates/pipeline/SH_SQL_Watermark_{IR}.json
new file: solution/DataFactory/Templates/trigger/Trigger on File End.json
deleted: solution/DataFactory/dataset/ADLSBinary_OnPrem_SH_IR.json
deleted: solution/DataFactory/dataset/ADLSParquet_OnPrem_SH_IR.json
deleted: solution/DataFactory/dataset/BlobBinary_OnPrem_SH_IR.json
deleted: solution/DataFactory/dataset/BlobParquet_OnPrem_SH_IR.json
deleted: solution/DataFactory/integrationRuntime/SelfHostedIntegrationRuntime-Azure-VNET.json
deleted: solution/DataFactory/integrationRuntime/SelfHostedIntegrationRuntime-OnPem-Net.json
deleted: solution/DataFactory/linkedService/GenericAzureDataLakeStorage.json
deleted: solution/DataFactory/linkedService/GenericBlobStorage.json
deleted: solution/DataFactory/pipeline/AZ-SQL-AZ-Storage-Parquet-AZ-IR.json
deleted: solution/DataFactory/pipeline/AZ-Storage-Parquet-AZ-SQL-AZ-IR.json
modified: solution/Database/ADSGoFastDbUp/AdsGoFastDbUp/0-1-0-beta/A-Journaled/006-SampleTasks_TaskMaster.sql
modified: solution/Database/ADSGoFastDbUp/AdsGoFastDbUp/0-1-0-beta/A-Journaled/008-SampleTasks_SourceAndTargetSystems.sql
new file: solution/Database/ADSGoFastDbUp/AdsGoFastDbUp/0-1-0-beta/A-Journaled/010-TaskTypeMapping_Update.sql
modified: solution/Deployment/.devcontainer/devcontainer.json
new file: solution/Deployment/arm/AppService_Func.bicep
new file: solution/Deployment/arm/AppService_Web.bicep
new file: solution/Deployment/arm/ApplicationInsights.bicep
modified: solution/Deployment/arm/AzureSQLServer.json
new file: solution/Deployment/arm/AzureSqlServer.bicep
new file: solution/Deployment/arm/DataFactory.bicep
new file: solution/Deployment/arm/FunctionApp.bicep
new file: solution/Deployment/arm/KeyVault.bicep
new file: solution/Deployment/arm/LogAnalytics.bicep
new file: solution/Deployment/arm/Networking.bicep
new file: solution/Deployment/arm/Storage_ADLS.bicep
new file: solution/Deployment/arm/Storage_Blob.bicep
new file: solution/Deployment/arm/Storage_Logging.bicep
new file: solution/Deployment/arm/VirtualMachine.bicep
new file: solution/Deployment/arm/WebApp.bicep
modified: solution/Deployment/arm/WebApp.json
new file: solution/Deployment/environments/EditSettings.html
new file: solution/Deployment/environments/Node/package-lock.json
new file: solution/Deployment/environments/Node/package.json
new file: solution/Deployment/environments/Node/server.js
modified: solution/Deployment/environments/development.json
new file: solution/Deployment/environments/development_bak.json
new file: solution/Deployment/environments/development_test.json
modified: solution/Deployment/environments/environment.schema.json
modified: solution/Deployment/workflows/CD_0a_CreateServicePrincipals_AAD_Elevated.ps1
modified: solution/Deployment/workflows/CD_1a_DeployServices.ps1
modified: solution/Deployment/workflows/CD_2a_CreateMSIs_AAD_Elevated.ps1
modified: solution/Deployment/workflows/CD_2b_ConfigureServices.ps1
modified: solution/Deployment/workflows/LocalDevOnly_EnvironmentSetUp.ps1
modified: solution/Deployment/workflows/Steps/CD_ConfigureADF.ps1
modified: solution/Deployment/workflows/Steps/CD_ConfigureAzureSQLServer.ps1
modified: solution/Deployment/workflows/Steps/CD_ConfigureAzureSqlServer_UpdateTaskTypeMappingJson.ps1
modified: solution/Deployment/workflows/Steps/CD_ConfigureFunctionApp.ps1
modified: solution/Deployment/workflows/Steps/CD_ConfigureKeyVault.ps1
new file: solution/Deployment/workflows/Steps/CD_ConfigureSampleData.ps1
modified: solution/Deployment/workflows/Steps/CD_ConfigureVnet.ps1
modified: solution/Deployment/workflows/Steps/CD_ConfigureWebApp.ps1
modified: solution/Deployment/workflows/Steps/CD_DeployADF.ps1
modified: solution/Deployment/workflows/Steps/CD_DeployADFOnPremSHIR.ps1
modified: solution/Deployment/workflows/Steps/CD_DeployAppInsights.ps1
modified: solution/Deployment/workflows/Steps/CD_DeployAppService.ps1
modified: solution/Deployment/workflows/Steps/CD_DeployAzureSqlServer.ps1
modified: solution/Deployment/workflows/Steps/CD_DeployFunctionApp.ps1
modified: solution/Deployment/workflows/Steps/CD_DeployKeyVault.ps1
modified: solution/Deployment/workflows/Steps/CD_DeployLogAnalytics.ps1
modified: solution/Deployment/workflows/Steps/CD_DeployResourceGroup.ps1
modified: solution/Deployment/workflows/Steps/CD_DeployStorageADLS.ps1
modified: solution/Deployment/workflows/Steps/CD_DeployStorageBlob.ps1
modified: solution/Deployment/workflows/Steps/CD_DeployStorageForLogging.ps1
modified: solution/Deployment/workflows/Steps/CD_DeployVnet.ps1
modified: solution/Deployment/workflows/Steps/CD_DeployWebSite.ps1
modified: solution/Deployment/workflows/Steps/CD_GrantRBAC.ps1
modified: solution/Deployment/workflows/Steps/CD_SetResourceGroupHash.ps1
modified: solution/Deployment/workflows/Steps/CI_BuildDataFactory.ps1
modified: solution/Deployment/workflows/Steps/InstallGatewayFunctions.ps1
modified: solution/Deployment/workflows/Steps/PushEnvFileIntoVariables.ps1
modified: solution/FunctionApp/FunctionApp.csproj
modified: solution/FunctionApp/Startup.cs
modified: solution/FunctionApp/TaskMetaData/TaskInstance.cs
new file: solution/SampleFiles/yellow_tripdata_2017-03.csv
new file: solution/SampleFiles/yellow_tripdata_2017-03.xlsx
renamed: solution/TaskTypeJson/AZ-SQL-AZ-Storage-Parquet-SH-IR.json -> solution/TaskTypeJson/AZ_SQL_AZ_Storage_Parquet_IRA.json
renamed: solution/TaskTypeJson/AZ-SQL-StoredProcedure-SH-IR.json -> solution/TaskTypeJson/AZ_SQL_StoredProcedure_IRA.json
renamed: solution/TaskTypeJson/AZ-Storage-Binary-AZ-Storage-Binary-SH-IR.json -> solution/TaskTypeJson/AZ_Storage_Binary_AZ_Storage_Binary_IRA.json
renamed: solution/TaskTypeJson/AZ-Storage-CSV-AZ-SQL-SH-IR.json -> solution/TaskTypeJson/AZ_Storage_CSV_AZ_SQL_IRA.json
renamed: solution/TaskTypeJson/AZ-Storage-Excel-AZ-SQL-SH-IR.json -> solution/TaskTypeJson/AZ_Storage_Excel_AZ_SQL_IRA.json
renamed: solution/TaskTypeJson/AZ-Storage-Excel-AZ-Storage-CSV-SH-IR.json -> solution/TaskTypeJson/AZ_Storage_Excel_AZ_Storage_CSV_IRA.json
renamed: solution/TaskTypeJson/AZ-Storage-JSON-AZ-SQL-SH-IR.json -> solution/TaskTypeJson/AZ_Storage_JSON_AZ_SQL_IRA.json
renamed: solution/TaskTypeJson/AZ-Storage-Parquet-AZ-SQL-SH-IR.json -> solution/TaskTypeJson/AZ_Storage_Parquet_AZ_SQL_IRA.json
renamed: solution/TaskTypeJson/GEN-File-Binary-AZ-Storage-Binary-OnP-SH-IR.json -> solution/TaskTypeJson/GEN_File_Binary_AZ_Storage_Binary_IRA.json
renamed: solution/TaskTypeJson/GEN-File-Binary-AZ-Storage-Binary-SH-IR.json -> solution/TaskTypeJson/GEN_File_Binary_AZ_Storage_Binary_IRB.json
renamed: solution/TaskTypeJson/OnP-SQL-GEN-File-Parquet-OnP-SH-IR.json -> solution/TaskTypeJson/OnP_SQL_AZ_Storage_Parquet_IRB.json
renamed: solution/TaskTypeJson/OnP-SQL-AZ-Storage-Parquet-OnP-SH-IR.json -> solution/TaskTypeJson/OnP_SQL_GEN_File_Parquet_IRB.json
modified: solution/WebApplication/WebApplication/Controllers/TaskMasterController.cs
modified: solution/WebApplication/WebApplication/Startup.cs
modified: solution/WebApplication/WebApplication/WebApplication.csproj
This commit is contained in:
John Rampono
Родитель
9f5aad425e
Коммит
66ad02966e
12
README.md
12
README.md
|
|
@ -43,7 +43,17 @@ Once you have your Prerequisite items, please move on to the Deployment Configur
|
|||
>- [Docker Desktop](https://www.docker.com/products/docker-desktop)
|
||||
>- [Windows Store Ubuntu 18.04 LTS](https://www.microsoft.com/store/productId/9N9TNGVNDL3Q)
|
||||
|
||||
The deployment uses a concept of **Developing inside a Container** to containerize all the necessary pre-requisite component without requiring them to be installed on the local machine. Follow our [Configuring your System for Development Containers](docs%2Fdeveloping_inside_a_container.md) guide.
|
||||
The deployment uses a concept of **Developing inside a Container** to containerize all the necessary pre-requisite component without requiring them to be installed on the local machine. Follow our [Configuring your System for Development Containers](https://code.visualstudio.com/docs/remote/containers) guide.
|
||||
|
||||
Once you have set up these pre-requisites you will then need to
|
||||
|
||||
>- [ ] [Clone](https://docs.github.com/en/enterprise-server@3.1/repositories/creating-and-managing-repositories/cloning-a-repository) this repository to your local machine
|
||||
>- [ ] Navigate to the root solution folder and then navigate to "solution/Deployment/environments". Within this folder you will find a file called "environment.schema.json".
|
||||
>- [ ] Open a web browser and navigate to the website [https://pmk65.github.io/jedemov2/dist/demo.html](https://pmk65.github.io/jedemov2/dist/demo.html)
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
---
|
||||
|
||||
|
|
|
|||
Двоичный файл не отображается.
|
После Ширина: | Высота: | Размер: 104 KiB |
|
|
@ -1,8 +1,8 @@
|
|||
{
|
||||
"name": "ADLSBinary_AZ_IR",
|
||||
"name": "ADLSBinary_IRA",
|
||||
"properties": {
|
||||
"linkedServiceName": {
|
||||
"referenceName": "GenericAzureDataLakeStorage",
|
||||
"referenceName": "GenericAzureDataLakeStorage_IRA",
|
||||
"type": "LinkedServiceReference",
|
||||
"parameters": {
|
||||
"StorageAccountEndpoint": {
|
||||
|
|
@ -26,7 +26,7 @@
|
|||
}
|
||||
},
|
||||
"folder": {
|
||||
"name": "ADS Go Fast/Generic/Azure IR"
|
||||
"name": "ADS Go Fast/Generic/IRA"
|
||||
},
|
||||
"annotations": [],
|
||||
"type": "Binary",
|
||||
|
|
@ -49,4 +49,4 @@
|
|||
}
|
||||
},
|
||||
"type": "Microsoft.DataFactory/factories/datasets"
|
||||
}
|
||||
}
|
||||
|
|
@ -1,8 +1,8 @@
|
|||
{
|
||||
"name": "ADLSDelimitedText_SH_IR",
|
||||
"name": "ADLSDelimitedText_IRA",
|
||||
"properties": {
|
||||
"linkedServiceName": {
|
||||
"referenceName": "GenericAzureDataLakeStorage_SH_IR",
|
||||
"referenceName": "GenericAzureDataLakeStorage_IRA",
|
||||
"type": "LinkedServiceReference",
|
||||
"parameters": {
|
||||
"StorageAccountEndpoint": {
|
||||
|
|
@ -29,7 +29,7 @@
|
|||
}
|
||||
},
|
||||
"folder": {
|
||||
"name": "ADS Go Fast/Generic/Azure SH IR"
|
||||
"name": "ADS Go Fast/Generic/IRA"
|
||||
},
|
||||
"annotations": [],
|
||||
"type": "DelimitedText",
|
||||
|
|
@ -60,4 +60,4 @@
|
|||
"schema": []
|
||||
},
|
||||
"type": "Microsoft.DataFactory/factories/datasets"
|
||||
}
|
||||
}
|
||||
|
|
@ -1,8 +1,8 @@
|
|||
{
|
||||
"name": "ADLSExcel_SH_IR",
|
||||
"name": "ADLSExcel_IRA",
|
||||
"properties": {
|
||||
"linkedServiceName": {
|
||||
"referenceName": "GenericAzureDataLakeStorage_SH_IR",
|
||||
"referenceName": "GenericAzureDataLakeStorage_IRA",
|
||||
"type": "LinkedServiceReference",
|
||||
"parameters": {
|
||||
"StorageAccountEndpoint": {
|
||||
|
|
@ -32,7 +32,7 @@
|
|||
}
|
||||
},
|
||||
"folder": {
|
||||
"name": "ADS Go Fast/Generic/Azure SH IR"
|
||||
"name": "ADS Go Fast/Generic/IRA"
|
||||
},
|
||||
"annotations": [],
|
||||
"type": "Excel",
|
||||
|
|
@ -64,4 +64,4 @@
|
|||
"schema": []
|
||||
},
|
||||
"type": "Microsoft.DataFactory/factories/datasets"
|
||||
}
|
||||
}
|
||||
|
|
@ -1,8 +1,8 @@
|
|||
{
|
||||
"name": "ADLSJson_SH_IR",
|
||||
"name": "ADLSJson_IRA",
|
||||
"properties": {
|
||||
"linkedServiceName": {
|
||||
"referenceName": "GenericAzureDataLakeStorage_SH_IR",
|
||||
"referenceName": "GenericAzureDataLakeStorage_IRA",
|
||||
"type": "LinkedServiceReference",
|
||||
"parameters": {
|
||||
"StorageAccountEndpoint": {
|
||||
|
|
@ -26,7 +26,7 @@
|
|||
}
|
||||
},
|
||||
"folder": {
|
||||
"name": "ADS Go Fast/Generic/Azure SH IR"
|
||||
"name": "ADS Go Fast/Generic/IRA"
|
||||
},
|
||||
"annotations": [],
|
||||
"type": "Json",
|
||||
|
|
@ -50,4 +50,4 @@
|
|||
"schema": {}
|
||||
},
|
||||
"type": "Microsoft.DataFactory/factories/datasets"
|
||||
}
|
||||
}
|
||||
|
|
@ -1,8 +1,8 @@
|
|||
{
|
||||
"name": "ADLSParquet_AZ_IR",
|
||||
"name": "ADLSParquet_IRA",
|
||||
"properties": {
|
||||
"linkedServiceName": {
|
||||
"referenceName": "GenericAzureDataLakeStorage",
|
||||
"referenceName": "GenericAzureDataLakeStorage_IRA",
|
||||
"type": "LinkedServiceReference",
|
||||
"parameters": {
|
||||
"StorageAccountEndpoint": {
|
||||
|
|
@ -26,7 +26,7 @@
|
|||
}
|
||||
},
|
||||
"folder": {
|
||||
"name": "ADS Go Fast/Generic/Azure IR"
|
||||
"name": "ADS Go Fast/Generic/IRA"
|
||||
},
|
||||
"annotations": [],
|
||||
"type": "Parquet",
|
||||
|
|
@ -51,4 +51,4 @@
|
|||
"schema": []
|
||||
},
|
||||
"type": "Microsoft.DataFactory/factories/datasets"
|
||||
}
|
||||
}
|
||||
|
|
@ -1,8 +1,8 @@
|
|||
{
|
||||
"name": "AzureSqlTable_AZ_IR",
|
||||
"name": "AzureSqlTable_IRA",
|
||||
"properties": {
|
||||
"linkedServiceName": {
|
||||
"referenceName": "GenericAzureSQL",
|
||||
"referenceName": "GenericAzureSQL_IRA",
|
||||
"type": "LinkedServiceReference",
|
||||
"parameters": {
|
||||
"Server": {
|
||||
|
|
@ -30,7 +30,7 @@
|
|||
}
|
||||
},
|
||||
"folder": {
|
||||
"name": "ADS Go Fast/Generic/Azure IR"
|
||||
"name": "ADS Go Fast/Generic/IRA"
|
||||
},
|
||||
"annotations": [],
|
||||
"type": "AzureSqlTable",
|
||||
|
|
@ -47,4 +47,4 @@
|
|||
}
|
||||
},
|
||||
"type": "Microsoft.DataFactory/factories/datasets"
|
||||
}
|
||||
}
|
||||
|
|
@ -1,8 +1,8 @@
|
|||
{
|
||||
"name": "BlobBinary_AZ_IR",
|
||||
"name": "BlobBinary_IRA",
|
||||
"properties": {
|
||||
"linkedServiceName": {
|
||||
"referenceName": "GenericBlobStorage",
|
||||
"referenceName": "GenericBlobStorage_IRA",
|
||||
"type": "LinkedServiceReference",
|
||||
"parameters": {
|
||||
"StorageAccountEndpoint": {
|
||||
|
|
@ -26,7 +26,7 @@
|
|||
}
|
||||
},
|
||||
"folder": {
|
||||
"name": "ADS Go Fast/Generic/Azure IR"
|
||||
"name": "ADS Go Fast/Generic/IRA"
|
||||
},
|
||||
"annotations": [],
|
||||
"type": "Binary",
|
||||
|
|
@ -49,4 +49,4 @@
|
|||
}
|
||||
},
|
||||
"type": "Microsoft.DataFactory/factories/datasets"
|
||||
}
|
||||
}
|
||||
|
|
@ -1,8 +1,8 @@
|
|||
{
|
||||
"name": "BlobDelimitedText_SH_IR",
|
||||
"name": "BlobDelimitedText_IRA",
|
||||
"properties": {
|
||||
"linkedServiceName": {
|
||||
"referenceName": "GenericBlobStorage_SH_IR",
|
||||
"referenceName": "GenericBlobStorage_IRA",
|
||||
"type": "LinkedServiceReference",
|
||||
"parameters": {
|
||||
"StorageAccountEndpoint": {
|
||||
|
|
@ -29,7 +29,7 @@
|
|||
}
|
||||
},
|
||||
"folder": {
|
||||
"name": "ADS Go Fast/Generic/Azure SH IR"
|
||||
"name": "ADS Go Fast/Generic/IRA"
|
||||
},
|
||||
"annotations": [],
|
||||
"type": "DelimitedText",
|
||||
|
|
@ -60,4 +60,4 @@
|
|||
"schema": []
|
||||
},
|
||||
"type": "Microsoft.DataFactory/factories/datasets"
|
||||
}
|
||||
}
|
||||
|
|
@ -1,8 +1,8 @@
|
|||
{
|
||||
"name": "BlobExcel_SH_IR",
|
||||
"name": "BlobExcel_IRA",
|
||||
"properties": {
|
||||
"linkedServiceName": {
|
||||
"referenceName": "GenericBlobStorage_SH_IR",
|
||||
"referenceName": "GenericBlobStorage_IRA",
|
||||
"type": "LinkedServiceReference",
|
||||
"parameters": {
|
||||
"StorageAccountEndpoint": {
|
||||
|
|
@ -32,7 +32,7 @@
|
|||
}
|
||||
},
|
||||
"folder": {
|
||||
"name": "ADS Go Fast/Generic/Azure SH IR"
|
||||
"name": "ADS Go Fast/Generic/IRA"
|
||||
},
|
||||
"annotations": [],
|
||||
"type": "Excel",
|
||||
|
|
@ -64,4 +64,4 @@
|
|||
"schema": []
|
||||
},
|
||||
"type": "Microsoft.DataFactory/factories/datasets"
|
||||
}
|
||||
}
|
||||
|
|
@ -1,8 +1,8 @@
|
|||
{
|
||||
"name": "BlobJson_SH_IR",
|
||||
"name": "BlobJson_IRA",
|
||||
"properties": {
|
||||
"linkedServiceName": {
|
||||
"referenceName": "GenericBlobStorage_SH_IR",
|
||||
"referenceName": "GenericBlobStorage_IRA",
|
||||
"type": "LinkedServiceReference",
|
||||
"parameters": {
|
||||
"StorageAccountEndpoint": {
|
||||
|
|
@ -26,7 +26,7 @@
|
|||
}
|
||||
},
|
||||
"folder": {
|
||||
"name": "ADS Go Fast/Generic/Azure SH IR"
|
||||
"name": "ADS Go Fast/Generic/IRA"
|
||||
},
|
||||
"annotations": [],
|
||||
"type": "Json",
|
||||
|
|
@ -50,4 +50,4 @@
|
|||
"schema": {}
|
||||
},
|
||||
"type": "Microsoft.DataFactory/factories/datasets"
|
||||
}
|
||||
}
|
||||
|
|
@ -1,8 +1,8 @@
|
|||
{
|
||||
"name": "BlobParquet_AZ_IR",
|
||||
"name": "BlobParquet_IRA",
|
||||
"properties": {
|
||||
"linkedServiceName": {
|
||||
"referenceName": "GenericBlobStorage",
|
||||
"referenceName": "GenericBlobStorage_IRA",
|
||||
"type": "LinkedServiceReference",
|
||||
"parameters": {
|
||||
"StorageAccountEndpoint": {
|
||||
|
|
@ -26,7 +26,7 @@
|
|||
}
|
||||
},
|
||||
"folder": {
|
||||
"name": "ADS Go Fast/Generic/Azure IR"
|
||||
"name": "ADS Go Fast/Generic/IRA"
|
||||
},
|
||||
"annotations": [],
|
||||
"type": "Parquet",
|
||||
|
|
@ -51,4 +51,4 @@
|
|||
"schema": []
|
||||
},
|
||||
"type": "Microsoft.DataFactory/factories/datasets"
|
||||
}
|
||||
}
|
||||
|
|
@ -1,8 +1,8 @@
|
|||
{
|
||||
"name": "FileBinary_SH_IR",
|
||||
"name": "FileBinary_IRA",
|
||||
"properties": {
|
||||
"linkedServiceName": {
|
||||
"referenceName": "GenericFileServer_SH_IR",
|
||||
"referenceName": "GenericFileServer_IRA",
|
||||
"type": "LinkedServiceReference",
|
||||
"parameters": {
|
||||
"Host": {
|
||||
|
|
@ -44,7 +44,7 @@
|
|||
}
|
||||
},
|
||||
"folder": {
|
||||
"name": "ADS Go Fast/Generic/Azure SH IR"
|
||||
"name": "ADS Go Fast/Generic/IRA"
|
||||
},
|
||||
"annotations": [],
|
||||
"type": "Binary",
|
||||
|
|
@ -63,4 +63,4 @@
|
|||
}
|
||||
},
|
||||
"type": "Microsoft.DataFactory/factories/datasets"
|
||||
}
|
||||
}
|
||||
|
|
@ -1,8 +1,8 @@
|
|||
{
|
||||
"name": "FileParquet_SH_IR",
|
||||
"name": "FileParquet_IRA",
|
||||
"properties": {
|
||||
"linkedServiceName": {
|
||||
"referenceName": "GenericFileServer_SH_IR",
|
||||
"referenceName": "GenericFileServer_IRA",
|
||||
"type": "LinkedServiceReference",
|
||||
"parameters": {
|
||||
"Host": {
|
||||
|
|
@ -44,7 +44,7 @@
|
|||
}
|
||||
},
|
||||
"folder": {
|
||||
"name": "ADS Go Fast/Generic/Azure SH IR"
|
||||
"name": "ADS Go Fast/Generic/IRA"
|
||||
},
|
||||
"annotations": [],
|
||||
"type": "Parquet",
|
||||
|
|
@ -65,4 +65,4 @@
|
|||
"schema": []
|
||||
},
|
||||
"type": "Microsoft.DataFactory/factories/datasets"
|
||||
}
|
||||
}
|
||||
|
|
@ -1,8 +1,8 @@
|
|||
{
|
||||
"name": "SqlServerTable_SH_IR",
|
||||
"name": "SqlServerTable_IRA",
|
||||
"properties": {
|
||||
"linkedServiceName": {
|
||||
"referenceName": "GenericMsSqlServer_SH_IR",
|
||||
"referenceName": "GenericMsSqlServer_IRA",
|
||||
"type": "LinkedServiceReference",
|
||||
"parameters": {
|
||||
"KeyVaultBaseUrl": {
|
||||
|
|
@ -51,7 +51,7 @@
|
|||
}
|
||||
},
|
||||
"folder": {
|
||||
"name": "ADS Go Fast/Generic/Azure SH IR"
|
||||
"name": "ADS Go Fast/Generic/IRA"
|
||||
},
|
||||
"annotations": [],
|
||||
"type": "SqlServerTable",
|
||||
|
|
@ -68,4 +68,4 @@
|
|||
}
|
||||
},
|
||||
"type": "Microsoft.DataFactory/factories/datasets"
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,22 @@
|
|||
{
|
||||
"name": "IRA",
|
||||
"properties": {
|
||||
"type": "Managed",
|
||||
"description": "Primary integration runtime that sits in the Azure cloud. Used to pull data into the \"Analytics Lockbox\", move data around the \"Analytics Lockbox\" and push data out of the \"Analytics Lockbox\".",
|
||||
"typeProperties": {
|
||||
"computeProperties": {
|
||||
"location": "AutoResolve",
|
||||
"dataFlowProperties": {
|
||||
"computeType": "General",
|
||||
"coreCount": 8,
|
||||
"timeToLive": 10,
|
||||
"cleanup": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"managedVirtualNetwork": {
|
||||
"type": "ManagedVirtualNetworkReference",
|
||||
"referenceName": "default"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
{
|
||||
"name": "GenericAzureDataLakeStorage_SH_IR",
|
||||
"name": "GenericAzureDataLakeStorage_IRA",
|
||||
"type": "Microsoft.DataFactory/factories/linkedservices",
|
||||
"properties": {
|
||||
"type": "AzureBlobFS",
|
||||
|
|
@ -14,8 +14,8 @@
|
|||
},
|
||||
"annotations": [],
|
||||
"connectVia": {
|
||||
"referenceName": "SelfHostedIntegrationRuntime-Azure-VNET",
|
||||
"referenceName": "IRA",
|
||||
"type": "IntegrationRuntimeReference"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -12,6 +12,10 @@
|
|||
"typeProperties": {
|
||||
"baseUrl": "@{linkedService().KeyVaultBaseUrl}"
|
||||
},
|
||||
"connectVia": {
|
||||
"referenceName": "IRA",
|
||||
"type": "IntegrationRuntimeReference"
|
||||
},
|
||||
"annotations": []
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
{
|
||||
"name": "GenericAzureSQL_SH_IR",
|
||||
"name": "GenericAzureSQL_IRA",
|
||||
"type": "Microsoft.DataFactory/factories/linkedservices",
|
||||
"properties": {
|
||||
"parameters": {
|
||||
|
|
@ -16,8 +16,8 @@
|
|||
"connectionString": "Integrated Security=False;Encrypt=True;Connection Timeout=30;Data Source=@{linkedService().Server};Initial Catalog=@{linkedService().Database}"
|
||||
},
|
||||
"connectVia": {
|
||||
"referenceName": "SelfHostedIntegrationRuntime-Azure-VNET",
|
||||
"referenceName": "IRA",
|
||||
"type": "IntegrationRuntimeReference"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
{
|
||||
"name": "GenericBlobStorage_SH_IR",
|
||||
"name": "GenericBlobStorage_IRA",
|
||||
"type": "Microsoft.DataFactory/factories/linkedservices",
|
||||
"properties": {
|
||||
"type": "AzureBlobStorage",
|
||||
|
|
@ -14,8 +14,8 @@
|
|||
},
|
||||
"annotations": [],
|
||||
"connectVia": {
|
||||
"referenceName": "SelfHostedIntegrationRuntime-Azure-VNET",
|
||||
"referenceName": "IRA",
|
||||
"type": "IntegrationRuntimeReference"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
{
|
||||
"name": "GenericFileServer_SH_IR",
|
||||
"name": "GenericFileServer_IRA",
|
||||
"type": "Microsoft.DataFactory/factories/linkedservices",
|
||||
"properties": {
|
||||
"type": "FileServer",
|
||||
|
|
@ -44,8 +44,8 @@
|
|||
},
|
||||
"annotations": [],
|
||||
"connectVia": {
|
||||
"referenceName": "SelfHostedIntegrationRuntime-Azure-VNET",
|
||||
"referenceName": "IRA",
|
||||
"type": "IntegrationRuntimeReference"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
{
|
||||
"name": "GenericMsSqlServer_SH_IR",
|
||||
"name": "GenericMsSqlServer_IRA",
|
||||
"type": "Microsoft.DataFactory/factories/linkedservices",
|
||||
"properties": {
|
||||
"parameters": {
|
||||
|
|
@ -42,8 +42,8 @@
|
|||
}
|
||||
},
|
||||
"connectVia": {
|
||||
"referenceName": "SelfHostedIntegrationRuntime-Azure-VNET",
|
||||
"referenceName": "IRA",
|
||||
"type": "IntegrationRuntimeReference"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,104 @@
|
|||
{
|
||||
"name": "AZ_Function_Generic",
|
||||
"properties": {
|
||||
"activities": [
|
||||
{
|
||||
"name": "Switch Method",
|
||||
"type": "Switch",
|
||||
"dependsOn": [],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"on": {
|
||||
"value": "@pipeline().parameters.Method",
|
||||
"type": "Expression"
|
||||
},
|
||||
"cases": [
|
||||
{
|
||||
"value": "Put",
|
||||
"activities": [
|
||||
{
|
||||
"name": "Azure Function Generic Put",
|
||||
"type": "AzureFunctionActivity",
|
||||
"dependsOn": [],
|
||||
"policy": {
|
||||
"timeout": "7.00:00:00",
|
||||
"retry": 0,
|
||||
"retryIntervalInSeconds": 30,
|
||||
"secureOutput": false,
|
||||
"secureInput": false
|
||||
},
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"functionName": {
|
||||
"value": "@pipeline().parameters.FunctionName",
|
||||
"type": "Expression"
|
||||
},
|
||||
"method": "PUT",
|
||||
"body": {
|
||||
"value": "@pipeline().parameters.Body",
|
||||
"type": "Expression"
|
||||
}
|
||||
},
|
||||
"linkedServiceName": {
|
||||
"referenceName": "AzureFunctionAdsGoFastDataLakeAccelFunApp",
|
||||
"type": "LinkedServiceReference"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"value": "Post",
|
||||
"activities": [
|
||||
{
|
||||
"name": "Azure Function Generic Post",
|
||||
"type": "AzureFunctionActivity",
|
||||
"dependsOn": [],
|
||||
"policy": {
|
||||
"timeout": "7.00:00:00",
|
||||
"retry": 0,
|
||||
"retryIntervalInSeconds": 30,
|
||||
"secureOutput": false,
|
||||
"secureInput": false
|
||||
},
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"functionName": {
|
||||
"value": "@pipeline().parameters.FunctionName",
|
||||
"type": "Expression"
|
||||
},
|
||||
"method": "POST",
|
||||
"body": {
|
||||
"value": "@pipeline().parameters.Body",
|
||||
"type": "Expression"
|
||||
}
|
||||
},
|
||||
"linkedServiceName": {
|
||||
"referenceName": "AzureFunctionAdsGoFastDataLakeAccelFunApp",
|
||||
"type": "LinkedServiceReference"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Body": {
|
||||
"type": "string"
|
||||
},
|
||||
"FunctionName": {
|
||||
"type": "string"
|
||||
},
|
||||
"Method": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"folder": {
|
||||
"name": "ADS Go Fast/Common"
|
||||
},
|
||||
"annotations": [],
|
||||
"lastPublishTime": "2020-07-14T10:58:30Z"
|
||||
},
|
||||
"type": "Microsoft.DataFactory/factories/pipelines"
|
||||
}
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
{
|
||||
"name": "AZ-SQL-AZ-Storage-Parquet-SH-IR",
|
||||
"name": "AZ_SQL_AZ_Storage_Parquet_IRA",
|
||||
"properties": {
|
||||
"activities": [
|
||||
{
|
||||
|
|
@ -16,7 +16,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-Function-Generic",
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
|
|
@ -60,7 +60,7 @@
|
|||
"partitionOption": "None"
|
||||
},
|
||||
"dataset": {
|
||||
"referenceName": "AzureSqlTable_SH_IR",
|
||||
"referenceName": "AzureSqlTable_IRA",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"Schema": {
|
||||
|
|
@ -163,13 +163,13 @@
|
|||
"value": "Full",
|
||||
"activities": [
|
||||
{
|
||||
"name": "Execute AZ-SQL-Full-Load",
|
||||
"name": "Execute AZ_SQL_Full_Load",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-SQL-Full-Load-SH-IR",
|
||||
"referenceName": "AZ_SQL_Full_Load_IRA",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": true,
|
||||
|
|
@ -193,7 +193,7 @@
|
|||
"value": "Watermark",
|
||||
"activities": [
|
||||
{
|
||||
"name": "Execute AZ-SQL-Watermark",
|
||||
"name": "Execute AZ_SQL_Watermark",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [
|
||||
{
|
||||
|
|
@ -206,7 +206,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-SQL-Watermark-SH-IR",
|
||||
"referenceName": "AZ_SQL_Watermark_IRA",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": true,
|
||||
|
|
@ -251,7 +251,7 @@
|
|||
"partitionOption": "None"
|
||||
},
|
||||
"dataset": {
|
||||
"referenceName": "AzureSqlTable_SH_IR",
|
||||
"referenceName": "AzureSqlTable_IRA",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"Schema": {
|
||||
|
|
@ -278,10 +278,10 @@
|
|||
]
|
||||
},
|
||||
{
|
||||
"value": "Full-Chunk",
|
||||
"value": "Full_Chunk",
|
||||
"activities": [
|
||||
{
|
||||
"name": "Execute AZ-SQL-Full-Load-Chunk",
|
||||
"name": "Execute AZ_SQL_Full_Load_Chunk",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [
|
||||
{
|
||||
|
|
@ -294,7 +294,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-SQL-Full-Load-Chunk-SH-IR",
|
||||
"referenceName": "AZ_SQL_Full_Load_Chunk_IRA",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": true,
|
||||
|
|
@ -337,7 +337,7 @@
|
|||
"partitionOption": "None"
|
||||
},
|
||||
"dataset": {
|
||||
"referenceName": "AzureSqlTable_SH_IR",
|
||||
"referenceName": "AzureSqlTable_IRA",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"Schema": {
|
||||
|
|
@ -364,10 +364,10 @@
|
|||
]
|
||||
},
|
||||
{
|
||||
"value": "Watermark-Chunk",
|
||||
"value": "Watermark_Chunk",
|
||||
"activities": [
|
||||
{
|
||||
"name": "Execute AZ-SQL-Watermark-Chunk",
|
||||
"name": "Execute AZ_SQL_Watermark_Chunk",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [
|
||||
{
|
||||
|
|
@ -380,7 +380,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-SQL-Watermark-Chunk-SH-IR",
|
||||
"referenceName": "AZ_SQL_Watermark_Chunk_IRA",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": true,
|
||||
|
|
@ -427,7 +427,7 @@
|
|||
"partitionOption": "None"
|
||||
},
|
||||
"dataset": {
|
||||
"referenceName": "AzureSqlTable_SH_IR",
|
||||
"referenceName": "AzureSqlTable_IRA",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"Schema": {
|
||||
|
|
@ -506,7 +506,7 @@
|
|||
"Name": "adsgofastdatakakeacceladf",
|
||||
"ResourceGroup": "AdsGoFastDataLakeAccel",
|
||||
"SubscriptionId": "035a1364-f00d-48e2-b582-4fe125905ee3",
|
||||
"ADFPipeline": "AZ-SQL-AZ-Storage-Parquet-SH-IR"
|
||||
"ADFPipeline": "AZ_SQL_AZ_Storage_Parquet_IRA"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -517,10 +517,10 @@
|
|||
}
|
||||
},
|
||||
"folder": {
|
||||
"name": "ADS Go Fast/Data Movement/Azure SH IR"
|
||||
"name": "ADS Go Fast/Data Movement/IRA"
|
||||
},
|
||||
"annotations": [],
|
||||
"lastPublishTime": "2020-08-04T12:40:45Z"
|
||||
},
|
||||
"type": "Microsoft.DataFactory/factories/pipelines"
|
||||
}
|
||||
}
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
{
|
||||
"name": "AZ-SQL-Create-Table-SH-IR",
|
||||
"name": "AZ_SQL_Create_Table_IRA",
|
||||
"properties": {
|
||||
"activities": [
|
||||
{
|
||||
|
|
@ -68,7 +68,7 @@
|
|||
"partitionOption": "None"
|
||||
},
|
||||
"dataset": {
|
||||
"referenceName": "AzureSqlTable_SH_IR",
|
||||
"referenceName": "AzureSqlTable_IRA",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"Schema": {
|
||||
|
|
@ -106,7 +106,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-Function-Generic",
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
|
|
@ -189,7 +189,7 @@
|
|||
"partitionOption": "None"
|
||||
},
|
||||
"dataset": {
|
||||
"referenceName": "AzureSqlTable_SH_IR",
|
||||
"referenceName": "AzureSqlTable_IRA",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"Schema": {
|
||||
|
|
@ -227,7 +227,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-Function-Generic",
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
|
|
@ -251,10 +251,10 @@
|
|||
}
|
||||
},
|
||||
"folder": {
|
||||
"name": "ADS Go Fast/Data Movement/Azure SH IR/Common"
|
||||
"name": "ADS Go Fast/Data Movement/IRA/Common"
|
||||
},
|
||||
"annotations": [],
|
||||
"lastPublishTime": "2020-08-04T13:09:30Z"
|
||||
},
|
||||
"type": "Microsoft.DataFactory/factories/pipelines"
|
||||
}
|
||||
}
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
{
|
||||
"name": "AZ-SQL-Full-Load-Chunk-SH-IR",
|
||||
"name": "AZ_SQL_Full_Load_Chunk_IRA",
|
||||
"properties": {
|
||||
"activities": [
|
||||
{
|
||||
|
|
@ -15,13 +15,13 @@
|
|||
"isSequential": true,
|
||||
"activities": [
|
||||
{
|
||||
"name": "Execute AZ-SQL-Full-Load",
|
||||
"name": "Execute AZ_SQL_Full_Load",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-SQL-Full-Load-SH-IR",
|
||||
"referenceName": "AZ_SQL_Full_Load_IRA",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": true,
|
||||
|
|
@ -98,7 +98,7 @@
|
|||
"Name": "adsgofastdatakakeacceladf",
|
||||
"ResourceGroup": "AdsGoFastDataLakeAccel",
|
||||
"SubscriptionId": "035a1364-f00d-48e2-b582-4fe125905ee3",
|
||||
"ADFPipeline": "AZ-SQL-AZ-Storage-Parquet-SH-IR"
|
||||
"ADFPipeline": "AZ_SQL_AZ_Storage_Parquet_IRA"
|
||||
}
|
||||
}
|
||||
},
|
||||
|
|
@ -110,10 +110,10 @@
|
|||
}
|
||||
},
|
||||
"folder": {
|
||||
"name": "ADS Go Fast/Data Movement/Azure SH IR/Components"
|
||||
"name": "ADS Go Fast/Data Movement/IRA/Components"
|
||||
},
|
||||
"annotations": [],
|
||||
"lastPublishTime": "2020-07-29T02:39:55Z"
|
||||
},
|
||||
"type": "Microsoft.DataFactory/factories/pipelines"
|
||||
}
|
||||
}
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
{
|
||||
"name": "AZ-SQL-Full-Load-SH-IR",
|
||||
"name": "AZ_SQL_Full_Load_IRA",
|
||||
"properties": {
|
||||
"activities": [
|
||||
{
|
||||
|
|
@ -70,7 +70,7 @@
|
|||
},
|
||||
"inputs": [
|
||||
{
|
||||
"referenceName": "AzureSqlTable_SH_IR",
|
||||
"referenceName": "AzureSqlTable_IRA",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"Schema": {
|
||||
|
|
@ -94,7 +94,7 @@
|
|||
],
|
||||
"outputs": [
|
||||
{
|
||||
"referenceName": "ADLSParquet_SH_IR",
|
||||
"referenceName": "ADLSParquet_IRA",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"RelativePath": {
|
||||
|
|
@ -131,7 +131,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-Function-Generic",
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
|
|
@ -198,7 +198,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"dataset": {
|
||||
"referenceName": "ADLSParquet_SH_IR",
|
||||
"referenceName": "ADLSParquet_IRA",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"RelativePath": {
|
||||
|
|
@ -235,7 +235,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-Function-Generic",
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
|
|
@ -263,7 +263,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-Function-Generic",
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
|
|
@ -329,7 +329,7 @@
|
|||
},
|
||||
"inputs": [
|
||||
{
|
||||
"referenceName": "AzureSqlTable_SH_IR",
|
||||
"referenceName": "AzureSqlTable_IRA",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"Schema": {
|
||||
|
|
@ -353,7 +353,7 @@
|
|||
],
|
||||
"outputs": [
|
||||
{
|
||||
"referenceName": "BlobParquet_SH_IR",
|
||||
"referenceName": "BlobParquet_IRA",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"RelativePath": {
|
||||
|
|
@ -390,7 +390,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-Function-Generic",
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
|
|
@ -457,7 +457,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"dataset": {
|
||||
"referenceName": "BlobParquet_SH_IR",
|
||||
"referenceName": "BlobParquet_IRA",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"RelativePath": {
|
||||
|
|
@ -494,7 +494,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-Function-Generic",
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
|
|
@ -522,7 +522,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-Function-Generic",
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
|
|
@ -604,7 +604,7 @@
|
|||
"Name": "adsgofastdatakakeacceladf",
|
||||
"ResourceGroup": "AdsGoFastDataLakeAccel",
|
||||
"SubscriptionId": "035a1364-f00d-48e2-b582-4fe125905ee3",
|
||||
"ADFPipeline": "AZ-SQL-AZ-Storage-Parquet-SH-IR"
|
||||
"ADFPipeline": "AZ_SQL_AZ_Storage_Parquet_IRA"
|
||||
}
|
||||
}
|
||||
},
|
||||
|
|
@ -624,10 +624,10 @@
|
|||
}
|
||||
},
|
||||
"folder": {
|
||||
"name": "ADS Go Fast/Data Movement/Azure SH IR/Components"
|
||||
"name": "ADS Go Fast/Data Movement/IRA/Components"
|
||||
},
|
||||
"annotations": [],
|
||||
"lastPublishTime": "2020-07-29T02:39:55Z"
|
||||
},
|
||||
"type": "Microsoft.DataFactory/factories/pipelines"
|
||||
}
|
||||
}
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
{
|
||||
"name": "AZ-SQL-Post-Copy-SH-IR",
|
||||
"name": "AZ_SQL_Post_Copy_IRA",
|
||||
"properties": {
|
||||
"activities": [
|
||||
{
|
||||
|
|
@ -36,7 +36,7 @@
|
|||
"partitionOption": "None"
|
||||
},
|
||||
"dataset": {
|
||||
"referenceName": "AzureSqlTable_SH_IR",
|
||||
"referenceName": "AzureSqlTable_IRA",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"Schema": {
|
||||
|
|
@ -73,7 +73,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-Function-Generic",
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
|
|
@ -131,7 +131,7 @@
|
|||
"partitionOption": "None"
|
||||
},
|
||||
"dataset": {
|
||||
"referenceName": "AzureSqlTable_SH_IR",
|
||||
"referenceName": "AzureSqlTable_IRA",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"Schema": {
|
||||
|
|
@ -168,7 +168,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-Function-Generic",
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
|
|
@ -233,7 +233,7 @@
|
|||
"partitionOption": "None"
|
||||
},
|
||||
"dataset": {
|
||||
"referenceName": "AzureSqlTable_SH_IR",
|
||||
"referenceName": "AzureSqlTable_IRA",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"Schema": {
|
||||
|
|
@ -350,7 +350,7 @@
|
|||
"partitionOption": "None"
|
||||
},
|
||||
"dataset": {
|
||||
"referenceName": "AzureSqlTable_SH_IR",
|
||||
"referenceName": "AzureSqlTable_IRA",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"Schema": {
|
||||
|
|
@ -404,7 +404,7 @@
|
|||
"partitionOption": "None"
|
||||
},
|
||||
"dataset": {
|
||||
"referenceName": "AzureSqlTable_SH_IR",
|
||||
"referenceName": "AzureSqlTable_IRA",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"Schema": {
|
||||
|
|
@ -467,7 +467,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-Function-Generic",
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
|
|
@ -495,7 +495,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-Function-Generic",
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
|
|
@ -523,7 +523,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-Function-Generic",
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
|
|
@ -547,10 +547,10 @@
|
|||
}
|
||||
},
|
||||
"folder": {
|
||||
"name": "ADS Go Fast/Data Movement/Azure SH IR/Common"
|
||||
"name": "ADS Go Fast/Data Movement/IRA/Common"
|
||||
},
|
||||
"annotations": [],
|
||||
"lastPublishTime": "2020-08-04T13:09:30Z"
|
||||
},
|
||||
"type": "Microsoft.DataFactory/factories/pipelines"
|
||||
}
|
||||
}
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
{
|
||||
"name": "AZ-SQL-StoredProcedure-SH-IR",
|
||||
"name": "AZ_SQL_StoredProcedure_IRA",
|
||||
"properties": {
|
||||
"activities": [
|
||||
{
|
||||
|
|
@ -25,7 +25,7 @@
|
|||
"partitionOption": "None"
|
||||
},
|
||||
"dataset": {
|
||||
"referenceName": "AzureSqlTable_SH_IR",
|
||||
"referenceName": "AzureSqlTable_IRA",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"Schema": {
|
||||
|
|
@ -62,7 +62,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-Function-Generic",
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
|
|
@ -83,10 +83,10 @@
|
|||
}
|
||||
},
|
||||
"folder": {
|
||||
"name": "ADS Go Fast/Data Movement/Azure SH IR"
|
||||
"name": "ADS Go Fast/Data Movement/IRA"
|
||||
},
|
||||
"annotations": [],
|
||||
"lastPublishTime": "2020-08-06T09:21:51Z"
|
||||
},
|
||||
"type": "Microsoft.DataFactory/factories/pipelines"
|
||||
}
|
||||
}
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
{
|
||||
"name": "AZ-SQL-Watermark-Chunk-SH-IR",
|
||||
"name": "AZ_SQL_Watermark_Chunk_IRA",
|
||||
"properties": {
|
||||
"activities": [
|
||||
{
|
||||
|
|
@ -16,7 +16,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-Function-Generic",
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
|
|
@ -43,13 +43,13 @@
|
|||
"isSequential": true,
|
||||
"activities": [
|
||||
{
|
||||
"name": "Execute AZ-SQL-Watermark",
|
||||
"name": "Execute AZ_SQL_Watermark",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-SQL-Watermark-SH-IR",
|
||||
"referenceName": "AZ_SQL_Watermark_IRA",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": true,
|
||||
|
|
@ -130,7 +130,7 @@
|
|||
"Name": "adsgofastdatakakeacceladf",
|
||||
"ResourceGroup": "AdsGoFastDataLakeAccel",
|
||||
"SubscriptionId": "035a1364-f00d-48e2-b582-4fe125905ee3",
|
||||
"ADFPipeline": "AZ-SQL-AZ-Storage-Parquet-SH-IR"
|
||||
"ADFPipeline": "AZ_SQL_AZ_Storage_Parquet_IRA"
|
||||
}
|
||||
}
|
||||
},
|
||||
|
|
@ -145,10 +145,10 @@
|
|||
}
|
||||
},
|
||||
"folder": {
|
||||
"name": "ADS Go Fast/Data Movement/Azure SH IR/Components"
|
||||
"name": "ADS Go Fast/Data Movement/IRA/Components"
|
||||
},
|
||||
"annotations": [],
|
||||
"lastPublishTime": "2020-07-29T02:39:55Z"
|
||||
},
|
||||
"type": "Microsoft.DataFactory/factories/pipelines"
|
||||
}
|
||||
}
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
{
|
||||
"name": "AZ-SQL-Watermark-SH-IR",
|
||||
"name": "AZ_SQL_Watermark_IRA",
|
||||
"properties": {
|
||||
"activities": [
|
||||
{
|
||||
|
|
@ -70,7 +70,7 @@
|
|||
},
|
||||
"inputs": [
|
||||
{
|
||||
"referenceName": "AzureSqlTable_SH_IR",
|
||||
"referenceName": "AzureSqlTable_IRA",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"Schema": {
|
||||
|
|
@ -94,7 +94,7 @@
|
|||
],
|
||||
"outputs": [
|
||||
{
|
||||
"referenceName": "ADLSParquet_SH_IR",
|
||||
"referenceName": "ADLSParquet_IRA",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"RelativePath": {
|
||||
|
|
@ -131,7 +131,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-Function-Generic",
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
|
|
@ -198,7 +198,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"dataset": {
|
||||
"referenceName": "ADLSParquet_SH_IR",
|
||||
"referenceName": "ADLSParquet_IRA",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"RelativePath": {
|
||||
|
|
@ -235,7 +235,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-Function-Generic",
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
|
|
@ -263,7 +263,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-Function-Generic",
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
|
|
@ -329,7 +329,7 @@
|
|||
},
|
||||
"inputs": [
|
||||
{
|
||||
"referenceName": "AzureSqlTable_SH_IR",
|
||||
"referenceName": "AzureSqlTable_IRA",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"Schema": {
|
||||
|
|
@ -353,7 +353,7 @@
|
|||
],
|
||||
"outputs": [
|
||||
{
|
||||
"referenceName": "BlobParquet_SH_IR",
|
||||
"referenceName": "BlobParquet_IRA",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"RelativePath": {
|
||||
|
|
@ -390,7 +390,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-Function-Generic",
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
|
|
@ -457,7 +457,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"dataset": {
|
||||
"referenceName": "BlobParquet_SH_IR",
|
||||
"referenceName": "BlobParquet_IRA",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"RelativePath": {
|
||||
|
|
@ -494,7 +494,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-Function-Generic",
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
|
|
@ -522,7 +522,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-Function-Generic",
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
|
|
@ -604,7 +604,7 @@
|
|||
"Name": "adsgofastdatakakeacceladf",
|
||||
"ResourceGroup": "AdsGoFastDataLakeAccel",
|
||||
"SubscriptionId": "035a1364-f00d-48e2-b582-4fe125905ee3",
|
||||
"ADFPipeline": "AZ-SQL-AZ-Storage-Parquet-SH-IR"
|
||||
"ADFPipeline": "AZ_SQL_AZ_Storage_Parquet_IRA"
|
||||
}
|
||||
}
|
||||
},
|
||||
|
|
@ -627,10 +627,10 @@
|
|||
}
|
||||
},
|
||||
"folder": {
|
||||
"name": "ADS Go Fast/Data Movement/Azure SH IR/Components"
|
||||
"name": "ADS Go Fast/Data Movement/IRA/Components"
|
||||
},
|
||||
"annotations": [],
|
||||
"lastPublishTime": "2020-07-29T02:39:55Z"
|
||||
},
|
||||
"type": "Microsoft.DataFactory/factories/pipelines"
|
||||
}
|
||||
}
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
{
|
||||
"name": "AZ-Storage-Binary-AZ-Storage-Binary-SH-IR",
|
||||
"name": "AZ_Storage_Binary_AZ_Storage_Binary_IRA",
|
||||
"properties": {
|
||||
"activities": [
|
||||
{
|
||||
|
|
@ -72,7 +72,7 @@
|
|||
},
|
||||
"inputs": [
|
||||
{
|
||||
"referenceName": "BlobBinary_SH_IR",
|
||||
"referenceName": "BlobBinary_IRA",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"StorageAccountEndpoint": {
|
||||
|
|
@ -96,7 +96,7 @@
|
|||
],
|
||||
"outputs": [
|
||||
{
|
||||
"referenceName": "BlobBinary_SH_IR",
|
||||
"referenceName": "BlobBinary_IRA",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"StorageAccountEndpoint": {
|
||||
|
|
@ -133,7 +133,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-Function-Generic",
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
|
|
@ -154,7 +154,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-Function-Generic",
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
|
|
@ -182,7 +182,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-Function-Generic",
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
|
|
@ -257,7 +257,7 @@
|
|||
},
|
||||
"inputs": [
|
||||
{
|
||||
"referenceName": "ADLSBinary_SH_IR",
|
||||
"referenceName": "ADLSBinary_IRA",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"StorageAccountEndpoint": {
|
||||
|
|
@ -281,7 +281,7 @@
|
|||
],
|
||||
"outputs": [
|
||||
{
|
||||
"referenceName": "ADLSBinary_SH_IR",
|
||||
"referenceName": "ADLSBinary_IRA",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"StorageAccountEndpoint": {
|
||||
|
|
@ -318,7 +318,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-Function-Generic",
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
|
|
@ -339,7 +339,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-Function-Generic",
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
|
|
@ -367,7 +367,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-Function-Generic",
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
|
|
@ -442,7 +442,7 @@
|
|||
},
|
||||
"inputs": [
|
||||
{
|
||||
"referenceName": "BlobBinary_SH_IR",
|
||||
"referenceName": "BlobBinary_IRA",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"StorageAccountEndpoint": {
|
||||
|
|
@ -466,7 +466,7 @@
|
|||
],
|
||||
"outputs": [
|
||||
{
|
||||
"referenceName": "ADLSBinary_SH_IR",
|
||||
"referenceName": "ADLSBinary_IRA",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"StorageAccountEndpoint": {
|
||||
|
|
@ -503,7 +503,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-Function-Generic",
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
|
|
@ -524,7 +524,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-Function-Generic",
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
|
|
@ -552,7 +552,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-Function-Generic",
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
|
|
@ -627,7 +627,7 @@
|
|||
},
|
||||
"inputs": [
|
||||
{
|
||||
"referenceName": "ADLSBinary_SH_IR",
|
||||
"referenceName": "ADLSBinary_IRA",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"StorageAccountEndpoint": {
|
||||
|
|
@ -651,7 +651,7 @@
|
|||
],
|
||||
"outputs": [
|
||||
{
|
||||
"referenceName": "BlobBinary_SH_IR",
|
||||
"referenceName": "BlobBinary_IRA",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"StorageAccountEndpoint": {
|
||||
|
|
@ -688,7 +688,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-Function-Generic",
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
|
|
@ -709,7 +709,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-Function-Generic",
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
|
|
@ -737,7 +737,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-Function-Generic",
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
|
|
@ -760,14 +760,14 @@
|
|||
"parameters": {
|
||||
"TaskObject": {
|
||||
"type": "object",
|
||||
"defaultValue": "[ { \"TaskInstanceId\": 2, \"TaskMasterId\": 1, \"TaskStatus\": \"Untried\", \"TaskType\": \"AzureStorageToAzureStorage-SH-IR\", \"Enabled\": true, \"ExecutionUid\": 1, \"KeyVaultBaseUrl\": \"https://adsgofastkeyvault.vault.azure.net/\", \"Source\": { \"StorageAccountName\": \"https://adsgofastdatalakeadls.dfs.core.windows.net\", \"Type\": \"ADLS\", \"StorageAccountContainer\": \"datalakelanding\", \"StorageAccountAccessMethod\": \"MSI\", \"StorageAccountSASUriKeyVaultSecretName\": null, \"RelativePath\": \"/Unprocessed/adsgofastdatakakeaccelsqlsvr/AWSample/SalesLT/2020/06/08/17/\", \"DataFileName\": \"Customer_Data.parquet\", \"SchemaFileName\": \"Customer_Schema.json\" }, \"Target\": { \"StorageAccountName\": \"https://adsgofastdatalakeadls.dfs.core.windows.net\", \"Type\": \"ADLS\", \"StorageAccountContainer\": \"datalakelanding\", \"StorageAccountAccessMethod\": \"MSI\", \"StorageAccountSASUriKeyVaultSecretName\": null, \"RelativePath\": \"/Processed/adsgofastdatakakeaccelsqlsvr/AWSample/SalesLT/2020/06/08/17/\", \"DataFileName\": \"Customer_Data.parquet\", \"SchemaFileName\": \"Customer_Schema.json\" }, \"DataFactory\": { \"Id\": 1, \"Name\": \"adsgofastdatakakeacceladf\", \"ResourceGroup\": \"AdsGoFastDataLakeAccel\", \"SubscriptionId\": \"035a1364-f00d-48e2-b582-4fe125905ee3\", } } ]"
|
||||
"defaultValue": "[ { \"TaskInstanceId\": 2, \"TaskMasterId\": 1, \"TaskStatus\": \"Untried\", \"TaskType\": \"AzureStorageToAzureStorage_IRA\", \"Enabled\": true, \"ExecutionUid\": 1, \"KeyVaultBaseUrl\": \"https://adsgofastkeyvault.vault.azure.net/\", \"Source\": { \"StorageAccountName\": \"https://adsgofastdatalakeadls.dfs.core.windows.net\", \"Type\": \"ADLS\", \"StorageAccountContainer\": \"datalakelanding\", \"StorageAccountAccessMethod\": \"MSI\", \"StorageAccountSASUriKeyVaultSecretName\": null, \"RelativePath\": \"/Unprocessed/adsgofastdatakakeaccelsqlsvr/AWSample/SalesLT/2020/06/08/17/\", \"DataFileName\": \"Customer_Data.parquet\", \"SchemaFileName\": \"Customer_Schema.json\" }, \"Target\": { \"StorageAccountName\": \"https://adsgofastdatalakeadls.dfs.core.windows.net\", \"Type\": \"ADLS\", \"StorageAccountContainer\": \"datalakelanding\", \"StorageAccountAccessMethod\": \"MSI\", \"StorageAccountSASUriKeyVaultSecretName\": null, \"RelativePath\": \"/Processed/adsgofastdatakakeaccelsqlsvr/AWSample/SalesLT/2020/06/08/17/\", \"DataFileName\": \"Customer_Data.parquet\", \"SchemaFileName\": \"Customer_Schema.json\" }, \"DataFactory\": { \"Id\": 1, \"Name\": \"adsgofastdatakakeacceladf\", \"ResourceGroup\": \"AdsGoFastDataLakeAccel\", \"SubscriptionId\": \"035a1364-f00d-48e2-b582-4fe125905ee3\", } } ]"
|
||||
}
|
||||
},
|
||||
"folder": {
|
||||
"name": "ADS Go Fast/Data Movement/Azure SH IR"
|
||||
"name": "ADS Go Fast/Data Movement/IRA"
|
||||
},
|
||||
"annotations": [],
|
||||
"lastPublishTime": "2020-08-05T04:14:00Z"
|
||||
},
|
||||
"type": "Microsoft.DataFactory/factories/pipelines"
|
||||
}
|
||||
}
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
{
|
||||
"name": "AZ-Storage-CSV-AZ-SQL-SH-IR",
|
||||
"name": "AZ_Storage_CSV_AZ_SQL_IRA",
|
||||
"properties": {
|
||||
"activities": [
|
||||
{
|
||||
|
|
@ -84,7 +84,7 @@
|
|||
},
|
||||
"inputs": [
|
||||
{
|
||||
"referenceName": "ADLSDelimitedText_SH_IR",
|
||||
"referenceName": "ADLSDelimitedText_IRA",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"RelativePath": {
|
||||
|
|
@ -112,7 +112,7 @@
|
|||
],
|
||||
"outputs": [
|
||||
{
|
||||
"referenceName": "AzureSqlTable_SH_IR",
|
||||
"referenceName": "AzureSqlTable_IRA",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"Schema": {
|
||||
|
|
@ -149,7 +149,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-Function-Generic",
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
|
|
@ -170,7 +170,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-Function-Generic",
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
|
|
@ -198,7 +198,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-Function-Generic",
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
|
|
@ -285,7 +285,7 @@
|
|||
},
|
||||
"inputs": [
|
||||
{
|
||||
"referenceName": "BlobDelimitedText_SH_IR",
|
||||
"referenceName": "BlobDelimitedText_IRA",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"RelativePath": {
|
||||
|
|
@ -313,7 +313,7 @@
|
|||
],
|
||||
"outputs": [
|
||||
{
|
||||
"referenceName": "AzureSqlTable_SH_IR",
|
||||
"referenceName": "AzureSqlTable_IRA",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"Schema": {
|
||||
|
|
@ -350,7 +350,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-Function-Generic",
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
|
|
@ -371,7 +371,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-Function-Generic",
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
|
|
@ -399,7 +399,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-Function-Generic",
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
|
|
@ -419,7 +419,7 @@
|
|||
}
|
||||
},
|
||||
{
|
||||
"name": "Execute AZ-SQL-Post-Copy",
|
||||
"name": "Execute AZ_SQL_Post-Copy",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [
|
||||
{
|
||||
|
|
@ -432,7 +432,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-SQL-Post-Copy-SH-IR",
|
||||
"referenceName": "AZ_SQL_Post_Copy_IRA",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": true,
|
||||
|
|
@ -451,10 +451,10 @@
|
|||
}
|
||||
},
|
||||
"folder": {
|
||||
"name": "ADS Go Fast/Data Movement/Azure SH IR"
|
||||
"name": "ADS Go Fast/Data Movement/IRA"
|
||||
},
|
||||
"annotations": [],
|
||||
"lastPublishTime": "2020-07-29T09:43:40Z"
|
||||
},
|
||||
"type": "Microsoft.DataFactory/factories/pipelines"
|
||||
}
|
||||
}
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
{
|
||||
"name": "AZ-Storage-Excel-AZ-SQL-SH-IR",
|
||||
"name": "AZ_Storage_Excel_AZ_SQL_IRA",
|
||||
"properties": {
|
||||
"activities": [
|
||||
{
|
||||
|
|
@ -60,7 +60,7 @@
|
|||
},
|
||||
"inputs": [
|
||||
{
|
||||
"referenceName": "ADLSExcel_SH_IR",
|
||||
"referenceName": "ADLSExcel_IRA",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"RelativePath": {
|
||||
|
|
@ -92,7 +92,7 @@
|
|||
],
|
||||
"outputs": [
|
||||
{
|
||||
"referenceName": "AzureSqlTable_SH_IR",
|
||||
"referenceName": "AzureSqlTable_IRA",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"Schema": {
|
||||
|
|
@ -129,7 +129,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-Function-Generic",
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
|
|
@ -150,7 +150,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-Function-Generic",
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
|
|
@ -178,7 +178,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-Function-Generic",
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
|
|
@ -241,7 +241,7 @@
|
|||
},
|
||||
"inputs": [
|
||||
{
|
||||
"referenceName": "BlobExcel_SH_IR",
|
||||
"referenceName": "BlobExcel_IRA",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"RelativePath": {
|
||||
|
|
@ -273,7 +273,7 @@
|
|||
],
|
||||
"outputs": [
|
||||
{
|
||||
"referenceName": "AzureSqlTable_SH_IR",
|
||||
"referenceName": "AzureSqlTable_IRA",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"Schema": {
|
||||
|
|
@ -310,7 +310,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-Function-Generic",
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
|
|
@ -331,7 +331,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-Function-Generic",
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
|
|
@ -359,7 +359,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-Function-Generic",
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
|
|
@ -379,7 +379,7 @@
|
|||
}
|
||||
},
|
||||
{
|
||||
"name": "Execute AZ-SQL-Post-Copy",
|
||||
"name": "Execute AZ_SQL_Post-Copy",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [
|
||||
{
|
||||
|
|
@ -392,7 +392,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-SQL-Post-Copy-SH-IR",
|
||||
"referenceName": "AZ_SQL_Post_Copy_IRA",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": true,
|
||||
|
|
@ -411,10 +411,10 @@
|
|||
}
|
||||
},
|
||||
"folder": {
|
||||
"name": "ADS Go Fast/Data Movement/Azure SH IR"
|
||||
"name": "ADS Go Fast/Data Movement/IRA"
|
||||
},
|
||||
"annotations": [],
|
||||
"lastPublishTime": "2020-07-24T01:51:29Z"
|
||||
},
|
||||
"type": "Microsoft.DataFactory/factories/pipelines"
|
||||
}
|
||||
}
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
{
|
||||
"name": "AZ-Storage-Excel-AZ-Storage-CSV-SH-IR",
|
||||
"name": "AZ_Storage_Excel_AZ_Storage_CSV_IRA",
|
||||
"properties": {
|
||||
"activities": [
|
||||
{
|
||||
|
|
@ -62,7 +62,7 @@
|
|||
},
|
||||
"inputs": [
|
||||
{
|
||||
"referenceName": "ADLSExcel_SH_IR",
|
||||
"referenceName": "ADLSExcel_IRA",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"RelativePath": {
|
||||
|
|
@ -94,7 +94,7 @@
|
|||
],
|
||||
"outputs": [
|
||||
{
|
||||
"referenceName": "ADLSDelimitedText_SH_IR",
|
||||
"referenceName": "ADLSDelimitedText_IRA",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"RelativePath": {
|
||||
|
|
@ -135,7 +135,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-Function-Generic",
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
|
|
@ -156,7 +156,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-Function-Generic",
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
|
|
@ -184,7 +184,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-Function-Generic",
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
|
|
@ -249,7 +249,7 @@
|
|||
},
|
||||
"inputs": [
|
||||
{
|
||||
"referenceName": "BlobExcel_SH_IR",
|
||||
"referenceName": "BlobExcel_IRA",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"RelativePath": {
|
||||
|
|
@ -281,7 +281,7 @@
|
|||
],
|
||||
"outputs": [
|
||||
{
|
||||
"referenceName": "BlobDelimitedText_SH_IR",
|
||||
"referenceName": "BlobDelimitedText_IRA",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"RelativePath": {
|
||||
|
|
@ -322,7 +322,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-Function-Generic",
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
|
|
@ -343,7 +343,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-Function-Generic",
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
|
|
@ -371,7 +371,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-Function-Generic",
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
|
|
@ -397,10 +397,10 @@
|
|||
}
|
||||
},
|
||||
"folder": {
|
||||
"name": "ADS Go Fast/Data Movement/Azure SH IR"
|
||||
"name": "ADS Go Fast/Data Movement/IRA"
|
||||
},
|
||||
"annotations": [],
|
||||
"lastPublishTime": "2020-08-05T08:58:04Z"
|
||||
},
|
||||
"type": "Microsoft.DataFactory/factories/pipelines"
|
||||
}
|
||||
}
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
{
|
||||
"name": "AZ-Storage-JSON-AZ-SQL-SH-IR",
|
||||
"name": "AZ_Storage_JSON_AZ_SQL_IRA",
|
||||
"properties": {
|
||||
"activities": [
|
||||
{
|
||||
|
|
@ -62,7 +62,7 @@
|
|||
},
|
||||
"inputs": [
|
||||
{
|
||||
"referenceName": "ADLSJson_SH_IR",
|
||||
"referenceName": "ADLSJson_IRA",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"RelativePath": {
|
||||
|
|
@ -86,7 +86,7 @@
|
|||
],
|
||||
"outputs": [
|
||||
{
|
||||
"referenceName": "AzureSqlTable_SH_IR",
|
||||
"referenceName": "AzureSqlTable_IRA",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"Schema": {
|
||||
|
|
@ -123,7 +123,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-Function-Generic",
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
|
|
@ -144,7 +144,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-Function-Generic",
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
|
|
@ -172,7 +172,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-Function-Generic",
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
|
|
@ -237,7 +237,7 @@
|
|||
},
|
||||
"inputs": [
|
||||
{
|
||||
"referenceName": "BlobJson_SH_IR",
|
||||
"referenceName": "BlobJson_IRA",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"RelativePath": {
|
||||
|
|
@ -261,7 +261,7 @@
|
|||
],
|
||||
"outputs": [
|
||||
{
|
||||
"referenceName": "AzureSqlTable_SH_IR",
|
||||
"referenceName": "AzureSqlTable_IRA",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"Schema": {
|
||||
|
|
@ -298,7 +298,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-Function-Generic",
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
|
|
@ -319,7 +319,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-Function-Generic",
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
|
|
@ -347,7 +347,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-Function-Generic",
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
|
|
@ -367,7 +367,7 @@
|
|||
}
|
||||
},
|
||||
{
|
||||
"name": "Execute AZ-SQL-Post-Copy",
|
||||
"name": "Execute AZ_SQL_Post-Copy",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [
|
||||
{
|
||||
|
|
@ -380,7 +380,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-SQL-Post-Copy-SH-IR",
|
||||
"referenceName": "AZ_SQL_Post_Copy_IRA",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": true,
|
||||
|
|
@ -399,10 +399,10 @@
|
|||
}
|
||||
},
|
||||
"folder": {
|
||||
"name": "ADS Go Fast/Data Movement/Azure SH IR"
|
||||
"name": "ADS Go Fast/Data Movement/IRA"
|
||||
},
|
||||
"annotations": [],
|
||||
"lastPublishTime": "2020-07-29T05:19:06Z"
|
||||
},
|
||||
"type": "Microsoft.DataFactory/factories/pipelines"
|
||||
}
|
||||
}
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
{
|
||||
"name": "AZ-Storage-Parquet-AZ-SQL-SH-IR",
|
||||
"name": "AZ_Storage_Parquet_AZ_SQL_IRA",
|
||||
"properties": {
|
||||
"activities": [
|
||||
{
|
||||
|
|
@ -76,7 +76,7 @@
|
|||
},
|
||||
"inputs": [
|
||||
{
|
||||
"referenceName": "ADLSParquet_SH_IR",
|
||||
"referenceName": "ADLSParquet_IRA",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"RelativePath": {
|
||||
|
|
@ -100,7 +100,7 @@
|
|||
],
|
||||
"outputs": [
|
||||
{
|
||||
"referenceName": "AzureSqlTable_SH_IR",
|
||||
"referenceName": "AzureSqlTable_IRA",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"Schema": {
|
||||
|
|
@ -137,7 +137,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-Function-Generic",
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
|
|
@ -158,7 +158,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-Function-Generic",
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
|
|
@ -186,7 +186,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-Function-Generic",
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
|
|
@ -258,7 +258,7 @@
|
|||
},
|
||||
"inputs": [
|
||||
{
|
||||
"referenceName": "BlobParquet_SH_IR",
|
||||
"referenceName": "BlobParquet_IRA",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"RelativePath": {
|
||||
|
|
@ -282,7 +282,7 @@
|
|||
],
|
||||
"outputs": [
|
||||
{
|
||||
"referenceName": "AzureSqlTable_SH_IR",
|
||||
"referenceName": "AzureSqlTable_IRA",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"Schema": {
|
||||
|
|
@ -319,7 +319,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-Function-Generic",
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
|
|
@ -340,7 +340,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-Function-Generic",
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
|
|
@ -368,7 +368,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-Function-Generic",
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
|
|
@ -405,7 +405,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-SQL-Create-Table-SH-IR",
|
||||
"referenceName": "AZ_SQL_Create_Table_IRA",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": true,
|
||||
|
|
@ -421,7 +421,7 @@
|
|||
}
|
||||
},
|
||||
{
|
||||
"name": "Execute AZ-SQL-Post-Copy",
|
||||
"name": "Execute AZ_SQL_Post-Copy",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [
|
||||
{
|
||||
|
|
@ -434,7 +434,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-SQL-Post-Copy-SH-IR",
|
||||
"referenceName": "AZ_SQL_Post_Copy_IRA",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": true,
|
||||
|
|
@ -485,10 +485,10 @@
|
|||
}
|
||||
},
|
||||
"folder": {
|
||||
"name": "ADS Go Fast/Data Movement/Azure SH IR"
|
||||
"name": "ADS Go Fast/Data Movement/IRA"
|
||||
},
|
||||
"annotations": [],
|
||||
"lastPublishTime": "2020-08-05T03:16:41Z"
|
||||
},
|
||||
"type": "Microsoft.DataFactory/factories/pipelines"
|
||||
}
|
||||
}
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
{
|
||||
"name": "Create-Task-Master-AZ-SQL-SH-IR",
|
||||
"name": "Create-Task_MASTER_AZ_SQL_IRA",
|
||||
"properties": {
|
||||
"activities": [
|
||||
{
|
||||
|
|
@ -16,7 +16,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-Function-Generic",
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
|
|
@ -37,7 +37,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-Function-Generic",
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
|
|
@ -65,7 +65,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-Function-Generic",
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
|
|
@ -109,7 +109,7 @@
|
|||
"partitionOption": "None"
|
||||
},
|
||||
"dataset": {
|
||||
"referenceName": "AzureSqlTable_SH_IR",
|
||||
"referenceName": "AzureSqlTable_IRA",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"Schema": {
|
||||
|
|
@ -236,10 +236,10 @@
|
|||
}
|
||||
},
|
||||
"folder": {
|
||||
"name": "ADS Go Fast/Create Task Master/Azure SH IR"
|
||||
"name": "ADS Go Fast/Create Task Master/IRA"
|
||||
},
|
||||
"annotations": [],
|
||||
"lastPublishTime": "2020-08-04T13:09:29Z"
|
||||
},
|
||||
"type": "Microsoft.DataFactory/factories/pipelines"
|
||||
}
|
||||
}
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
{
|
||||
"name": "GEN-File-Binary-AZ-Storage-Binary-SH-IR",
|
||||
"name": "GEN_File_Binary_AZ_Storage_Binary_IRA",
|
||||
"properties": {
|
||||
"activities": [
|
||||
{
|
||||
|
|
@ -61,7 +61,7 @@
|
|||
},
|
||||
"inputs": [
|
||||
{
|
||||
"referenceName": "FileBinary_SH_IR",
|
||||
"referenceName": "FileBinary_IRA",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"Host": {
|
||||
|
|
@ -93,7 +93,7 @@
|
|||
],
|
||||
"outputs": [
|
||||
{
|
||||
"referenceName": "ADLSBinary_SH_IR",
|
||||
"referenceName": "ADLSBinary_IRA",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"StorageAccountEndpoint": {
|
||||
|
|
@ -130,7 +130,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-Function-Generic",
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
|
|
@ -151,7 +151,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-Function-Generic",
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
|
|
@ -179,7 +179,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-Function-Generic",
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
|
|
@ -243,7 +243,7 @@
|
|||
},
|
||||
"inputs": [
|
||||
{
|
||||
"referenceName": "FileBinary_SH_IR",
|
||||
"referenceName": "FileBinary_IRA",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"Host": {
|
||||
|
|
@ -275,7 +275,7 @@
|
|||
],
|
||||
"outputs": [
|
||||
{
|
||||
"referenceName": "BlobBinary_SH_IR",
|
||||
"referenceName": "BlobBinary_IRA",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"StorageAccountEndpoint": {
|
||||
|
|
@ -312,7 +312,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-Function-Generic",
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
|
|
@ -333,7 +333,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-Function-Generic",
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
|
|
@ -361,7 +361,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-Function-Generic",
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
|
|
@ -384,14 +384,14 @@
|
|||
"parameters": {
|
||||
"TaskObject": {
|
||||
"type": "object",
|
||||
"defaultValue": " [ { \"TaskInstanceId\": 2, \"TaskMasterId\": 1, \"TaskStatus\": \"Untried\", \"TaskType\": \"FiletoAzureStorage-Azure-SH-IR\", \"Enabled\": true, \"ExecutionUid\": 1, \"KeyVaultBaseUrl\": \"https://adsgofastkeyvault.vault.azure.net/\", \"Source\": { \"Type\": \"File\", \"Host\": \"\\\\\\\\10.2.0.4\\\\D\\\\dataingestion\\\\\", \"UserId\": \"adsgofastonprem\\\\AzureUser\", \"PasswordKeyVaultSecretName\": \"adsgofast-onpre-file-password\", \"Directory\": \"AdventureWorks2017/2020/06/15/14/22/\", \"File\": \"Sales.SalesOrderDetails.parquet\" }, \"Target\": { \"StorageAccountName\": \"https://adsgofastdatalakeaccelst.blob.core.windows.net\", \"Type\": \"Blob\", \"StorageAccountContainer\": \"datalakelanding\", \"StorageAccountAccessMethod\": \"MSI\", \"StorageAccountSASUriKeyVaultSecretName\": null, \"RelativePath\": \"/Unprocessed/adsgofastdatakakeaccelsqlsvr/AWSample/SalesLT/2020/06/08/17/\", \"DataFileName\": \"Customer_Data.parquet\", \"SchemaFileName\": \"Customer_Schema.json\" }, \"DataFactory\": { \"Id\": 1, \"Name\": \"adsgofastdatakakeacceladf\", \"ResourceGroup\": \"AdsGoFastDataLakeAccel\", \"SubscriptionId\": \"035a1364-f00d-48e2-b582-4fe125905ee3\", } } ]"
|
||||
"defaultValue": " [ { \"TaskInstanceId\": 2, \"TaskMasterId\": 1, \"TaskStatus\": \"Untried\", \"TaskType\": \"FiletoAzureStorage-Azure_IRA\", \"Enabled\": true, \"ExecutionUid\": 1, \"KeyVaultBaseUrl\": \"https://adsgofastkeyvault.vault.azure.net/\", \"Source\": { \"Type\": \"File\", \"Host\": \"\\\\\\\\10.2.0.4\\\\D\\\\dataingestion\\\\\", \"UserId\": \"adsgofastonprem\\\\AzureUser\", \"PasswordKeyVaultSecretName\": \"adsgofast-onpre_File_password\", \"Directory\": \"AdventureWorks2017/2020/06/15/14/22/\", \"File\": \"Sales.SalesOrderDetails.parquet\" }, \"Target\": { \"StorageAccountName\": \"https://adsgofastdatalakeaccelst.blob.core.windows.net\", \"Type\": \"Blob\", \"StorageAccountContainer\": \"datalakelanding\", \"StorageAccountAccessMethod\": \"MSI\", \"StorageAccountSASUriKeyVaultSecretName\": null, \"RelativePath\": \"/Unprocessed/adsgofastdatakakeaccelsqlsvr/AWSample/SalesLT/2020/06/08/17/\", \"DataFileName\": \"Customer_Data.parquet\", \"SchemaFileName\": \"Customer_Schema.json\" }, \"DataFactory\": { \"Id\": 1, \"Name\": \"adsgofastdatakakeacceladf\", \"ResourceGroup\": \"AdsGoFastDataLakeAccel\", \"SubscriptionId\": \"035a1364-f00d-48e2-b582-4fe125905ee3\", } } ]"
|
||||
}
|
||||
},
|
||||
"folder": {
|
||||
"name": "ADS Go Fast/Data Movement/Azure SH IR"
|
||||
"name": "ADS Go Fast/Data Movement/IRA"
|
||||
},
|
||||
"annotations": [],
|
||||
"lastPublishTime": "2020-07-24T01:51:29Z"
|
||||
},
|
||||
"type": "Microsoft.DataFactory/factories/pipelines"
|
||||
}
|
||||
}
|
||||
|
|
@ -14,16 +14,16 @@
|
|||
},
|
||||
"cases": [
|
||||
{
|
||||
"value": "AZ-SQL-AZ-Storage-Parquet-SH-IR",
|
||||
"value": "AZ_SQL_AZ_Storage_Parquet_IRA",
|
||||
"activities": [
|
||||
{
|
||||
"name": "Execute AZ-SQL-AZ-Storage-Parquet-SH-IR",
|
||||
"name": "Execute AZ_SQL_AZ_Storage_Parquet_IRA",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-SQL-AZ-Storage-Parquet-SH-IR",
|
||||
"referenceName": "AZ_SQL_AZ_Storage_Parquet_IRA",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": true,
|
||||
|
|
@ -38,16 +38,16 @@
|
|||
]
|
||||
},
|
||||
{
|
||||
"value": "AZ-Storage-Binary-AZ-Storage-Binary-SH-IR",
|
||||
"value": "AZ_Storage_Binary_AZ_Storage_Binary_IRA",
|
||||
"activities": [
|
||||
{
|
||||
"name": "Execute AZ-Storage-Binary-AZ-Storage-Binary-SH-IR",
|
||||
"name": "Execute AZ_Storage_Binary_AZ_Storage_Binary_IRA",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-Storage-Binary-AZ-Storage-Binary-SH-IR",
|
||||
"referenceName": "AZ_Storage_Binary_AZ_Storage_Binary_IRA",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": true,
|
||||
|
|
@ -62,16 +62,16 @@
|
|||
]
|
||||
},
|
||||
{
|
||||
"value": "AZ-Storage-CSV-AZ-SQL-SH-IR",
|
||||
"value": "AZ_Storage_CSV_AZ_SQL_IRA",
|
||||
"activities": [
|
||||
{
|
||||
"name": "Execute AZ-Storage-CSV-AZ-SQL-SH-IR",
|
||||
"name": "Execute AZ_Storage_CSV_AZ_SQL_IRA",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-Storage-CSV-AZ-SQL-SH-IR",
|
||||
"referenceName": "AZ_Storage_CSV_AZ_SQL_IRA",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": true,
|
||||
|
|
@ -86,16 +86,16 @@
|
|||
]
|
||||
},
|
||||
{
|
||||
"value": "AZ-Storage-Excel-AZ-SQL-SH-IR",
|
||||
"value": "AZ_Storage_Excel_AZ_SQL_IRA",
|
||||
"activities": [
|
||||
{
|
||||
"name": "Execute AZ-Storage-Excel-AZ-SQL-SH-IR",
|
||||
"name": "Execute AZ_Storage_Excel_AZ_SQL_IRA",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-Storage-Excel-AZ-SQL-SH-IR",
|
||||
"referenceName": "AZ_Storage_Excel_AZ_SQL_IRA",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": true,
|
||||
|
|
@ -110,16 +110,16 @@
|
|||
]
|
||||
},
|
||||
{
|
||||
"value": "AZ-Storage-JSON-AZ-SQL-SH-IR",
|
||||
"value": "AZ_Storage_JSON_AZ_SQL_IRA",
|
||||
"activities": [
|
||||
{
|
||||
"name": "Execute AZ-Storage-JSON-AZ-SQL-SH-IR",
|
||||
"name": "Execute AZ_Storage_JSON_AZ_SQL_IRA",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-Storage-JSON-AZ-SQL-SH-IR",
|
||||
"referenceName": "AZ_Storage_JSON_AZ_SQL_IRA",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": true,
|
||||
|
|
@ -134,16 +134,16 @@
|
|||
]
|
||||
},
|
||||
{
|
||||
"value": "AZ-Storage-Parquet-AZ-SQL-SH-IR",
|
||||
"value": "AZ_Storage_Parquet_AZ_SQL_IRA",
|
||||
"activities": [
|
||||
{
|
||||
"name": "Execute AZ-Storage-Parquet-AZ-SQL-SH-IR",
|
||||
"name": "Execute AZ_Storage_Parquet_AZ_SQL_IRA",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-Storage-Parquet-AZ-SQL-SH-IR",
|
||||
"referenceName": "AZ_Storage_Parquet_AZ_SQL_IRA",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": true,
|
||||
|
|
@ -158,16 +158,40 @@
|
|||
]
|
||||
},
|
||||
{
|
||||
"value": "GN-File-Binary-AZ-Storage-Binary-SH-IR",
|
||||
"value": "GEN_File_Binary_AZ_Storage_Binary_IRA",
|
||||
"activities": [
|
||||
{
|
||||
"name": "Execute GEN-File-Binary-AZ-Storage-Binary-SH-IR",
|
||||
"name": "Execute GEN_File_Binary_AZ_Storage_Binary_IRA",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "GEN-File-Binary-AZ-Storage-Binary-SH-IR",
|
||||
"referenceName": "GEN_File_Binary_AZ_Storage_Binary_IRA",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": true,
|
||||
"parameters": {
|
||||
"TaskObject": {
|
||||
"value": "@pipeline().parameters.TaskObject",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"value": "SH_SQL_AZ_Storage_Parquet_IRA",
|
||||
"activities": [
|
||||
{
|
||||
"name": "Execute SH_SQL_AZ_Storage_Parquet_IRA",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "SH_SQL_AZ_Storage_Parquet_IRA",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": true,
|
||||
|
|
@ -182,16 +206,16 @@
|
|||
]
|
||||
},
|
||||
{
|
||||
"value": "GEN-File-Binary-AZ-Storage-Binary-OnP-SH-IR",
|
||||
"value": "SH_SQL_GEN_File_Parquet_IRA",
|
||||
"activities": [
|
||||
{
|
||||
"name": "Execute GEN-File-Binary-AZ-Storage-Binary-OnP-SH-IR",
|
||||
"name": "Execute SH_SQL_GEN_File_Parquet_IRA",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "GEN-File-Binary-AZ-Storage-Binary-OnP-SH-IR",
|
||||
"referenceName": "SH_SQL_GEN_File_Parquet_IRA",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": true,
|
||||
|
|
@ -206,16 +230,16 @@
|
|||
]
|
||||
},
|
||||
{
|
||||
"value": "OnP-SQL-AZ-Storage-Parquet-OnP-SH-IR",
|
||||
"value": "Create-Task_MASTER_AZ_SQL_IRA",
|
||||
"activities": [
|
||||
{
|
||||
"name": "Execute OnP-SQL-AZ-Storage-Parquet-OnP-SH-IR",
|
||||
"name": "Execute Create-Task_MASTER_AZ_SQL_IRA",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "OnP-SQL-AZ-Storage-Parquet-OnP-SH-IR",
|
||||
"referenceName": "Create-Task_MASTER_AZ_SQL_IRA",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": true,
|
||||
|
|
@ -230,16 +254,16 @@
|
|||
]
|
||||
},
|
||||
{
|
||||
"value": "OnP-SQL-GEN-File-Parquet-OnP-SH-IR",
|
||||
"value": "AZ_Storage_Excel_AZ_Storage_CSV_IRA",
|
||||
"activities": [
|
||||
{
|
||||
"name": "Execute OnP-SQL-GEN-File-Parquet-OnP-SH-IR",
|
||||
"name": "Execute AZ_Storage_Excel_AZ_Storage_CSV_IRA",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "OnP-SQL-GEN-File-Parquet-OnP-SH-IR",
|
||||
"referenceName": "AZ_Storage_Excel_AZ_Storage_CSV_IRA",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": true,
|
||||
|
|
@ -254,88 +278,16 @@
|
|||
]
|
||||
},
|
||||
{
|
||||
"value": "Create-Task-Master-AZ-SQL-SH-IR",
|
||||
"value": "AZ_SQL_StoredProcedure_IRA",
|
||||
"activities": [
|
||||
{
|
||||
"name": "Execute Create-Task-Master-AZ-SQL-SH-IR",
|
||||
"name": "Execute AZ_SQL_StoredProcedure_IRA",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "Create-Task-Master-AZ-SQL-SH-IR",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": true,
|
||||
"parameters": {
|
||||
"TaskObject": {
|
||||
"value": "@pipeline().parameters.TaskObject",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"value": "Create-Task-Master-AZ-SQL-OnP-SH-IR",
|
||||
"activities": [
|
||||
{
|
||||
"name": "Create-Task-Master-AZ-SQL-OnP-SH-IR",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "Create-Task-Master-AZ-SQL-OnP-SH-IR",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": true,
|
||||
"parameters": {
|
||||
"TaskObject": {
|
||||
"value": "@pipeline().parameters.TaskObject",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"value": "AZ-Storage-Excel-AZ-Storage-CSV-SH-IR",
|
||||
"activities": [
|
||||
{
|
||||
"name": "Execute AZ-Storage-Excel-AZ-Storage-CSV-SH-IR",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-Storage-Excel-AZ-Storage-CSV-SH-IR",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": true,
|
||||
"parameters": {
|
||||
"TaskObject": {
|
||||
"value": "@pipeline().parameters.TaskObject",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"value": "AZ-SQL-StoredProcedure-SH-IR",
|
||||
"activities": [
|
||||
{
|
||||
"name": "Execute AZ-SQL-StoredProcedure-SH-IR",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-SQL-StoredProcedure-SH-IR",
|
||||
"referenceName": "AZ_SQL_StoredProcedure_IRA",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": true,
|
||||
|
|
@ -358,7 +310,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-Function-Generic",
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
|
|
@ -389,7 +341,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-Function-Generic",
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": true,
|
||||
|
|
@ -417,7 +369,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-Function-Generic",
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": true,
|
||||
|
|
@ -483,7 +435,7 @@
|
|||
"Name": "adsgofastdatakakeacceladf",
|
||||
"ResourceGroup": "AdsGoFastDataLakeAccel",
|
||||
"SubscriptionId": "035a1364-f00d-48e2-b582-4fe125905ee3",
|
||||
"ADFPipeline": "OnP-SQL-AZ-Storage-Parquet-OnP-SH-IR"
|
||||
"ADFPipeline": "OnP_SQL_AZ_Storage_Parquet_IRA"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -495,4 +447,4 @@
|
|||
"lastPublishTime": "2020-08-06T06:27:14Z"
|
||||
},
|
||||
"type": "Microsoft.DataFactory/factories/pipelines"
|
||||
}
|
||||
}
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
{
|
||||
"name": "SH-SQL-AZ-Storage-Parquet-SH-IR",
|
||||
"name": "SH_SQL_AZ_Storage_Parquet_IRA",
|
||||
"properties": {
|
||||
"activities": [
|
||||
{
|
||||
|
|
@ -16,7 +16,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-Function-Generic",
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
|
|
@ -117,7 +117,7 @@
|
|||
"partitionOption": "None"
|
||||
},
|
||||
"dataset": {
|
||||
"referenceName": "SqlServerTable_SH_IR",
|
||||
"referenceName": "SqlServerTable_IRA",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"TableSchema": {
|
||||
|
|
@ -175,13 +175,13 @@
|
|||
"value": "Full",
|
||||
"activities": [
|
||||
{
|
||||
"name": "Execute SH-SQL-Full-Load-SH-IR",
|
||||
"name": "Execute SH_SQL_Full_Load_IRA",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "SH-SQL-Full-Load-SH-IR",
|
||||
"referenceName": "SH_SQL_Full_Load_IRA",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": true,
|
||||
|
|
@ -205,7 +205,7 @@
|
|||
"value": "Watermark",
|
||||
"activities": [
|
||||
{
|
||||
"name": "Execute SH-SQL-Watermark-SH-IR",
|
||||
"name": "Execute SH_SQL_Watermark_IRA",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [
|
||||
{
|
||||
|
|
@ -218,7 +218,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "SH-SQL-Watermark-SH-IR",
|
||||
"referenceName": "SH_SQL_Watermark_IRA",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": true,
|
||||
|
|
@ -263,7 +263,7 @@
|
|||
"partitionOption": "None"
|
||||
},
|
||||
"dataset": {
|
||||
"referenceName": "SqlServerTable_SH_IR",
|
||||
"referenceName": "SqlServerTable_IRA",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"TableSchema": {
|
||||
|
|
@ -302,10 +302,10 @@
|
|||
]
|
||||
},
|
||||
{
|
||||
"value": "Full-Chunk",
|
||||
"value": "Full_Chunk",
|
||||
"activities": [
|
||||
{
|
||||
"name": "Execute SH-SQL-Full-Load-Chunk-SH-IR",
|
||||
"name": "Execute SH_SQL_Full_Load_Chunk_IRA",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [
|
||||
{
|
||||
|
|
@ -318,7 +318,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "SH-SQL-Full-Load-Chunk-SH-IR",
|
||||
"referenceName": "SH_SQL_Full_Load_Chunk_IRA",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": true,
|
||||
|
|
@ -361,7 +361,7 @@
|
|||
"partitionOption": "None"
|
||||
},
|
||||
"dataset": {
|
||||
"referenceName": "SqlServerTable_SH_IR",
|
||||
"referenceName": "SqlServerTable_IRA",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"TableSchema": {
|
||||
|
|
@ -400,10 +400,10 @@
|
|||
]
|
||||
},
|
||||
{
|
||||
"value": "Watermark-Chunk",
|
||||
"value": "Watermark_Chunk",
|
||||
"activities": [
|
||||
{
|
||||
"name": "Execute SH-SQL-Watermark-Chunk-SH-IR",
|
||||
"name": "Execute SH_SQL_Watermark_Chunk_IRA",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [
|
||||
{
|
||||
|
|
@ -416,7 +416,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "SH-SQL-Watermark-Chunk-SH-IR",
|
||||
"referenceName": "SH_SQL_Watermark_Chunk_IRA",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": true,
|
||||
|
|
@ -463,7 +463,7 @@
|
|||
"partitionOption": "None"
|
||||
},
|
||||
"dataset": {
|
||||
"referenceName": "SqlServerTable_SH_IR",
|
||||
"referenceName": "SqlServerTable_IRA",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"TableSchema": {
|
||||
|
|
@ -508,14 +508,14 @@
|
|||
"parameters": {
|
||||
"TaskObject": {
|
||||
"type": "object",
|
||||
"defaultValue": " [ { \"TaskInstanceId\": 2, \"TaskMasterId\": 1, \"TaskStatus\": \"Untried\", \"TaskType\": \"SQLtoAzureStorage-OnPrem-SH-IR\", \"Enabled\": true, \"ExecutionUid\": 1, \"KeyVaultBaseUrl\": \"https://adsgofastkeyvault.vault.azure.net/\", \"Source\": { \"Type\": \"MsSqlServer\", \"Database\": { \"Name\": \"AdventureWorks2017\", \"SystemName\": \"adsgofast-onpre\", \"Type\": \"MsSqlServer\", \"AuthenticationType\": \"SQLAuth\", \"Username\": \"sqladfir\", \"PasswordKeyVaultSecretName\": \"adsgofast-onpre-sqladfir-password\" }, \"Extraction\": { \"Type\": \"Table\", \"FullOrIncremental\": true, \"IncrementalType\": null, \"TableSchema\": \"Sales\", \"TableName\": \"SalesOrderDetail\" } }, \"Target\": { \"StorageAccountName\": \"https://adsgofastdatalakeadls.dfs.core.windows.net\", \"Type\": \"ADLS\", \"StorageAccountContainer\": \"datalakelanding\", \"StorageAccountAccessMethod\": \"MSI\", \"StorageAccountSASUriKeyVaultSecretName\": null, \"RelativePath\": \"/Unprocessed/adsgofastdatakakeaccelsqlsvr/AWSample/SalesLT/2020/06/08/17/\", \"DataFileName\": \"Customer_Data.parquet\", \"SchemaFileName\": \"Customer_Schema.json\" }, \"DataFactory\": { \"Id\": 1, \"Name\": \"adsgofastdatakakeacceladf\", \"ResourceGroup\": \"AdsGoFastDataLakeAccel\", \"SubscriptionId\": \"035a1364-f00d-48e2-b582-4fe125905ee3\", } } ]"
|
||||
"defaultValue": " [ { \"TaskInstanceId\": 2, \"TaskMasterId\": 1, \"TaskStatus\": \"Untried\", \"TaskType\": \"SQLtoAzureStorage_IRA\", \"Enabled\": true, \"ExecutionUid\": 1, \"KeyVaultBaseUrl\": \"https://adsgofastkeyvault.vault.azure.net/\", \"Source\": { \"Type\": \"MsSqlServer\", \"Database\": { \"Name\": \"AdventureWorks2017\", \"SystemName\": \"adsgofast-onpre\", \"Type\": \"MsSqlServer\", \"AuthenticationType\": \"SQLAuth\", \"Username\": \"sqladfir\", \"PasswordKeyVaultSecretName\": \"adsgofast-onpre-sqladfir-password\" }, \"Extraction\": { \"Type\": \"Table\", \"FullOrIncremental\": true, \"IncrementalType\": null, \"TableSchema\": \"Sales\", \"TableName\": \"SalesOrderDetail\" } }, \"Target\": { \"StorageAccountName\": \"https://adsgofastdatalakeadls.dfs.core.windows.net\", \"Type\": \"ADLS\", \"StorageAccountContainer\": \"datalakelanding\", \"StorageAccountAccessMethod\": \"MSI\", \"StorageAccountSASUriKeyVaultSecretName\": null, \"RelativePath\": \"/Unprocessed/adsgofastdatakakeaccelsqlsvr/AWSample/SalesLT/2020/06/08/17/\", \"DataFileName\": \"Customer_Data.parquet\", \"SchemaFileName\": \"Customer_Schema.json\" }, \"DataFactory\": { \"Id\": 1, \"Name\": \"adsgofastdatakakeacceladf\", \"ResourceGroup\": \"AdsGoFastDataLakeAccel\", \"SubscriptionId\": \"035a1364-f00d-48e2-b582-4fe125905ee3\", } } ]"
|
||||
}
|
||||
},
|
||||
"folder": {
|
||||
"name": "ADS Go Fast/Data Movement/SH-IR"
|
||||
"name": "ADS Go Fast/Data Movement/IRA"
|
||||
},
|
||||
"annotations": [],
|
||||
"lastPublishTime": "2020-08-04T12:40:45Z"
|
||||
},
|
||||
"type": "Microsoft.DataFactory/factories/pipelines"
|
||||
}
|
||||
}
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
{
|
||||
"name": "SH-SQL-Full-Load-Chunk-SH-IR",
|
||||
"name": "SH_SQL_Full_Load_Chunk_IRA",
|
||||
"properties": {
|
||||
"activities": [
|
||||
{
|
||||
|
|
@ -15,13 +15,13 @@
|
|||
"isSequential": true,
|
||||
"activities": [
|
||||
{
|
||||
"name": "Execute SH-SQL-Full-Load",
|
||||
"name": "Execute SH_SQL_Full_Load",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "SH-SQL-Full-Load-SH-IR",
|
||||
"referenceName": "SH_SQL_Full_Load_IRA",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": true,
|
||||
|
|
@ -98,7 +98,7 @@
|
|||
"Name": "adsgofastdatakakeacceladf",
|
||||
"ResourceGroup": "AdsGoFastDataLakeAccel",
|
||||
"SubscriptionId": "035a1364-f00d-48e2-b582-4fe125905ee3",
|
||||
"ADFPipeline": "SH-AZ-Storage-Parquet-SH-IR"
|
||||
"ADFPipeline": "SH-AZ_Storage_Parquet_IRA"
|
||||
}
|
||||
}
|
||||
},
|
||||
|
|
@ -110,7 +110,7 @@
|
|||
}
|
||||
},
|
||||
"folder": {
|
||||
"name": "ADS Go Fast/Data Movement/SH-IR/Components"
|
||||
"name": "ADS Go Fast/Data Movement/IRA/Components"
|
||||
},
|
||||
"annotations": []
|
||||
},
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
{
|
||||
"name": "SH-SQL-Full-Load-SH-IR",
|
||||
"name": "SH_SQL_Full_Load_IRA",
|
||||
"properties": {
|
||||
"activities": [
|
||||
{
|
||||
|
|
@ -82,7 +82,7 @@
|
|||
},
|
||||
"inputs": [
|
||||
{
|
||||
"referenceName": "SqlServerTable_SH_IR",
|
||||
"referenceName": "SqlServerTable_IRA",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"TableSchema": {
|
||||
|
|
@ -118,7 +118,7 @@
|
|||
],
|
||||
"outputs": [
|
||||
{
|
||||
"referenceName": "ADLSParquet_SH_IR",
|
||||
"referenceName": "ADLSParquet_IRA",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"RelativePath": {
|
||||
|
|
@ -148,7 +148,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-Function-Generic",
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
|
|
@ -215,7 +215,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"dataset": {
|
||||
"referenceName": "ADLSParquet_SH_IR",
|
||||
"referenceName": "ADLSParquet_IRA",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"RelativePath": {
|
||||
|
|
@ -259,7 +259,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-Function-Generic",
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
|
|
@ -287,7 +287,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-Function-Generic",
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
|
|
@ -352,7 +352,7 @@
|
|||
},
|
||||
"inputs": [
|
||||
{
|
||||
"referenceName": "SqlServerTable_SH_IR",
|
||||
"referenceName": "SqlServerTable_IRA",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"TableSchema": {
|
||||
|
|
@ -388,7 +388,7 @@
|
|||
],
|
||||
"outputs": [
|
||||
{
|
||||
"referenceName": "BlobParquet_SH_IR",
|
||||
"referenceName": "BlobParquet_IRA",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"RelativePath": {
|
||||
|
|
@ -418,7 +418,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-Function-Generic",
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
|
|
@ -485,7 +485,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"dataset": {
|
||||
"referenceName": "BlobParquet_SH_IR",
|
||||
"referenceName": "BlobParquet_IRA",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"RelativePath": {
|
||||
|
|
@ -529,7 +529,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-Function-Generic",
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
|
|
@ -557,7 +557,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-Function-Generic",
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
|
|
@ -626,7 +626,7 @@
|
|||
"Name": "adsgofastdatakakeacceladf",
|
||||
"ResourceGroup": "AdsGoFastDataLakeAccel",
|
||||
"SubscriptionId": "035a1364-f00d-48e2-b582-4fe125905ee3",
|
||||
"ADFPipeline": "AZ-SQL-AZ-Storage-Parquet-SH-IR"
|
||||
"ADFPipeline": "AZ_SQL_AZ_Storage_Parquet_IRA"
|
||||
}
|
||||
}
|
||||
},
|
||||
|
|
@ -646,7 +646,7 @@
|
|||
}
|
||||
},
|
||||
"folder": {
|
||||
"name": "ADS Go Fast/Data Movement/SH-IR/Components"
|
||||
"name": "ADS Go Fast/Data Movement/IRA/Components"
|
||||
},
|
||||
"annotations": []
|
||||
},
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
{
|
||||
"name": "OnP-SQL-GEN-File-Parquet-OnP-SH-IR",
|
||||
"name": "SH_SQL_GEN_File_Parquet_IRA",
|
||||
"properties": {
|
||||
"activities": [
|
||||
{
|
||||
|
|
@ -45,7 +45,7 @@
|
|||
},
|
||||
"inputs": [
|
||||
{
|
||||
"referenceName": "SqlServerTable_OnPrem_SH_IR",
|
||||
"referenceName": "SqlServerTable_IRA",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"TableSchema": {
|
||||
|
|
@ -81,7 +81,7 @@
|
|||
],
|
||||
"outputs": [
|
||||
{
|
||||
"referenceName": "FileParquet_OnPrem_SH_IR",
|
||||
"referenceName": "FileParquet_IRA",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"Host": {
|
||||
|
|
@ -126,7 +126,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-Function-Generic",
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
|
|
@ -154,7 +154,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-Function-Generic",
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
|
|
@ -182,7 +182,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-Function-Generic",
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
|
|
@ -210,7 +210,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-Function-Generic",
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
|
|
@ -311,7 +311,7 @@
|
|||
"partitionOption": "None"
|
||||
},
|
||||
"dataset": {
|
||||
"referenceName": "SqlServerTable_OnPrem_SH_IR",
|
||||
"referenceName": "SqlServerTable_IRA",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"TableSchema": {
|
||||
|
|
@ -351,14 +351,14 @@
|
|||
"parameters": {
|
||||
"TaskObject": {
|
||||
"type": "object",
|
||||
"defaultValue": "[ { \"TaskInstanceId\": 2, \"TaskMasterId\": 1, \"TaskStatus\": \"Untried\", \"TaskType\": \"RelationalDataExtractToDataLake\", \"Enabled\": true, \"ExecutionUid\": 1, \"KeyVaultBaseUrl\" : \"https://adsgofastkeyvault.vault.azure.net/\", \"Source\": { \"Type\":\"MsSqlServer\", \"Database\": { \"Name\": \"AdventureWorks2017\", \"SystemName\": \"adsgofast-onpre\", \"Type\": \"MsSqlServer\", \"AuthenticationType\": \"SQLAuth\", \"Username\": \"sqladfir\", \"PasswordKeyVaultSecretName\": \"adsgofast-onpre-sqladfir-password\" }, \"Extraction\": { \"Type\": \"Table\", \"FullOrIncremental\": true, \"IncrementalType\": null, \"TableSchema\": \"Sales\", \"TableName\": \"SalesOrderDetail\" } }, \"Target\": { \"Type\" : \"File\", \"Host\": \"D:\\\\dataingestion\\\\\", \"UserId\": \"AzureUser\", \"PasswordKeyVaultSecretName\": \"adsgofast-onpre-file-password\", \"Directory\": \"AdventureWorks2017/2020/06/15/14/22/\", \"File\": \"Sales.SalesOrderDetails.parquet\" }, \"DataFactory\": { \"Id\": 1, \"Name\": \"adsgofastdatakakeacceladf\", \"ResourceGroup\": \"AdsGoFastDataLakeAccel\", \"SubscriptionId\": \"035a1364-f00d-48e2-b582-4fe125905ee3\", } } ]"
|
||||
"defaultValue": "[ { \"TaskInstanceId\": 2, \"TaskMasterId\": 1, \"TaskStatus\": \"Untried\", \"TaskType\": \"RelationalDataExtractToDataLake\", \"Enabled\": true, \"ExecutionUid\": 1, \"KeyVaultBaseUrl\" : \"https://adsgofastkeyvault.vault.azure.net/\", \"Source\": { \"Type\":\"MsSqlServer\", \"Database\": { \"Name\": \"AdventureWorks2017\", \"SystemName\": \"adsgofast-onpre\", \"Type\": \"MsSqlServer\", \"AuthenticationType\": \"SQLAuth\", \"Username\": \"sqladfir\", \"PasswordKeyVaultSecretName\": \"adsgofast-onpre-sqladfir-password\" }, \"Extraction\": { \"Type\": \"Table\", \"FullOrIncremental\": true, \"IncrementalType\": null, \"TableSchema\": \"Sales\", \"TableName\": \"SalesOrderDetail\" } }, \"Target\": { \"Type\" : \"File\", \"Host\": \"D:\\\\dataingestion\\\\\", \"UserId\": \"AzureUser\", \"PasswordKeyVaultSecretName\": \"adsgofast-onpre_File_password\", \"Directory\": \"AdventureWorks2017/2020/06/15/14/22/\", \"File\": \"Sales.SalesOrderDetails.parquet\" }, \"DataFactory\": { \"Id\": 1, \"Name\": \"adsgofastdatakakeacceladf\", \"ResourceGroup\": \"AdsGoFastDataLakeAccel\", \"SubscriptionId\": \"035a1364-f00d-48e2-b582-4fe125905ee3\", } } ]"
|
||||
}
|
||||
},
|
||||
"folder": {
|
||||
"name": "ADS Go Fast/Data Movement/OnPrem SH IR"
|
||||
"name": "ADS Go Fast/Data Movement/IRA"
|
||||
},
|
||||
"annotations": [],
|
||||
"lastPublishTime": "2020-07-27T03:24:22Z"
|
||||
},
|
||||
"type": "Microsoft.DataFactory/factories/pipelines"
|
||||
}
|
||||
}
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
{
|
||||
"name": "SH-SQL-Watermark-Chunk-SH-IR",
|
||||
"name": "SH_SQL_Watermark_Chunk_IRA",
|
||||
"properties": {
|
||||
"activities": [
|
||||
{
|
||||
|
|
@ -16,7 +16,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-Function-Generic",
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
|
|
@ -43,13 +43,13 @@
|
|||
"isSequential": true,
|
||||
"activities": [
|
||||
{
|
||||
"name": "Execute SH-SQL-Watermark",
|
||||
"name": "Execute SH_SQL_Watermark",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "SH-SQL-Watermark-SH-IR",
|
||||
"referenceName": "SH_SQL_Watermark_IRA",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": true,
|
||||
|
|
@ -130,7 +130,7 @@
|
|||
"Name": "adsgofastdatakakeacceladf",
|
||||
"ResourceGroup": "AdsGoFastDataLakeAccel",
|
||||
"SubscriptionId": "035a1364-f00d-48e2-b582-4fe125905ee3",
|
||||
"ADFPipeline": "AZ-SQL-AZ-Storage-Parquet-SH-IR"
|
||||
"ADFPipeline": "AZ_SQL_AZ_Storage_Parquet_IRA"
|
||||
}
|
||||
}
|
||||
},
|
||||
|
|
@ -145,7 +145,7 @@
|
|||
}
|
||||
},
|
||||
"folder": {
|
||||
"name": "ADS Go Fast/Data Movement/SH-IR/Components"
|
||||
"name": "ADS Go Fast/Data Movement/IRA/Components"
|
||||
},
|
||||
"annotations": []
|
||||
},
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
{
|
||||
"name": "SH-SQL-Watermark-SH-IR",
|
||||
"name": "SH_SQL_Watermark_IRA",
|
||||
"properties": {
|
||||
"activities": [
|
||||
{
|
||||
|
|
@ -82,7 +82,7 @@
|
|||
},
|
||||
"inputs": [
|
||||
{
|
||||
"referenceName": "SqlServerTable_SH_IR",
|
||||
"referenceName": "SqlServerTable_IRA",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"TableSchema": {
|
||||
|
|
@ -118,7 +118,7 @@
|
|||
],
|
||||
"outputs": [
|
||||
{
|
||||
"referenceName": "ADLSParquet_SH_IR",
|
||||
"referenceName": "ADLSParquet_IRA",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"RelativePath": {
|
||||
|
|
@ -148,7 +148,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-Function-Generic",
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
|
|
@ -215,7 +215,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"dataset": {
|
||||
"referenceName": "ADLSParquet_SH_IR",
|
||||
"referenceName": "ADLSParquet_IRA",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"RelativePath": {
|
||||
|
|
@ -259,7 +259,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-Function-Generic",
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
|
|
@ -287,7 +287,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-Function-Generic",
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
|
|
@ -348,7 +348,7 @@
|
|||
},
|
||||
"inputs": [
|
||||
{
|
||||
"referenceName": "SqlServerTable_SH_IR",
|
||||
"referenceName": "SqlServerTable_IRA",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"TableSchema": {
|
||||
|
|
@ -384,7 +384,7 @@
|
|||
],
|
||||
"outputs": [
|
||||
{
|
||||
"referenceName": "BlobParquet_SH_IR",
|
||||
"referenceName": "BlobParquet_IRA",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"RelativePath": {
|
||||
|
|
@ -414,7 +414,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-Function-Generic",
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
|
|
@ -481,7 +481,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"dataset": {
|
||||
"referenceName": "BlobParquet_SH_IR",
|
||||
"referenceName": "BlobParquet_IRA",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"RelativePath": {
|
||||
|
|
@ -525,7 +525,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-Function-Generic",
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
|
|
@ -553,7 +553,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-Function-Generic",
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
|
|
@ -622,7 +622,7 @@
|
|||
"Name": "adsgofastdatakakeacceladf",
|
||||
"ResourceGroup": "AdsGoFastDataLakeAccel",
|
||||
"SubscriptionId": "035a1364-f00d-48e2-b582-4fe125905ee3",
|
||||
"ADFPipeline": "AZ-SQL-AZ-Storage-Parquet-SH-IR"
|
||||
"ADFPipeline": "AZ_SQL_AZ_Storage_Parquet_IRA"
|
||||
}
|
||||
}
|
||||
},
|
||||
|
|
@ -645,7 +645,7 @@
|
|||
}
|
||||
},
|
||||
"folder": {
|
||||
"name": "ADS Go Fast/Data Movement/SH-IR/Components"
|
||||
"name": "ADS Go Fast/Data Movement/IRA/Components"
|
||||
},
|
||||
"annotations": []
|
||||
},
|
||||
|
|
@ -0,0 +1,29 @@
|
|||
|
||||
|
||||
function Generate($IR, $targetdir)
|
||||
{
|
||||
|
||||
Get-ChildItem -Path ".\Templates\$targetdir" -Filter "*.json" |
|
||||
ForEach-Object {
|
||||
$targetfile = ".\ADF\$targetdir" + "\" + ($_.BaseName).Replace("{IR}",$IR) + ".json"
|
||||
$fileName = $_.FullName
|
||||
$jsonobject = ($_ | Get-Content).Replace("@GF{IR}",$IR)
|
||||
$jsonobject | set-content $targetfile
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
$Irc = Get-Content .\IRConfig.json | ConvertFrom-Json
|
||||
foreach($Ir in $Irc.IRList)
|
||||
{
|
||||
if ($Ir.Enable)
|
||||
{
|
||||
Write-Output ("Generating Files for Integration Runtime: " + $Ir.ADFPostFix)
|
||||
Generate -IR $Ir.ADFPostFix -targetdir "linkedService"
|
||||
Generate -IR $Ir.ADFPostFix -targetdir "dataset"
|
||||
Generate -IR $Ir.ADFPostFix -targetdir "pipeline"
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
|
@ -0,0 +1,114 @@
|
|||
{
|
||||
"IRList": [
|
||||
{
|
||||
"NameInMetaDataDB": "IRA",
|
||||
"SelfHosted": false,
|
||||
"NameInADF": "IRA",
|
||||
"ADFPostFix": "IRA",
|
||||
"Enable": true,
|
||||
"ADFTemplate": "IRA"
|
||||
},
|
||||
{
|
||||
"NameInMetaDataDB": "IRB",
|
||||
"SelfHosted": false,
|
||||
"NameInADF": "IRB",
|
||||
"ADFPostFix": "IRB",
|
||||
"Enable": true,
|
||||
"ADFTemplate": "IRB"
|
||||
},
|
||||
{
|
||||
"NameInMetaDataDB": "IRC",
|
||||
"SelfHosted": false,
|
||||
"NameInADF": "IRC",
|
||||
"ADFPostFix": "IRC",
|
||||
"Enable": false,
|
||||
"ADFTemplate": "IRC"
|
||||
}
|
||||
],
|
||||
"ADFTemplates": [
|
||||
{
|
||||
"Name": "IRA",
|
||||
"dataset": [
|
||||
"ADLSBinary_SH_IR.json",
|
||||
"ADLSDelimitedText_SH_IR.json",
|
||||
"ADLSExcel_SH_IR.json",
|
||||
"ADLSJson_SH_IR.json",
|
||||
"ADLSParquet_SH_IR.json",
|
||||
"AzureSqlTable_SH_IR.json",
|
||||
"BlobBinary_SH_IR.json",
|
||||
"BlobDelimitedText_SH_IR.json",
|
||||
"BlobExcel_SH_IR.json",
|
||||
"BlobJson_SH_IR.json",
|
||||
"BlobParquet_SH_IR.json",
|
||||
"FileBinary_SH_IR.json",
|
||||
"FileParquet_SH_IR.json",
|
||||
"SqlServerTable_SH_IR.json"
|
||||
],
|
||||
"linkedService": [
|
||||
"GenericAzureDataLakeStorage_{IR}.json",
|
||||
"GenericAzureSQL_{IR}.json",
|
||||
"GenericBlobStorage_{IR}.json",
|
||||
"GenericFileServer_{IR}.json",
|
||||
"GenericMsSqlServer_{IR}.json"
|
||||
],
|
||||
"pipeline": [
|
||||
"AZ_SQL_AZ_Storage_Parquet_SH_IR.json",
|
||||
"AZ-SQL-Create-Table-SH-IR.json",
|
||||
"AZ-SQL-Full-Load-Chunk-SH-IR.json",
|
||||
"AZ-SQL-Full-Load-SH-IR.json",
|
||||
"AZ-SQL-Post-Copy-SH-IR.json",
|
||||
"AZ-SQL-StoredProcedure-SH-IR.json",
|
||||
"AZ-SQL-Watermark-Chunk-SH-IR.json",
|
||||
"AZ-SQL-Watermark-SH-IR.json",
|
||||
"AZ-Storage-Binary-AZ-Storage-Binary-SH-IR.json",
|
||||
"AZ-Storage-CSV-AZ-SQL-SH-IR.json",
|
||||
"AZ-Storage-Excel-AZ-SQL-SH-IR.json",
|
||||
"AZ-Storage-Excel-AZ-Storage-CSV-SH-IR.json",
|
||||
"AZ-Storage-JSON-AZ-SQL-SH-IR.json",
|
||||
"AZ-Storage-Parquet-AZ-SQL-SH-IR.json",
|
||||
"Create-Task-Master-AZ-SQL-OnP-SH-IR.json",
|
||||
"Create-Task-Master-AZ-SQL-SH-IR.json",
|
||||
"GEN-File-Binary-AZ-Storage-Binary-OnP-SH-IR.json",
|
||||
"GEN-File-Binary-AZ-Storage-Binary-SH-IR.json",
|
||||
"OnP-SQL-AZ-Storage-Parquet-OnP-SH-IR.json",
|
||||
"OnP-SQL-Full-Load-Chunk-OnP-SH-IR.json",
|
||||
"OnP-SQL-Full-Load-OnP-SH-IR.json",
|
||||
"OnP-SQL-GEN-File-Parquet-OnP-SH-IR.json",
|
||||
"OnP-SQL-Watermark-Chunk-OnP-SH-IR.json",
|
||||
"OnP-SQL-Watermark-OnP-SH-IR.json",
|
||||
"SH-SQL-AZ-Storage-Parquet-SH-IR.json",
|
||||
"SH-SQL-Full-Load-Chunk-SH-IR.json",
|
||||
"SH-SQL-Full-Load-SH-IR.json",
|
||||
"SH-SQL-Watermark-Chunk-SH-IR.json",
|
||||
"SH-SQL-Watermark-SH-IR.json"
|
||||
]
|
||||
},
|
||||
{
|
||||
"Name": "OnPrem_Net",
|
||||
"dataset": [
|
||||
"ADLSBinary_OnPrem_Net",
|
||||
"ADLSParquet_OnPrem_Net",
|
||||
"BlobParquet_OnPrem_Net.json",
|
||||
"FileBinary_OnPrem_Net.json",
|
||||
"FileParquet_OnPrem_Net.json",
|
||||
"SqlServerTable_OnPrem_Net.json"
|
||||
],
|
||||
"linkedService": [
|
||||
"GenericMsSqlServer_OnPrem_Net.json",
|
||||
"GenericFileServer_OnPrem_Net.json",
|
||||
"GenericBlobStorage_OnPrem_Net.json",
|
||||
"GenericAzureDataLakeStorage_OnPrem_Net.json"
|
||||
],
|
||||
"pipeline": [
|
||||
"Create-Task-Master-AZ-SQL-OnPrem_Net.json",
|
||||
"GEN-File-Binary-AZ-Storage-Binary-OnPrem_Net.json",
|
||||
"OnP-SQL-AZ-Storage-Parquet-OnPrem_Net.json",
|
||||
"OnP-SQL-Full-Load-Chunk-OnP-SH-IR.json",
|
||||
"OnP-SQL-Full-Load-OnP-SH-IR.json",
|
||||
"OnP-SQL-GEN-File-Parquet-OnP-SH-IR.json",
|
||||
"OnP-SQL-Watermark-Chunk-OnP-SH-IR.json",
|
||||
"OnP-SQL-Watermark-OnP-SH-IR.json"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -1,8 +1,8 @@
|
|||
{
|
||||
"name": "ADLSBinary_SH_IR",
|
||||
"name": "ADLSBinary_@GF{IR}",
|
||||
"properties": {
|
||||
"linkedServiceName": {
|
||||
"referenceName": "GenericAzureDataLakeStorage_SH_IR",
|
||||
"referenceName": "GenericAzureDataLakeStorage_@GF{IR}",
|
||||
"type": "LinkedServiceReference",
|
||||
"parameters": {
|
||||
"StorageAccountEndpoint": {
|
||||
|
|
@ -26,7 +26,7 @@
|
|||
}
|
||||
},
|
||||
"folder": {
|
||||
"name": "ADS Go Fast/Generic/Azure SH IR"
|
||||
"name": "ADS Go Fast/Generic/@GF{IR}"
|
||||
},
|
||||
"annotations": [],
|
||||
"type": "Binary",
|
||||
|
|
@ -0,0 +1,63 @@
|
|||
{
|
||||
"name": "ADLSDelimitedText_@GF{IR}",
|
||||
"properties": {
|
||||
"linkedServiceName": {
|
||||
"referenceName": "GenericAzureDataLakeStorage_@GF{IR}",
|
||||
"type": "LinkedServiceReference",
|
||||
"parameters": {
|
||||
"StorageAccountEndpoint": {
|
||||
"value": "@dataset().StorageAccountEndpoint",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
},
|
||||
"parameters": {
|
||||
"RelativePath": {
|
||||
"type": "string"
|
||||
},
|
||||
"FileName": {
|
||||
"type": "string"
|
||||
},
|
||||
"StorageAccountEndpoint": {
|
||||
"type": "string"
|
||||
},
|
||||
"StorageAccountContainerName": {
|
||||
"type": "string"
|
||||
},
|
||||
"FirstRownAsHeader": {
|
||||
"type": "bool"
|
||||
}
|
||||
},
|
||||
"folder": {
|
||||
"name": "ADS Go Fast/Generic/@GF{IR}"
|
||||
},
|
||||
"annotations": [],
|
||||
"type": "DelimitedText",
|
||||
"typeProperties": {
|
||||
"location": {
|
||||
"type": "AzureBlobFSLocation",
|
||||
"fileName": {
|
||||
"value": "@dataset().FileName",
|
||||
"type": "Expression"
|
||||
},
|
||||
"folderPath": {
|
||||
"value": "@dataset().RelativePath",
|
||||
"type": "Expression"
|
||||
},
|
||||
"fileSystem": {
|
||||
"value": "@dataset().StorageAccountContainerName",
|
||||
"type": "Expression"
|
||||
}
|
||||
},
|
||||
"columnDelimiter": ",",
|
||||
"escapeChar": "\\",
|
||||
"firstRowAsHeader": {
|
||||
"value": "@dataset().FirstRownAsHeader",
|
||||
"type": "Expression"
|
||||
},
|
||||
"quoteChar": "\""
|
||||
},
|
||||
"schema": []
|
||||
},
|
||||
"type": "Microsoft.DataFactory/factories/datasets"
|
||||
}
|
||||
|
|
@ -0,0 +1,67 @@
|
|||
{
|
||||
"name": "ADLSExcel_@GF{IR}",
|
||||
"properties": {
|
||||
"linkedServiceName": {
|
||||
"referenceName": "GenericAzureDataLakeStorage_@GF{IR}",
|
||||
"type": "LinkedServiceReference",
|
||||
"parameters": {
|
||||
"StorageAccountEndpoint": {
|
||||
"value": "@dataset().StorageAccountEndpoint",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
},
|
||||
"parameters": {
|
||||
"RelativePath": {
|
||||
"type": "string"
|
||||
},
|
||||
"FileName": {
|
||||
"type": "string"
|
||||
},
|
||||
"StorageAccountEndpoint": {
|
||||
"type": "string"
|
||||
},
|
||||
"StorageAccountContainerName": {
|
||||
"type": "string"
|
||||
},
|
||||
"SheetName": {
|
||||
"type": "string"
|
||||
},
|
||||
"FirstRowAsHeader": {
|
||||
"type": "bool"
|
||||
}
|
||||
},
|
||||
"folder": {
|
||||
"name": "ADS Go Fast/Generic/@GF{IR}"
|
||||
},
|
||||
"annotations": [],
|
||||
"type": "Excel",
|
||||
"typeProperties": {
|
||||
"sheetName": {
|
||||
"value": "@dataset().SheetName",
|
||||
"type": "Expression"
|
||||
},
|
||||
"location": {
|
||||
"type": "AzureBlobFSLocation",
|
||||
"fileName": {
|
||||
"value": "@dataset().FileName",
|
||||
"type": "Expression"
|
||||
},
|
||||
"folderPath": {
|
||||
"value": "@dataset().RelativePath",
|
||||
"type": "Expression"
|
||||
},
|
||||
"fileSystem": {
|
||||
"value": "@dataset().StorageAccountContainerName",
|
||||
"type": "Expression"
|
||||
}
|
||||
},
|
||||
"firstRowAsHeader": {
|
||||
"value": "@dataset().FirstRowAsHeader",
|
||||
"type": "Expression"
|
||||
}
|
||||
},
|
||||
"schema": []
|
||||
},
|
||||
"type": "Microsoft.DataFactory/factories/datasets"
|
||||
}
|
||||
|
|
@ -0,0 +1,53 @@
|
|||
{
|
||||
"name": "ADLSJson_@GF{IR}",
|
||||
"properties": {
|
||||
"linkedServiceName": {
|
||||
"referenceName": "GenericAzureDataLakeStorage_@GF{IR}",
|
||||
"type": "LinkedServiceReference",
|
||||
"parameters": {
|
||||
"StorageAccountEndpoint": {
|
||||
"value": "@dataset().StorageAccountEndpoint",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
},
|
||||
"parameters": {
|
||||
"RelativePath": {
|
||||
"type": "string"
|
||||
},
|
||||
"FileName": {
|
||||
"type": "string"
|
||||
},
|
||||
"StorageAccountEndpoint": {
|
||||
"type": "string"
|
||||
},
|
||||
"StorageAccountContainerName": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"folder": {
|
||||
"name": "ADS Go Fast/Generic/@GF{IR}"
|
||||
},
|
||||
"annotations": [],
|
||||
"type": "Json",
|
||||
"typeProperties": {
|
||||
"location": {
|
||||
"type": "AzureBlobFSLocation",
|
||||
"fileName": {
|
||||
"value": "@dataset().FileName",
|
||||
"type": "Expression"
|
||||
},
|
||||
"folderPath": {
|
||||
"value": "@dataset().RelativePath",
|
||||
"type": "Expression"
|
||||
},
|
||||
"fileSystem": {
|
||||
"value": "@dataset().StorageAccountContainerName",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
},
|
||||
"schema": {}
|
||||
},
|
||||
"type": "Microsoft.DataFactory/factories/datasets"
|
||||
}
|
||||
|
|
@ -1,8 +1,8 @@
|
|||
{
|
||||
"name": "ADLSParquet_SH_IR",
|
||||
"name": "ADLSParquet_@GF{IR}",
|
||||
"properties": {
|
||||
"linkedServiceName": {
|
||||
"referenceName": "GenericAzureDataLakeStorage_SH_IR",
|
||||
"referenceName": "GenericAzureDataLakeStorage_@GF{IR}",
|
||||
"type": "LinkedServiceReference",
|
||||
"parameters": {
|
||||
"StorageAccountEndpoint": {
|
||||
|
|
@ -26,7 +26,7 @@
|
|||
}
|
||||
},
|
||||
"folder": {
|
||||
"name": "ADS Go Fast/Generic/Azure SH IR"
|
||||
"name": "ADS Go Fast/Generic/@GF{IR}"
|
||||
},
|
||||
"annotations": [],
|
||||
"type": "Parquet",
|
||||
|
|
@ -1,8 +1,8 @@
|
|||
{
|
||||
"name": "AzureSqlTable_SH_IR",
|
||||
"name": "AzureSqlTable_@GF{IR}",
|
||||
"properties": {
|
||||
"linkedServiceName": {
|
||||
"referenceName": "GenericAzureSQL_SH_IR",
|
||||
"referenceName": "GenericAzureSQL_@GF{IR}",
|
||||
"type": "LinkedServiceReference",
|
||||
"parameters": {
|
||||
"Server": {
|
||||
|
|
@ -30,7 +30,7 @@
|
|||
}
|
||||
},
|
||||
"folder": {
|
||||
"name": "ADS Go Fast/Generic/Azure SH IR"
|
||||
"name": "ADS Go Fast/Generic/@GF{IR}"
|
||||
},
|
||||
"annotations": [],
|
||||
"type": "AzureSqlTable",
|
||||
|
|
@ -1,8 +1,8 @@
|
|||
{
|
||||
"name": "BlobBinary_SH_IR",
|
||||
"name": "BlobBinary_@GF{IR}",
|
||||
"properties": {
|
||||
"linkedServiceName": {
|
||||
"referenceName": "GenericBlobStorage_SH_IR",
|
||||
"referenceName": "GenericBlobStorage_@GF{IR}",
|
||||
"type": "LinkedServiceReference",
|
||||
"parameters": {
|
||||
"StorageAccountEndpoint": {
|
||||
|
|
@ -26,7 +26,7 @@
|
|||
}
|
||||
},
|
||||
"folder": {
|
||||
"name": "ADS Go Fast/Generic/Azure SH IR"
|
||||
"name": "ADS Go Fast/Generic/@GF{IR}"
|
||||
},
|
||||
"annotations": [],
|
||||
"type": "Binary",
|
||||
|
|
@ -0,0 +1,63 @@
|
|||
{
|
||||
"name": "BlobDelimitedText_@GF{IR}",
|
||||
"properties": {
|
||||
"linkedServiceName": {
|
||||
"referenceName": "GenericBlobStorage_@GF{IR}",
|
||||
"type": "LinkedServiceReference",
|
||||
"parameters": {
|
||||
"StorageAccountEndpoint": {
|
||||
"value": "@dataset().StorageAccountEndpoint",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
},
|
||||
"parameters": {
|
||||
"RelativePath": {
|
||||
"type": "string"
|
||||
},
|
||||
"FileName": {
|
||||
"type": "string"
|
||||
},
|
||||
"StorageAccountEndpoint": {
|
||||
"type": "string"
|
||||
},
|
||||
"StorageAccountContainerName": {
|
||||
"type": "string"
|
||||
},
|
||||
"FirstRowAsHeader": {
|
||||
"type": "bool"
|
||||
}
|
||||
},
|
||||
"folder": {
|
||||
"name": "ADS Go Fast/Generic/@GF{IR}"
|
||||
},
|
||||
"annotations": [],
|
||||
"type": "DelimitedText",
|
||||
"typeProperties": {
|
||||
"location": {
|
||||
"type": "AzureBlobStorageLocation",
|
||||
"fileName": {
|
||||
"value": "@dataset().FileName",
|
||||
"type": "Expression"
|
||||
},
|
||||
"folderPath": {
|
||||
"value": "@dataset().RelativePath",
|
||||
"type": "Expression"
|
||||
},
|
||||
"container": {
|
||||
"value": "@dataset().StorageAccountContainerName",
|
||||
"type": "Expression"
|
||||
}
|
||||
},
|
||||
"columnDelimiter": ",",
|
||||
"escapeChar": "\\",
|
||||
"firstRowAsHeader": {
|
||||
"value": "@dataset().FirstRowAsHeader",
|
||||
"type": "Expression"
|
||||
},
|
||||
"quoteChar": "\""
|
||||
},
|
||||
"schema": []
|
||||
},
|
||||
"type": "Microsoft.DataFactory/factories/datasets"
|
||||
}
|
||||
|
|
@ -0,0 +1,67 @@
|
|||
{
|
||||
"name": "BlobExcel_@GF{IR}",
|
||||
"properties": {
|
||||
"linkedServiceName": {
|
||||
"referenceName": "GenericBlobStorage_@GF{IR}",
|
||||
"type": "LinkedServiceReference",
|
||||
"parameters": {
|
||||
"StorageAccountEndpoint": {
|
||||
"value": "@dataset().StorageAccountEndpoint",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
},
|
||||
"parameters": {
|
||||
"RelativePath": {
|
||||
"type": "string"
|
||||
},
|
||||
"FileName": {
|
||||
"type": "string"
|
||||
},
|
||||
"StorageAccountEndpoint": {
|
||||
"type": "string"
|
||||
},
|
||||
"StorageAccountContainerName": {
|
||||
"type": "string"
|
||||
},
|
||||
"SheetName": {
|
||||
"type": "string"
|
||||
},
|
||||
"FirstRowAsHeader": {
|
||||
"type": "bool"
|
||||
}
|
||||
},
|
||||
"folder": {
|
||||
"name": "ADS Go Fast/Generic/@GF{IR}"
|
||||
},
|
||||
"annotations": [],
|
||||
"type": "Excel",
|
||||
"typeProperties": {
|
||||
"sheetName": {
|
||||
"value": "@dataset().SheetName",
|
||||
"type": "Expression"
|
||||
},
|
||||
"location": {
|
||||
"type": "AzureBlobStorageLocation",
|
||||
"fileName": {
|
||||
"value": "@dataset().FileName",
|
||||
"type": "Expression"
|
||||
},
|
||||
"folderPath": {
|
||||
"value": "@dataset().RelativePath",
|
||||
"type": "Expression"
|
||||
},
|
||||
"container": {
|
||||
"value": "@dataset().StorageAccountContainerName",
|
||||
"type": "Expression"
|
||||
}
|
||||
},
|
||||
"firstRowAsHeader": {
|
||||
"value": "@dataset().FirstRowAsHeader",
|
||||
"type": "Expression"
|
||||
}
|
||||
},
|
||||
"schema": []
|
||||
},
|
||||
"type": "Microsoft.DataFactory/factories/datasets"
|
||||
}
|
||||
|
|
@ -0,0 +1,53 @@
|
|||
{
|
||||
"name": "BlobJson_@GF{IR}",
|
||||
"properties": {
|
||||
"linkedServiceName": {
|
||||
"referenceName": "GenericBlobStorage_@GF{IR}",
|
||||
"type": "LinkedServiceReference",
|
||||
"parameters": {
|
||||
"StorageAccountEndpoint": {
|
||||
"value": "@dataset().StorageAccountEndpoint",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
},
|
||||
"parameters": {
|
||||
"RelativePath": {
|
||||
"type": "string"
|
||||
},
|
||||
"FileName": {
|
||||
"type": "string"
|
||||
},
|
||||
"StorageAccountEndpoint": {
|
||||
"type": "string"
|
||||
},
|
||||
"StorageAccountContainerName": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"folder": {
|
||||
"name": "ADS Go Fast/Generic/@GF{IR}"
|
||||
},
|
||||
"annotations": [],
|
||||
"type": "Json",
|
||||
"typeProperties": {
|
||||
"location": {
|
||||
"type": "AzureBlobStorageLocation",
|
||||
"fileName": {
|
||||
"value": "@dataset().FileName",
|
||||
"type": "Expression"
|
||||
},
|
||||
"folderPath": {
|
||||
"value": "@dataset().RelativePath",
|
||||
"type": "Expression"
|
||||
},
|
||||
"container": {
|
||||
"value": "@dataset().StorageAccountContainerName",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
},
|
||||
"schema": {}
|
||||
},
|
||||
"type": "Microsoft.DataFactory/factories/datasets"
|
||||
}
|
||||
|
|
@ -1,8 +1,8 @@
|
|||
{
|
||||
"name": "BlobParquet_SH_IR",
|
||||
"name": "BlobParquet_@GF{IR}",
|
||||
"properties": {
|
||||
"linkedServiceName": {
|
||||
"referenceName": "GenericBlobStorage_SH_IR",
|
||||
"referenceName": "GenericBlobStorage_@GF{IR}",
|
||||
"type": "LinkedServiceReference",
|
||||
"parameters": {
|
||||
"StorageAccountEndpoint": {
|
||||
|
|
@ -26,7 +26,7 @@
|
|||
}
|
||||
},
|
||||
"folder": {
|
||||
"name": "ADS Go Fast/Generic/Azure SH IR"
|
||||
"name": "ADS Go Fast/Generic/@GF{IR}"
|
||||
},
|
||||
"annotations": [],
|
||||
"type": "Parquet",
|
||||
|
|
@ -1,8 +1,8 @@
|
|||
{
|
||||
"name": "FileBinary_OnPrem_SH_IR",
|
||||
"name": "FileBinary_@GF{IR}",
|
||||
"properties": {
|
||||
"linkedServiceName": {
|
||||
"referenceName": "GenericFileServer_OnPrem_Net",
|
||||
"referenceName": "GenericFileServer_@GF{IR}",
|
||||
"type": "LinkedServiceReference",
|
||||
"parameters": {
|
||||
"Host": {
|
||||
|
|
@ -44,7 +44,7 @@
|
|||
}
|
||||
},
|
||||
"folder": {
|
||||
"name": "ADS Go Fast/Generic/OnPrem SH IR"
|
||||
"name": "ADS Go Fast/Generic/@GF{IR}"
|
||||
},
|
||||
"annotations": [],
|
||||
"type": "Binary",
|
||||
|
|
@ -1,8 +1,8 @@
|
|||
{
|
||||
"name": "FileParquet_OnPrem_SH_IR",
|
||||
"name": "FileParquet_@GF{IR}",
|
||||
"properties": {
|
||||
"linkedServiceName": {
|
||||
"referenceName": "GenericFileServer_OnPrem_Net",
|
||||
"referenceName": "GenericFileServer_@GF{IR}",
|
||||
"type": "LinkedServiceReference",
|
||||
"parameters": {
|
||||
"Host": {
|
||||
|
|
@ -44,7 +44,7 @@
|
|||
}
|
||||
},
|
||||
"folder": {
|
||||
"name": "ADS Go Fast/Generic/OnPrem SH IR"
|
||||
"name": "ADS Go Fast/Generic/@GF{IR}"
|
||||
},
|
||||
"annotations": [],
|
||||
"type": "Parquet",
|
||||
|
|
@ -1,8 +1,8 @@
|
|||
{
|
||||
"name": "SqlServerTable_OnPrem_SH_IR",
|
||||
"name": "SqlServerTable_@GF{IR}",
|
||||
"properties": {
|
||||
"linkedServiceName": {
|
||||
"referenceName": "GenericMsSqlServer_OnPrem_Net",
|
||||
"referenceName": "GenericMsSqlServer_@GF{IR}",
|
||||
"type": "LinkedServiceReference",
|
||||
"parameters": {
|
||||
"KeyVaultBaseUrl": {
|
||||
|
|
@ -51,7 +51,7 @@
|
|||
}
|
||||
},
|
||||
"folder": {
|
||||
"name": "ADS Go Fast/Generic/OnPrem SH IR"
|
||||
"name": "ADS Go Fast/Generic/@GF{IR}"
|
||||
},
|
||||
"annotations": [],
|
||||
"type": "SqlServerTable",
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
{
|
||||
"name": "adsgofastdatakakeacceladf",
|
||||
"location": "australiaeast"
|
||||
}
|
||||
|
|
@ -0,0 +1,22 @@
|
|||
{
|
||||
"name": "IRA",
|
||||
"properties": {
|
||||
"type": "Managed",
|
||||
"description": "Primary integration runtime that sits in the Azure cloud. Used to pull data into the \"Analytics Lockbox\", move data around the \"Analytics Lockbox\" and push data out of the \"Analytics Lockbox\".",
|
||||
"typeProperties": {
|
||||
"computeProperties": {
|
||||
"location": "AutoResolve",
|
||||
"dataFlowProperties": {
|
||||
"computeType": "General",
|
||||
"coreCount": 8,
|
||||
"timeToLive": 10,
|
||||
"cleanup": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"managedVirtualNetwork": {
|
||||
"type": "ManagedVirtualNetworkReference",
|
||||
"referenceName": "default"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,22 @@
|
|||
{
|
||||
"name": "IRB",
|
||||
"properties": {
|
||||
"type": "Managed",
|
||||
"description": "Mock On-Premise Network IR For Testing.",
|
||||
"typeProperties": {
|
||||
"computeProperties": {
|
||||
"location": "AutoResolve",
|
||||
"dataFlowProperties": {
|
||||
"computeType": "General",
|
||||
"coreCount": 8,
|
||||
"timeToLive": 10,
|
||||
"cleanup": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"managedVirtualNetwork": {
|
||||
"type": "ManagedVirtualNetworkReference",
|
||||
"referenceName": "default"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
{
|
||||
"name": "GenericAzureDataLakeStorage_OnPrem_Net",
|
||||
"name": "GenericAzureDataLakeStorage_IRA",
|
||||
"type": "Microsoft.DataFactory/factories/linkedservices",
|
||||
"properties": {
|
||||
"type": "AzureBlobFS",
|
||||
|
|
@ -14,7 +14,7 @@
|
|||
},
|
||||
"annotations": [],
|
||||
"connectVia": {
|
||||
"referenceName": "SelfHostedIntegrationRuntime-OnPem-Net",
|
||||
"referenceName": "@GF{IR}",
|
||||
"type": "IntegrationRuntimeReference"
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,21 @@
|
|||
{
|
||||
"name": "GenericAzureKeyVault",
|
||||
"type": "Microsoft.DataFactory/factories/linkedservices",
|
||||
"properties": {
|
||||
"type": "AzureKeyVault",
|
||||
"parameters": {
|
||||
"KeyVaultBaseUrl": {
|
||||
"type": "String",
|
||||
"defaultValue": "https://adsgofastkeyvault.vault.azure.net/"
|
||||
}
|
||||
},
|
||||
"typeProperties": {
|
||||
"baseUrl": "@{linkedService().KeyVaultBaseUrl}"
|
||||
},
|
||||
"connectVia": {
|
||||
"referenceName": "@GF{IR}",
|
||||
"type": "IntegrationRuntimeReference"
|
||||
},
|
||||
"annotations": []
|
||||
}
|
||||
}
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
{
|
||||
"name": "GenericAzureSQL",
|
||||
"name": "GenericAzureSQL_IRA",
|
||||
"type": "Microsoft.DataFactory/factories/linkedservices",
|
||||
"properties": {
|
||||
"parameters": {
|
||||
|
|
@ -14,6 +14,10 @@
|
|||
"type": "AzureSqlDatabase",
|
||||
"typeProperties": {
|
||||
"connectionString": "Integrated Security=False;Encrypt=True;Connection Timeout=30;Data Source=@{linkedService().Server};Initial Catalog=@{linkedService().Database}"
|
||||
},
|
||||
"connectVia": {
|
||||
"referenceName": "@GF{IR}",
|
||||
"type": "IntegrationRuntimeReference"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
{
|
||||
"name": "GenericBlobStorage_OnPrem_Net",
|
||||
"name": "GenericBlobStorage_IRA",
|
||||
"type": "Microsoft.DataFactory/factories/linkedservices",
|
||||
"properties": {
|
||||
"type": "AzureBlobStorage",
|
||||
|
|
@ -14,7 +14,7 @@
|
|||
},
|
||||
"annotations": [],
|
||||
"connectVia": {
|
||||
"referenceName": "SelfHostedIntegrationRuntime-OnPem-Net",
|
||||
"referenceName": "@GF{IR}",
|
||||
"type": "IntegrationRuntimeReference"
|
||||
}
|
||||
}
|
||||
|
|
@ -1,16 +1,16 @@
|
|||
{
|
||||
"name": "GenericFileServer_OnPrem_Net",
|
||||
"name": "GenericFileServer_IRA",
|
||||
"type": "Microsoft.DataFactory/factories/linkedservices",
|
||||
"properties": {
|
||||
"type": "FileServer",
|
||||
"parameters": {
|
||||
"Host": {
|
||||
"type": "String",
|
||||
"defaultValue": "d:\\dataingestion"
|
||||
"defaultValue": ""
|
||||
},
|
||||
"UserId": {
|
||||
"type": "String",
|
||||
"defaultValue": "AzureUser"
|
||||
"defaultValue": ""
|
||||
},
|
||||
"Secret": {
|
||||
"type": "String",
|
||||
|
|
@ -44,7 +44,7 @@
|
|||
},
|
||||
"annotations": [],
|
||||
"connectVia": {
|
||||
"referenceName": "SelfHostedIntegrationRuntime-OnPem-Net",
|
||||
"referenceName": "@GF{IR}",
|
||||
"type": "IntegrationRuntimeReference"
|
||||
}
|
||||
}
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
{
|
||||
"name": "GenericMsSqlServer_OnPrem_Net",
|
||||
"name": "GenericMsSqlServer_IRA",
|
||||
"type": "Microsoft.DataFactory/factories/linkedservices",
|
||||
"properties": {
|
||||
"parameters": {
|
||||
|
|
@ -42,7 +42,7 @@
|
|||
}
|
||||
},
|
||||
"connectVia": {
|
||||
"referenceName": "SelfHostedIntegrationRuntime-OnPem-Net",
|
||||
"referenceName": "@GF{IR}",
|
||||
"type": "IntegrationRuntimeReference"
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1 @@
|
|||
Generic{DataSourceType}_{IntegrationRuntimename}.json
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
{
|
||||
"name": "AZ-Function-Generic",
|
||||
"name": "AZ_Function_Generic",
|
||||
"properties": {
|
||||
"activities": [
|
||||
{
|
||||
|
|
@ -0,0 +1,526 @@
|
|||
{
|
||||
"name": "AZ_SQL_AZ_Storage_Parquet_@GF{IR}",
|
||||
"properties": {
|
||||
"activities": [
|
||||
{
|
||||
"name": "AF Log - Get Metadata Failed",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "Lookup Get SQL Metadata",
|
||||
"dependencyConditions": [
|
||||
"Failed"
|
||||
]
|
||||
}
|
||||
],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
"parameters": {
|
||||
"Body": {
|
||||
"value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":1,\"LogSource\":\"ADF\",\"ActivityType\":\"Get Metadata\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"EndDateTimeOffSet\":\"', string(utcnow()), '\",\"Comment\":\"', string(activity('Lookup Get SQL Metadata').error.message), '\",\"Status\":\"Failed\"}'))",
|
||||
"type": "Expression"
|
||||
},
|
||||
"FunctionName": "Log",
|
||||
"Method": "Post"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Lookup Get SQL Metadata",
|
||||
"type": "Lookup",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "AF Get Information Schema SQL",
|
||||
"dependencyConditions": [
|
||||
"Succeeded"
|
||||
]
|
||||
}
|
||||
],
|
||||
"policy": {
|
||||
"timeout": "7.00:00:00",
|
||||
"retry": 0,
|
||||
"retryIntervalInSeconds": 30,
|
||||
"secureOutput": false,
|
||||
"secureInput": false
|
||||
},
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"source": {
|
||||
"type": "AzureSqlSource",
|
||||
"sqlReaderQuery": {
|
||||
"value": "@activity('AF Get Information Schema SQL').output.InformationSchemaSQL",
|
||||
"type": "Expression"
|
||||
},
|
||||
"queryTimeout": "02:00:00",
|
||||
"partitionOption": "None"
|
||||
},
|
||||
"dataset": {
|
||||
"referenceName": "AzureSqlTable_@GF{IR}",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"Schema": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.Extraction.TableSchema",
|
||||
"type": "Expression"
|
||||
},
|
||||
"Table": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.Extraction.TableName",
|
||||
"type": "Expression"
|
||||
},
|
||||
"Server": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.Database.SystemName",
|
||||
"type": "Expression"
|
||||
},
|
||||
"Database": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.Database.Name",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
},
|
||||
"firstRowOnly": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "AF Get Information Schema SQL",
|
||||
"type": "AzureFunctionActivity",
|
||||
"dependsOn": [],
|
||||
"policy": {
|
||||
"timeout": "7.00:00:00",
|
||||
"retry": 0,
|
||||
"retryIntervalInSeconds": 30,
|
||||
"secureOutput": false,
|
||||
"secureInput": false
|
||||
},
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"functionName": "GetInformationSchemaSQL",
|
||||
"method": "POST",
|
||||
"body": {
|
||||
"value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"TableSchema\":\"', string(pipeline().parameters.TaskObject.Source.Extraction.TableSchema), '\",\"TableName\":\"', string(pipeline().parameters.TaskObject.Source.Extraction.TableName),'\"}'))",
|
||||
"type": "Expression"
|
||||
}
|
||||
},
|
||||
"linkedServiceName": {
|
||||
"referenceName": "AzureFunctionAdsGoFastDataLakeAccelFunApp",
|
||||
"type": "LinkedServiceReference"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "AF Persist Metadata and Get Mapping",
|
||||
"type": "AzureFunctionActivity",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "Lookup Get SQL Metadata",
|
||||
"dependencyConditions": [
|
||||
"Succeeded"
|
||||
]
|
||||
}
|
||||
],
|
||||
"policy": {
|
||||
"timeout": "7.00:00:00",
|
||||
"retry": 0,
|
||||
"retryIntervalInSeconds": 30,
|
||||
"secureOutput": false,
|
||||
"secureInput": false
|
||||
},
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"functionName": "TaskExecutionSchemaFile",
|
||||
"method": "POST",
|
||||
"body": {
|
||||
"value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"StorageAccountName\":\"', string(pipeline().parameters.TaskObject.Target.StorageAccountName), '\",\"StorageAccountContainer\":\"', string(pipeline().parameters.TaskObject.Target.StorageAccountContainer), '\",\"RelativePath\":\"', string(pipeline().parameters.TaskObject.Target.RelativePath), '\",\"SchemaFileName\":\"', string(pipeline().parameters.TaskObject.Target.SchemaFileName), '\",\"SourceType\":\"', string(pipeline().parameters.TaskObject.Source.Type), '\",\"TargetType\":\"', string(pipeline().parameters.TaskObject.Target.Type), '\",\"Data\":',string(activity('Lookup Get SQL Metadata').output),',\"MetadataType\":\"SQL\"}'))",
|
||||
"type": "Expression"
|
||||
}
|
||||
},
|
||||
"linkedServiceName": {
|
||||
"referenceName": "AzureFunctionAdsGoFastDataLakeAccelFunApp",
|
||||
"type": "LinkedServiceReference"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Switch Load Type",
|
||||
"type": "Switch",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "AF Persist Metadata and Get Mapping",
|
||||
"dependencyConditions": [
|
||||
"Succeeded"
|
||||
]
|
||||
}
|
||||
],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"on": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.Extraction.IncrementalType",
|
||||
"type": "Expression"
|
||||
},
|
||||
"cases": [
|
||||
{
|
||||
"value": "Full",
|
||||
"activities": [
|
||||
{
|
||||
"name": "Execute AZ_SQL_Full_Load",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ_SQL_Full_Load_@GF{IR}",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": true,
|
||||
"parameters": {
|
||||
"TaskObject": {
|
||||
"value": "@pipeline().parameters.TaskObject",
|
||||
"type": "Expression"
|
||||
},
|
||||
"Mapping": {
|
||||
"value": "@activity('AF Persist Metadata and Get Mapping').output.value",
|
||||
"type": "Expression"
|
||||
},
|
||||
"BatchCount": "1",
|
||||
"Item": "1"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"value": "Watermark",
|
||||
"activities": [
|
||||
{
|
||||
"name": "Execute AZ_SQL_Watermark",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "Lookup New Watermark",
|
||||
"dependencyConditions": [
|
||||
"Succeeded"
|
||||
]
|
||||
}
|
||||
],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ_SQL_Watermark_@GF{IR}",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": true,
|
||||
"parameters": {
|
||||
"TaskObject": {
|
||||
"value": "@pipeline().parameters.TaskObject",
|
||||
"type": "Expression"
|
||||
},
|
||||
"Mapping": {
|
||||
"value": "@activity('AF Persist Metadata and Get Mapping').output.value",
|
||||
"type": "Expression"
|
||||
},
|
||||
"NewWaterMark": {
|
||||
"value": "@activity('Lookup New Watermark').output.firstRow.newWatermark",
|
||||
"type": "Expression"
|
||||
},
|
||||
"Item": "1",
|
||||
"BatchCount": "1"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Lookup New Watermark",
|
||||
"type": "Lookup",
|
||||
"dependsOn": [],
|
||||
"policy": {
|
||||
"timeout": "0.00:30:00",
|
||||
"retry": 0,
|
||||
"retryIntervalInSeconds": 30,
|
||||
"secureOutput": false,
|
||||
"secureInput": false
|
||||
},
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"source": {
|
||||
"type": "AzureSqlSource",
|
||||
"sqlReaderQuery": {
|
||||
"value": "@{pipeline().parameters.TaskObject.Source.Extraction.IncrementalSQLStatement}",
|
||||
"type": "Expression"
|
||||
},
|
||||
"queryTimeout": "02:00:00",
|
||||
"partitionOption": "None"
|
||||
},
|
||||
"dataset": {
|
||||
"referenceName": "AzureSqlTable_@GF{IR}",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"Schema": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.Extraction.TableSchema",
|
||||
"type": "Expression"
|
||||
},
|
||||
"Table": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.Extraction.TableName",
|
||||
"type": "Expression"
|
||||
},
|
||||
"Server": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.Database.SystemName",
|
||||
"type": "Expression"
|
||||
},
|
||||
"Database": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.Database.Name",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
},
|
||||
"firstRowOnly": true
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"value": "Full_Chunk",
|
||||
"activities": [
|
||||
{
|
||||
"name": "Execute AZ_SQL_Full_Load_Chunk",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "Lookup Chunk",
|
||||
"dependencyConditions": [
|
||||
"Succeeded"
|
||||
]
|
||||
}
|
||||
],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ_SQL_Full_Load_Chunk_@GF{IR}",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": true,
|
||||
"parameters": {
|
||||
"TaskObject": {
|
||||
"value": "@pipeline().parameters.TaskObject",
|
||||
"type": "Expression"
|
||||
},
|
||||
"Mapping": {
|
||||
"value": "@activity('AF Persist Metadata and Get Mapping').output.value",
|
||||
"type": "Expression"
|
||||
},
|
||||
"BatchCount": {
|
||||
"value": "@activity('Lookup Chunk').output.firstRow.batchcount",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Lookup Chunk",
|
||||
"type": "Lookup",
|
||||
"dependsOn": [],
|
||||
"policy": {
|
||||
"timeout": "0.00:30:00",
|
||||
"retry": 0,
|
||||
"retryIntervalInSeconds": 30,
|
||||
"secureOutput": false,
|
||||
"secureInput": false
|
||||
},
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"source": {
|
||||
"type": "AzureSqlSource",
|
||||
"sqlReaderQuery": {
|
||||
"value": "@{pipeline().parameters.TaskObject.Source.Extraction.IncrementalSQLStatement}",
|
||||
"type": "Expression"
|
||||
},
|
||||
"queryTimeout": "02:00:00",
|
||||
"partitionOption": "None"
|
||||
},
|
||||
"dataset": {
|
||||
"referenceName": "AzureSqlTable_@GF{IR}",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"Schema": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.Extraction.TableSchema",
|
||||
"type": "Expression"
|
||||
},
|
||||
"Table": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.Extraction.TableName",
|
||||
"type": "Expression"
|
||||
},
|
||||
"Server": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.Database.SystemName",
|
||||
"type": "Expression"
|
||||
},
|
||||
"Database": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.Database.Name",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
},
|
||||
"firstRowOnly": true
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"value": "Watermark_Chunk",
|
||||
"activities": [
|
||||
{
|
||||
"name": "Execute AZ_SQL_Watermark_Chunk",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "Lookup New Watermark and Chunk",
|
||||
"dependencyConditions": [
|
||||
"Succeeded"
|
||||
]
|
||||
}
|
||||
],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ_SQL_Watermark_Chunk_@GF{IR}",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": true,
|
||||
"parameters": {
|
||||
"TaskObject": {
|
||||
"value": "@pipeline().parameters.TaskObject",
|
||||
"type": "Expression"
|
||||
},
|
||||
"Mapping": {
|
||||
"value": "@activity('AF Persist Metadata and Get Mapping').output.value",
|
||||
"type": "Expression"
|
||||
},
|
||||
"NewWatermark": {
|
||||
"value": "@activity('Lookup New Watermark and Chunk').output.firstRow.newWatermark",
|
||||
"type": "Expression"
|
||||
},
|
||||
"BatchCount": {
|
||||
"value": "@activity('Lookup New Watermark and Chunk').output.firstRow.batchcount",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Lookup New Watermark and Chunk",
|
||||
"type": "Lookup",
|
||||
"dependsOn": [],
|
||||
"policy": {
|
||||
"timeout": "0.00:30:00",
|
||||
"retry": 0,
|
||||
"retryIntervalInSeconds": 30,
|
||||
"secureOutput": false,
|
||||
"secureInput": false
|
||||
},
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"source": {
|
||||
"type": "AzureSqlSource",
|
||||
"sqlReaderQuery": {
|
||||
"value": "@{pipeline().parameters.TaskObject.Source.Extraction.IncrementalSQLStatement}",
|
||||
"type": "Expression"
|
||||
},
|
||||
"queryTimeout": "02:00:00",
|
||||
"partitionOption": "None"
|
||||
},
|
||||
"dataset": {
|
||||
"referenceName": "AzureSqlTable_@GF{IR}",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"Schema": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.Extraction.TableSchema",
|
||||
"type": "Expression"
|
||||
},
|
||||
"Table": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.Extraction.TableName",
|
||||
"type": "Expression"
|
||||
},
|
||||
"Server": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.Database.SystemName",
|
||||
"type": "Expression"
|
||||
},
|
||||
"Database": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.Database.Name",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
},
|
||||
"firstRowOnly": true
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"TaskObject": {
|
||||
"type": "object",
|
||||
"defaultValue": {
|
||||
"TaskInstanceId": 75,
|
||||
"TaskMasterId": 12,
|
||||
"TaskStatus": "Untried",
|
||||
"TaskType": "SQL Database to Azure Storage",
|
||||
"Enabled": 1,
|
||||
"ExecutionUid": "2c5924ee-b855-4d2b-bb7e-4f5dde4c4dd3",
|
||||
"NumberOfRetries": 111,
|
||||
"DegreeOfCopyParallelism": 1,
|
||||
"KeyVaultBaseUrl": "https://adsgofastkeyvault.vault.azure.net/",
|
||||
"ScheduleMasterId": 2,
|
||||
"TaskGroupConcurrency": 10,
|
||||
"TaskGroupPriority": 0,
|
||||
"Source": {
|
||||
"Type": "Azure SQL",
|
||||
"Database": {
|
||||
"SystemName": "adsgofastdatakakeaccelsqlsvr.database.windows.net",
|
||||
"Name": "AWSample",
|
||||
"AuthenticationType": "MSI"
|
||||
},
|
||||
"Extraction": {
|
||||
"Type": "Table",
|
||||
"FullOrIncremental": "Full",
|
||||
"IncrementalType": null,
|
||||
"TableSchema": "SalesLT",
|
||||
"TableName": "SalesOrderHeader"
|
||||
}
|
||||
},
|
||||
"Target": {
|
||||
"Type": "Azure Blob",
|
||||
"StorageAccountName": "https://adsgofastdatalakeaccelst.blob.core.windows.net",
|
||||
"StorageAccountContainer": "datalakeraw",
|
||||
"StorageAccountAccessMethod": "MSI",
|
||||
"RelativePath": "/AwSample/SalesLT/SalesOrderHeader/2020/7/9/14/12/",
|
||||
"DataFileName": "SalesLT.SalesOrderHeader.parquet",
|
||||
"SchemaFileName": "SalesLT.SalesOrderHeader",
|
||||
"FirstRowAsHeader": null,
|
||||
"SheetName": null,
|
||||
"SkipLineCount": null,
|
||||
"MaxConcorrentConnections": null
|
||||
},
|
||||
"DataFactory": {
|
||||
"Id": 1,
|
||||
"Name": "adsgofastdatakakeacceladf",
|
||||
"ResourceGroup": "AdsGoFastDataLakeAccel",
|
||||
"SubscriptionId": "035a1364-f00d-48e2-b582-4fe125905ee3",
|
||||
"ADFPipeline": "AZ_SQL_AZ_Storage_Parquet_@GF{IR}"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"variables": {
|
||||
"SQLStatement": {
|
||||
"type": "String"
|
||||
}
|
||||
},
|
||||
"folder": {
|
||||
"name": "ADS Go Fast/Data Movement/@GF{IR}"
|
||||
},
|
||||
"annotations": [],
|
||||
"lastPublishTime": "2020-08-04T12:40:45Z"
|
||||
},
|
||||
"type": "Microsoft.DataFactory/factories/pipelines"
|
||||
}
|
||||
|
|
@ -0,0 +1,260 @@
|
|||
{
|
||||
"name": "AZ_SQL_Create_Table_@GF{IR}",
|
||||
"properties": {
|
||||
"activities": [
|
||||
{
|
||||
"name": "If exist Target TableName",
|
||||
"type": "IfCondition",
|
||||
"dependsOn": [],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"expression": {
|
||||
"value": "@not(empty(pipeline().parameters.TaskObject.Target.TableName))",
|
||||
"type": "Expression"
|
||||
},
|
||||
"ifTrueActivities": [
|
||||
{
|
||||
"name": "AF Get SQL Create Statement Staging",
|
||||
"type": "AzureFunctionActivity",
|
||||
"dependsOn": [],
|
||||
"policy": {
|
||||
"timeout": "7.00:00:00",
|
||||
"retry": 0,
|
||||
"retryIntervalInSeconds": 30,
|
||||
"secureOutput": false,
|
||||
"secureInput": false
|
||||
},
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"functionName": "GetSQLCreateStatementFromSchema",
|
||||
"method": "POST",
|
||||
"body": {
|
||||
"value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"TableSchema\":\"',string(pipeline().parameters.TaskObject.Target.TableSchema), '\",\"TableName\":\"', string(pipeline().parameters.TaskObject.Target.TableName),'\",\"StorageAccountName\":\"', string(pipeline().parameters.TaskObject.Source.StorageAccountName), '\",\"StorageAccountContainer\":\"', string(pipeline().parameters.TaskObject.Source.StorageAccountContainer), '\",\"RelativePath\":\"', string(pipeline().parameters.TaskObject.Source.RelativePath), '\",\"SchemaFileName\":\"', string(pipeline().parameters.TaskObject.Source.SchemaFileName), '\"}'))",
|
||||
"type": "Expression"
|
||||
}
|
||||
},
|
||||
"linkedServiceName": {
|
||||
"referenceName": "AzureFunctionAdsGoFastDataLakeAccelFunApp",
|
||||
"type": "LinkedServiceReference"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Lookup Create Staging Table",
|
||||
"type": "Lookup",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "AF Get SQL Create Statement Staging",
|
||||
"dependencyConditions": [
|
||||
"Succeeded"
|
||||
]
|
||||
}
|
||||
],
|
||||
"policy": {
|
||||
"timeout": "7.00:00:00",
|
||||
"retry": 0,
|
||||
"retryIntervalInSeconds": 30,
|
||||
"secureOutput": false,
|
||||
"secureInput": false
|
||||
},
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"source": {
|
||||
"type": "AzureSqlSource",
|
||||
"sqlReaderQuery": {
|
||||
"value": "@activity('AF Get SQL Create Statement Staging').output.CreateStatement",
|
||||
"type": "Expression"
|
||||
},
|
||||
"queryTimeout": "02:00:00",
|
||||
"partitionOption": "None"
|
||||
},
|
||||
"dataset": {
|
||||
"referenceName": "AzureSqlTable_@GF{IR}",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"Schema": {
|
||||
"value": "@pipeline().parameters.TaskObject.Target.StagingTableSchema",
|
||||
"type": "Expression"
|
||||
},
|
||||
"Table": {
|
||||
"value": "@pipeline().parameters.TaskObject.Target.StagingTableName",
|
||||
"type": "Expression"
|
||||
},
|
||||
"Server": {
|
||||
"value": "@pipeline().parameters.TaskObject.Target.Database.SystemName",
|
||||
"type": "Expression"
|
||||
},
|
||||
"Database": {
|
||||
"value": "@pipeline().parameters.TaskObject.Target.Database.Name",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
},
|
||||
"firstRowOnly": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "AF Log - Create Staging Table Failed",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "Lookup Create Staging Table",
|
||||
"dependencyConditions": [
|
||||
"Failed"
|
||||
]
|
||||
}
|
||||
],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
"parameters": {
|
||||
"Body": {
|
||||
"value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":1,\"LogSource\":\"ADF\",\"ActivityType\":\"Create Staging Table\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"EndDateTimeOffSet\":\"', string(utcnow()), '\",\"Comment\":\"', string(activity('Lookup Create Staging Table').error.message), '\",\"Status\":\"Failed\"}'))",
|
||||
"type": "Expression"
|
||||
},
|
||||
"FunctionName": "Log",
|
||||
"Method": "Post"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "If exist Staging TableName",
|
||||
"type": "IfCondition",
|
||||
"dependsOn": [],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"expression": {
|
||||
"value": "@not(empty(pipeline().parameters.TaskObject.Target.StagingTableName))",
|
||||
"type": "Expression"
|
||||
},
|
||||
"ifTrueActivities": [
|
||||
{
|
||||
"name": "AF Get SQL Create Statement Target",
|
||||
"type": "AzureFunctionActivity",
|
||||
"dependsOn": [],
|
||||
"policy": {
|
||||
"timeout": "7.00:00:00",
|
||||
"retry": 0,
|
||||
"retryIntervalInSeconds": 30,
|
||||
"secureOutput": false,
|
||||
"secureInput": false
|
||||
},
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"functionName": "GetSQLCreateStatementFromSchema",
|
||||
"method": "POST",
|
||||
"body": {
|
||||
"value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"TableSchema\":\"',string(pipeline().parameters.TaskObject.Target.StagingTableSchema), '\",\"TableName\":\"', string(pipeline().parameters.TaskObject.Target.StagingTableName),'\",\"StorageAccountName\":\"', string(pipeline().parameters.TaskObject.Source.StorageAccountName), '\",\"StorageAccountContainer\":\"', string(pipeline().parameters.TaskObject.Source.StorageAccountContainer), '\",\"RelativePath\":\"', string(pipeline().parameters.TaskObject.Source.RelativePath), '\",\"SchemaFileName\":\"', string(pipeline().parameters.TaskObject.Source.SchemaFileName), '\",\"DropIfExist\":\"True\"}'))",
|
||||
"type": "Expression"
|
||||
}
|
||||
},
|
||||
"linkedServiceName": {
|
||||
"referenceName": "AzureFunctionAdsGoFastDataLakeAccelFunApp",
|
||||
"type": "LinkedServiceReference"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Lookup Create Target Table",
|
||||
"type": "Lookup",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "AF Get SQL Create Statement Target",
|
||||
"dependencyConditions": [
|
||||
"Succeeded"
|
||||
]
|
||||
}
|
||||
],
|
||||
"policy": {
|
||||
"timeout": "7.00:00:00",
|
||||
"retry": 0,
|
||||
"retryIntervalInSeconds": 30,
|
||||
"secureOutput": false,
|
||||
"secureInput": false
|
||||
},
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"source": {
|
||||
"type": "AzureSqlSource",
|
||||
"sqlReaderQuery": {
|
||||
"value": "@activity('AF Get SQL Create Statement Target').output.CreateStatement",
|
||||
"type": "Expression"
|
||||
},
|
||||
"queryTimeout": "02:00:00",
|
||||
"partitionOption": "None"
|
||||
},
|
||||
"dataset": {
|
||||
"referenceName": "AzureSqlTable_@GF{IR}",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"Schema": {
|
||||
"value": "@pipeline().parameters.TaskObject.Target.StagingTableSchema",
|
||||
"type": "Expression"
|
||||
},
|
||||
"Table": {
|
||||
"value": "@pipeline().parameters.TaskObject.Target.StagingTableName",
|
||||
"type": "Expression"
|
||||
},
|
||||
"Server": {
|
||||
"value": "@pipeline().parameters.TaskObject.Target.Database.SystemName",
|
||||
"type": "Expression"
|
||||
},
|
||||
"Database": {
|
||||
"value": "@pipeline().parameters.TaskObject.Target.Database.Name",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
},
|
||||
"firstRowOnly": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "AF Log - Create Target Table Failed",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "Lookup Create Target Table",
|
||||
"dependencyConditions": [
|
||||
"Failed"
|
||||
]
|
||||
}
|
||||
],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
"parameters": {
|
||||
"Body": {
|
||||
"value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":1,\"LogSource\":\"ADF\",\"ActivityType\":\"Create Target Table\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"EndDateTimeOffSet\":\"', string(utcnow()), '\",\"Comment\":\"', string(activity('Lookup Create Target Table').error.message), '\",\"Status\":\"Failed\"}'))",
|
||||
"type": "Expression"
|
||||
},
|
||||
"FunctionName": "Log",
|
||||
"Method": "Post"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"TaskObject": {
|
||||
"type": "object"
|
||||
}
|
||||
},
|
||||
"folder": {
|
||||
"name": "ADS Go Fast/Data Movement/@GF{IR}/Common"
|
||||
},
|
||||
"annotations": [],
|
||||
"lastPublishTime": "2020-08-04T13:09:30Z"
|
||||
},
|
||||
"type": "Microsoft.DataFactory/factories/pipelines"
|
||||
}
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
{
|
||||
"name": "OnP-SQL-Full-Load-Chunk-OnP-SH-IR",
|
||||
"name": "AZ_SQL_Full_Load_Chunk_@GF{IR}",
|
||||
"properties": {
|
||||
"activities": [
|
||||
{
|
||||
|
|
@ -15,13 +15,13 @@
|
|||
"isSequential": true,
|
||||
"activities": [
|
||||
{
|
||||
"name": "Execute OnP-SQL-Full-Load",
|
||||
"name": "Execute AZ_SQL_Full_Load",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "OnP-SQL-Full-Load-OnP-SH-IR",
|
||||
"referenceName": "AZ_SQL_Full_Load_@GF{IR}",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": true,
|
||||
|
|
@ -98,7 +98,7 @@
|
|||
"Name": "adsgofastdatakakeacceladf",
|
||||
"ResourceGroup": "AdsGoFastDataLakeAccel",
|
||||
"SubscriptionId": "035a1364-f00d-48e2-b582-4fe125905ee3",
|
||||
"ADFPipeline": "AZ-SQL-AZ-Storage-Parquet-SH-IR"
|
||||
"ADFPipeline": "AZ_SQL_AZ_Storage_Parquet_@GF{IR}"
|
||||
}
|
||||
}
|
||||
},
|
||||
|
|
@ -110,10 +110,10 @@
|
|||
}
|
||||
},
|
||||
"folder": {
|
||||
"name": "ADS Go Fast/Data Movement/OnPrem SH IR/Components"
|
||||
"name": "ADS Go Fast/Data Movement/@GF{IR}/Components"
|
||||
},
|
||||
"annotations": [],
|
||||
"lastPublishTime": "2020-08-04T07:52:08Z"
|
||||
"lastPublishTime": "2020-07-29T02:39:55Z"
|
||||
},
|
||||
"type": "Microsoft.DataFactory/factories/pipelines"
|
||||
}
|
||||
|
|
@ -0,0 +1,633 @@
|
|||
{
|
||||
"name": "AZ_SQL_Full_Load_@GF{IR}",
|
||||
"properties": {
|
||||
"activities": [
|
||||
{
|
||||
"name": "Switch Storage Type",
|
||||
"type": "Switch",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "Set SQLStatement",
|
||||
"dependencyConditions": [
|
||||
"Succeeded"
|
||||
]
|
||||
}
|
||||
],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"on": {
|
||||
"value": "@pipeline().parameters.TaskObject.Target.Type",
|
||||
"type": "Expression"
|
||||
},
|
||||
"cases": [
|
||||
{
|
||||
"value": "ADLS",
|
||||
"activities": [
|
||||
{
|
||||
"name": "Copy Azure SQL to ADLS",
|
||||
"type": "Copy",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "Pipeline AF Log - Azure SQL to ADLS Start",
|
||||
"dependencyConditions": [
|
||||
"Succeeded"
|
||||
]
|
||||
}
|
||||
],
|
||||
"policy": {
|
||||
"timeout": "7.00:00:00",
|
||||
"retry": 0,
|
||||
"retryIntervalInSeconds": 30,
|
||||
"secureOutput": false,
|
||||
"secureInput": false
|
||||
},
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"source": {
|
||||
"type": "AzureSqlSource",
|
||||
"sqlReaderQuery": {
|
||||
"value": "@variables('SQLStatement1')",
|
||||
"type": "Expression"
|
||||
},
|
||||
"queryTimeout": "02:00:00",
|
||||
"partitionOption": "None"
|
||||
},
|
||||
"sink": {
|
||||
"type": "ParquetSink",
|
||||
"storeSettings": {
|
||||
"type": "AzureBlobFSWriteSettings"
|
||||
}
|
||||
},
|
||||
"enableStaging": false,
|
||||
"parallelCopies": {
|
||||
"value": "@pipeline().parameters.TaskObject.DegreeOfCopyParallelism",
|
||||
"type": "Expression"
|
||||
},
|
||||
"translator": {
|
||||
"value": "@pipeline().parameters.Mapping",
|
||||
"type": "Expression"
|
||||
}
|
||||
},
|
||||
"inputs": [
|
||||
{
|
||||
"referenceName": "AzureSqlTable_@GF{IR}",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"Schema": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.Extraction.TableSchema",
|
||||
"type": "Expression"
|
||||
},
|
||||
"Table": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.Extraction.TableName",
|
||||
"type": "Expression"
|
||||
},
|
||||
"Server": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.Database.SystemName",
|
||||
"type": "Expression"
|
||||
},
|
||||
"Database": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.Database.Name",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"referenceName": "ADLSParquet_@GF{IR}",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"RelativePath": {
|
||||
"value": "@pipeline().parameters.TaskObject.Target.RelativePath",
|
||||
"type": "Expression"
|
||||
},
|
||||
"FileName": {
|
||||
"value": "@replace(pipeline().parameters.TaskObject.Target.DataFileName,'.parquet',concat('.chunk_', string(pipeline().parameters.Item),'.parquet'))",
|
||||
"type": "Expression"
|
||||
},
|
||||
"StorageAccountEndpoint": {
|
||||
"value": "@pipeline().parameters.TaskObject.Target.StorageAccountName",
|
||||
"type": "Expression"
|
||||
},
|
||||
"StorageAccountContainerName": {
|
||||
"value": "@pipeline().parameters.TaskObject.Target.StorageAccountContainer",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "Pipeline AF Log - Azure SQL to ADLS Failed",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "Copy Azure SQL to ADLS",
|
||||
"dependencyConditions": [
|
||||
"Failed"
|
||||
]
|
||||
}
|
||||
],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
"parameters": {
|
||||
"Body": {
|
||||
"value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":1,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy Azure SQL to ADLS\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"EndDateTimeOffSet\":\"', string(utcnow()), '\",\"Comment\":\"', string(activity('Copy Azure SQL to ADLS').error.message), '\",\"Status\":\"Failed\"}'))",
|
||||
"type": "Expression"
|
||||
},
|
||||
"FunctionName": "Log",
|
||||
"Method": "Post"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "AF Persist Parquet Metadata - ADLS",
|
||||
"type": "AzureFunctionActivity",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "Get Parquet Metadata ADLS",
|
||||
"dependencyConditions": [
|
||||
"Succeeded"
|
||||
]
|
||||
}
|
||||
],
|
||||
"policy": {
|
||||
"timeout": "7.00:00:00",
|
||||
"retry": 0,
|
||||
"retryIntervalInSeconds": 30,
|
||||
"secureOutput": false,
|
||||
"secureInput": false
|
||||
},
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"functionName": "TaskExecutionSchemaFile",
|
||||
"method": "POST",
|
||||
"body": {
|
||||
"value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"StorageAccountName\":\"', string(pipeline().parameters.TaskObject.Target.StorageAccountName), '\",\"StorageAccountContainer\":\"', string(pipeline().parameters.TaskObject.Target.StorageAccountContainer), '\",\"RelativePath\":\"', string(pipeline().parameters.TaskObject.Target.RelativePath), '\",\"SchemaFileName\":\"', string(pipeline().parameters.TaskObject.Target.SchemaFileName), '\",\"SourceType\":\"', string(pipeline().parameters.TaskObject.Source.Type), '\",\"TargetType\":\"', string(pipeline().parameters.TaskObject.Target.Type), '\",\"Data\":',string(activity('Get Parquet Metadata ADLS').output),',\"MetadataType\":\"Parquet\"}'))",
|
||||
"type": "Expression"
|
||||
}
|
||||
},
|
||||
"linkedServiceName": {
|
||||
"referenceName": "AzureFunctionAdsGoFastDataLakeAccelFunApp",
|
||||
"type": "LinkedServiceReference"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Get Parquet Metadata ADLS",
|
||||
"type": "GetMetadata",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "Copy Azure SQL to ADLS",
|
||||
"dependencyConditions": [
|
||||
"Succeeded"
|
||||
]
|
||||
}
|
||||
],
|
||||
"policy": {
|
||||
"timeout": "7.00:00:00",
|
||||
"retry": 0,
|
||||
"retryIntervalInSeconds": 30,
|
||||
"secureOutput": false,
|
||||
"secureInput": false
|
||||
},
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"dataset": {
|
||||
"referenceName": "ADLSParquet_@GF{IR}",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"RelativePath": {
|
||||
"value": "@pipeline().parameters.TaskObject.Target.RelativePath",
|
||||
"type": "Expression"
|
||||
},
|
||||
"FileName": {
|
||||
"value": "@replace(pipeline().parameters.TaskObject.Target.DataFileName,'.parquet',concat('.chunk_', string(pipeline().parameters.Item),'.parquet'))",
|
||||
"type": "Expression"
|
||||
},
|
||||
"StorageAccountEndpoint": {
|
||||
"value": "@pipeline().parameters.TaskObject.Target.StorageAccountName",
|
||||
"type": "Expression"
|
||||
},
|
||||
"StorageAccountContainerName": {
|
||||
"value": "@pipeline().parameters.TaskObject.Target.StorageAccountContainer",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
},
|
||||
"fieldList": [
|
||||
"structure"
|
||||
],
|
||||
"storeSettings": {
|
||||
"type": "AzureBlobFSReadSettings",
|
||||
"recursive": true
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Pipeline AF Log - Azure SQL to ADLS Start",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
"parameters": {
|
||||
"Body": {
|
||||
"value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":3,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy Azure SQL to ADLS\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"Status\":\"Started\"}'))",
|
||||
"type": "Expression"
|
||||
},
|
||||
"FunctionName": "Log",
|
||||
"Method": "Post"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Pipeline AF Log - Azure SQL ADLS to Succeed",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "Copy Azure SQL to ADLS",
|
||||
"dependencyConditions": [
|
||||
"Succeeded"
|
||||
]
|
||||
}
|
||||
],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
"parameters": {
|
||||
"Body": {
|
||||
"value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":1,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy Azure SQL to ADLS\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"EndDateTimeOffSet\":\"', string(utcnow()), '\",\"RowsInserted\":\"', string(activity('Copy Azure SQL to ADLS').output.rowsCopied), '\",\"Comment\":\"\",\"Status\":\"Complete\"}'))",
|
||||
"type": "Expression"
|
||||
},
|
||||
"FunctionName": "Log",
|
||||
"Method": "Post"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"value": "Azure Blob",
|
||||
"activities": [
|
||||
{
|
||||
"name": "Copy Azure SQL to Blob",
|
||||
"type": "Copy",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "Pipeline AF Log - Azure SQL to Blob Start",
|
||||
"dependencyConditions": [
|
||||
"Succeeded"
|
||||
]
|
||||
}
|
||||
],
|
||||
"policy": {
|
||||
"timeout": "7.00:00:00",
|
||||
"retry": 0,
|
||||
"retryIntervalInSeconds": 30,
|
||||
"secureOutput": false,
|
||||
"secureInput": false
|
||||
},
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"source": {
|
||||
"type": "AzureSqlSource",
|
||||
"sqlReaderQuery": {
|
||||
"value": "@variables('SQLStatement1')",
|
||||
"type": "Expression"
|
||||
},
|
||||
"queryTimeout": "02:00:00",
|
||||
"partitionOption": "None"
|
||||
},
|
||||
"sink": {
|
||||
"type": "ParquetSink",
|
||||
"storeSettings": {
|
||||
"type": "AzureBlobStorageWriteSettings"
|
||||
}
|
||||
},
|
||||
"enableStaging": false,
|
||||
"parallelCopies": {
|
||||
"value": "@pipeline().parameters.TaskObject.DegreeOfCopyParallelism",
|
||||
"type": "Expression"
|
||||
},
|
||||
"translator": {
|
||||
"value": "@pipeline().parameters.Mapping",
|
||||
"type": "Expression"
|
||||
}
|
||||
},
|
||||
"inputs": [
|
||||
{
|
||||
"referenceName": "AzureSqlTable_@GF{IR}",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"Schema": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.Extraction.TableSchema",
|
||||
"type": "Expression"
|
||||
},
|
||||
"Table": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.Extraction.TableName",
|
||||
"type": "Expression"
|
||||
},
|
||||
"Server": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.Database.SystemName",
|
||||
"type": "Expression"
|
||||
},
|
||||
"Database": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.Database.Name",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"referenceName": "BlobParquet_@GF{IR}",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"RelativePath": {
|
||||
"value": "@pipeline().parameters.TaskObject.Target.RelativePath",
|
||||
"type": "Expression"
|
||||
},
|
||||
"FileName": {
|
||||
"value": "@replace(pipeline().parameters.TaskObject.Target.DataFileName,'.parquet',concat('.chunk_', string(pipeline().parameters.Item),'.parquet'))",
|
||||
"type": "Expression"
|
||||
},
|
||||
"StorageAccountEndpoint": {
|
||||
"value": "@pipeline().parameters.TaskObject.Target.StorageAccountName",
|
||||
"type": "Expression"
|
||||
},
|
||||
"StorageAccountContainerName": {
|
||||
"value": "@pipeline().parameters.TaskObject.Target.StorageAccountContainer",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "Pipeline AF Log - Azure SQL to Blob Failed",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "Copy Azure SQL to Blob",
|
||||
"dependencyConditions": [
|
||||
"Failed"
|
||||
]
|
||||
}
|
||||
],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
"parameters": {
|
||||
"Body": {
|
||||
"value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":1,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy Azure SQL to Blob\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"EndDateTimeOffSet\":\"', string(utcnow()), '\",\"Comment\":\"', string(activity('Copy Azure SQL to Blob').error.message), '\",\"Status\":\"Failed\"}'))",
|
||||
"type": "Expression"
|
||||
},
|
||||
"FunctionName": "Log",
|
||||
"Method": "Post"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "AF Persist Parquet Metadata - Blob",
|
||||
"type": "AzureFunctionActivity",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "Get Parquet Metadata Blob",
|
||||
"dependencyConditions": [
|
||||
"Succeeded"
|
||||
]
|
||||
}
|
||||
],
|
||||
"policy": {
|
||||
"timeout": "7.00:00:00",
|
||||
"retry": 0,
|
||||
"retryIntervalInSeconds": 30,
|
||||
"secureOutput": false,
|
||||
"secureInput": false
|
||||
},
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"functionName": "TaskExecutionSchemaFile",
|
||||
"method": "POST",
|
||||
"body": {
|
||||
"value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"StorageAccountName\":\"', string(pipeline().parameters.TaskObject.Target.StorageAccountName), '\",\"StorageAccountContainer\":\"', string(pipeline().parameters.TaskObject.Target.StorageAccountContainer), '\",\"RelativePath\":\"', string(pipeline().parameters.TaskObject.Target.RelativePath), '\",\"SchemaFileName\":\"', string(pipeline().parameters.TaskObject.Target.SchemaFileName), '\",\"SourceType\":\"', string(pipeline().parameters.TaskObject.Source.Type), '\",\"TargetType\":\"', string(pipeline().parameters.TaskObject.Target.Type), '\",\"Data\":',string(activity('Get Parquet Metadata Blob').output),',\"MetadataType\":\"Parquet\"}'))",
|
||||
"type": "Expression"
|
||||
}
|
||||
},
|
||||
"linkedServiceName": {
|
||||
"referenceName": "AzureFunctionAdsGoFastDataLakeAccelFunApp",
|
||||
"type": "LinkedServiceReference"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Get Parquet Metadata Blob",
|
||||
"type": "GetMetadata",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "Copy Azure SQL to Blob",
|
||||
"dependencyConditions": [
|
||||
"Succeeded"
|
||||
]
|
||||
}
|
||||
],
|
||||
"policy": {
|
||||
"timeout": "7.00:00:00",
|
||||
"retry": 0,
|
||||
"retryIntervalInSeconds": 30,
|
||||
"secureOutput": false,
|
||||
"secureInput": false
|
||||
},
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"dataset": {
|
||||
"referenceName": "BlobParquet_@GF{IR}",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"RelativePath": {
|
||||
"value": "@pipeline().parameters.TaskObject.Target.RelativePath",
|
||||
"type": "Expression"
|
||||
},
|
||||
"FileName": {
|
||||
"value": "@replace(pipeline().parameters.TaskObject.Target.DataFileName,'.parquet',concat('.chunk_', string(pipeline().parameters.Item),'.parquet'))",
|
||||
"type": "Expression"
|
||||
},
|
||||
"StorageAccountEndpoint": {
|
||||
"value": "@pipeline().parameters.TaskObject.Target.StorageAccountName",
|
||||
"type": "Expression"
|
||||
},
|
||||
"StorageAccountContainerName": {
|
||||
"value": "@pipeline().parameters.TaskObject.Target.StorageAccountContainer",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
},
|
||||
"fieldList": [
|
||||
"structure"
|
||||
],
|
||||
"storeSettings": {
|
||||
"type": "AzureBlobStorageReadSettings",
|
||||
"recursive": true
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Pipeline AF Log - Azure SQL to Blob Start",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
"parameters": {
|
||||
"Body": {
|
||||
"value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":3,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy Azure SQL to Blob\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"Status\":\"Started\"}'))",
|
||||
"type": "Expression"
|
||||
},
|
||||
"FunctionName": "Log",
|
||||
"Method": "Post"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Pipeline AF Log - Azure SQL Blob to Succeed",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "Copy Azure SQL to Blob",
|
||||
"dependencyConditions": [
|
||||
"Succeeded"
|
||||
]
|
||||
}
|
||||
],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
"parameters": {
|
||||
"Body": {
|
||||
"value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":1,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy Azure SQL to Blob\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"EndDateTimeOffSet\":\"', string(utcnow()), '\",\"RowsInserted\":\"', string(activity('Copy Azure SQL to Blob').output.rowsCopied), '\",\"Comment\":\"\",\"Status\":\"Complete\"}'))",
|
||||
"type": "Expression"
|
||||
},
|
||||
"FunctionName": "Log",
|
||||
"Method": "Post"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Set SQLStatement",
|
||||
"type": "SetVariable",
|
||||
"dependsOn": [],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"variableName": "SQLStatement1",
|
||||
"value": {
|
||||
"value": "@replace(replace(pipeline().parameters.TaskObject.Source.Extraction.SQLStatement,'<batchcount>',string(pipeline().parameters.BatchCount)),'<item>',string(pipeline().parameters.Item))",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"TaskObject": {
|
||||
"type": "object",
|
||||
"defaultValue": {
|
||||
"TaskInstanceId": 75,
|
||||
"TaskMasterId": 12,
|
||||
"TaskStatus": "Untried",
|
||||
"TaskType": "SQL Database to Azure Storage",
|
||||
"Enabled": 1,
|
||||
"ExecutionUid": "2c5924ee-b855-4d2b-bb7e-4f5dde4c4dd3",
|
||||
"NumberOfRetries": 111,
|
||||
"DegreeOfCopyParallelism": 1,
|
||||
"KeyVaultBaseUrl": "https://adsgofastkeyvault.vault.azure.net/",
|
||||
"ScheduleMasterId": 2,
|
||||
"TaskGroupConcurrency": 10,
|
||||
"TaskGroupPriority": 0,
|
||||
"Source": {
|
||||
"Type": "Azure SQL",
|
||||
"Database": {
|
||||
"SystemName": "adsgofastdatakakeaccelsqlsvr.database.windows.net",
|
||||
"Name": "AWSample",
|
||||
"AuthenticationType": "MSI"
|
||||
},
|
||||
"Extraction": {
|
||||
"Type": "Table",
|
||||
"FullOrIncremental": "Full",
|
||||
"IncrementalType": null,
|
||||
"TableSchema": "SalesLT",
|
||||
"TableName": "SalesOrderHeader"
|
||||
}
|
||||
},
|
||||
"Target": {
|
||||
"Type": "Azure Blob",
|
||||
"StorageAccountName": "https://adsgofastdatalakeaccelst.blob.core.windows.net",
|
||||
"StorageAccountContainer": "datalakeraw",
|
||||
"StorageAccountAccessMethod": "MSI",
|
||||
"RelativePath": "/AwSample/SalesLT/SalesOrderHeader/2020/7/9/14/12/",
|
||||
"DataFileName": "SalesLT.SalesOrderHeader.parquet",
|
||||
"SchemaFileName": "SalesLT.SalesOrderHeader",
|
||||
"FirstRowAsHeader": null,
|
||||
"SheetName": null,
|
||||
"SkipLineCount": null,
|
||||
"MaxConcorrentConnections": null
|
||||
},
|
||||
"DataFactory": {
|
||||
"Id": 1,
|
||||
"Name": "adsgofastdatakakeacceladf",
|
||||
"ResourceGroup": "AdsGoFastDataLakeAccel",
|
||||
"SubscriptionId": "035a1364-f00d-48e2-b582-4fe125905ee3",
|
||||
"ADFPipeline": "AZ_SQL_AZ_Storage_Parquet_@GF{IR}"
|
||||
}
|
||||
}
|
||||
},
|
||||
"Mapping": {
|
||||
"type": "object"
|
||||
},
|
||||
"BatchCount": {
|
||||
"type": "int"
|
||||
},
|
||||
"Item": {
|
||||
"type": "int"
|
||||
}
|
||||
},
|
||||
"variables": {
|
||||
"SQLStatement1": {
|
||||
"type": "String"
|
||||
}
|
||||
},
|
||||
"folder": {
|
||||
"name": "ADS Go Fast/Data Movement/@GF{IR}/Components"
|
||||
},
|
||||
"annotations": [],
|
||||
"lastPublishTime": "2020-07-29T02:39:55Z"
|
||||
},
|
||||
"type": "Microsoft.DataFactory/factories/pipelines"
|
||||
}
|
||||
|
|
@ -0,0 +1,556 @@
|
|||
{
|
||||
"name": "AZ_SQL_Post_Copy_@GF{IR}",
|
||||
"properties": {
|
||||
"activities": [
|
||||
{
|
||||
"name": "If Exist PostCopySQL",
|
||||
"type": "IfCondition",
|
||||
"dependsOn": [],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"expression": {
|
||||
"value": "@not(empty(pipeline().parameters.TaskObject.Target.PostCopySQL))",
|
||||
"type": "Expression"
|
||||
},
|
||||
"ifTrueActivities": [
|
||||
{
|
||||
"name": "Run PostCopySQL",
|
||||
"type": "Lookup",
|
||||
"dependsOn": [],
|
||||
"policy": {
|
||||
"timeout": "7.00:00:00",
|
||||
"retry": 0,
|
||||
"retryIntervalInSeconds": 30,
|
||||
"secureOutput": false,
|
||||
"secureInput": false
|
||||
},
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"source": {
|
||||
"type": "AzureSqlSource",
|
||||
"sqlReaderQuery": {
|
||||
"value": "@pipeline().parameters.TaskObject.Target.PostCopySQL",
|
||||
"type": "Expression"
|
||||
},
|
||||
"queryTimeout": "02:00:00",
|
||||
"partitionOption": "None"
|
||||
},
|
||||
"dataset": {
|
||||
"referenceName": "AzureSqlTable_@GF{IR}",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"Schema": {
|
||||
"value": "@pipeline().parameters.TaskObject.Target.StagingTableSchema",
|
||||
"type": "Expression"
|
||||
},
|
||||
"Table": {
|
||||
"value": "@pipeline().parameters.TaskObject.Target.StagingTableName",
|
||||
"type": "Expression"
|
||||
},
|
||||
"Server": {
|
||||
"value": "@pipeline().parameters.TaskObject.Target.Database.SystemName",
|
||||
"type": "Expression"
|
||||
},
|
||||
"Database": {
|
||||
"value": "@pipeline().parameters.TaskObject.Target.Database.Name",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "AF Log - Run PostCopySQL Failed",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "Run PostCopySQL",
|
||||
"dependencyConditions": [
|
||||
"Failed"
|
||||
]
|
||||
}
|
||||
],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
"parameters": {
|
||||
"Body": {
|
||||
"value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":1,\"LogSource\":\"ADF\",\"ActivityType\":\"Run PostCopySQL\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"EndDateTimeOffSet\":\"', string(utcnow()), '\",\"Comment\":\"', string(activity('Run PostCopySQL').error.message), '\",\"Status\":\"Failed\"}'))",
|
||||
"type": "Expression"
|
||||
},
|
||||
"FunctionName": "Log",
|
||||
"Method": "Post"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "If Exist MergeSQL",
|
||||
"type": "IfCondition",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "If Exist PostCopySQL",
|
||||
"dependencyConditions": [
|
||||
"Succeeded"
|
||||
]
|
||||
}
|
||||
],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"expression": {
|
||||
"value": "@not(empty(pipeline().parameters.TaskObject.Target.MergeSQL))",
|
||||
"type": "Expression"
|
||||
},
|
||||
"ifTrueActivities": [
|
||||
{
|
||||
"name": "Run MergeSQL",
|
||||
"type": "Lookup",
|
||||
"dependsOn": [],
|
||||
"policy": {
|
||||
"timeout": "7.00:00:00",
|
||||
"retry": 0,
|
||||
"retryIntervalInSeconds": 30,
|
||||
"secureOutput": false,
|
||||
"secureInput": false
|
||||
},
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"source": {
|
||||
"type": "AzureSqlSource",
|
||||
"sqlReaderQuery": {
|
||||
"value": "@pipeline().parameters.TaskObject.Target.MergeSQL",
|
||||
"type": "Expression"
|
||||
},
|
||||
"queryTimeout": "02:00:00",
|
||||
"partitionOption": "None"
|
||||
},
|
||||
"dataset": {
|
||||
"referenceName": "AzureSqlTable_@GF{IR}",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"Schema": {
|
||||
"value": "@pipeline().parameters.TaskObject.Target.StagingTableSchema",
|
||||
"type": "Expression"
|
||||
},
|
||||
"Table": {
|
||||
"value": "@pipeline().parameters.TaskObject.Target.StagingTableName",
|
||||
"type": "Expression"
|
||||
},
|
||||
"Server": {
|
||||
"value": "@pipeline().parameters.TaskObject.Target.Database.SystemName",
|
||||
"type": "Expression"
|
||||
},
|
||||
"Database": {
|
||||
"value": "@pipeline().parameters.TaskObject.Target.Database.Name",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "AF Log - Run MergeSQL Failed",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "Run MergeSQL",
|
||||
"dependencyConditions": [
|
||||
"Failed"
|
||||
]
|
||||
}
|
||||
],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
"parameters": {
|
||||
"Body": {
|
||||
"value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":1,\"LogSource\":\"ADF\",\"ActivityType\":\"Run MergeSQL\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"EndDateTimeOffSet\":\"', string(utcnow()), '\",\"Comment\":\"', string(activity('Run MergeSQL').error.message), '\",\"Status\":\"Failed\"}'))",
|
||||
"type": "Expression"
|
||||
},
|
||||
"FunctionName": "Log",
|
||||
"Method": "Post"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "If AutoGenerateMerge",
|
||||
"type": "IfCondition",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "If Exist PostCopySQL",
|
||||
"dependencyConditions": [
|
||||
"Succeeded"
|
||||
]
|
||||
}
|
||||
],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"expression": {
|
||||
"value": "@bool(pipeline().parameters.TaskObject.Target.AutoGenerateMerge)",
|
||||
"type": "Expression"
|
||||
},
|
||||
"ifTrueActivities": [
|
||||
{
|
||||
"name": "Run MergeStatement",
|
||||
"type": "Lookup",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "AF Get Merge Statement",
|
||||
"dependencyConditions": [
|
||||
"Succeeded"
|
||||
]
|
||||
}
|
||||
],
|
||||
"policy": {
|
||||
"timeout": "7.00:00:00",
|
||||
"retry": 0,
|
||||
"retryIntervalInSeconds": 30,
|
||||
"secureOutput": false,
|
||||
"secureInput": false
|
||||
},
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"source": {
|
||||
"type": "AzureSqlSource",
|
||||
"sqlReaderQuery": {
|
||||
"value": "@activity('AF Get Merge Statement').output.MergeStatement",
|
||||
"type": "Expression"
|
||||
},
|
||||
"queryTimeout": "02:00:00",
|
||||
"partitionOption": "None"
|
||||
},
|
||||
"dataset": {
|
||||
"referenceName": "AzureSqlTable_@GF{IR}",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"Schema": {
|
||||
"value": "@pipeline().parameters.TaskObject.Target.StagingTableSchema",
|
||||
"type": "Expression"
|
||||
},
|
||||
"Table": {
|
||||
"value": "@pipeline().parameters.TaskObject.Target.StagingTableName",
|
||||
"type": "Expression"
|
||||
},
|
||||
"Server": {
|
||||
"value": "@pipeline().parameters.TaskObject.Target.Database.SystemName",
|
||||
"type": "Expression"
|
||||
},
|
||||
"Database": {
|
||||
"value": "@pipeline().parameters.TaskObject.Target.Database.Name",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
},
|
||||
"firstRowOnly": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "AF Get Information Schema SQL Stage",
|
||||
"type": "AzureFunctionActivity",
|
||||
"dependsOn": [],
|
||||
"policy": {
|
||||
"timeout": "7.00:00:00",
|
||||
"retry": 0,
|
||||
"retryIntervalInSeconds": 30,
|
||||
"secureOutput": false,
|
||||
"secureInput": false
|
||||
},
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"functionName": "GetInformationSchemaSQL",
|
||||
"method": "POST",
|
||||
"body": {
|
||||
"value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"TableSchema\":\"', string(pipeline().parameters.TaskObject.Target.StagingTableSchema), '\",\"TableName\":\"', string(pipeline().parameters.TaskObject.Target.StagingTableName),'\"}'))",
|
||||
"type": "Expression"
|
||||
}
|
||||
},
|
||||
"linkedServiceName": {
|
||||
"referenceName": "AzureFunctionAdsGoFastDataLakeAccelFunApp",
|
||||
"type": "LinkedServiceReference"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "AF Get Merge Statement",
|
||||
"type": "AzureFunctionActivity",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "Lookup Get Metadata Stage",
|
||||
"dependencyConditions": [
|
||||
"Succeeded"
|
||||
]
|
||||
},
|
||||
{
|
||||
"activity": "Lookup Get Metadata Target",
|
||||
"dependencyConditions": [
|
||||
"Succeeded"
|
||||
]
|
||||
}
|
||||
],
|
||||
"policy": {
|
||||
"timeout": "7.00:00:00",
|
||||
"retry": 0,
|
||||
"retryIntervalInSeconds": 30,
|
||||
"secureOutput": false,
|
||||
"secureInput": false
|
||||
},
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"functionName": "GetSQLMergeStatement",
|
||||
"method": "POST",
|
||||
"body": {
|
||||
"value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId),'\",\"TargetTableSchema\":\"',string(pipeline().parameters.TaskObject.Target.TableSchema),'\",\"TargetTableName\":\"',string(pipeline().parameters.TaskObject.Target.TableName),'\",\"StagingTableSchema\":\"',string(pipeline().parameters.TaskObject.Target.StagingTableSchema),'\",\"StagingTableName\":\"',string(pipeline().parameters.TaskObject.Target.StagingTableName),'\",\"Stage\":', string(activity('Lookup Get Metadata Stage').output.value), ',\"Target\":', string(activity('Lookup Get Metadata Target').output.value),'}'))",
|
||||
"type": "Expression"
|
||||
}
|
||||
},
|
||||
"linkedServiceName": {
|
||||
"referenceName": "AzureFunctionAdsGoFastDataLakeAccelFunApp",
|
||||
"type": "LinkedServiceReference"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Lookup Get Metadata Stage",
|
||||
"type": "Lookup",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "AF Get Information Schema SQL Stage",
|
||||
"dependencyConditions": [
|
||||
"Succeeded"
|
||||
]
|
||||
}
|
||||
],
|
||||
"policy": {
|
||||
"timeout": "7.00:00:00",
|
||||
"retry": 0,
|
||||
"retryIntervalInSeconds": 30,
|
||||
"secureOutput": false,
|
||||
"secureInput": false
|
||||
},
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"source": {
|
||||
"type": "AzureSqlSource",
|
||||
"sqlReaderQuery": {
|
||||
"value": "@activity('AF Get Information Schema SQL Stage').output.InformationSchemaSQL",
|
||||
"type": "Expression"
|
||||
},
|
||||
"queryTimeout": "02:00:00",
|
||||
"partitionOption": "None"
|
||||
},
|
||||
"dataset": {
|
||||
"referenceName": "AzureSqlTable_@GF{IR}",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"Schema": {
|
||||
"value": "@pipeline().parameters.TaskObject.Target.StagingTableSchema",
|
||||
"type": "Expression"
|
||||
},
|
||||
"Table": {
|
||||
"value": "@pipeline().parameters.TaskObject.Target.StagingTableName",
|
||||
"type": "Expression"
|
||||
},
|
||||
"Server": {
|
||||
"value": "@pipeline().parameters.TaskObject.Target.Database.SystemName",
|
||||
"type": "Expression"
|
||||
},
|
||||
"Database": {
|
||||
"value": "@pipeline().parameters.TaskObject.Target.Database.Name",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
},
|
||||
"firstRowOnly": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Lookup Get Metadata Target",
|
||||
"type": "Lookup",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "AF Get Information Schema SQL Target",
|
||||
"dependencyConditions": [
|
||||
"Succeeded"
|
||||
]
|
||||
}
|
||||
],
|
||||
"policy": {
|
||||
"timeout": "7.00:00:00",
|
||||
"retry": 0,
|
||||
"retryIntervalInSeconds": 30,
|
||||
"secureOutput": false,
|
||||
"secureInput": false
|
||||
},
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"source": {
|
||||
"type": "AzureSqlSource",
|
||||
"sqlReaderQuery": {
|
||||
"value": "@activity('AF Get Information Schema SQL Target').output.InformationSchemaSQL",
|
||||
"type": "Expression"
|
||||
},
|
||||
"queryTimeout": "02:00:00",
|
||||
"partitionOption": "None"
|
||||
},
|
||||
"dataset": {
|
||||
"referenceName": "AzureSqlTable_@GF{IR}",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"Schema": {
|
||||
"value": "@pipeline().parameters.TaskObject.Target.TableSchema",
|
||||
"type": "Expression"
|
||||
},
|
||||
"Table": {
|
||||
"value": "@pipeline().parameters.TaskObject.Target.TableName",
|
||||
"type": "Expression"
|
||||
},
|
||||
"Server": {
|
||||
"value": "@pipeline().parameters.TaskObject.Target.Database.SystemName",
|
||||
"type": "Expression"
|
||||
},
|
||||
"Database": {
|
||||
"value": "@pipeline().parameters.TaskObject.Target.Database.Name",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
},
|
||||
"firstRowOnly": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "AF Get Information Schema SQL Target",
|
||||
"type": "AzureFunctionActivity",
|
||||
"dependsOn": [],
|
||||
"policy": {
|
||||
"timeout": "7.00:00:00",
|
||||
"retry": 0,
|
||||
"retryIntervalInSeconds": 30,
|
||||
"secureOutput": false,
|
||||
"secureInput": false
|
||||
},
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"functionName": "GetInformationSchemaSQL",
|
||||
"method": "POST",
|
||||
"body": {
|
||||
"value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"TableSchema\":\"', string(pipeline().parameters.TaskObject.Target.TableSchema), '\",\"TableName\":\"', string(pipeline().parameters.TaskObject.Target.TableName),'\"}'))",
|
||||
"type": "Expression"
|
||||
}
|
||||
},
|
||||
"linkedServiceName": {
|
||||
"referenceName": "AzureFunctionAdsGoFastDataLakeAccelFunApp",
|
||||
"type": "LinkedServiceReference"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "AF Log - Run AutoMerge Failed",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "Run MergeStatement",
|
||||
"dependencyConditions": [
|
||||
"Failed"
|
||||
]
|
||||
}
|
||||
],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
"parameters": {
|
||||
"Body": {
|
||||
"value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":1,\"LogSource\":\"ADF\",\"ActivityType\":\"Run AutoMerge\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"EndDateTimeOffSet\":\"', string(utcnow()), '\",\"Comment\":\"', string(activity('Run MergeStatement').error.message), '\",\"Status\":\"Failed\"}'))",
|
||||
"type": "Expression"
|
||||
},
|
||||
"FunctionName": "Log",
|
||||
"Method": "Post"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "AF Log - Run Lookup Get Metadata Target Failed",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "Lookup Get Metadata Target",
|
||||
"dependencyConditions": [
|
||||
"Failed"
|
||||
]
|
||||
}
|
||||
],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
"parameters": {
|
||||
"Body": {
|
||||
"value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":1,\"LogSource\":\"ADF\",\"ActivityType\":\"Lookup Get Metadata Target\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"EndDateTimeOffSet\":\"', string(utcnow()), '\",\"Comment\":\"', string(activity('Lookup Get Metadata Target').error.message), '\",\"Status\":\"Failed\"}'))",
|
||||
"type": "Expression"
|
||||
},
|
||||
"FunctionName": "Log",
|
||||
"Method": "Post"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "AF Log - Lookup Get Metadata Stage Failed",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "Lookup Get Metadata Stage",
|
||||
"dependencyConditions": [
|
||||
"Failed"
|
||||
]
|
||||
}
|
||||
],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
"parameters": {
|
||||
"Body": {
|
||||
"value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":1,\"LogSource\":\"ADF\",\"ActivityType\":\"Lookup Get Metadata Stage\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"EndDateTimeOffSet\":\"', string(utcnow()), '\",\"Comment\":\"', string(activity('Lookup Get Metadata Stage').error.message), '\",\"Status\":\"Failed\"}'))",
|
||||
"type": "Expression"
|
||||
},
|
||||
"FunctionName": "Log",
|
||||
"Method": "Post"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"TaskObject": {
|
||||
"type": "object"
|
||||
}
|
||||
},
|
||||
"folder": {
|
||||
"name": "ADS Go Fast/Data Movement/@GF{IR}/Common"
|
||||
},
|
||||
"annotations": [],
|
||||
"lastPublishTime": "2020-08-04T13:09:30Z"
|
||||
},
|
||||
"type": "Microsoft.DataFactory/factories/pipelines"
|
||||
}
|
||||
|
|
@ -0,0 +1,92 @@
|
|||
{
|
||||
"name": "AZ_SQL_StoredProcedure_@GF{IR}",
|
||||
"properties": {
|
||||
"activities": [
|
||||
{
|
||||
"name": "Execute Stored Procedure",
|
||||
"type": "Lookup",
|
||||
"dependsOn": [],
|
||||
"policy": {
|
||||
"timeout": "7.00:00:00",
|
||||
"retry": 0,
|
||||
"retryIntervalInSeconds": 30,
|
||||
"secureOutput": false,
|
||||
"secureInput": false
|
||||
},
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"source": {
|
||||
"type": "AzureSqlSource",
|
||||
"sqlReaderQuery": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.Execute.StoredProcedure",
|
||||
"type": "Expression"
|
||||
},
|
||||
"queryTimeout": "02:00:00",
|
||||
"partitionOption": "None"
|
||||
},
|
||||
"dataset": {
|
||||
"referenceName": "AzureSqlTable_@GF{IR}",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"Schema": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.Extraction.TableSchema",
|
||||
"type": "Expression"
|
||||
},
|
||||
"Table": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.Extraction.TableName",
|
||||
"type": "Expression"
|
||||
},
|
||||
"Server": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.Database.SystemName",
|
||||
"type": "Expression"
|
||||
},
|
||||
"Database": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.Database.Name",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Pipeline AF Log - Lookup Failed",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "Execute Stored Procedure",
|
||||
"dependencyConditions": [
|
||||
"Failed"
|
||||
]
|
||||
}
|
||||
],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
"parameters": {
|
||||
"Body": {
|
||||
"value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":1,\"LogSource\":\"ADF\",\"ActivityType\":\"Execute Stored Procedure\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"EndDateTimeOffSet\":\"', string(utcnow()), '\",\"Comment\":\"', string(activity('Execute Stored Procedure').error.message), '\",\"Status\":\"Failed\"}'))",
|
||||
"type": "Expression"
|
||||
},
|
||||
"FunctionName": "Log",
|
||||
"Method": "Post"
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"TaskObject": {
|
||||
"type": "object"
|
||||
}
|
||||
},
|
||||
"folder": {
|
||||
"name": "ADS Go Fast/Data Movement/@GF{IR}"
|
||||
},
|
||||
"annotations": [],
|
||||
"lastPublishTime": "2020-08-06T09:21:51Z"
|
||||
},
|
||||
"type": "Microsoft.DataFactory/factories/pipelines"
|
||||
}
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
{
|
||||
"name": "OnP-SQL-Watermark-Chunk-OnP-SH-IR",
|
||||
"name": "AZ_SQL_Watermark_Chunk_@GF{IR}",
|
||||
"properties": {
|
||||
"activities": [
|
||||
{
|
||||
|
|
@ -16,7 +16,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-Function-Generic",
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
|
|
@ -43,13 +43,13 @@
|
|||
"isSequential": true,
|
||||
"activities": [
|
||||
{
|
||||
"name": "Execute OnP-SQL-Watermark",
|
||||
"name": "Execute AZ_SQL_Watermark",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "OnP-SQL-Watermark-OnP-SH-IR",
|
||||
"referenceName": "AZ_SQL_Watermark_@GF{IR}",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": true,
|
||||
|
|
@ -130,7 +130,7 @@
|
|||
"Name": "adsgofastdatakakeacceladf",
|
||||
"ResourceGroup": "AdsGoFastDataLakeAccel",
|
||||
"SubscriptionId": "035a1364-f00d-48e2-b582-4fe125905ee3",
|
||||
"ADFPipeline": "AZ-SQL-AZ-Storage-Parquet-SH-IR"
|
||||
"ADFPipeline": "AZ_SQL_AZ_Storage_Parquet_@GF{IR}"
|
||||
}
|
||||
}
|
||||
},
|
||||
|
|
@ -145,10 +145,10 @@
|
|||
}
|
||||
},
|
||||
"folder": {
|
||||
"name": "ADS Go Fast/Data Movement/OnPrem SH IR/Components"
|
||||
"name": "ADS Go Fast/Data Movement/@GF{IR}/Components"
|
||||
},
|
||||
"annotations": [],
|
||||
"lastPublishTime": "2020-08-04T07:52:08Z"
|
||||
"lastPublishTime": "2020-07-29T02:39:55Z"
|
||||
},
|
||||
"type": "Microsoft.DataFactory/factories/pipelines"
|
||||
}
|
||||
|
|
@ -0,0 +1,636 @@
|
|||
{
|
||||
"name": "AZ_SQL_Watermark_@GF{IR}",
|
||||
"properties": {
|
||||
"activities": [
|
||||
{
|
||||
"name": "Switch Storage Type",
|
||||
"type": "Switch",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "Set SQLStatement",
|
||||
"dependencyConditions": [
|
||||
"Succeeded"
|
||||
]
|
||||
}
|
||||
],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"on": {
|
||||
"value": "@pipeline().parameters.TaskObject.Target.Type",
|
||||
"type": "Expression"
|
||||
},
|
||||
"cases": [
|
||||
{
|
||||
"value": "ADLS",
|
||||
"activities": [
|
||||
{
|
||||
"name": "Copy Azure SQL to ADLS",
|
||||
"type": "Copy",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "Pipeline AF Log - Azure SQL to ADLS Start",
|
||||
"dependencyConditions": [
|
||||
"Succeeded"
|
||||
]
|
||||
}
|
||||
],
|
||||
"policy": {
|
||||
"timeout": "7.00:00:00",
|
||||
"retry": 0,
|
||||
"retryIntervalInSeconds": 30,
|
||||
"secureOutput": false,
|
||||
"secureInput": false
|
||||
},
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"source": {
|
||||
"type": "AzureSqlSource",
|
||||
"sqlReaderQuery": {
|
||||
"value": "@variables('SQLStatement')",
|
||||
"type": "Expression"
|
||||
},
|
||||
"queryTimeout": "02:00:00",
|
||||
"partitionOption": "None"
|
||||
},
|
||||
"sink": {
|
||||
"type": "ParquetSink",
|
||||
"storeSettings": {
|
||||
"type": "AzureBlobFSWriteSettings"
|
||||
}
|
||||
},
|
||||
"enableStaging": false,
|
||||
"parallelCopies": {
|
||||
"value": "@pipeline().parameters.TaskObject.DegreeOfCopyParallelism",
|
||||
"type": "Expression"
|
||||
},
|
||||
"translator": {
|
||||
"value": "@pipeline().parameters.Mapping",
|
||||
"type": "Expression"
|
||||
}
|
||||
},
|
||||
"inputs": [
|
||||
{
|
||||
"referenceName": "AzureSqlTable_@GF{IR}",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"Schema": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.Extraction.TableSchema",
|
||||
"type": "Expression"
|
||||
},
|
||||
"Table": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.Extraction.TableName",
|
||||
"type": "Expression"
|
||||
},
|
||||
"Server": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.Database.SystemName",
|
||||
"type": "Expression"
|
||||
},
|
||||
"Database": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.Database.Name",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"referenceName": "ADLSParquet_@GF{IR}",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"RelativePath": {
|
||||
"value": "@pipeline().parameters.TaskObject.Target.RelativePath",
|
||||
"type": "Expression"
|
||||
},
|
||||
"FileName": {
|
||||
"value": "@replace(pipeline().parameters.TaskObject.Target.DataFileName,'.parquet',concat('.chunk_', string(pipeline().parameters.Item),'.parquet'))",
|
||||
"type": "Expression"
|
||||
},
|
||||
"StorageAccountEndpoint": {
|
||||
"value": "@pipeline().parameters.TaskObject.Target.StorageAccountName",
|
||||
"type": "Expression"
|
||||
},
|
||||
"StorageAccountContainerName": {
|
||||
"value": "@pipeline().parameters.TaskObject.Target.StorageAccountContainer",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "Pipeline AF Log - Azure SQL to ADLS Failed",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "Copy Azure SQL to ADLS",
|
||||
"dependencyConditions": [
|
||||
"Failed"
|
||||
]
|
||||
}
|
||||
],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
"parameters": {
|
||||
"Body": {
|
||||
"value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":1,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy Azure SQL to ADLS\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"EndDateTimeOffSet\":\"', string(utcnow()), '\",\"Comment\":\"', string(activity('Copy Azure SQL to ADLS').error.message), '\",\"Status\":\"Failed\"}'))",
|
||||
"type": "Expression"
|
||||
},
|
||||
"FunctionName": "Log",
|
||||
"Method": "Post"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "AF Persist Parquet Metadata - ADLS",
|
||||
"type": "AzureFunctionActivity",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "Get Parquet Metadata ADLS",
|
||||
"dependencyConditions": [
|
||||
"Succeeded"
|
||||
]
|
||||
}
|
||||
],
|
||||
"policy": {
|
||||
"timeout": "7.00:00:00",
|
||||
"retry": 0,
|
||||
"retryIntervalInSeconds": 30,
|
||||
"secureOutput": false,
|
||||
"secureInput": false
|
||||
},
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"functionName": "TaskExecutionSchemaFile",
|
||||
"method": "POST",
|
||||
"body": {
|
||||
"value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"StorageAccountName\":\"', string(pipeline().parameters.TaskObject.Target.StorageAccountName), '\",\"StorageAccountContainer\":\"', string(pipeline().parameters.TaskObject.Target.StorageAccountContainer), '\",\"RelativePath\":\"', string(pipeline().parameters.TaskObject.Target.RelativePath), '\",\"SchemaFileName\":\"', string(pipeline().parameters.TaskObject.Target.SchemaFileName), '\",\"SourceType\":\"', string(pipeline().parameters.TaskObject.Source.Type), '\",\"TargetType\":\"', string(pipeline().parameters.TaskObject.Target.Type), '\",\"Data\":',string(activity('Get Parquet Metadata ADLS').output),',\"MetadataType\":\"Parquet\"}'))",
|
||||
"type": "Expression"
|
||||
}
|
||||
},
|
||||
"linkedServiceName": {
|
||||
"referenceName": "AzureFunctionAdsGoFastDataLakeAccelFunApp",
|
||||
"type": "LinkedServiceReference"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Get Parquet Metadata ADLS",
|
||||
"type": "GetMetadata",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "Copy Azure SQL to ADLS",
|
||||
"dependencyConditions": [
|
||||
"Succeeded"
|
||||
]
|
||||
}
|
||||
],
|
||||
"policy": {
|
||||
"timeout": "7.00:00:00",
|
||||
"retry": 0,
|
||||
"retryIntervalInSeconds": 30,
|
||||
"secureOutput": false,
|
||||
"secureInput": false
|
||||
},
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"dataset": {
|
||||
"referenceName": "ADLSParquet_@GF{IR}",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"RelativePath": {
|
||||
"value": "@pipeline().parameters.TaskObject.Target.RelativePath",
|
||||
"type": "Expression"
|
||||
},
|
||||
"FileName": {
|
||||
"value": "@replace(pipeline().parameters.TaskObject.Target.DataFileName,'.parquet',concat('.chunk_', string(pipeline().parameters.Item),'.parquet'))",
|
||||
"type": "Expression"
|
||||
},
|
||||
"StorageAccountEndpoint": {
|
||||
"value": "@pipeline().parameters.TaskObject.Target.StorageAccountName",
|
||||
"type": "Expression"
|
||||
},
|
||||
"StorageAccountContainerName": {
|
||||
"value": "@pipeline().parameters.TaskObject.Target.StorageAccountContainer",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
},
|
||||
"fieldList": [
|
||||
"structure"
|
||||
],
|
||||
"storeSettings": {
|
||||
"type": "AzureBlobFSReadSettings",
|
||||
"recursive": true
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Pipeline AF Log - Azure SQL to ADLS Start",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
"parameters": {
|
||||
"Body": {
|
||||
"value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":3,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy Azure SQL to ADLS\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"Status\":\"Started\"}'))",
|
||||
"type": "Expression"
|
||||
},
|
||||
"FunctionName": "Log",
|
||||
"Method": "Post"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Pipeline AF Log - Azure SQL ADLS to Succeed",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "Copy Azure SQL to ADLS",
|
||||
"dependencyConditions": [
|
||||
"Succeeded"
|
||||
]
|
||||
}
|
||||
],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
"parameters": {
|
||||
"Body": {
|
||||
"value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":1,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy Azure SQL to ADLS\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"EndDateTimeOffSet\":\"', string(utcnow()), '\",\"RowsInserted\":\"', string(activity('Copy Azure SQL to ADLS').output.rowsCopied), '\",\"Comment\":\"\",\"Status\":\"Complete\"}'))",
|
||||
"type": "Expression"
|
||||
},
|
||||
"FunctionName": "Log",
|
||||
"Method": "Post"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"value": "Azure Blob",
|
||||
"activities": [
|
||||
{
|
||||
"name": "Copy Azure SQL to Blob",
|
||||
"type": "Copy",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "Pipeline AF Log - Azure SQL to Blob Start",
|
||||
"dependencyConditions": [
|
||||
"Succeeded"
|
||||
]
|
||||
}
|
||||
],
|
||||
"policy": {
|
||||
"timeout": "7.00:00:00",
|
||||
"retry": 0,
|
||||
"retryIntervalInSeconds": 30,
|
||||
"secureOutput": false,
|
||||
"secureInput": false
|
||||
},
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"source": {
|
||||
"type": "AzureSqlSource",
|
||||
"sqlReaderQuery": {
|
||||
"value": "@variables('SQLStatement')",
|
||||
"type": "Expression"
|
||||
},
|
||||
"queryTimeout": "02:00:00",
|
||||
"partitionOption": "None"
|
||||
},
|
||||
"sink": {
|
||||
"type": "ParquetSink",
|
||||
"storeSettings": {
|
||||
"type": "AzureBlobStorageWriteSettings"
|
||||
}
|
||||
},
|
||||
"enableStaging": false,
|
||||
"parallelCopies": {
|
||||
"value": "@pipeline().parameters.TaskObject.DegreeOfCopyParallelism",
|
||||
"type": "Expression"
|
||||
},
|
||||
"translator": {
|
||||
"value": "@pipeline().parameters.Mapping",
|
||||
"type": "Expression"
|
||||
}
|
||||
},
|
||||
"inputs": [
|
||||
{
|
||||
"referenceName": "AzureSqlTable_@GF{IR}",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"Schema": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.Extraction.TableSchema",
|
||||
"type": "Expression"
|
||||
},
|
||||
"Table": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.Extraction.TableName",
|
||||
"type": "Expression"
|
||||
},
|
||||
"Server": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.Database.SystemName",
|
||||
"type": "Expression"
|
||||
},
|
||||
"Database": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.Database.Name",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"referenceName": "BlobParquet_@GF{IR}",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"RelativePath": {
|
||||
"value": "@pipeline().parameters.TaskObject.Target.RelativePath",
|
||||
"type": "Expression"
|
||||
},
|
||||
"FileName": {
|
||||
"value": "@replace(pipeline().parameters.TaskObject.Target.DataFileName,'.parquet',concat('.chunk_', string(pipeline().parameters.Item),'.parquet'))",
|
||||
"type": "Expression"
|
||||
},
|
||||
"StorageAccountEndpoint": {
|
||||
"value": "@pipeline().parameters.TaskObject.Target.StorageAccountName",
|
||||
"type": "Expression"
|
||||
},
|
||||
"StorageAccountContainerName": {
|
||||
"value": "@pipeline().parameters.TaskObject.Target.StorageAccountContainer",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "Pipeline AF Log - Azure SQL to Blob Failed",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "Copy Azure SQL to Blob",
|
||||
"dependencyConditions": [
|
||||
"Failed"
|
||||
]
|
||||
}
|
||||
],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
"parameters": {
|
||||
"Body": {
|
||||
"value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":1,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy Azure SQL to Blob\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"EndDateTimeOffSet\":\"', string(utcnow()), '\",\"Comment\":\"', string(activity('Copy Azure SQL to Blob').error.message), '\",\"Status\":\"Failed\"}'))",
|
||||
"type": "Expression"
|
||||
},
|
||||
"FunctionName": "Log",
|
||||
"Method": "Post"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "AF Persist Parquet Metadata - Blob",
|
||||
"type": "AzureFunctionActivity",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "Get Parquet Metadata Blob",
|
||||
"dependencyConditions": [
|
||||
"Succeeded"
|
||||
]
|
||||
}
|
||||
],
|
||||
"policy": {
|
||||
"timeout": "7.00:00:00",
|
||||
"retry": 0,
|
||||
"retryIntervalInSeconds": 30,
|
||||
"secureOutput": false,
|
||||
"secureInput": false
|
||||
},
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"functionName": "TaskExecutionSchemaFile",
|
||||
"method": "POST",
|
||||
"body": {
|
||||
"value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"StorageAccountName\":\"', string(pipeline().parameters.TaskObject.Target.StorageAccountName), '\",\"StorageAccountContainer\":\"', string(pipeline().parameters.TaskObject.Target.StorageAccountContainer), '\",\"RelativePath\":\"', string(pipeline().parameters.TaskObject.Target.RelativePath), '\",\"SchemaFileName\":\"', string(pipeline().parameters.TaskObject.Target.SchemaFileName), '\",\"SourceType\":\"', string(pipeline().parameters.TaskObject.Source.Type), '\",\"TargetType\":\"', string(pipeline().parameters.TaskObject.Target.Type), '\",\"Data\":',string(activity('Get Parquet Metadata Blob').output),',\"MetadataType\":\"Parquet\"}'))",
|
||||
"type": "Expression"
|
||||
}
|
||||
},
|
||||
"linkedServiceName": {
|
||||
"referenceName": "AzureFunctionAdsGoFastDataLakeAccelFunApp",
|
||||
"type": "LinkedServiceReference"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Get Parquet Metadata Blob",
|
||||
"type": "GetMetadata",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "Copy Azure SQL to Blob",
|
||||
"dependencyConditions": [
|
||||
"Succeeded"
|
||||
]
|
||||
}
|
||||
],
|
||||
"policy": {
|
||||
"timeout": "7.00:00:00",
|
||||
"retry": 0,
|
||||
"retryIntervalInSeconds": 30,
|
||||
"secureOutput": false,
|
||||
"secureInput": false
|
||||
},
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"dataset": {
|
||||
"referenceName": "BlobParquet_@GF{IR}",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"RelativePath": {
|
||||
"value": "@pipeline().parameters.TaskObject.Target.RelativePath",
|
||||
"type": "Expression"
|
||||
},
|
||||
"FileName": {
|
||||
"value": "@replace(pipeline().parameters.TaskObject.Target.DataFileName,'.parquet',concat('.chunk_', string(pipeline().parameters.Item),'.parquet'))",
|
||||
"type": "Expression"
|
||||
},
|
||||
"StorageAccountEndpoint": {
|
||||
"value": "@pipeline().parameters.TaskObject.Target.StorageAccountName",
|
||||
"type": "Expression"
|
||||
},
|
||||
"StorageAccountContainerName": {
|
||||
"value": "@pipeline().parameters.TaskObject.Target.StorageAccountContainer",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
},
|
||||
"fieldList": [
|
||||
"structure"
|
||||
],
|
||||
"storeSettings": {
|
||||
"type": "AzureBlobStorageReadSettings",
|
||||
"recursive": true
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Pipeline AF Log - Azure SQL to Blob Start",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
"parameters": {
|
||||
"Body": {
|
||||
"value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":3,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy Azure SQL to Blob\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"Status\":\"Started\"}'))",
|
||||
"type": "Expression"
|
||||
},
|
||||
"FunctionName": "Log",
|
||||
"Method": "Post"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Pipeline AF Log - Azure SQL Blob to Succeed",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "Copy Azure SQL to Blob",
|
||||
"dependencyConditions": [
|
||||
"Succeeded"
|
||||
]
|
||||
}
|
||||
],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
"parameters": {
|
||||
"Body": {
|
||||
"value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":1,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy Azure SQL to Blob\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"EndDateTimeOffSet\":\"', string(utcnow()), '\",\"RowsInserted\":\"', string(activity('Copy Azure SQL to Blob').output.rowsCopied), '\",\"Comment\":\"\",\"Status\":\"Complete\"}'))",
|
||||
"type": "Expression"
|
||||
},
|
||||
"FunctionName": "Log",
|
||||
"Method": "Post"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Set SQLStatement",
|
||||
"type": "SetVariable",
|
||||
"dependsOn": [],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"variableName": "SQLStatement",
|
||||
"value": {
|
||||
"value": "@replace(replace(replace(pipeline().parameters.TaskObject.Source.Extraction.SQLStatement,'<batchcount>',string(pipeline().parameters.BatchCount)),'<item>',string(pipeline().parameters.Item)),'<newWatermark>',pipeline().parameters.NewWaterMark)",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"TaskObject": {
|
||||
"type": "object",
|
||||
"defaultValue": {
|
||||
"TaskInstanceId": 75,
|
||||
"TaskMasterId": 12,
|
||||
"TaskStatus": "Untried",
|
||||
"TaskType": "SQL Database to Azure Storage",
|
||||
"Enabled": 1,
|
||||
"ExecutionUid": "2c5924ee-b855-4d2b-bb7e-4f5dde4c4dd3",
|
||||
"NumberOfRetries": 111,
|
||||
"DegreeOfCopyParallelism": 1,
|
||||
"KeyVaultBaseUrl": "https://adsgofastkeyvault.vault.azure.net/",
|
||||
"ScheduleMasterId": 2,
|
||||
"TaskGroupConcurrency": 10,
|
||||
"TaskGroupPriority": 0,
|
||||
"Source": {
|
||||
"Type": "Azure SQL",
|
||||
"Database": {
|
||||
"SystemName": "adsgofastdatakakeaccelsqlsvr.database.windows.net",
|
||||
"Name": "AWSample",
|
||||
"AuthenticationType": "MSI"
|
||||
},
|
||||
"Extraction": {
|
||||
"Type": "Table",
|
||||
"FullOrIncremental": "Full",
|
||||
"IncrementalType": null,
|
||||
"TableSchema": "SalesLT",
|
||||
"TableName": "SalesOrderHeader"
|
||||
}
|
||||
},
|
||||
"Target": {
|
||||
"Type": "Azure Blob",
|
||||
"StorageAccountName": "https://adsgofastdatalakeaccelst.blob.core.windows.net",
|
||||
"StorageAccountContainer": "datalakeraw",
|
||||
"StorageAccountAccessMethod": "MSI",
|
||||
"RelativePath": "/AwSample/SalesLT/SalesOrderHeader/2020/7/9/14/12/",
|
||||
"DataFileName": "SalesLT.SalesOrderHeader.parquet",
|
||||
"SchemaFileName": "SalesLT.SalesOrderHeader",
|
||||
"FirstRowAsHeader": null,
|
||||
"SheetName": null,
|
||||
"SkipLineCount": null,
|
||||
"MaxConcorrentConnections": null
|
||||
},
|
||||
"DataFactory": {
|
||||
"Id": 1,
|
||||
"Name": "adsgofastdatakakeacceladf",
|
||||
"ResourceGroup": "AdsGoFastDataLakeAccel",
|
||||
"SubscriptionId": "035a1364-f00d-48e2-b582-4fe125905ee3",
|
||||
"ADFPipeline": "AZ_SQL_AZ_Storage_Parquet_@GF{IR}"
|
||||
}
|
||||
}
|
||||
},
|
||||
"Mapping": {
|
||||
"type": "object"
|
||||
},
|
||||
"NewWaterMark": {
|
||||
"type": "string"
|
||||
},
|
||||
"Item": {
|
||||
"type": "int"
|
||||
},
|
||||
"BatchCount": {
|
||||
"type": "int"
|
||||
}
|
||||
},
|
||||
"variables": {
|
||||
"SQLStatement": {
|
||||
"type": "String"
|
||||
}
|
||||
},
|
||||
"folder": {
|
||||
"name": "ADS Go Fast/Data Movement/@GF{IR}/Components"
|
||||
},
|
||||
"annotations": [],
|
||||
"lastPublishTime": "2020-07-29T02:39:55Z"
|
||||
},
|
||||
"type": "Microsoft.DataFactory/factories/pipelines"
|
||||
}
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
{
|
||||
"name": "AZ-Storage-Binary-AZ-Storage-Binary-AZ-IR",
|
||||
"name": "AZ_Storage_Binary_AZ_Storage_Binary_@GF{IR}",
|
||||
"properties": {
|
||||
"activities": [
|
||||
{
|
||||
|
|
@ -14,7 +14,7 @@
|
|||
},
|
||||
"cases": [
|
||||
{
|
||||
"value": "AzureBlobAzureBlob",
|
||||
"value": "Azure BlobAzure Blob",
|
||||
"activities": [
|
||||
{
|
||||
"name": "Copy Blob to Blob",
|
||||
|
|
@ -40,7 +40,18 @@
|
|||
"type": "BinarySource",
|
||||
"storeSettings": {
|
||||
"type": "AzureBlobStorageReadSettings",
|
||||
"recursive": true
|
||||
"recursive": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.Recursively",
|
||||
"type": "Expression"
|
||||
},
|
||||
"wildcardFileName": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.DataFileName",
|
||||
"type": "Expression"
|
||||
},
|
||||
"deleteFilesAfterCompletion": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.DeleteAfterCompletion",
|
||||
"type": "Expression"
|
||||
}
|
||||
},
|
||||
"formatSettings": {
|
||||
"type": "BinaryReadSettings"
|
||||
|
|
@ -49,7 +60,8 @@
|
|||
"sink": {
|
||||
"type": "BinarySink",
|
||||
"storeSettings": {
|
||||
"type": "AzureBlobStorageWriteSettings"
|
||||
"type": "AzureBlobStorageWriteSettings",
|
||||
"copyBehavior": "PreserveHierarchy"
|
||||
}
|
||||
},
|
||||
"enableStaging": false,
|
||||
|
|
@ -60,7 +72,7 @@
|
|||
},
|
||||
"inputs": [
|
||||
{
|
||||
"referenceName": "BlobBinary_AZ_IR",
|
||||
"referenceName": "BlobBinary_@GF{IR}",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"StorageAccountEndpoint": {
|
||||
|
|
@ -84,7 +96,7 @@
|
|||
],
|
||||
"outputs": [
|
||||
{
|
||||
"referenceName": "BlobBinary_AZ_IR",
|
||||
"referenceName": "BlobBinary_@GF{IR}",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"StorageAccountEndpoint": {
|
||||
|
|
@ -107,6 +119,34 @@
|
|||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "Pipeline AF Log - Blob to Blob Failed",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "Copy Blob to Blob",
|
||||
"dependencyConditions": [
|
||||
"Failed"
|
||||
]
|
||||
}
|
||||
],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
"parameters": {
|
||||
"Body": {
|
||||
"value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":1,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy Blob to Blob\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"EndDateTimeOffSet\":\"', string(utcnow()), '\",\"Comment\":\"', string(activity('Copy Blob to Blob').error.message), '\",\"Status\":\"Failed\"}'))",
|
||||
"type": "Expression"
|
||||
},
|
||||
"FunctionName": "Log",
|
||||
"Method": "Post"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Pipeline AF Log - Blob to Blob Start",
|
||||
"type": "ExecutePipeline",
|
||||
|
|
@ -114,17 +154,17 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-Function-Generic",
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
"parameters": {
|
||||
"Body": {
|
||||
"value": "[\n{\n \"TaskInstanceId\":@{pipeline().parameters.TaskObject.TaskInstanceId},\n \"ExecutionUid\":@{pipeline().parameters.TaskObject.ExecutionUid},\n \"RunId\":@{pipeline().RunId}, \n \"StartTime\":@{utcnow()},\n \"Status\":\"Processing\",\n \"Activity\":\"Copy Blob to Blob\",\n \"Comments\":\"\"\n}\n]",
|
||||
"value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":3,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy Blob to Blob\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"Status\":\"Started\"}'))",
|
||||
"type": "Expression"
|
||||
},
|
||||
"FunctionName": "Log",
|
||||
"Method": "Put"
|
||||
"Method": "Post"
|
||||
}
|
||||
}
|
||||
},
|
||||
|
|
@ -142,45 +182,17 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-Function-Generic",
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
"parameters": {
|
||||
"Body": {
|
||||
"value": "[\n{\n \"TaskInstanceId\":@{pipeline().parameters.TaskObject.TaskInstanceId},\n \"ExecutionUid\":@{pipeline().parameters.TaskObject.ExecutionUid},\n \"RunId\":@{pipeline().RunId}, \n \"EndTime\":@{utcnow()},\n \"Status\":\"Completed\",\n \"Activity\":\"Copy Blob to Blob\",\n \"Comments\":\"\"\n}\n]",
|
||||
"value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":1,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy Blob to Blob\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"EndDateTimeOffSet\":\"', string(utcnow()), '\",\"RowsInserted\":\"', string(activity('Copy Blob to Blob').output.filesWritten), '\",\"Comment\":\"\",\"Status\":\"Complete\"}'))",
|
||||
"type": "Expression"
|
||||
},
|
||||
"FunctionName": "Log",
|
||||
"Method": "Put"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Pipeline AF Log - Blob to Blob Failed",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "Copy Blob to Blob",
|
||||
"dependencyConditions": [
|
||||
"Failed"
|
||||
]
|
||||
}
|
||||
],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-Function-Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
"parameters": {
|
||||
"Body": {
|
||||
"value": "[\n{\n \"TaskInstanceId\":@{pipeline().parameters.TaskObject.TaskInstanceId},\n \"ExecutionUid\":@{pipeline().parameters.TaskObject.ExecutionUid},\n \"RunId\":@{pipeline().RunId}, \n \"EndTime\":@{utcnow()},\n \"Status\":\"Failed\",\n \"Activity\":\"Copy Blob to Blob\",\n \"Comments\":@{activity('Copy Blob to Blob').errors}\n}\n]",
|
||||
"type": "Expression"
|
||||
},
|
||||
"FunctionName": "Log",
|
||||
"Method": "Put"
|
||||
"Method": "Post"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -213,8 +225,18 @@
|
|||
"type": "BinarySource",
|
||||
"storeSettings": {
|
||||
"type": "AzureBlobFSReadSettings",
|
||||
"recursive": true,
|
||||
"deleteFilesAfterCompletion": true
|
||||
"recursive": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.Recursively",
|
||||
"type": "Expression"
|
||||
},
|
||||
"wildcardFileName": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.DataFileName",
|
||||
"type": "Expression"
|
||||
},
|
||||
"deleteFilesAfterCompletion": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.DeleteAfterCompletion",
|
||||
"type": "Expression"
|
||||
}
|
||||
},
|
||||
"formatSettings": {
|
||||
"type": "BinaryReadSettings"
|
||||
|
|
@ -223,7 +245,8 @@
|
|||
"sink": {
|
||||
"type": "BinarySink",
|
||||
"storeSettings": {
|
||||
"type": "AzureBlobFSWriteSettings"
|
||||
"type": "AzureBlobFSWriteSettings",
|
||||
"copyBehavior": "PreserveHierarchy"
|
||||
}
|
||||
},
|
||||
"enableStaging": false,
|
||||
|
|
@ -234,7 +257,7 @@
|
|||
},
|
||||
"inputs": [
|
||||
{
|
||||
"referenceName": "ADLSBinary_AZ_IR",
|
||||
"referenceName": "ADLSBinary_@GF{IR}",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"StorageAccountEndpoint": {
|
||||
|
|
@ -258,7 +281,7 @@
|
|||
],
|
||||
"outputs": [
|
||||
{
|
||||
"referenceName": "ADLSBinary_AZ_IR",
|
||||
"referenceName": "ADLSBinary_@GF{IR}",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"StorageAccountEndpoint": {
|
||||
|
|
@ -281,6 +304,34 @@
|
|||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "Pipeline AF Log - ADLS to ADLS Failed",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "Copy ADLS to ADLS",
|
||||
"dependencyConditions": [
|
||||
"Failed"
|
||||
]
|
||||
}
|
||||
],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
"parameters": {
|
||||
"Body": {
|
||||
"value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":1,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy ADLS to ADLS\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"EndDateTimeOffSet\":\"', string(utcnow()), '\",\"Comment\":\"', string(activity('Copy ADLS to ADLS').error.message), '\",\"Status\":\"Failed\"}'))",
|
||||
"type": "Expression"
|
||||
},
|
||||
"FunctionName": "Log",
|
||||
"Method": "Post"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Pipeline AF Log - ADLS to ADLS Start",
|
||||
"type": "ExecutePipeline",
|
||||
|
|
@ -288,17 +339,17 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-Function-Generic",
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
"parameters": {
|
||||
"Body": {
|
||||
"value": "[\n{\n \"TaskInstanceId\":@{pipeline().parameters.TaskObject.TaskInstanceId},\n \"ExecutionUid\":@{pipeline().parameters.TaskObject.ExecutionUid},\n \"RunId\":@{pipeline().RunId}, \n \"StartTime\":@{utcnow()},\n \"Status\":\"Processing\",\n \"Activity\":\"Copy ADLS to ADLS\",\n \"Comments\":\"\"\n}\n]",
|
||||
"value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":3,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy ADLS to ADLS\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"Status\":\"Started\"}'))",
|
||||
"type": "Expression"
|
||||
},
|
||||
"FunctionName": "Log",
|
||||
"Method": "Put"
|
||||
"Method": "Post"
|
||||
}
|
||||
}
|
||||
},
|
||||
|
|
@ -316,52 +367,24 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-Function-Generic",
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
"parameters": {
|
||||
"Body": {
|
||||
"value": "[\n{\n \"TaskInstanceId\":@{pipeline().parameters.TaskObject.TaskInstanceId},\n \"ExecutionUid\":@{pipeline().parameters.TaskObject.ExecutionUid},\n \"RunId\":@{pipeline().RunId}, \n \"EndTime\":@{utcnow()},\n \"Status\":\"Completed\",\n \"Activity\":\"Copy ADLS to ADLS\",\n \"Comments\":\"\"\n}\n]",
|
||||
"value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":1,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy ADLS to ADLS\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"EndDateTimeOffSet\":\"', string(utcnow()), '\",\"RowsInserted\":\"', string(activity('Copy ADLS to ADLS').output.filesWritten), '\",\"Comment\":\"\",\"Status\":\"Complete\"}'))",
|
||||
"type": "Expression"
|
||||
},
|
||||
"FunctionName": "Log",
|
||||
"Method": "Put"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Pipeline AF Log - ADLS to ADLS Failed",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "Copy ADLS to ADLS",
|
||||
"dependencyConditions": [
|
||||
"Failed"
|
||||
]
|
||||
}
|
||||
],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-Function-Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
"parameters": {
|
||||
"Body": {
|
||||
"value": "[\n{\n \"TaskInstanceId\":@{pipeline().parameters.TaskObject.TaskInstanceId},\n \"ExecutionUid\":@{pipeline().parameters.TaskObject.ExecutionUid},\n \"RunId\":@{pipeline().RunId}, \n \"EndTime\":@{utcnow()},\n \"Status\":\"Failed\",\n \"Activity\":\"Copy ADLS to ADLS\",\n \"Comments\":@{activity('Copy ADLS to ADLS').errors}\n}\n]",
|
||||
"type": "Expression"
|
||||
},
|
||||
"FunctionName": "Log",
|
||||
"Method": "Put"
|
||||
"Method": "Post"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"value": "AzureBlobADLS",
|
||||
"value": "Azure BlobADLS",
|
||||
"activities": [
|
||||
{
|
||||
"name": "Copy Blob to ADLS",
|
||||
|
|
@ -387,7 +410,18 @@
|
|||
"type": "BinarySource",
|
||||
"storeSettings": {
|
||||
"type": "AzureBlobStorageReadSettings",
|
||||
"recursive": true
|
||||
"recursive": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.Recursively",
|
||||
"type": "Expression"
|
||||
},
|
||||
"wildcardFileName": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.DataFileName",
|
||||
"type": "Expression"
|
||||
},
|
||||
"deleteFilesAfterCompletion": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.DeleteAfterCompletion",
|
||||
"type": "Expression"
|
||||
}
|
||||
},
|
||||
"formatSettings": {
|
||||
"type": "BinaryReadSettings"
|
||||
|
|
@ -396,7 +430,8 @@
|
|||
"sink": {
|
||||
"type": "BinarySink",
|
||||
"storeSettings": {
|
||||
"type": "AzureBlobFSWriteSettings"
|
||||
"type": "AzureBlobFSWriteSettings",
|
||||
"copyBehavior": "PreserveHierarchy"
|
||||
}
|
||||
},
|
||||
"enableStaging": false,
|
||||
|
|
@ -407,7 +442,7 @@
|
|||
},
|
||||
"inputs": [
|
||||
{
|
||||
"referenceName": "BlobBinary_AZ_IR",
|
||||
"referenceName": "BlobBinary_@GF{IR}",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"StorageAccountEndpoint": {
|
||||
|
|
@ -431,7 +466,7 @@
|
|||
],
|
||||
"outputs": [
|
||||
{
|
||||
"referenceName": "ADLSBinary_AZ_IR",
|
||||
"referenceName": "ADLSBinary_@GF{IR}",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"StorageAccountEndpoint": {
|
||||
|
|
@ -454,6 +489,34 @@
|
|||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "Pipeline AF Log - Blob to ADLS Failed",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "Copy Blob to ADLS",
|
||||
"dependencyConditions": [
|
||||
"Failed"
|
||||
]
|
||||
}
|
||||
],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
"parameters": {
|
||||
"Body": {
|
||||
"value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":1,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy Blob to ADLS\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"EndDateTimeOffSet\":\"', string(utcnow()), '\",\"Comment\":\"', string(activity('Copy Blob to ADLS').error.message), '\",\"Status\":\"Failed\"}'))",
|
||||
"type": "Expression"
|
||||
},
|
||||
"FunctionName": "Log",
|
||||
"Method": "Post"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Pipeline AF Log - Blob to ADLS Start",
|
||||
"type": "ExecutePipeline",
|
||||
|
|
@ -461,17 +524,17 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-Function-Generic",
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
"parameters": {
|
||||
"Body": {
|
||||
"value": "[\n{\n \"TaskInstanceId\":@{pipeline().parameters.TaskObject.TaskInstanceId},\n \"ExecutionUid\":@{pipeline().parameters.TaskObject.ExecutionUid},\n \"RunId\":@{pipeline().RunId}, \n \"StartTime\":@{utcnow()},\n \"Status\":\"Processing\",\n \"Activity\":\"Copy Blob to ADLS\",\n \"Comments\":\"\"\n}\n]",
|
||||
"value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":3,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy Blob to ADLS\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"Status\":\"Started\"}'))",
|
||||
"type": "Expression"
|
||||
},
|
||||
"FunctionName": "Log",
|
||||
"Method": "Put"
|
||||
"Method": "Post"
|
||||
}
|
||||
}
|
||||
},
|
||||
|
|
@ -489,52 +552,24 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-Function-Generic",
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
"parameters": {
|
||||
"Body": {
|
||||
"value": "[\n{\n \"TaskInstanceId\":@{pipeline().parameters.TaskObject.TaskInstanceId},\n \"ExecutionUid\":@{pipeline().parameters.TaskObject.ExecutionUid},\n \"RunId\":@{pipeline().RunId}, \n \"EndTime\":@{utcnow()},\n \"Status\":\"Completed\",\n \"Activity\":\"Copy Blob to ADLS\",\n \"Comments\":\"\"\n}\n]",
|
||||
"value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":1,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy Blob to ADLS\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"EndDateTimeOffSet\":\"', string(utcnow()), '\",\"RowsInserted\":\"', string(activity('Copy Blob to ADLS').output.filesWritten), '\",\"Comment\":\"\",\"Status\":\"Complete\"}'))",
|
||||
"type": "Expression"
|
||||
},
|
||||
"FunctionName": "Log",
|
||||
"Method": "Put"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Pipeline AF Log - Blob to ADLS Failed",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "Copy Blob to ADLS",
|
||||
"dependencyConditions": [
|
||||
"Failed"
|
||||
]
|
||||
}
|
||||
],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-Function-Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
"parameters": {
|
||||
"Body": {
|
||||
"value": "[\n{\n \"TaskInstanceId\":@{pipeline().parameters.TaskObject.TaskInstanceId},\n \"ExecutionUid\":@{pipeline().parameters.TaskObject.ExecutionUid},\n \"RunId\":@{pipeline().RunId}, \n \"EndTime\":@{utcnow()},\n \"Status\":\"Failed\",\n \"Activity\":\"Copy Blob to ADLS\",\n \"Comments\":@{activity('Copy Blob to ADLS').errors}\n}\n]",
|
||||
"type": "Expression"
|
||||
},
|
||||
"FunctionName": "Log",
|
||||
"Method": "Put"
|
||||
"Method": "Post"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"value": "ADLSAzureBlob",
|
||||
"value": "ADLSAzure Blob",
|
||||
"activities": [
|
||||
{
|
||||
"name": "Copy ADLS to Blob",
|
||||
|
|
@ -560,8 +595,18 @@
|
|||
"type": "BinarySource",
|
||||
"storeSettings": {
|
||||
"type": "AzureBlobFSReadSettings",
|
||||
"recursive": true,
|
||||
"deleteFilesAfterCompletion": true
|
||||
"recursive": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.Recursively",
|
||||
"type": "Expression"
|
||||
},
|
||||
"wildcardFileName": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.DataFileName",
|
||||
"type": "Expression"
|
||||
},
|
||||
"deleteFilesAfterCompletion": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.DeleteAfterCompletion",
|
||||
"type": "Expression"
|
||||
}
|
||||
},
|
||||
"formatSettings": {
|
||||
"type": "BinaryReadSettings"
|
||||
|
|
@ -570,7 +615,8 @@
|
|||
"sink": {
|
||||
"type": "BinarySink",
|
||||
"storeSettings": {
|
||||
"type": "AzureBlobStorageWriteSettings"
|
||||
"type": "AzureBlobStorageWriteSettings",
|
||||
"copyBehavior": "PreserveHierarchy"
|
||||
}
|
||||
},
|
||||
"enableStaging": false,
|
||||
|
|
@ -581,7 +627,7 @@
|
|||
},
|
||||
"inputs": [
|
||||
{
|
||||
"referenceName": "ADLSBinary_AZ_IR",
|
||||
"referenceName": "ADLSBinary_@GF{IR}",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"StorageAccountEndpoint": {
|
||||
|
|
@ -605,7 +651,7 @@
|
|||
],
|
||||
"outputs": [
|
||||
{
|
||||
"referenceName": "BlobBinary_AZ_IR",
|
||||
"referenceName": "BlobBinary_@GF{IR}",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"StorageAccountEndpoint": {
|
||||
|
|
@ -628,6 +674,34 @@
|
|||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "Pipeline AF Log - ADLS to Blob Failed",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "Copy ADLS to Blob",
|
||||
"dependencyConditions": [
|
||||
"Failed"
|
||||
]
|
||||
}
|
||||
],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
"parameters": {
|
||||
"Body": {
|
||||
"value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":1,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy ADLS to Blob\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"EndDateTimeOffSet\":\"', string(utcnow()), '\",\"Comment\":\"', string(activity('Copy ADLS to Blob').error.message), '\",\"Status\":\"Failed\"}'))",
|
||||
"type": "Expression"
|
||||
},
|
||||
"FunctionName": "Log",
|
||||
"Method": "Post"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Pipeline AF Log - ADLS to Blob Start",
|
||||
"type": "ExecutePipeline",
|
||||
|
|
@ -635,17 +709,17 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-Function-Generic",
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
"parameters": {
|
||||
"Body": {
|
||||
"value": "[\n{\n \"TaskInstanceId\":@{pipeline().parameters.TaskObject.TaskInstanceId},\n \"ExecutionUid\":@{pipeline().parameters.TaskObject.ExecutionUid},\n \"RunId\":@{pipeline().RunId}, \n \"StartTime\":@{utcnow()},\n \"Status\":\"Processing\",\n \"Activity\":\"Copy ADLS to Blob\",\n \"Comments\":\"\"\n}\n]",
|
||||
"value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":3,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy ADLS to Blob\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"Status\":\"Started\"}'))",
|
||||
"type": "Expression"
|
||||
},
|
||||
"FunctionName": "Log",
|
||||
"Method": "Put"
|
||||
"Method": "Post"
|
||||
}
|
||||
}
|
||||
},
|
||||
|
|
@ -663,45 +737,17 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-Function-Generic",
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
"parameters": {
|
||||
"Body": {
|
||||
"value": "[\n{\n \"TaskInstanceId\":@{pipeline().parameters.TaskObject.TaskInstanceId},\n \"ExecutionUid\":@{pipeline().parameters.TaskObject.ExecutionUid},\n \"RunId\":@{pipeline().RunId}, \n \"EndTime\":@{utcnow()},\n \"Status\":\"Completed\",\n \"Activity\":\"Copy ADLS to Blob\",\n \"Comments\":\"\"\n}\n]",
|
||||
"value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":1,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy ADLS to Blob\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"EndDateTimeOffSet\":\"', string(utcnow()), '\",\"RowsInserted\":\"', string(activity('Copy ADLS to Blob').output.filesWritten), '\",\"Comment\":\"\",\"Status\":\"Complete\"}'))",
|
||||
"type": "Expression"
|
||||
},
|
||||
"FunctionName": "Log",
|
||||
"Method": "Put"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Pipeline AF Log - ADLS to Blob Failed",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "Copy ADLS to Blob",
|
||||
"dependencyConditions": [
|
||||
"Failed"
|
||||
]
|
||||
}
|
||||
],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-Function-Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
"parameters": {
|
||||
"Body": {
|
||||
"value": "[\n{\n \"TaskInstanceId\":@{pipeline().parameters.TaskObject.TaskInstanceId},\n \"ExecutionUid\":@{pipeline().parameters.TaskObject.ExecutionUid},\n \"RunId\":@{pipeline().RunId}, \n \"EndTime\":@{utcnow()},\n \"Status\":\"Failed\",\n \"Activity\":\"Copy ADLS to Blob\",\n \"Comments\":@{activity('Copy ADLS to Blob').errors}\n}\n]",
|
||||
"type": "Expression"
|
||||
},
|
||||
"FunctionName": "Log",
|
||||
"Method": "Put"
|
||||
"Method": "Post"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -714,14 +760,14 @@
|
|||
"parameters": {
|
||||
"TaskObject": {
|
||||
"type": "object",
|
||||
"defaultValue": "[ { \"TaskInstanceId\": 2, \"TaskMasterId\": 1, \"TaskStatus\": \"Untried\", \"TaskType\": \"RelationalDataExtractToDataLake\", \"Enabled\": true, \"ExecutionUid\": 1, \"Target\": { \"Database\": { \"Name\": \"MetadataDB\", \"SystemName\" : \"adsgofastdatakakeaccelsqlsvr.database.windows.net\", \"Type\": \"AzureSQL\", \"AuthenticationType\": \"MSI\", \"UsernameKeyVaultSecretName\": \"\", \"PasswordKeyVaultSecretName\": \"\" }, \"Extraction\": { \"Type\": \"Table\", \"FullOrIncremental\": true, \"IncrementalType\": null, \"TableSchema\" : \"stage\", \"TableName\": \"Customer\" } }, \"Source\": { \"StorageAccountName\": \"https://adsgofastdatalakeaccelst.dfs.core.windows.net\", \"Type\" : \"ADLS\", \"StorageAccountContainer\": \"Container\", \"StorageAccountContainerName\": \"datalakelanding\", \"StorageAccountAccessMethod\": \"MSI\", \"StorageAccountSASUriKeyVaultSecretName\": null, \"RelativePath\": \"/Unprocessed/adsgofastdatakakeaccelsqlsvr/AWSample/SalesLT/2020/06/08/17/\", \"DataFileName\": \"Customer_Data.parquet\", \"SchemaFileName\": \"Customer_Schema.json\" }, \"DataFactory\": { \"Id\": 1, \"Name\": \"adsgofastdatakakeacceladf\", \"ResourceGroup\": \"AdsGoFastDataLakeAccel\", \"SubscriptionId\": \"035a1364-f00d-48e2-b582-4fe125905ee3\", } } ] "
|
||||
"defaultValue": "[ { \"TaskInstanceId\": 2, \"TaskMasterId\": 1, \"TaskStatus\": \"Untried\", \"TaskType\": \"AzureStorageToAzureStorage_@GF{IR}\", \"Enabled\": true, \"ExecutionUid\": 1, \"KeyVaultBaseUrl\": \"https://adsgofastkeyvault.vault.azure.net/\", \"Source\": { \"StorageAccountName\": \"https://adsgofastdatalakeadls.dfs.core.windows.net\", \"Type\": \"ADLS\", \"StorageAccountContainer\": \"datalakelanding\", \"StorageAccountAccessMethod\": \"MSI\", \"StorageAccountSASUriKeyVaultSecretName\": null, \"RelativePath\": \"/Unprocessed/adsgofastdatakakeaccelsqlsvr/AWSample/SalesLT/2020/06/08/17/\", \"DataFileName\": \"Customer_Data.parquet\", \"SchemaFileName\": \"Customer_Schema.json\" }, \"Target\": { \"StorageAccountName\": \"https://adsgofastdatalakeadls.dfs.core.windows.net\", \"Type\": \"ADLS\", \"StorageAccountContainer\": \"datalakelanding\", \"StorageAccountAccessMethod\": \"MSI\", \"StorageAccountSASUriKeyVaultSecretName\": null, \"RelativePath\": \"/Processed/adsgofastdatakakeaccelsqlsvr/AWSample/SalesLT/2020/06/08/17/\", \"DataFileName\": \"Customer_Data.parquet\", \"SchemaFileName\": \"Customer_Schema.json\" }, \"DataFactory\": { \"Id\": 1, \"Name\": \"adsgofastdatakakeacceladf\", \"ResourceGroup\": \"AdsGoFastDataLakeAccel\", \"SubscriptionId\": \"035a1364-f00d-48e2-b582-4fe125905ee3\", } } ]"
|
||||
}
|
||||
},
|
||||
"folder": {
|
||||
"name": "ADS Go Fast/Data Movement/Auto Resolve IR"
|
||||
"name": "ADS Go Fast/Data Movement/@GF{IR}"
|
||||
},
|
||||
"annotations": [],
|
||||
"lastPublishTime": "2020-07-23T08:15:14Z"
|
||||
"lastPublishTime": "2020-08-05T04:14:00Z"
|
||||
},
|
||||
"type": "Microsoft.DataFactory/factories/pipelines"
|
||||
}
|
||||
|
|
@ -0,0 +1,460 @@
|
|||
{
|
||||
"name": "AZ_Storage_CSV_AZ_SQL_@GF{IR}",
|
||||
"properties": {
|
||||
"activities": [
|
||||
{
|
||||
"name": "Switch Storage Type",
|
||||
"type": "Switch",
|
||||
"dependsOn": [],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"on": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.Type",
|
||||
"type": "Expression"
|
||||
},
|
||||
"cases": [
|
||||
{
|
||||
"value": "ADLS",
|
||||
"activities": [
|
||||
{
|
||||
"name": "Copy ADLS to Azure SQL",
|
||||
"type": "Copy",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "Pipeline AF Log - ADLS to Azure SQL Start",
|
||||
"dependencyConditions": [
|
||||
"Succeeded"
|
||||
]
|
||||
}
|
||||
],
|
||||
"policy": {
|
||||
"timeout": "7.00:00:00",
|
||||
"retry": 0,
|
||||
"retryIntervalInSeconds": 30,
|
||||
"secureOutput": false,
|
||||
"secureInput": false
|
||||
},
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"source": {
|
||||
"type": "DelimitedTextSource",
|
||||
"storeSettings": {
|
||||
"type": "AzureBlobFSReadSettings",
|
||||
"maxConcurrentConnections": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.MaxConcorrentConnections",
|
||||
"type": "Expression"
|
||||
},
|
||||
"recursive": true,
|
||||
"wildcardFolderPath": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.RelativePath",
|
||||
"type": "Expression"
|
||||
},
|
||||
"wildcardFileName": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.DataFileName",
|
||||
"type": "Expression"
|
||||
},
|
||||
"enablePartitionDiscovery": false
|
||||
},
|
||||
"formatSettings": {
|
||||
"type": "DelimitedTextReadSettings",
|
||||
"skipLineCount": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.SkipLineCount",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
},
|
||||
"sink": {
|
||||
"type": "AzureSqlSink",
|
||||
"preCopyScript": {
|
||||
"value": "@{pipeline().parameters.TaskObject.Target.PreCopySQL}",
|
||||
"type": "Expression"
|
||||
},
|
||||
"tableOption": "autoCreate",
|
||||
"disableMetricsCollection": false
|
||||
},
|
||||
"enableStaging": false,
|
||||
"parallelCopies": {
|
||||
"value": "@pipeline().parameters.TaskObject.DegreeOfCopyParallelism",
|
||||
"type": "Expression"
|
||||
},
|
||||
"translator": {
|
||||
"value": "@pipeline().parameters.TaskObject.Target.DynamicMapping",
|
||||
"type": "Expression"
|
||||
}
|
||||
},
|
||||
"inputs": [
|
||||
{
|
||||
"referenceName": "ADLSDelimitedText_@GF{IR}",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"RelativePath": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.RelativePath",
|
||||
"type": "Expression"
|
||||
},
|
||||
"FileName": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.DataFileName",
|
||||
"type": "Expression"
|
||||
},
|
||||
"StorageAccountEndpoint": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.StorageAccountName",
|
||||
"type": "Expression"
|
||||
},
|
||||
"StorageAccountContainerName": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.StorageAccountContainer",
|
||||
"type": "Expression"
|
||||
},
|
||||
"FirstRownAsHeader": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.FirstRowAsHeader",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"referenceName": "AzureSqlTable_@GF{IR}",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"Schema": {
|
||||
"value": "@pipeline().parameters.TaskObject.Target.StagingTableSchema",
|
||||
"type": "Expression"
|
||||
},
|
||||
"Table": {
|
||||
"value": "@pipeline().parameters.TaskObject.Target.StagingTableName",
|
||||
"type": "Expression"
|
||||
},
|
||||
"Server": {
|
||||
"value": "@pipeline().parameters.TaskObject.Target.Database.SystemName",
|
||||
"type": "Expression"
|
||||
},
|
||||
"Database": {
|
||||
"value": "@pipeline().parameters.TaskObject.Target.Database.Name",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "Pipeline AF Log - ADLS to Azure SQL Failed",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "Copy ADLS to Azure SQL",
|
||||
"dependencyConditions": [
|
||||
"Failed"
|
||||
]
|
||||
}
|
||||
],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
"parameters": {
|
||||
"Body": {
|
||||
"value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":1,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy ADLS to Azure SQL\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"EndDateTimeOffSet\":\"', string(utcnow()), '\",\"Comment\":\"', string(activity('Copy ADLS to Azure SQL').error.message), '\",\"Status\":\"Failed\"}'))",
|
||||
"type": "Expression"
|
||||
},
|
||||
"FunctionName": "Log",
|
||||
"Method": "Post"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Pipeline AF Log - ADLS to Azure SQL Start",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
"parameters": {
|
||||
"Body": {
|
||||
"value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":3,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy ADLS to Azure SQL\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"Status\":\"Started\"}'))",
|
||||
"type": "Expression"
|
||||
},
|
||||
"FunctionName": "Log",
|
||||
"Method": "Post"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Pipeline AF Log - ADLS to Azure SQL Succeed",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "Copy ADLS to Azure SQL",
|
||||
"dependencyConditions": [
|
||||
"Succeeded"
|
||||
]
|
||||
}
|
||||
],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
"parameters": {
|
||||
"Body": {
|
||||
"value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":1,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy ADLS to Azure SQL\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"EndDateTimeOffSet\":\"', string(utcnow()), '\",\"RowsInserted\":\"', string(activity('Copy ADLS to Azure SQL').output.rowsCopied), '\",\"Comment\":\"\",\"Status\":\"Complete\"}'))",
|
||||
"type": "Expression"
|
||||
},
|
||||
"FunctionName": "Log",
|
||||
"Method": "Post"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"value": "Azure Blob",
|
||||
"activities": [
|
||||
{
|
||||
"name": "Copy Blob to Azure SQL",
|
||||
"type": "Copy",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "Pipeline AF Log - BLOB to Azure SQL Start",
|
||||
"dependencyConditions": [
|
||||
"Succeeded"
|
||||
]
|
||||
}
|
||||
],
|
||||
"policy": {
|
||||
"timeout": "7.00:00:00",
|
||||
"retry": 0,
|
||||
"retryIntervalInSeconds": 30,
|
||||
"secureOutput": false,
|
||||
"secureInput": false
|
||||
},
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"source": {
|
||||
"type": "DelimitedTextSource",
|
||||
"storeSettings": {
|
||||
"type": "AzureBlobStorageReadSettings",
|
||||
"maxConcurrentConnections": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.MaxConcorrentConnections",
|
||||
"type": "Expression"
|
||||
},
|
||||
"recursive": true,
|
||||
"wildcardFolderPath": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.RelativePath",
|
||||
"type": "Expression"
|
||||
},
|
||||
"wildcardFileName": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.DataFileName",
|
||||
"type": "Expression"
|
||||
},
|
||||
"enablePartitionDiscovery": false
|
||||
},
|
||||
"formatSettings": {
|
||||
"type": "DelimitedTextReadSettings",
|
||||
"skipLineCount": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.SkipLineCount",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
},
|
||||
"sink": {
|
||||
"type": "AzureSqlSink",
|
||||
"preCopyScript": {
|
||||
"value": "@{pipeline().parameters.TaskObject.Target.PreCopySQL}",
|
||||
"type": "Expression"
|
||||
},
|
||||
"tableOption": "autoCreate",
|
||||
"disableMetricsCollection": false
|
||||
},
|
||||
"enableStaging": false,
|
||||
"parallelCopies": {
|
||||
"value": "@pipeline().parameters.TaskObject.DegreeOfCopyParallelism",
|
||||
"type": "Expression"
|
||||
},
|
||||
"translator": {
|
||||
"value": "@pipeline().parameters.TaskObject.Target.DynamicMapping",
|
||||
"type": "Expression"
|
||||
}
|
||||
},
|
||||
"inputs": [
|
||||
{
|
||||
"referenceName": "BlobDelimitedText_@GF{IR}",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"RelativePath": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.RelativePath",
|
||||
"type": "Expression"
|
||||
},
|
||||
"FileName": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.DataFileName",
|
||||
"type": "Expression"
|
||||
},
|
||||
"StorageAccountEndpoint": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.StorageAccountName",
|
||||
"type": "Expression"
|
||||
},
|
||||
"StorageAccountContainerName": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.StorageAccountContainer",
|
||||
"type": "Expression"
|
||||
},
|
||||
"FirstRowAsHeader": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.FirstRowAsHeader",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"referenceName": "AzureSqlTable_@GF{IR}",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"Schema": {
|
||||
"value": "@pipeline().parameters.TaskObject.Target.StagingTableSchema",
|
||||
"type": "Expression"
|
||||
},
|
||||
"Table": {
|
||||
"value": "@pipeline().parameters.TaskObject.Target.StagingTableName",
|
||||
"type": "Expression"
|
||||
},
|
||||
"Server": {
|
||||
"value": "@pipeline().parameters.TaskObject.Target.Database.SystemName",
|
||||
"type": "Expression"
|
||||
},
|
||||
"Database": {
|
||||
"value": "@pipeline().parameters.TaskObject.Target.Database.Name",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "Pipeline AF Log - BLOB to Azure SQL Failed",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "Copy Blob to Azure SQL",
|
||||
"dependencyConditions": [
|
||||
"Failed"
|
||||
]
|
||||
}
|
||||
],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
"parameters": {
|
||||
"Body": {
|
||||
"value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":1,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy Blob to Azure SQL\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"EndDateTimeOffSet\":\"', string(utcnow()), '\",\"Comment\":\"', string(activity('Copy Blob to Azure SQL').error.message), '\",\"Status\":\"Failed\"}'))",
|
||||
"type": "Expression"
|
||||
},
|
||||
"FunctionName": "Log",
|
||||
"Method": "Post"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Pipeline AF Log - BLOB to Azure SQL Start",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
"parameters": {
|
||||
"Body": {
|
||||
"value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":3,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy Blob to Azure SQL\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"Status\":\"Started\"}'))",
|
||||
"type": "Expression"
|
||||
},
|
||||
"FunctionName": "Log",
|
||||
"Method": "Post"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Pipeline AF Log - Blob to Azure SQL Succeed",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "Copy Blob to Azure SQL",
|
||||
"dependencyConditions": [
|
||||
"Succeeded"
|
||||
]
|
||||
}
|
||||
],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
"parameters": {
|
||||
"Body": {
|
||||
"value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":1,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy Blob to Azure SQL\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"EndDateTimeOffSet\":\"', string(utcnow()), '\",\"RowsInserted\":\"', string(activity('Copy Blob to Azure SQL').output.rowsCopied), '\",\"Comment\":\"\",\"Status\":\"Complete\"}'))",
|
||||
"type": "Expression"
|
||||
},
|
||||
"FunctionName": "Log",
|
||||
"Method": "Post"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Execute AZ_SQL_Post-Copy",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "Switch Storage Type",
|
||||
"dependencyConditions": [
|
||||
"Succeeded"
|
||||
]
|
||||
}
|
||||
],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ_SQL_Post_Copy_@GF{IR}",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": true,
|
||||
"parameters": {
|
||||
"TaskObject": {
|
||||
"value": "@pipeline().parameters.TaskObject",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"TaskObject": {
|
||||
"type": "object"
|
||||
}
|
||||
},
|
||||
"folder": {
|
||||
"name": "ADS Go Fast/Data Movement/@GF{IR}"
|
||||
},
|
||||
"annotations": [],
|
||||
"lastPublishTime": "2020-07-29T09:43:40Z"
|
||||
},
|
||||
"type": "Microsoft.DataFactory/factories/pipelines"
|
||||
}
|
||||
|
|
@ -0,0 +1,420 @@
|
|||
{
|
||||
"name": "AZ_Storage_Excel_AZ_SQL_@GF{IR}",
|
||||
"properties": {
|
||||
"activities": [
|
||||
{
|
||||
"name": "Switch Storage Type",
|
||||
"type": "Switch",
|
||||
"dependsOn": [],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"on": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.Type",
|
||||
"type": "Expression"
|
||||
},
|
||||
"cases": [
|
||||
{
|
||||
"value": "ADLS",
|
||||
"activities": [
|
||||
{
|
||||
"name": "Copy ADLS to Azure SQL",
|
||||
"type": "Copy",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "Pipeline AF Log - ADLS to Azure SQL Start",
|
||||
"dependencyConditions": [
|
||||
"Succeeded"
|
||||
]
|
||||
}
|
||||
],
|
||||
"policy": {
|
||||
"timeout": "7.00:00:00",
|
||||
"retry": 0,
|
||||
"retryIntervalInSeconds": 30,
|
||||
"secureOutput": false,
|
||||
"secureInput": false
|
||||
},
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"source": {
|
||||
"type": "ExcelSource",
|
||||
"storeSettings": {
|
||||
"type": "AzureBlobFSReadSettings",
|
||||
"recursive": true
|
||||
}
|
||||
},
|
||||
"sink": {
|
||||
"type": "AzureSqlSink",
|
||||
"preCopyScript": {
|
||||
"value": "@{pipeline().parameters.TaskObject.Target.PreCopySQL}",
|
||||
"type": "Expression"
|
||||
},
|
||||
"tableOption": "autoCreate",
|
||||
"disableMetricsCollection": false
|
||||
},
|
||||
"enableStaging": false,
|
||||
"parallelCopies": {
|
||||
"value": "@pipeline().parameters.TaskObject.DegreeOfCopyParallelism",
|
||||
"type": "Expression"
|
||||
}
|
||||
},
|
||||
"inputs": [
|
||||
{
|
||||
"referenceName": "ADLSExcel_@GF{IR}",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"RelativePath": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.RelativePath",
|
||||
"type": "Expression"
|
||||
},
|
||||
"FileName": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.DataFileName",
|
||||
"type": "Expression"
|
||||
},
|
||||
"StorageAccountEndpoint": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.StorageAccountName",
|
||||
"type": "Expression"
|
||||
},
|
||||
"StorageAccountContainerName": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.StorageAccountContainer",
|
||||
"type": "Expression"
|
||||
},
|
||||
"SheetName": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.SheetName",
|
||||
"type": "Expression"
|
||||
},
|
||||
"FirstRowAsHeader": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.FirstRowAsHeader",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"referenceName": "AzureSqlTable_@GF{IR}",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"Schema": {
|
||||
"value": "@pipeline().parameters.TaskObject.Target.StagingTableSchema",
|
||||
"type": "Expression"
|
||||
},
|
||||
"Table": {
|
||||
"value": "@pipeline().parameters.TaskObject.Target.StagingTableName",
|
||||
"type": "Expression"
|
||||
},
|
||||
"Server": {
|
||||
"value": "@pipeline().parameters.TaskObject.Target.Database.SystemName",
|
||||
"type": "Expression"
|
||||
},
|
||||
"Database": {
|
||||
"value": "@pipeline().parameters.TaskObject.Target.Database.Name",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "Pipeline AF Log - ADLS to Azure SQL Failed",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "Copy ADLS to Azure SQL",
|
||||
"dependencyConditions": [
|
||||
"Failed"
|
||||
]
|
||||
}
|
||||
],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
"parameters": {
|
||||
"Body": {
|
||||
"value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":1,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy ADLS to Azure SQL\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"EndDateTimeOffSet\":\"', string(utcnow()), '\",\"Comment\":\"', string(activity('Copy ADLS to Azure SQL').error.message), '\",\"Status\":\"Failed\"}'))",
|
||||
"type": "Expression"
|
||||
},
|
||||
"FunctionName": "Log",
|
||||
"Method": "Post"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Pipeline AF Log - ADLS to Azure SQL Start",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
"parameters": {
|
||||
"Body": {
|
||||
"value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":3,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy ADLS to Azure SQL\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"Status\":\"Started\"}'))",
|
||||
"type": "Expression"
|
||||
},
|
||||
"FunctionName": "Log",
|
||||
"Method": "Post"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Pipeline AF Log - ADLS to Azure SQL Succeed",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "Copy ADLS to Azure SQL",
|
||||
"dependencyConditions": [
|
||||
"Succeeded"
|
||||
]
|
||||
}
|
||||
],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
"parameters": {
|
||||
"Body": {
|
||||
"value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":1,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy ADLS to Azure SQL\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"EndDateTimeOffSet\":\"', string(utcnow()), '\",\"RowsInserted\":\"', string(activity('Copy ADLS to Azure SQL').output.rowsCopied), '\",\"Comment\":\"\",\"Status\":\"Complete\"}'))",
|
||||
"type": "Expression"
|
||||
},
|
||||
"FunctionName": "Log",
|
||||
"Method": "Post"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"value": "Azure Blob",
|
||||
"activities": [
|
||||
{
|
||||
"name": "Copy Blob to Azure SQL",
|
||||
"type": "Copy",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "Pipeline AF Log - BLOB to Azure SQL Start",
|
||||
"dependencyConditions": [
|
||||
"Succeeded"
|
||||
]
|
||||
}
|
||||
],
|
||||
"policy": {
|
||||
"timeout": "7.00:00:00",
|
||||
"retry": 0,
|
||||
"retryIntervalInSeconds": 30,
|
||||
"secureOutput": false,
|
||||
"secureInput": false
|
||||
},
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"source": {
|
||||
"type": "ExcelSource",
|
||||
"storeSettings": {
|
||||
"type": "AzureBlobStorageReadSettings",
|
||||
"recursive": true
|
||||
}
|
||||
},
|
||||
"sink": {
|
||||
"type": "AzureSqlSink",
|
||||
"preCopyScript": {
|
||||
"value": "@{pipeline().parameters.TaskObject.Target.PreCopySQL}",
|
||||
"type": "Expression"
|
||||
},
|
||||
"tableOption": "autoCreate",
|
||||
"disableMetricsCollection": false
|
||||
},
|
||||
"enableStaging": false,
|
||||
"parallelCopies": {
|
||||
"value": "@pipeline().parameters.TaskObject.DegreeOfCopyParallelism",
|
||||
"type": "Expression"
|
||||
}
|
||||
},
|
||||
"inputs": [
|
||||
{
|
||||
"referenceName": "BlobExcel_@GF{IR}",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"RelativePath": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.RelativePath",
|
||||
"type": "Expression"
|
||||
},
|
||||
"FileName": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.DataFileName",
|
||||
"type": "Expression"
|
||||
},
|
||||
"StorageAccountEndpoint": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.StorageAccountName",
|
||||
"type": "Expression"
|
||||
},
|
||||
"StorageAccountContainerName": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.StorageAccountContainer",
|
||||
"type": "Expression"
|
||||
},
|
||||
"SheetName": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.SheetName",
|
||||
"type": "Expression"
|
||||
},
|
||||
"FirstRowAsHeader": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.FirstRowAsHeader",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"referenceName": "AzureSqlTable_@GF{IR}",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"Schema": {
|
||||
"value": "@pipeline().parameters.TaskObject.Target.StagingTableSchema",
|
||||
"type": "Expression"
|
||||
},
|
||||
"Table": {
|
||||
"value": "@pipeline().parameters.TaskObject.Target.StagingTableName",
|
||||
"type": "Expression"
|
||||
},
|
||||
"Server": {
|
||||
"value": "@pipeline().parameters.TaskObject.Target.Database.SystemName",
|
||||
"type": "Expression"
|
||||
},
|
||||
"Database": {
|
||||
"value": "@pipeline().parameters.TaskObject.Target.Database.Name",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "Pipeline AF Log - BLOB to Azure SQL Failed",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "Copy Blob to Azure SQL",
|
||||
"dependencyConditions": [
|
||||
"Failed"
|
||||
]
|
||||
}
|
||||
],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
"parameters": {
|
||||
"Body": {
|
||||
"value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":1,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy Blob to Azure SQL\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"EndDateTimeOffSet\":\"', string(utcnow()), '\",\"Comment\":\"', string(activity('Copy Blob to Azure SQL').error.message), '\",\"Status\":\"Failed\"}'))",
|
||||
"type": "Expression"
|
||||
},
|
||||
"FunctionName": "Log",
|
||||
"Method": "Post"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Pipeline AF Log - BLOB to Azure SQL Start",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
"parameters": {
|
||||
"Body": {
|
||||
"value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":3,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy Blob to Azure SQL\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"Status\":\"Started\"}'))",
|
||||
"type": "Expression"
|
||||
},
|
||||
"FunctionName": "Log",
|
||||
"Method": "Post"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Pipeline AF Log - Blob to Azure SQL Succeed",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "Copy Blob to Azure SQL",
|
||||
"dependencyConditions": [
|
||||
"Succeeded"
|
||||
]
|
||||
}
|
||||
],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
"parameters": {
|
||||
"Body": {
|
||||
"value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":1,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy Blob to Azure SQL\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"EndDateTimeOffSet\":\"', string(utcnow()), '\",\"RowsInserted\":\"', string(activity('Copy Blob to Azure SQL').output.rowsCopied), '\",\"Comment\":\"\",\"Status\":\"Complete\"}'))",
|
||||
"type": "Expression"
|
||||
},
|
||||
"FunctionName": "Log",
|
||||
"Method": "Post"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Execute AZ_SQL_Post-Copy",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "Switch Storage Type",
|
||||
"dependencyConditions": [
|
||||
"Succeeded"
|
||||
]
|
||||
}
|
||||
],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ_SQL_Post_Copy_@GF{IR}",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": true,
|
||||
"parameters": {
|
||||
"TaskObject": {
|
||||
"value": "@pipeline().parameters.TaskObject",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"TaskObject": {
|
||||
"type": "object"
|
||||
}
|
||||
},
|
||||
"folder": {
|
||||
"name": "ADS Go Fast/Data Movement/@GF{IR}"
|
||||
},
|
||||
"annotations": [],
|
||||
"lastPublishTime": "2020-07-24T01:51:29Z"
|
||||
},
|
||||
"type": "Microsoft.DataFactory/factories/pipelines"
|
||||
}
|
||||
|
|
@ -0,0 +1,406 @@
|
|||
{
|
||||
"name": "AZ_Storage_Excel_AZ_Storage_CSV_@GF{IR}",
|
||||
"properties": {
|
||||
"activities": [
|
||||
{
|
||||
"name": "Switch Storage Type",
|
||||
"type": "Switch",
|
||||
"dependsOn": [],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"on": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.Type",
|
||||
"type": "Expression"
|
||||
},
|
||||
"cases": [
|
||||
{
|
||||
"value": "ADLS",
|
||||
"activities": [
|
||||
{
|
||||
"name": "Copy ADLS to ADLS",
|
||||
"type": "Copy",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "Pipeline AF Log - ADLS to ADLS Start",
|
||||
"dependencyConditions": [
|
||||
"Succeeded"
|
||||
]
|
||||
}
|
||||
],
|
||||
"policy": {
|
||||
"timeout": "7.00:00:00",
|
||||
"retry": 0,
|
||||
"retryIntervalInSeconds": 30,
|
||||
"secureOutput": false,
|
||||
"secureInput": false
|
||||
},
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"source": {
|
||||
"type": "ExcelSource",
|
||||
"storeSettings": {
|
||||
"type": "AzureBlobFSReadSettings",
|
||||
"recursive": true
|
||||
}
|
||||
},
|
||||
"sink": {
|
||||
"type": "DelimitedTextSink",
|
||||
"storeSettings": {
|
||||
"type": "AzureBlobFSWriteSettings"
|
||||
},
|
||||
"formatSettings": {
|
||||
"type": "DelimitedTextWriteSettings",
|
||||
"quoteAllText": true,
|
||||
"fileExtension": ".txt"
|
||||
}
|
||||
},
|
||||
"enableStaging": false,
|
||||
"parallelCopies": {
|
||||
"value": "@pipeline().parameters.TaskObject.DegreeOfCopyParallelism",
|
||||
"type": "Expression"
|
||||
}
|
||||
},
|
||||
"inputs": [
|
||||
{
|
||||
"referenceName": "ADLSExcel_@GF{IR}",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"RelativePath": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.RelativePath",
|
||||
"type": "Expression"
|
||||
},
|
||||
"FileName": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.DataFileName",
|
||||
"type": "Expression"
|
||||
},
|
||||
"StorageAccountEndpoint": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.StorageAccountName",
|
||||
"type": "Expression"
|
||||
},
|
||||
"StorageAccountContainerName": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.StorageAccountContainer",
|
||||
"type": "Expression"
|
||||
},
|
||||
"SheetName": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.SheetName",
|
||||
"type": "Expression"
|
||||
},
|
||||
"FirstRowAsHeader": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.FirstRowAsHeader",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"referenceName": "ADLSDelimitedText_@GF{IR}",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"RelativePath": {
|
||||
"value": "@pipeline().parameters.TaskObject.Target.RelativePath",
|
||||
"type": "Expression"
|
||||
},
|
||||
"FileName": {
|
||||
"value": "@pipeline().parameters.TaskObject.Target.DataFileName",
|
||||
"type": "Expression"
|
||||
},
|
||||
"StorageAccountEndpoint": {
|
||||
"value": "@pipeline().parameters.TaskObject.Target.StorageAccountName",
|
||||
"type": "Expression"
|
||||
},
|
||||
"StorageAccountContainerName": {
|
||||
"value": "@pipeline().parameters.TaskObject.Target.StorageAccountContainer",
|
||||
"type": "Expression"
|
||||
},
|
||||
"FirstRownAsHeader": {
|
||||
"value": "@pipeline().parameters.TaskObject.Target.FirstRowAsHeader",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "Pipeline AF Log - ADLS to ADLS Failed",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "Copy ADLS to ADLS",
|
||||
"dependencyConditions": [
|
||||
"Failed"
|
||||
]
|
||||
}
|
||||
],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
"parameters": {
|
||||
"Body": {
|
||||
"value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":1,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy ADLS to ADLS\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"EndDateTimeOffSet\":\"', string(utcnow()), '\",\"Comment\":\"', string(activity('Copy ADLS to ADLS').error.message), '\",\"Status\":\"Failed\"}'))",
|
||||
"type": "Expression"
|
||||
},
|
||||
"FunctionName": "Log",
|
||||
"Method": "Post"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Pipeline AF Log - ADLS to ADLS Start",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
"parameters": {
|
||||
"Body": {
|
||||
"value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":3,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy ADLS to ADLS\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"Status\":\"Started\"}'))",
|
||||
"type": "Expression"
|
||||
},
|
||||
"FunctionName": "Log",
|
||||
"Method": "Post"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Pipeline AF Log - ADLS to ADLS Succeed",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "Copy ADLS to ADLS",
|
||||
"dependencyConditions": [
|
||||
"Succeeded"
|
||||
]
|
||||
}
|
||||
],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
"parameters": {
|
||||
"Body": {
|
||||
"value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":1,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy ADLS to ADLS\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"EndDateTimeOffSet\":\"', string(utcnow()), '\",\"RowsInserted\":\"', string(activity('Copy ADLS to ADLS').output.rowsCopied), '\",\"Comment\":\"\",\"Status\":\"Complete\"}'))",
|
||||
"type": "Expression"
|
||||
},
|
||||
"FunctionName": "Log",
|
||||
"Method": "Post"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"value": "Azure Blob",
|
||||
"activities": [
|
||||
{
|
||||
"name": "Copy Blob to Blob",
|
||||
"type": "Copy",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "Pipeline AF Log - BLOB to Blob Start",
|
||||
"dependencyConditions": [
|
||||
"Succeeded"
|
||||
]
|
||||
}
|
||||
],
|
||||
"policy": {
|
||||
"timeout": "7.00:00:00",
|
||||
"retry": 0,
|
||||
"retryIntervalInSeconds": 30,
|
||||
"secureOutput": false,
|
||||
"secureInput": false
|
||||
},
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"source": {
|
||||
"type": "ExcelSource",
|
||||
"storeSettings": {
|
||||
"type": "AzureBlobStorageReadSettings",
|
||||
"recursive": true
|
||||
}
|
||||
},
|
||||
"sink": {
|
||||
"type": "DelimitedTextSink",
|
||||
"storeSettings": {
|
||||
"type": "AzureBlobStorageWriteSettings"
|
||||
},
|
||||
"formatSettings": {
|
||||
"type": "DelimitedTextWriteSettings",
|
||||
"quoteAllText": true,
|
||||
"fileExtension": ".txt"
|
||||
}
|
||||
},
|
||||
"enableStaging": false,
|
||||
"parallelCopies": {
|
||||
"value": "@pipeline().parameters.TaskObject.DegreeOfCopyParallelism",
|
||||
"type": "Expression"
|
||||
}
|
||||
},
|
||||
"inputs": [
|
||||
{
|
||||
"referenceName": "BlobExcel_@GF{IR}",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"RelativePath": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.RelativePath",
|
||||
"type": "Expression"
|
||||
},
|
||||
"FileName": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.DataFileName",
|
||||
"type": "Expression"
|
||||
},
|
||||
"StorageAccountEndpoint": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.StorageAccountName",
|
||||
"type": "Expression"
|
||||
},
|
||||
"StorageAccountContainerName": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.StorageAccountContainer",
|
||||
"type": "Expression"
|
||||
},
|
||||
"SheetName": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.SheetName",
|
||||
"type": "Expression"
|
||||
},
|
||||
"FirstRowAsHeader": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.FirstRowAsHeader",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"referenceName": "BlobDelimitedText_@GF{IR}",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"RelativePath": {
|
||||
"value": "@pipeline().parameters.TaskObject.Target.RelativePath",
|
||||
"type": "Expression"
|
||||
},
|
||||
"FileName": {
|
||||
"value": "@pipeline().parameters.TaskObject.Target.DataFileName",
|
||||
"type": "Expression"
|
||||
},
|
||||
"StorageAccountEndpoint": {
|
||||
"value": "@pipeline().parameters.TaskObject.Target.StorageAccountName",
|
||||
"type": "Expression"
|
||||
},
|
||||
"StorageAccountContainerName": {
|
||||
"value": "@pipeline().parameters.TaskObject.Target.StorageAccountContainer",
|
||||
"type": "Expression"
|
||||
},
|
||||
"FirstRowAsHeader": {
|
||||
"value": "@pipeline().parameters.TaskObject.Target.FirstRowAsHeader",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "Pipeline AF Log - BLOB to Azure SQL Failed",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "Copy Blob to Blob",
|
||||
"dependencyConditions": [
|
||||
"Failed"
|
||||
]
|
||||
}
|
||||
],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
"parameters": {
|
||||
"Body": {
|
||||
"value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":1,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy Blob to Blob\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"EndDateTimeOffSet\":\"', string(utcnow()), '\",\"Comment\":\"', string(activity('Copy Blob to Blob').error.message), '\",\"Status\":\"Failed\"}'))",
|
||||
"type": "Expression"
|
||||
},
|
||||
"FunctionName": "Log",
|
||||
"Method": "Post"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Pipeline AF Log - BLOB to Blob Start",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
"parameters": {
|
||||
"Body": {
|
||||
"value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":3,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy Blob to Blob\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"Status\":\"Started\"}'))",
|
||||
"type": "Expression"
|
||||
},
|
||||
"FunctionName": "Log",
|
||||
"Method": "Post"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Pipeline AF Log - Blob to Azure SQL Succeed",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "Copy Blob to Blob",
|
||||
"dependencyConditions": [
|
||||
"Succeeded"
|
||||
]
|
||||
}
|
||||
],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
"parameters": {
|
||||
"Body": {
|
||||
"value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":1,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy Blob to Blob\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"EndDateTimeOffSet\":\"', string(utcnow()), '\",\"RowsInserted\":\"', string(activity('Copy Blob to Blob').output.rowsCopied), '\",\"Comment\":\"\",\"Status\":\"Complete\"}'))",
|
||||
"type": "Expression"
|
||||
},
|
||||
"FunctionName": "Log",
|
||||
"Method": "Post"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"TaskObject": {
|
||||
"type": "object"
|
||||
}
|
||||
},
|
||||
"folder": {
|
||||
"name": "ADS Go Fast/Data Movement/@GF{IR}"
|
||||
},
|
||||
"annotations": [],
|
||||
"lastPublishTime": "2020-08-05T08:58:04Z"
|
||||
},
|
||||
"type": "Microsoft.DataFactory/factories/pipelines"
|
||||
}
|
||||
|
|
@ -0,0 +1,408 @@
|
|||
{
|
||||
"name": "AZ_Storage_JSON_AZ_SQL_@GF{IR}",
|
||||
"properties": {
|
||||
"activities": [
|
||||
{
|
||||
"name": "Switch Storage Type",
|
||||
"type": "Switch",
|
||||
"dependsOn": [],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"on": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.Type",
|
||||
"type": "Expression"
|
||||
},
|
||||
"cases": [
|
||||
{
|
||||
"value": "ADLS",
|
||||
"activities": [
|
||||
{
|
||||
"name": "Copy ADLS to Azure SQL",
|
||||
"type": "Copy",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "Pipeline AF Log - ADLS to Azure SQL Start",
|
||||
"dependencyConditions": [
|
||||
"Succeeded"
|
||||
]
|
||||
}
|
||||
],
|
||||
"policy": {
|
||||
"timeout": "7.00:00:00",
|
||||
"retry": 0,
|
||||
"retryIntervalInSeconds": 30,
|
||||
"secureOutput": false,
|
||||
"secureInput": false
|
||||
},
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"source": {
|
||||
"type": "JsonSource",
|
||||
"storeSettings": {
|
||||
"type": "AzureBlobFSReadSettings",
|
||||
"recursive": true
|
||||
},
|
||||
"formatSettings": {
|
||||
"type": "JsonReadSettings"
|
||||
}
|
||||
},
|
||||
"sink": {
|
||||
"type": "AzureSqlSink",
|
||||
"preCopyScript": {
|
||||
"value": "@{pipeline().parameters.TaskObject.Target.PreCopySQL}",
|
||||
"type": "Expression"
|
||||
},
|
||||
"disableMetricsCollection": false
|
||||
},
|
||||
"enableStaging": false,
|
||||
"parallelCopies": {
|
||||
"value": "@pipeline().parameters.TaskObject.DegreeOfCopyParallelism",
|
||||
"type": "Expression"
|
||||
}
|
||||
},
|
||||
"inputs": [
|
||||
{
|
||||
"referenceName": "ADLSJson_@GF{IR}",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"RelativePath": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.RelativePath",
|
||||
"type": "Expression"
|
||||
},
|
||||
"FileName": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.DataFileName",
|
||||
"type": "Expression"
|
||||
},
|
||||
"StorageAccountEndpoint": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.StorageAccountName",
|
||||
"type": "Expression"
|
||||
},
|
||||
"StorageAccountContainerName": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.StorageAccountContainer",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"referenceName": "AzureSqlTable_@GF{IR}",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"Schema": {
|
||||
"value": "@pipeline().parameters.TaskObject.Target.StagingTableSchema",
|
||||
"type": "Expression"
|
||||
},
|
||||
"Table": {
|
||||
"value": "@pipeline().parameters.TaskObject.Target.StagingTableName",
|
||||
"type": "Expression"
|
||||
},
|
||||
"Server": {
|
||||
"value": "@pipeline().parameters.TaskObject.Target.Database.SystemName",
|
||||
"type": "Expression"
|
||||
},
|
||||
"Database": {
|
||||
"value": "@pipeline().parameters.TaskObject.Target.Database.Name",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "Pipeline AF Log - ADLS to Azure SQL Failed",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "Copy ADLS to Azure SQL",
|
||||
"dependencyConditions": [
|
||||
"Failed"
|
||||
]
|
||||
}
|
||||
],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
"parameters": {
|
||||
"Body": {
|
||||
"value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":1,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy ADLS to Azure SQL\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"EndDateTimeOffSet\":\"', string(utcnow()), '\",\"Comment\":\"', string(activity('Copy ADLS to Azure SQL').error.message), '\",\"Status\":\"Failed\"}'))",
|
||||
"type": "Expression"
|
||||
},
|
||||
"FunctionName": "Log",
|
||||
"Method": "Post"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Pipeline AF Log - ADLS to Azure SQL Start",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
"parameters": {
|
||||
"Body": {
|
||||
"value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":3,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy ADLS to Azure SQL\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"Status\":\"Started\"}'))",
|
||||
"type": "Expression"
|
||||
},
|
||||
"FunctionName": "Log",
|
||||
"Method": "Post"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Pipeline AF Log - ADLS to Azure SQL Succeed",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "Copy ADLS to Azure SQL",
|
||||
"dependencyConditions": [
|
||||
"Succeeded"
|
||||
]
|
||||
}
|
||||
],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
"parameters": {
|
||||
"Body": {
|
||||
"value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":1,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy ADLS to Azure SQL\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"EndDateTimeOffSet\":\"', string(utcnow()), '\",\"RowsInserted\":\"', string(activity('Copy ADLS to Azure SQL').output.rowsCopied), '\",\"Comment\":\"\",\"Status\":\"Complete\"}'))",
|
||||
"type": "Expression"
|
||||
},
|
||||
"FunctionName": "Log",
|
||||
"Method": "Post"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"value": "Azure Blob",
|
||||
"activities": [
|
||||
{
|
||||
"name": "Copy Blob to Azure SQL",
|
||||
"type": "Copy",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "Pipeline AF Log - BLOB to Azure SQL Start",
|
||||
"dependencyConditions": [
|
||||
"Succeeded"
|
||||
]
|
||||
}
|
||||
],
|
||||
"policy": {
|
||||
"timeout": "7.00:00:00",
|
||||
"retry": 0,
|
||||
"retryIntervalInSeconds": 30,
|
||||
"secureOutput": false,
|
||||
"secureInput": false
|
||||
},
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"source": {
|
||||
"type": "JsonSource",
|
||||
"storeSettings": {
|
||||
"type": "AzureBlobStorageReadSettings",
|
||||
"recursive": true
|
||||
},
|
||||
"formatSettings": {
|
||||
"type": "JsonReadSettings"
|
||||
}
|
||||
},
|
||||
"sink": {
|
||||
"type": "AzureSqlSink",
|
||||
"preCopyScript": {
|
||||
"value": "@{pipeline().parameters.TaskObject.Target.PreCopySQL}",
|
||||
"type": "Expression"
|
||||
},
|
||||
"disableMetricsCollection": false
|
||||
},
|
||||
"enableStaging": false,
|
||||
"parallelCopies": {
|
||||
"value": "@pipeline().parameters.TaskObject.DegreeOfCopyParallelism",
|
||||
"type": "Expression"
|
||||
}
|
||||
},
|
||||
"inputs": [
|
||||
{
|
||||
"referenceName": "BlobJson_@GF{IR}",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"RelativePath": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.RelativePath",
|
||||
"type": "Expression"
|
||||
},
|
||||
"FileName": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.DataFileName",
|
||||
"type": "Expression"
|
||||
},
|
||||
"StorageAccountEndpoint": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.StorageAccountName",
|
||||
"type": "Expression"
|
||||
},
|
||||
"StorageAccountContainerName": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.StorageAccountContainer",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"referenceName": "AzureSqlTable_@GF{IR}",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"Schema": {
|
||||
"value": "@pipeline().parameters.TaskObject.Target.StagingTableSchema",
|
||||
"type": "Expression"
|
||||
},
|
||||
"Table": {
|
||||
"value": "@pipeline().parameters.TaskObject.Target.StagingTableName",
|
||||
"type": "Expression"
|
||||
},
|
||||
"Server": {
|
||||
"value": "@pipeline().parameters.TaskObject.Target.Database.SystemName",
|
||||
"type": "Expression"
|
||||
},
|
||||
"Database": {
|
||||
"value": "@pipeline().parameters.TaskObject.Target.Database.Name",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "Pipeline AF Log - BLOB to Azure SQL Failed",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "Copy Blob to Azure SQL",
|
||||
"dependencyConditions": [
|
||||
"Failed"
|
||||
]
|
||||
}
|
||||
],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
"parameters": {
|
||||
"Body": {
|
||||
"value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":1,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy Blob to Azure SQL\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"EndDateTimeOffSet\":\"', string(utcnow()), '\",\"Comment\":\"', string(activity('Copy Blob to Azure SQL').error.message), '\",\"Status\":\"Failed\"}'))",
|
||||
"type": "Expression"
|
||||
},
|
||||
"FunctionName": "Log",
|
||||
"Method": "Post"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Pipeline AF Log - BLOB to Azure SQL Start",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
"parameters": {
|
||||
"Body": {
|
||||
"value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":3,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy Blob to Azure SQL\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"Status\":\"Started\"}'))",
|
||||
"type": "Expression"
|
||||
},
|
||||
"FunctionName": "Log",
|
||||
"Method": "Post"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Pipeline AF Log - Blob to Azure SQL Succeed",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "Copy Blob to Azure SQL",
|
||||
"dependencyConditions": [
|
||||
"Succeeded"
|
||||
]
|
||||
}
|
||||
],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
"parameters": {
|
||||
"Body": {
|
||||
"value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":1,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy Blob to Azure SQL\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"EndDateTimeOffSet\":\"', string(utcnow()), '\",\"RowsInserted\":\"', string(activity('Copy Blob to Azure SQL').output.rowsCopied), '\",\"Comment\":\"\",\"Status\":\"Complete\"}'))",
|
||||
"type": "Expression"
|
||||
},
|
||||
"FunctionName": "Log",
|
||||
"Method": "Post"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Execute AZ_SQL_Post-Copy",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "Switch Storage Type",
|
||||
"dependencyConditions": [
|
||||
"Succeeded"
|
||||
]
|
||||
}
|
||||
],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ_SQL_Post_Copy_@GF{IR}",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": true,
|
||||
"parameters": {
|
||||
"TaskObject": {
|
||||
"value": "@pipeline().parameters.TaskObject",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"TaskObject": {
|
||||
"type": "object"
|
||||
}
|
||||
},
|
||||
"folder": {
|
||||
"name": "ADS Go Fast/Data Movement/@GF{IR}"
|
||||
},
|
||||
"annotations": [],
|
||||
"lastPublishTime": "2020-07-29T05:19:06Z"
|
||||
},
|
||||
"type": "Microsoft.DataFactory/factories/pipelines"
|
||||
}
|
||||
|
|
@ -0,0 +1,494 @@
|
|||
{
|
||||
"name": "AZ_Storage_Parquet_AZ_SQL_@GF{IR}",
|
||||
"properties": {
|
||||
"activities": [
|
||||
{
|
||||
"name": "Switch Storage Type",
|
||||
"type": "Switch",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "AF Get Mapping",
|
||||
"dependencyConditions": [
|
||||
"Succeeded"
|
||||
]
|
||||
}
|
||||
],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"on": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.Type",
|
||||
"type": "Expression"
|
||||
},
|
||||
"cases": [
|
||||
{
|
||||
"value": "ADLS",
|
||||
"activities": [
|
||||
{
|
||||
"name": "Copy ADLS to Azure SQL",
|
||||
"type": "Copy",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "Pipeline AF Log - ADLS to Azure SQL Start",
|
||||
"dependencyConditions": [
|
||||
"Succeeded"
|
||||
]
|
||||
}
|
||||
],
|
||||
"policy": {
|
||||
"timeout": "7.00:00:00",
|
||||
"retry": 0,
|
||||
"retryIntervalInSeconds": 30,
|
||||
"secureOutput": false,
|
||||
"secureInput": false
|
||||
},
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"source": {
|
||||
"type": "ParquetSource",
|
||||
"storeSettings": {
|
||||
"type": "AzureBlobFSReadSettings",
|
||||
"recursive": false,
|
||||
"wildcardFolderPath": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.RelativePath",
|
||||
"type": "Expression"
|
||||
},
|
||||
"wildcardFileName": "*.parquet",
|
||||
"enablePartitionDiscovery": false
|
||||
}
|
||||
},
|
||||
"sink": {
|
||||
"type": "AzureSqlSink",
|
||||
"preCopyScript": {
|
||||
"value": "@{pipeline().parameters.TaskObject.Target.PreCopySQL}",
|
||||
"type": "Expression"
|
||||
},
|
||||
"disableMetricsCollection": false
|
||||
},
|
||||
"enableStaging": false,
|
||||
"parallelCopies": {
|
||||
"value": "@pipeline().parameters.TaskObject.DegreeOfCopyParallelism",
|
||||
"type": "Expression"
|
||||
},
|
||||
"translator": {
|
||||
"value": "@activity('AF Get Mapping').output.value",
|
||||
"type": "Expression"
|
||||
}
|
||||
},
|
||||
"inputs": [
|
||||
{
|
||||
"referenceName": "ADLSParquet_@GF{IR}",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"RelativePath": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.RelativePath",
|
||||
"type": "Expression"
|
||||
},
|
||||
"FileName": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.DataFileName",
|
||||
"type": "Expression"
|
||||
},
|
||||
"StorageAccountEndpoint": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.StorageAccountName",
|
||||
"type": "Expression"
|
||||
},
|
||||
"StorageAccountContainerName": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.StorageAccountContainer",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"referenceName": "AzureSqlTable_@GF{IR}",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"Schema": {
|
||||
"value": "@pipeline().parameters.TaskObject.Target.StagingTableSchema",
|
||||
"type": "Expression"
|
||||
},
|
||||
"Table": {
|
||||
"value": "@pipeline().parameters.TaskObject.Target.StagingTableName",
|
||||
"type": "Expression"
|
||||
},
|
||||
"Server": {
|
||||
"value": "@pipeline().parameters.TaskObject.Target.Database.SystemName",
|
||||
"type": "Expression"
|
||||
},
|
||||
"Database": {
|
||||
"value": "@pipeline().parameters.TaskObject.Target.Database.Name",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "Pipeline AF Log - ADLS to Azure SQL Failed",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "Copy ADLS to Azure SQL",
|
||||
"dependencyConditions": [
|
||||
"Failed"
|
||||
]
|
||||
}
|
||||
],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
"parameters": {
|
||||
"Body": {
|
||||
"value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":1,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy ADLS to Azure SQL\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"EndDateTimeOffSet\":\"', string(utcnow()), '\",\"Comment\":\"', string(activity('Copy ADLS to Azure SQL').error.message), '\",\"Status\":\"Failed\"}'))",
|
||||
"type": "Expression"
|
||||
},
|
||||
"FunctionName": "Log",
|
||||
"Method": "Post"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Pipeline AF Log - ADLS to Azure SQL Start",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
"parameters": {
|
||||
"Body": {
|
||||
"value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":3,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy ADLS to Azure SQL\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"Status\":\"Started\"}'))",
|
||||
"type": "Expression"
|
||||
},
|
||||
"FunctionName": "Log",
|
||||
"Method": "Post"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Pipeline AF Log - ADLS to Azure SQL Succeed",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "Copy ADLS to Azure SQL",
|
||||
"dependencyConditions": [
|
||||
"Succeeded"
|
||||
]
|
||||
}
|
||||
],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
"parameters": {
|
||||
"Body": {
|
||||
"value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":1,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy ADLS to Azure SQL\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"EndDateTimeOffSet\":\"', string(utcnow()), '\",\"RowsInserted\":\"', string(activity('Copy ADLS to Azure SQL').output.rowsCopied), '\",\"Comment\":\"\",\"Status\":\"Complete\"}'))",
|
||||
"type": "Expression"
|
||||
},
|
||||
"FunctionName": "Log",
|
||||
"Method": "Post"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"value": "Azure Blob",
|
||||
"activities": [
|
||||
{
|
||||
"name": "Copy Blob to Azure SQL",
|
||||
"type": "Copy",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "Pipeline AF Log - BLOB to Azure SQL Start",
|
||||
"dependencyConditions": [
|
||||
"Succeeded"
|
||||
]
|
||||
}
|
||||
],
|
||||
"policy": {
|
||||
"timeout": "7.00:00:00",
|
||||
"retry": 0,
|
||||
"retryIntervalInSeconds": 30,
|
||||
"secureOutput": false,
|
||||
"secureInput": false
|
||||
},
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"source": {
|
||||
"type": "ParquetSource",
|
||||
"storeSettings": {
|
||||
"type": "AzureBlobStorageReadSettings",
|
||||
"recursive": false,
|
||||
"wildcardFolderPath": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.RelativePath",
|
||||
"type": "Expression"
|
||||
},
|
||||
"wildcardFileName": "*.parquet",
|
||||
"enablePartitionDiscovery": false
|
||||
}
|
||||
},
|
||||
"sink": {
|
||||
"type": "AzureSqlSink",
|
||||
"preCopyScript": {
|
||||
"value": "@{pipeline().parameters.TaskObject.Target.PreCopySQL}",
|
||||
"type": "Expression"
|
||||
},
|
||||
"disableMetricsCollection": false
|
||||
},
|
||||
"enableStaging": false,
|
||||
"parallelCopies": {
|
||||
"value": "@pipeline().parameters.TaskObject.DegreeOfCopyParallelism",
|
||||
"type": "Expression"
|
||||
},
|
||||
"translator": {
|
||||
"value": "@activity('AF Get Mapping').output.value",
|
||||
"type": "Expression"
|
||||
}
|
||||
},
|
||||
"inputs": [
|
||||
{
|
||||
"referenceName": "BlobParquet_@GF{IR}",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"RelativePath": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.RelativePath",
|
||||
"type": "Expression"
|
||||
},
|
||||
"FileName": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.DataFileName",
|
||||
"type": "Expression"
|
||||
},
|
||||
"StorageAccountEndpoint": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.StorageAccountName",
|
||||
"type": "Expression"
|
||||
},
|
||||
"StorageAccountContainerName": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.StorageAccountContainer",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"referenceName": "AzureSqlTable_@GF{IR}",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"Schema": {
|
||||
"value": "@pipeline().parameters.TaskObject.Target.StagingTableSchema",
|
||||
"type": "Expression"
|
||||
},
|
||||
"Table": {
|
||||
"value": "@pipeline().parameters.TaskObject.Target.StagingTableName",
|
||||
"type": "Expression"
|
||||
},
|
||||
"Server": {
|
||||
"value": "@pipeline().parameters.TaskObject.Target.Database.SystemName",
|
||||
"type": "Expression"
|
||||
},
|
||||
"Database": {
|
||||
"value": "@pipeline().parameters.TaskObject.Target.Database.Name",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "Pipeline AF Log - BLOB to Azure SQL Failed",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "Copy Blob to Azure SQL",
|
||||
"dependencyConditions": [
|
||||
"Failed"
|
||||
]
|
||||
}
|
||||
],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
"parameters": {
|
||||
"Body": {
|
||||
"value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":1,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy Blob to Azure SQL\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"EndDateTimeOffSet\":\"', string(utcnow()), '\",\"Comment\":\"', string(activity('Copy Blob to Azure SQL').error.message), '\",\"Status\":\"Failed\"}'))",
|
||||
"type": "Expression"
|
||||
},
|
||||
"FunctionName": "Log",
|
||||
"Method": "Post"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Pipeline AF Log - BLOB to Azure SQL Start",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
"parameters": {
|
||||
"Body": {
|
||||
"value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":3,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy Blob to Azure SQL\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"Status\":\"Started\"}'))",
|
||||
"type": "Expression"
|
||||
},
|
||||
"FunctionName": "Log",
|
||||
"Method": "Post"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Pipeline AF Log - Blob to Azure SQL Succeed",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "Copy Blob to Azure SQL",
|
||||
"dependencyConditions": [
|
||||
"Succeeded"
|
||||
]
|
||||
}
|
||||
],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
"parameters": {
|
||||
"Body": {
|
||||
"value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":1,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy Blob to Azure SQL\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"EndDateTimeOffSet\":\"', string(utcnow()), '\",\"RowsInserted\":\"', string(activity('Copy Blob to Azure SQL').output.rowsCopied), '\",\"Comment\":\"\",\"Status\":\"Complete\"}'))",
|
||||
"type": "Expression"
|
||||
},
|
||||
"FunctionName": "Log",
|
||||
"Method": "Post"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "If Auto Create Table",
|
||||
"type": "IfCondition",
|
||||
"dependsOn": [],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"expression": {
|
||||
"value": "@bool(pipeline().parameters.TaskObject.Target.AutoCreateTable)",
|
||||
"type": "Expression"
|
||||
},
|
||||
"ifTrueActivities": [
|
||||
{
|
||||
"name": "Execute Create Table",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ_SQL_Create_Table_@GF{IR}",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": true,
|
||||
"parameters": {
|
||||
"TaskObject": {
|
||||
"value": "@pipeline().parameters.TaskObject",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Execute AZ_SQL_Post-Copy",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "Switch Storage Type",
|
||||
"dependencyConditions": [
|
||||
"Succeeded"
|
||||
]
|
||||
}
|
||||
],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ_SQL_Post_Copy_@GF{IR}",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": true,
|
||||
"parameters": {
|
||||
"TaskObject": {
|
||||
"value": "@pipeline().parameters.TaskObject",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "AF Get Mapping",
|
||||
"type": "AzureFunctionActivity",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "If Auto Create Table",
|
||||
"dependencyConditions": [
|
||||
"Succeeded"
|
||||
]
|
||||
}
|
||||
],
|
||||
"policy": {
|
||||
"timeout": "7.00:00:00",
|
||||
"retry": 0,
|
||||
"retryIntervalInSeconds": 30,
|
||||
"secureOutput": false,
|
||||
"secureInput": false
|
||||
},
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"functionName": "GetSourceTargetMapping",
|
||||
"method": "POST",
|
||||
"body": {
|
||||
"value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"StorageAccountName\":\"', string(pipeline().parameters.TaskObject.Source.StorageAccountName), '\",\"StorageAccountContainer\":\"', string(pipeline().parameters.TaskObject.Source.StorageAccountContainer), '\",\"RelativePath\":\"', string(pipeline().parameters.TaskObject.Source.RelativePath), '\",\"SchemaFileName\":\"', string(pipeline().parameters.TaskObject.Source.SchemaFileName), '\",\"SourceType\":\"', string(pipeline().parameters.TaskObject.Source.Type), '\",\"TargetType\":\"', string(pipeline().parameters.TaskObject.Target.Type), '\",\"MetadataType\":\"Parquet\"}'))",
|
||||
"type": "Expression"
|
||||
}
|
||||
},
|
||||
"linkedServiceName": {
|
||||
"referenceName": "AzureFunctionAdsGoFastDataLakeAccelFunApp",
|
||||
"type": "LinkedServiceReference"
|
||||
}
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"TaskObject": {
|
||||
"type": "object"
|
||||
}
|
||||
},
|
||||
"folder": {
|
||||
"name": "ADS Go Fast/Data Movement/@GF{IR}"
|
||||
},
|
||||
"annotations": [],
|
||||
"lastPublishTime": "2020-08-05T03:16:41Z"
|
||||
},
|
||||
"type": "Microsoft.DataFactory/factories/pipelines"
|
||||
}
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
{
|
||||
"name": "Create-Task-Master-AZ-SQL-OnP-SH-IR",
|
||||
"name": "Create-Task_MASTER_AZ_SQL_@GF{IR}",
|
||||
"properties": {
|
||||
"activities": [
|
||||
{
|
||||
|
|
@ -16,7 +16,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-Function-Generic",
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
|
|
@ -37,7 +37,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-Function-Generic",
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
|
|
@ -65,7 +65,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-Function-Generic",
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
|
|
@ -109,25 +109,17 @@
|
|||
"partitionOption": "None"
|
||||
},
|
||||
"dataset": {
|
||||
"referenceName": "SqlServerTable_OnPrem_SH_IR",
|
||||
"referenceName": "AzureSqlTable_@GF{IR}",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"TableSchema": {
|
||||
"Schema": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.Extraction.TableSchema",
|
||||
"type": "Expression"
|
||||
},
|
||||
"TableName": {
|
||||
"Table": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.Extraction.TableName",
|
||||
"type": "Expression"
|
||||
},
|
||||
"KeyVaultBaseUrl": {
|
||||
"value": "@pipeline().parameters.TaskObject.KeyVaultBaseUrl",
|
||||
"type": "Expression"
|
||||
},
|
||||
"PasswordSecret": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.Database.PasswordKeyVaultSecretName",
|
||||
"type": "Expression"
|
||||
},
|
||||
"Server": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.Database.SystemName",
|
||||
"type": "Expression"
|
||||
|
|
@ -135,10 +127,6 @@
|
|||
"Database": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.Database.Name",
|
||||
"type": "Expression"
|
||||
},
|
||||
"UserName": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.Database.Username",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
},
|
||||
|
|
@ -248,7 +236,7 @@
|
|||
}
|
||||
},
|
||||
"folder": {
|
||||
"name": "ADS Go Fast/Create Task Master/OnPrem SH IR"
|
||||
"name": "ADS Go Fast/Create Task Master/@GF{IR}"
|
||||
},
|
||||
"annotations": [],
|
||||
"lastPublishTime": "2020-08-04T13:09:29Z"
|
||||
|
|
@ -1,9 +1,9 @@
|
|||
{
|
||||
"name": "GEN-File-Binary-AZ-Storage-Binary-OnP-SH-IR",
|
||||
"name": "GEN_File_Binary_AZ_Storage_Binary_@GF{IR}",
|
||||
"properties": {
|
||||
"activities": [
|
||||
{
|
||||
"name": "Switch Azure Storage",
|
||||
"name": "Switch Storage Type",
|
||||
"type": "Switch",
|
||||
"dependsOn": [],
|
||||
"userProperties": [],
|
||||
|
|
@ -40,18 +40,8 @@
|
|||
"type": "BinarySource",
|
||||
"storeSettings": {
|
||||
"type": "FileServerReadSettings",
|
||||
"recursive": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.Recursively",
|
||||
"type": "Expression"
|
||||
},
|
||||
"wildcardFileName": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.DataFileName",
|
||||
"type": "Expression"
|
||||
},
|
||||
"deleteFilesAfterCompletion": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.DeleteAfterCompletion",
|
||||
"type": "Expression"
|
||||
}
|
||||
"recursive": true,
|
||||
"deleteFilesAfterCompletion": true
|
||||
},
|
||||
"formatSettings": {
|
||||
"type": "BinaryReadSettings"
|
||||
|
|
@ -60,8 +50,7 @@
|
|||
"sink": {
|
||||
"type": "BinarySink",
|
||||
"storeSettings": {
|
||||
"type": "AzureBlobFSWriteSettings",
|
||||
"copyBehavior": "PreserveHierarchy"
|
||||
"type": "AzureBlobFSWriteSettings"
|
||||
}
|
||||
},
|
||||
"enableStaging": false,
|
||||
|
|
@ -72,7 +61,7 @@
|
|||
},
|
||||
"inputs": [
|
||||
{
|
||||
"referenceName": "FileBinary_OnPrem_SH_IR",
|
||||
"referenceName": "FileBinary_@GF{IR}",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"Host": {
|
||||
|
|
@ -88,11 +77,11 @@
|
|||
"type": "Expression"
|
||||
},
|
||||
"Directory": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.RelativePath",
|
||||
"value": "@pipeline().parameters.TaskObject.Source.Directory",
|
||||
"type": "Expression"
|
||||
},
|
||||
"File": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.DataFileName",
|
||||
"value": "@pipeline().parameters.TaskObject.Source.File",
|
||||
"type": "Expression"
|
||||
},
|
||||
"KeyVaultBaseUrl": {
|
||||
|
|
@ -104,21 +93,21 @@
|
|||
],
|
||||
"outputs": [
|
||||
{
|
||||
"referenceName": "ADLSBinary_OnPrem_SH_IR",
|
||||
"referenceName": "ADLSBinary_@GF{IR}",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"StorageAccountEndpoint": {
|
||||
"value": "@pipeline().parameters.TaskObject.Target.StorageAccountName",
|
||||
"type": "Expression"
|
||||
},
|
||||
"FileSystem": {
|
||||
"value": "@pipeline().parameters.TaskObject.Target.StorageAccountContainer",
|
||||
"type": "Expression"
|
||||
},
|
||||
"Directory": {
|
||||
"value": "@pipeline().parameters.TaskObject.Target.RelativePath",
|
||||
"type": "Expression"
|
||||
},
|
||||
"FileSystem": {
|
||||
"value": "@pipeline().parameters.TaskObject.Target.StorageAccountContainer",
|
||||
"type": "Expression"
|
||||
},
|
||||
"File": {
|
||||
"value": "@pipeline().parameters.TaskObject.Target.DataFileName",
|
||||
"type": "Expression"
|
||||
|
|
@ -127,6 +116,34 @@
|
|||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "Pipeline AF Log - File to ADLS Failed",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "Copy File to ADLS",
|
||||
"dependencyConditions": [
|
||||
"Failed"
|
||||
]
|
||||
}
|
||||
],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
"parameters": {
|
||||
"Body": {
|
||||
"value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":1,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy File to ADLS\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"EndDateTimeOffSet\":\"', string(utcnow()), '\",\"Comment\":\"', string(activity('Copy File to ADLS').error.message), '\",\"Status\":\"Failed\"}'))",
|
||||
"type": "Expression"
|
||||
},
|
||||
"FunctionName": "Log",
|
||||
"Method": "Post"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Pipeline AF Log - File to ADLS Start",
|
||||
"type": "ExecutePipeline",
|
||||
|
|
@ -134,13 +151,13 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-Function-Generic",
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
"parameters": {
|
||||
"Body": {
|
||||
"value": "[\n{\n \"TaskInstanceId\":@{pipeline().parameters.TaskObject.TaskInstanceId},\n \"ExecutionUid\":@{pipeline().parameters.TaskObject.ExecutionUid},\n \"RunId\":@{pipeline().RunId}, \n \"StartTime\":@{utcnow()},\n \"Status\":\"Processing\",\n \"Activity\":\"Copy File to ADLS\",\n \"Comments\":\"\"\n}\n]",
|
||||
"value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":3,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy File to ADLS\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"Status\":\"Started\"}'))",
|
||||
"type": "Expression"
|
||||
},
|
||||
"FunctionName": "Log",
|
||||
|
|
@ -162,41 +179,13 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-Function-Generic",
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
"parameters": {
|
||||
"Body": {
|
||||
"value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":1,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy File to ADLS\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"EndDateTimeOffSet\":\"', string(utcnow()), '\",\"RowsInserted\":\"', string(activity('Copy File to ADLS').output.filesWritten), '\",\"Comment\":\"\",\"Status\":\"Complete\"}'))",
|
||||
"type": "Expression"
|
||||
},
|
||||
"FunctionName": "Log",
|
||||
"Method": "Post"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Pipeline AF Log - File to ADLS Failed",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "Copy File to ADLS",
|
||||
"dependencyConditions": [
|
||||
"Failed"
|
||||
]
|
||||
}
|
||||
],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-Function-Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
"parameters": {
|
||||
"Body": {
|
||||
"value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":1,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy File to ADLS\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"EndDateTimeOffSet\":\"', string(utcnow()), '\",\"Comment\":\"', string(activity('Copy File to ADLS').error.message), '\",\"Status\":\"Failed\"}'))",
|
||||
"value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":1,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy File to ADLS\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"EndDateTimeOffSet\":\"', string(utcnow()), '\",\"RowsInserted\":\"', string(activity('Copy File to ADLS').output.rowsCopied), '\",\"Comment\":\"\",\"Status\":\"Complete\"}'))",
|
||||
"type": "Expression"
|
||||
},
|
||||
"FunctionName": "Log",
|
||||
|
|
@ -233,18 +222,8 @@
|
|||
"type": "BinarySource",
|
||||
"storeSettings": {
|
||||
"type": "FileServerReadSettings",
|
||||
"recursive": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.Recursively",
|
||||
"type": "Expression"
|
||||
},
|
||||
"wildcardFileName": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.DataFileName",
|
||||
"type": "Expression"
|
||||
},
|
||||
"deleteFilesAfterCompletion": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.DeleteAfterCompletion",
|
||||
"type": "Expression"
|
||||
}
|
||||
"recursive": true,
|
||||
"deleteFilesAfterCompletion": true
|
||||
},
|
||||
"formatSettings": {
|
||||
"type": "BinaryReadSettings"
|
||||
|
|
@ -253,8 +232,7 @@
|
|||
"sink": {
|
||||
"type": "BinarySink",
|
||||
"storeSettings": {
|
||||
"type": "AzureBlobStorageWriteSettings",
|
||||
"copyBehavior": "PreserveHierarchy"
|
||||
"type": "AzureBlobStorageWriteSettings"
|
||||
}
|
||||
},
|
||||
"enableStaging": false,
|
||||
|
|
@ -265,7 +243,7 @@
|
|||
},
|
||||
"inputs": [
|
||||
{
|
||||
"referenceName": "FileBinary_OnPrem_SH_IR",
|
||||
"referenceName": "FileBinary_@GF{IR}",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"Host": {
|
||||
|
|
@ -281,11 +259,11 @@
|
|||
"type": "Expression"
|
||||
},
|
||||
"Directory": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.RelativePath",
|
||||
"value": "@pipeline().parameters.TaskObject.Source.Directory",
|
||||
"type": "Expression"
|
||||
},
|
||||
"File": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.DataFileName",
|
||||
"value": "@pipeline().parameters.TaskObject.Source.File",
|
||||
"type": "Expression"
|
||||
},
|
||||
"KeyVaultBaseUrl": {
|
||||
|
|
@ -297,7 +275,7 @@
|
|||
],
|
||||
"outputs": [
|
||||
{
|
||||
"referenceName": "BlobBinary_OnPrem_SH_IR",
|
||||
"referenceName": "BlobBinary_@GF{IR}",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"StorageAccountEndpoint": {
|
||||
|
|
@ -320,6 +298,34 @@
|
|||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "Pipeline AF Log - File to Blob Failed",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "Copy File to Blob",
|
||||
"dependencyConditions": [
|
||||
"Failed"
|
||||
]
|
||||
}
|
||||
],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
"parameters": {
|
||||
"Body": {
|
||||
"value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":1,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy File to Blob\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"EndDateTimeOffSet\":\"', string(utcnow()), '\",\"Comment\":\"', string(activity('Copy File to Blob').error.message), '\",\"Status\":\"Failed\"}'))",
|
||||
"type": "Expression"
|
||||
},
|
||||
"FunctionName": "Log",
|
||||
"Method": "Post"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Pipeline AF Log - File to Blob Start",
|
||||
"type": "ExecutePipeline",
|
||||
|
|
@ -327,13 +333,13 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-Function-Generic",
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
"parameters": {
|
||||
"Body": {
|
||||
"value": "[\n{\n \"TaskInstanceId\":@{pipeline().parameters.TaskObject.TaskInstanceId},\n \"ExecutionUid\":@{pipeline().parameters.TaskObject.ExecutionUid},\n \"RunId\":@{pipeline().RunId}, \n \"StartTime\":@{utcnow()},\n \"Status\":\"Processing\",\n \"Activity\":\"Copy File to Blob\",\n \"Comments\":\"\"\n}\n]",
|
||||
"value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":3,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy File to Blob\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"Status\":\"Started\"}'))",
|
||||
"type": "Expression"
|
||||
},
|
||||
"FunctionName": "Log",
|
||||
|
|
@ -355,41 +361,13 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-Function-Generic",
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
"parameters": {
|
||||
"Body": {
|
||||
"value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":1,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy File to Blob\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"EndDateTimeOffSet\":\"', string(utcnow()), '\",\"RowsInserted\":\"', string(activity('Copy File to Blob').output.filesWritten), '\",\"Comment\":\"\",\"Status\":\"Complete\"}'))",
|
||||
"type": "Expression"
|
||||
},
|
||||
"FunctionName": "Log",
|
||||
"Method": "Post"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Pipeline AF Log - File to Blob Failed",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "Copy File to Blob",
|
||||
"dependencyConditions": [
|
||||
"Failed"
|
||||
]
|
||||
}
|
||||
],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-Function-Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
"parameters": {
|
||||
"Body": {
|
||||
"value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":1,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy File to Blob\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"EndDateTimeOffSet\":\"', string(utcnow()), '\",\"Comment\":\"', string(activity('Copy File to Blob').error.message), '\",\"Status\":\"Failed\"}'))",
|
||||
"value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":1,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy File to Blob\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"EndDateTimeOffSet\":\"', string(utcnow()), '\",\"RowsInserted\":\"', string(activity('Copy File to Blob').output.rowsCopied), '\",\"Comment\":\"\",\"Status\":\"Complete\"}'))",
|
||||
"type": "Expression"
|
||||
},
|
||||
"FunctionName": "Log",
|
||||
|
|
@ -406,14 +384,14 @@
|
|||
"parameters": {
|
||||
"TaskObject": {
|
||||
"type": "object",
|
||||
"defaultValue": "[ { \"TaskInstanceId\": 2, \"TaskMasterId\": 1, \"TaskStatus\": \"Untried\", \"TaskType\": \"FiletoAzureStorage-OnPrem-SH-IR\", \"Enabled\": true, \"ExecutionUid\": 1, \"KeyVaultBaseUrl\": \"https://adsgofastkeyvault.vault.azure.net/\", \"Source\": { \"Type\": \"File\", \"Host\": \"D:\\\\dataingestion\\\\\", \"UserId\": \"AzureUser\", \"PasswordKeyVaultSecretName\": \"adsgofast-onpre-file-password\", \"Directory\": \"AdventureWorks2017/2020/06/15/14/22/\", \"File\": \"Sales.SalesOrderDetails.parquet\" }, \"Target\": { \"StorageAccountName\": \"https://adsgofastdatalakeadls.dfs.core.windows.net\", \"Type\": \"ADLS\", \"StorageAccountContainer\": \"datalakelanding\", \"StorageAccountAccessMethod\": \"MSI\", \"StorageAccountSASUriKeyVaultSecretName\": null, \"RelativePath\": \"/Unprocessed/adsgofastdatakakeaccelsqlsvr/AWSample/SalesLT/2020/06/08/17/\", \"DataFileName\": \"Customer_Data.parquet\", \"SchemaFileName\": \"Customer_Schema.json\" }, \"DataFactory\": { \"Id\": 1, \"Name\": \"adsgofastdatakakeacceladf\", \"ResourceGroup\": \"AdsGoFastDataLakeAccel\", \"SubscriptionId\": \"035a1364-f00d-48e2-b582-4fe125905ee3\", } } ]"
|
||||
"defaultValue": " [ { \"TaskInstanceId\": 2, \"TaskMasterId\": 1, \"TaskStatus\": \"Untried\", \"TaskType\": \"FiletoAzureStorage-Azure_@GF{IR}\", \"Enabled\": true, \"ExecutionUid\": 1, \"KeyVaultBaseUrl\": \"https://adsgofastkeyvault.vault.azure.net/\", \"Source\": { \"Type\": \"File\", \"Host\": \"\\\\\\\\10.2.0.4\\\\D\\\\dataingestion\\\\\", \"UserId\": \"adsgofastonprem\\\\AzureUser\", \"PasswordKeyVaultSecretName\": \"adsgofast-onpre_File_password\", \"Directory\": \"AdventureWorks2017/2020/06/15/14/22/\", \"File\": \"Sales.SalesOrderDetails.parquet\" }, \"Target\": { \"StorageAccountName\": \"https://adsgofastdatalakeaccelst.blob.core.windows.net\", \"Type\": \"Blob\", \"StorageAccountContainer\": \"datalakelanding\", \"StorageAccountAccessMethod\": \"MSI\", \"StorageAccountSASUriKeyVaultSecretName\": null, \"RelativePath\": \"/Unprocessed/adsgofastdatakakeaccelsqlsvr/AWSample/SalesLT/2020/06/08/17/\", \"DataFileName\": \"Customer_Data.parquet\", \"SchemaFileName\": \"Customer_Schema.json\" }, \"DataFactory\": { \"Id\": 1, \"Name\": \"adsgofastdatakakeacceladf\", \"ResourceGroup\": \"AdsGoFastDataLakeAccel\", \"SubscriptionId\": \"035a1364-f00d-48e2-b582-4fe125905ee3\", } } ]"
|
||||
}
|
||||
},
|
||||
"folder": {
|
||||
"name": "ADS Go Fast/Data Movement/OnPrem SH IR"
|
||||
"name": "ADS Go Fast/Data Movement/@GF{IR}"
|
||||
},
|
||||
"annotations": [],
|
||||
"lastPublishTime": "2020-08-04T02:49:50Z"
|
||||
"lastPublishTime": "2020-07-24T01:51:29Z"
|
||||
},
|
||||
"type": "Microsoft.DataFactory/factories/pipelines"
|
||||
}
|
||||
|
|
@ -0,0 +1,470 @@
|
|||
{
|
||||
"name": "Master",
|
||||
"properties": {
|
||||
"activities": [
|
||||
{
|
||||
"name": "Switch Pipelines",
|
||||
"type": "Switch",
|
||||
"dependsOn": [],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"on": {
|
||||
"value": "@pipeline().parameters.TaskObject.DataFactory.ADFPipeline",
|
||||
"type": "Expression"
|
||||
},
|
||||
"cases": [
|
||||
{
|
||||
"value": "AZ_SQL_AZ_Storage_Parquet_@GF{IR}",
|
||||
"activities": [
|
||||
{
|
||||
"name": "Execute AZ_SQL_AZ_Storage_Parquet_@GF{IR}",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ_SQL_AZ_Storage_Parquet_@GF{IR}",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": true,
|
||||
"parameters": {
|
||||
"TaskObject": {
|
||||
"value": "@pipeline().parameters.TaskObject",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"value": "AZ_Storage_Binary_AZ_Storage_Binary_@GF{IR}",
|
||||
"activities": [
|
||||
{
|
||||
"name": "Execute AZ_Storage_Binary_AZ_Storage_Binary_@GF{IR}",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ_Storage_Binary_AZ_Storage_Binary_@GF{IR}",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": true,
|
||||
"parameters": {
|
||||
"TaskObject": {
|
||||
"value": "@pipeline().parameters.TaskObject",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"value": "AZ_Storage_CSV_AZ_SQL_@GF{IR}",
|
||||
"activities": [
|
||||
{
|
||||
"name": "Execute AZ_Storage_CSV_AZ_SQL_@GF{IR}",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ_Storage_CSV_AZ_SQL_@GF{IR}",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": true,
|
||||
"parameters": {
|
||||
"TaskObject": {
|
||||
"value": "@pipeline().parameters.TaskObject",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"value": "AZ_Storage_Excel_AZ_SQL_@GF{IR}",
|
||||
"activities": [
|
||||
{
|
||||
"name": "Execute AZ_Storage_Excel_AZ_SQL_@GF{IR}",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ_Storage_Excel_AZ_SQL_@GF{IR}",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": true,
|
||||
"parameters": {
|
||||
"TaskObject": {
|
||||
"value": "@pipeline().parameters.TaskObject",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"value": "AZ_Storage_JSON_AZ_SQL_@GF{IR}",
|
||||
"activities": [
|
||||
{
|
||||
"name": "Execute AZ_Storage_JSON_AZ_SQL_@GF{IR}",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ_Storage_JSON_AZ_SQL_@GF{IR}",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": true,
|
||||
"parameters": {
|
||||
"TaskObject": {
|
||||
"value": "@pipeline().parameters.TaskObject",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"value": "AZ_Storage_Parquet_AZ_SQL_@GF{IR}",
|
||||
"activities": [
|
||||
{
|
||||
"name": "Execute AZ_Storage_Parquet_AZ_SQL_@GF{IR}",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ_Storage_Parquet_AZ_SQL_@GF{IR}",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": true,
|
||||
"parameters": {
|
||||
"TaskObject": {
|
||||
"value": "@pipeline().parameters.TaskObject",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"value": "GEN_File_Binary_AZ_Storage_Binary_@GF{IR}",
|
||||
"activities": [
|
||||
{
|
||||
"name": "Execute GEN_File_Binary_AZ_Storage_Binary_@GF{IR}",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "GEN_File_Binary_AZ_Storage_Binary_@GF{IR}",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": true,
|
||||
"parameters": {
|
||||
"TaskObject": {
|
||||
"value": "@pipeline().parameters.TaskObject",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"value": "SH_SQL_AZ_Storage_Parquet_@GF{IR}",
|
||||
"activities": [
|
||||
{
|
||||
"name": "Execute SH_SQL_AZ_Storage_Parquet_@GF{IR}",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "SH_SQL_AZ_Storage_Parquet_@GF{IR}",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": true,
|
||||
"parameters": {
|
||||
"TaskObject": {
|
||||
"value": "@pipeline().parameters.TaskObject",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"value": "SH_SQL_GEN_File_Parquet_@GF{IR}",
|
||||
"activities": [
|
||||
{
|
||||
"name": "Execute SH_SQL_GEN_File_Parquet_@GF{IR}",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "SH_SQL_GEN_File_Parquet_@GF{IR}",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": true,
|
||||
"parameters": {
|
||||
"TaskObject": {
|
||||
"value": "@pipeline().parameters.TaskObject",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"value": "Create-Task_MASTER_AZ_SQL_@GF{IR}",
|
||||
"activities": [
|
||||
{
|
||||
"name": "Execute Create-Task_MASTER_AZ_SQL_@GF{IR}",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "Create-Task_MASTER_AZ_SQL_@GF{IR}",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": true,
|
||||
"parameters": {
|
||||
"TaskObject": {
|
||||
"value": "@pipeline().parameters.TaskObject",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"value": "AZ_Storage_Excel_AZ_Storage_CSV_@GF{IR}",
|
||||
"activities": [
|
||||
{
|
||||
"name": "Execute AZ_Storage_Excel_AZ_Storage_CSV_@GF{IR}",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ_Storage_Excel_AZ_Storage_CSV_@GF{IR}",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": true,
|
||||
"parameters": {
|
||||
"TaskObject": {
|
||||
"value": "@pipeline().parameters.TaskObject",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"value": "AZ_SQL_StoredProcedure_@GF{IR}",
|
||||
"activities": [
|
||||
{
|
||||
"name": "Execute AZ_SQL_StoredProcedure_@GF{IR}",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ_SQL_StoredProcedure_@GF{IR}",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": true,
|
||||
"parameters": {
|
||||
"TaskObject": {
|
||||
"value": "@pipeline().parameters.TaskObject",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"defaultActivities": [
|
||||
{
|
||||
"name": "Pipeline AF Log - Invalid Pipeline",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
"parameters": {
|
||||
"Body": {
|
||||
"value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":1,\"LogSource\":\"ADF\",\"ActivityType\":\"Data-Movement-Master\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"EndDateTimeOffSet\":\"', string(utcnow()), '\",\"Comment\":\"Invalid TaskType: ', string(pipeline().parameters.TaskObject.TaskType), '\",\"Status\":\"Failed\",\"NumberOfRetries\":\"', string(pipeline().parameters.TaskObject.NumberOfRetries),'\"}'))",
|
||||
"type": "Expression"
|
||||
},
|
||||
"FunctionName": "Log",
|
||||
"Method": "Post"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Fail Due to Invalid Pipeline",
|
||||
"type": "Fail",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "Pipeline AF Log - Invalid Pipeline",
|
||||
"dependencyConditions": [
|
||||
"Succeeded"
|
||||
]
|
||||
}
|
||||
],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"message": {
|
||||
"value": "@concat('Invalid Pipeline:',pipeline().parameters.TaskObject.DataFactory.ADFPipeline)",
|
||||
"type": "Expression"
|
||||
},
|
||||
"errorCode": "500"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Pipeline AF Log - Failed",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "Switch Pipelines",
|
||||
"dependencyConditions": [
|
||||
"Failed"
|
||||
]
|
||||
}
|
||||
],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": true,
|
||||
"parameters": {
|
||||
"Body": {
|
||||
"value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":1,\"LogSource\":\"ADF\",\"ActivityType\":\"Data-Movement-Master\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"EndDateTimeOffSet\":\"', string(utcnow()), '\",\"Comment\":\"', string(activity('Switch Pipelines').error.message), '\",\"Status\":\"Failed\",\"NumberOfRetries\":\"', string(pipeline().parameters.TaskObject.NumberOfRetries),'\"}'))",
|
||||
"type": "Expression"
|
||||
},
|
||||
"FunctionName": "Log",
|
||||
"Method": "Post"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Pipeline AF Log - Succeed",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "Switch Pipelines",
|
||||
"dependencyConditions": [
|
||||
"Succeeded"
|
||||
]
|
||||
}
|
||||
],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": true,
|
||||
"parameters": {
|
||||
"Body": {
|
||||
"value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":1,\"LogSource\":\"ADF\",\"ActivityType\":\"Data-Movement-Master\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"EndDateTimeOffSet\":\"', string(utcnow()), '\",\"Comment\":\"\",\"Status\":\"Complete\",\"NumberOfRetries\":\"', string(pipeline().parameters.TaskObject.NumberOfRetries),'\"}'))",
|
||||
"type": "Expression"
|
||||
},
|
||||
"FunctionName": "Log",
|
||||
"Method": "Post"
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"TaskObject": {
|
||||
"type": "object",
|
||||
"defaultValue": {
|
||||
"TaskInstanceId": 865,
|
||||
"TaskMasterId": 102,
|
||||
"TaskStatus": "Untried",
|
||||
"TaskType": "SQL Database to Azure Storage",
|
||||
"Enabled": 1,
|
||||
"ExecutionUid": "d954cfee-44c1-419f-83d5-51f75eaf809c",
|
||||
"NumberOfRetries": 0,
|
||||
"DegreeOfCopyParallelism": 1,
|
||||
"KeyVaultBaseUrl": "https://adsgofastkeyvault.vault.azure.net/",
|
||||
"ScheduleMasterId": 2,
|
||||
"TaskGroupConcurrency": 10,
|
||||
"TaskGroupPriority": 0,
|
||||
"Source": {
|
||||
"Type": "SQL Server",
|
||||
"Database": {
|
||||
"SystemName": "adsgofast-onpre",
|
||||
"Name": "AdventureWorks2017",
|
||||
"AuthenticationType": "SQLAuth",
|
||||
"Username": "sqladfir",
|
||||
"PasswordKeyVaultSecretName": "adsgofast-onpre-sqladfir-password"
|
||||
},
|
||||
"Extraction": {
|
||||
"Type": "Table",
|
||||
"FullOrIncremental": "Full",
|
||||
"IncrementalType": null,
|
||||
"TableSchema": "Sales",
|
||||
"TableName": "SalesOrderHeader"
|
||||
}
|
||||
},
|
||||
"Target": {
|
||||
"Type": "Azure Blob",
|
||||
"StorageAccountName": "https://adsgofastdatalakeaccelst.blob.core.windows.net",
|
||||
"StorageAccountContainer": "datalakeraw",
|
||||
"StorageAccountAccessMethod": "MSI",
|
||||
"RelativePath": "AdventureWorks2017/Sales/SalesOrderHeader/2020/7/23/9/0/",
|
||||
"DataFileName": "Sales.SalesOrderHeader.parquet",
|
||||
"SchemaFileName": "Sales.SalesOrderHeader.json",
|
||||
"FirstRowAsHeader": null,
|
||||
"SheetName": null,
|
||||
"SkipLineCount": null,
|
||||
"MaxConcorrentConnections": null
|
||||
},
|
||||
"DataFactory": {
|
||||
"Id": 1,
|
||||
"Name": "adsgofastdatakakeacceladf",
|
||||
"ResourceGroup": "AdsGoFastDataLakeAccel",
|
||||
"SubscriptionId": "035a1364-f00d-48e2-b582-4fe125905ee3",
|
||||
"ADFPipeline": "OnP_SQL_AZ_Storage_Parquet_@GF{IR}"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"folder": {
|
||||
"name": "ADS Go Fast"
|
||||
},
|
||||
"annotations": [],
|
||||
"lastPublishTime": "2020-08-06T06:27:14Z"
|
||||
},
|
||||
"type": "Microsoft.DataFactory/factories/pipelines"
|
||||
}
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
{
|
||||
"name": "OnP-SQL-AZ-Storage-Parquet-OnP-SH-IR",
|
||||
"name": "SH_SQL_AZ_Storage_Parquet_@GF{IR}",
|
||||
"properties": {
|
||||
"activities": [
|
||||
{
|
||||
|
|
@ -16,7 +16,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-Function-Generic",
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
|
|
@ -117,7 +117,7 @@
|
|||
"partitionOption": "None"
|
||||
},
|
||||
"dataset": {
|
||||
"referenceName": "SqlServerTable_OnPrem_SH_IR",
|
||||
"referenceName": "SqlServerTable_@GF{IR}",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"TableSchema": {
|
||||
|
|
@ -175,13 +175,13 @@
|
|||
"value": "Full",
|
||||
"activities": [
|
||||
{
|
||||
"name": "Execute OnP-SQL-Full-Load-OnP-SH-IR",
|
||||
"name": "Execute SH_SQL_Full_Load_@GF{IR}",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "OnP-SQL-Full-Load-OnP-SH-IR",
|
||||
"referenceName": "SH_SQL_Full_Load_@GF{IR}",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": true,
|
||||
|
|
@ -205,7 +205,7 @@
|
|||
"value": "Watermark",
|
||||
"activities": [
|
||||
{
|
||||
"name": "Execute OnP-SQL-Watermark-OnP-SH-IR",
|
||||
"name": "Execute SH_SQL_Watermark_@GF{IR}",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [
|
||||
{
|
||||
|
|
@ -218,7 +218,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "OnP-SQL-Watermark-OnP-SH-IR",
|
||||
"referenceName": "SH_SQL_Watermark_@GF{IR}",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": true,
|
||||
|
|
@ -263,7 +263,7 @@
|
|||
"partitionOption": "None"
|
||||
},
|
||||
"dataset": {
|
||||
"referenceName": "SqlServerTable_OnPrem_SH_IR",
|
||||
"referenceName": "SqlServerTable_@GF{IR}",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"TableSchema": {
|
||||
|
|
@ -302,10 +302,10 @@
|
|||
]
|
||||
},
|
||||
{
|
||||
"value": "Full-Chunk",
|
||||
"value": "Full_Chunk",
|
||||
"activities": [
|
||||
{
|
||||
"name": "Execute OnP-SQL-Full-Load-Chunk-OnP-SH-IR",
|
||||
"name": "Execute SH_SQL_Full_Load_Chunk_@GF{IR}",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [
|
||||
{
|
||||
|
|
@ -318,7 +318,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "OnP-SQL-Full-Load-Chunk-OnP-SH-IR",
|
||||
"referenceName": "SH_SQL_Full_Load_Chunk_@GF{IR}",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": true,
|
||||
|
|
@ -361,7 +361,7 @@
|
|||
"partitionOption": "None"
|
||||
},
|
||||
"dataset": {
|
||||
"referenceName": "SqlServerTable_OnPrem_SH_IR",
|
||||
"referenceName": "SqlServerTable_@GF{IR}",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"TableSchema": {
|
||||
|
|
@ -400,10 +400,10 @@
|
|||
]
|
||||
},
|
||||
{
|
||||
"value": "Watermark-Chunk",
|
||||
"value": "Watermark_Chunk",
|
||||
"activities": [
|
||||
{
|
||||
"name": "Execute OnP-SQL-Watermark-Chunk-OnP-SH-IR",
|
||||
"name": "Execute SH_SQL_Watermark_Chunk_@GF{IR}",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [
|
||||
{
|
||||
|
|
@ -416,7 +416,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "OnP-SQL-Watermark-Chunk-OnP-SH-IR",
|
||||
"referenceName": "SH_SQL_Watermark_Chunk_@GF{IR}",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": true,
|
||||
|
|
@ -463,7 +463,7 @@
|
|||
"partitionOption": "None"
|
||||
},
|
||||
"dataset": {
|
||||
"referenceName": "SqlServerTable_OnPrem_SH_IR",
|
||||
"referenceName": "SqlServerTable_@GF{IR}",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"TableSchema": {
|
||||
|
|
@ -508,11 +508,11 @@
|
|||
"parameters": {
|
||||
"TaskObject": {
|
||||
"type": "object",
|
||||
"defaultValue": " [ { \"TaskInstanceId\": 2, \"TaskMasterId\": 1, \"TaskStatus\": \"Untried\", \"TaskType\": \"SQLtoAzureStorage-OnPrem-SH-IR\", \"Enabled\": true, \"ExecutionUid\": 1, \"KeyVaultBaseUrl\": \"https://adsgofastkeyvault.vault.azure.net/\", \"Source\": { \"Type\": \"MsSqlServer\", \"Database\": { \"Name\": \"AdventureWorks2017\", \"SystemName\": \"adsgofast-onpre\", \"Type\": \"MsSqlServer\", \"AuthenticationType\": \"SQLAuth\", \"Username\": \"sqladfir\", \"PasswordKeyVaultSecretName\": \"adsgofast-onpre-sqladfir-password\" }, \"Extraction\": { \"Type\": \"Table\", \"FullOrIncremental\": true, \"IncrementalType\": null, \"TableSchema\": \"Sales\", \"TableName\": \"SalesOrderDetail\" } }, \"Target\": { \"StorageAccountName\": \"https://adsgofastdatalakeadls.dfs.core.windows.net\", \"Type\": \"ADLS\", \"StorageAccountContainer\": \"datalakelanding\", \"StorageAccountAccessMethod\": \"MSI\", \"StorageAccountSASUriKeyVaultSecretName\": null, \"RelativePath\": \"/Unprocessed/adsgofastdatakakeaccelsqlsvr/AWSample/SalesLT/2020/06/08/17/\", \"DataFileName\": \"Customer_Data.parquet\", \"SchemaFileName\": \"Customer_Schema.json\" }, \"DataFactory\": { \"Id\": 1, \"Name\": \"adsgofastdatakakeacceladf\", \"ResourceGroup\": \"AdsGoFastDataLakeAccel\", \"SubscriptionId\": \"035a1364-f00d-48e2-b582-4fe125905ee3\", } } ]"
|
||||
"defaultValue": " [ { \"TaskInstanceId\": 2, \"TaskMasterId\": 1, \"TaskStatus\": \"Untried\", \"TaskType\": \"SQLtoAzureStorage_@GF{IR}\", \"Enabled\": true, \"ExecutionUid\": 1, \"KeyVaultBaseUrl\": \"https://adsgofastkeyvault.vault.azure.net/\", \"Source\": { \"Type\": \"MsSqlServer\", \"Database\": { \"Name\": \"AdventureWorks2017\", \"SystemName\": \"adsgofast-onpre\", \"Type\": \"MsSqlServer\", \"AuthenticationType\": \"SQLAuth\", \"Username\": \"sqladfir\", \"PasswordKeyVaultSecretName\": \"adsgofast-onpre-sqladfir-password\" }, \"Extraction\": { \"Type\": \"Table\", \"FullOrIncremental\": true, \"IncrementalType\": null, \"TableSchema\": \"Sales\", \"TableName\": \"SalesOrderDetail\" } }, \"Target\": { \"StorageAccountName\": \"https://adsgofastdatalakeadls.dfs.core.windows.net\", \"Type\": \"ADLS\", \"StorageAccountContainer\": \"datalakelanding\", \"StorageAccountAccessMethod\": \"MSI\", \"StorageAccountSASUriKeyVaultSecretName\": null, \"RelativePath\": \"/Unprocessed/adsgofastdatakakeaccelsqlsvr/AWSample/SalesLT/2020/06/08/17/\", \"DataFileName\": \"Customer_Data.parquet\", \"SchemaFileName\": \"Customer_Schema.json\" }, \"DataFactory\": { \"Id\": 1, \"Name\": \"adsgofastdatakakeacceladf\", \"ResourceGroup\": \"AdsGoFastDataLakeAccel\", \"SubscriptionId\": \"035a1364-f00d-48e2-b582-4fe125905ee3\", } } ]"
|
||||
}
|
||||
},
|
||||
"folder": {
|
||||
"name": "ADS Go Fast/Data Movement/OnPrem SH IR"
|
||||
"name": "ADS Go Fast/Data Movement/@GF{IR}"
|
||||
},
|
||||
"annotations": [],
|
||||
"lastPublishTime": "2020-08-04T12:40:45Z"
|
||||
|
|
@ -0,0 +1,118 @@
|
|||
{
|
||||
"name": "SH_SQL_Full_Load_Chunk_@GF{IR}",
|
||||
"properties": {
|
||||
"activities": [
|
||||
{
|
||||
"name": "ForEach Chunk",
|
||||
"type": "ForEach",
|
||||
"dependsOn": [],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"items": {
|
||||
"value": "@range(1, pipeline().parameters.BatchCount)",
|
||||
"type": "Expression"
|
||||
},
|
||||
"isSequential": true,
|
||||
"activities": [
|
||||
{
|
||||
"name": "Execute SH_SQL_Full_Load",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "SH_SQL_Full_Load_@GF{IR}",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": true,
|
||||
"parameters": {
|
||||
"TaskObject": {
|
||||
"value": "@pipeline().parameters.TaskObject",
|
||||
"type": "Expression"
|
||||
},
|
||||
"Mapping": {
|
||||
"value": "@pipeline().parameters.Mapping",
|
||||
"type": "Expression"
|
||||
},
|
||||
"BatchCount": {
|
||||
"value": "@pipeline().parameters.BatchCount",
|
||||
"type": "Expression"
|
||||
},
|
||||
"Item": {
|
||||
"value": "@item()",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"TaskObject": {
|
||||
"type": "object",
|
||||
"defaultValue": {
|
||||
"TaskInstanceId": 75,
|
||||
"TaskMasterId": 12,
|
||||
"TaskStatus": "Untried",
|
||||
"TaskType": "SQL Database to Azure Storage",
|
||||
"Enabled": 1,
|
||||
"ExecutionUid": "2c5924ee-b855-4d2b-bb7e-4f5dde4c4dd3",
|
||||
"NumberOfRetries": 111,
|
||||
"DegreeOfCopyParallelism": 1,
|
||||
"KeyVaultBaseUrl": "https://adsgofastkeyvault.vault.azure.net/",
|
||||
"ScheduleMasterId": 2,
|
||||
"TaskGroupConcurrency": 10,
|
||||
"TaskGroupPriority": 0,
|
||||
"Source": {
|
||||
"Type": "Azure SQL",
|
||||
"Database": {
|
||||
"SystemName": "adsgofastdatakakeaccelsqlsvr.database.windows.net",
|
||||
"Name": "AWSample",
|
||||
"AuthenticationType": "MSI"
|
||||
},
|
||||
"Extraction": {
|
||||
"Type": "Table",
|
||||
"FullOrIncremental": "Full",
|
||||
"IncrementalType": null,
|
||||
"TableSchema": "SalesLT",
|
||||
"TableName": "SalesOrderHeader"
|
||||
}
|
||||
},
|
||||
"Target": {
|
||||
"Type": "Azure Blob",
|
||||
"StorageAccountName": "https://adsgofastdatalakeaccelst.blob.core.windows.net",
|
||||
"StorageAccountContainer": "datalakeraw",
|
||||
"StorageAccountAccessMethod": "MSI",
|
||||
"RelativePath": "/AwSample/SalesLT/SalesOrderHeader/2020/7/9/14/12/",
|
||||
"DataFileName": "SalesLT.SalesOrderHeader.parquet",
|
||||
"SchemaFileName": "SalesLT.SalesOrderHeader",
|
||||
"FirstRowAsHeader": null,
|
||||
"SheetName": null,
|
||||
"SkipLineCount": null,
|
||||
"MaxConcorrentConnections": null
|
||||
},
|
||||
"DataFactory": {
|
||||
"Id": 1,
|
||||
"Name": "adsgofastdatakakeacceladf",
|
||||
"ResourceGroup": "AdsGoFastDataLakeAccel",
|
||||
"SubscriptionId": "035a1364-f00d-48e2-b582-4fe125905ee3",
|
||||
"ADFPipeline": "SH-AZ_Storage_Parquet_@GF{IR}"
|
||||
}
|
||||
}
|
||||
},
|
||||
"Mapping": {
|
||||
"type": "object"
|
||||
},
|
||||
"BatchCount": {
|
||||
"type": "int"
|
||||
}
|
||||
},
|
||||
"folder": {
|
||||
"name": "ADS Go Fast/Data Movement/@GF{IR}/Components"
|
||||
},
|
||||
"annotations": []
|
||||
},
|
||||
"type": "Microsoft.DataFactory/factories/pipelines"
|
||||
}
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
{
|
||||
"name": "OnP-SQL-Full-Load-OnP-SH-IR",
|
||||
"name": "SH_SQL_Full_Load_@GF{IR}",
|
||||
"properties": {
|
||||
"activities": [
|
||||
{
|
||||
|
|
@ -62,8 +62,7 @@
|
|||
"value": "@variables('SQLStatement')",
|
||||
"type": "Expression"
|
||||
},
|
||||
"queryTimeout": "02:00:00",
|
||||
"partitionOption": "None"
|
||||
"queryTimeout": "02:00:00"
|
||||
},
|
||||
"sink": {
|
||||
"type": "ParquetSink",
|
||||
|
|
@ -83,7 +82,7 @@
|
|||
},
|
||||
"inputs": [
|
||||
{
|
||||
"referenceName": "SqlServerTable_OnPrem_SH_IR",
|
||||
"referenceName": "SqlServerTable_@GF{IR}",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"TableSchema": {
|
||||
|
|
@ -119,7 +118,7 @@
|
|||
],
|
||||
"outputs": [
|
||||
{
|
||||
"referenceName": "ADLSParquet_OnPrem_SH_IR",
|
||||
"referenceName": "ADLSParquet_@GF{IR}",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"RelativePath": {
|
||||
|
|
@ -149,7 +148,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-Function-Generic",
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
|
|
@ -216,7 +215,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"dataset": {
|
||||
"referenceName": "ADLSParquet_SH_IR",
|
||||
"referenceName": "ADLSParquet_@GF{IR}",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"RelativePath": {
|
||||
|
|
@ -260,7 +259,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-Function-Generic",
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
|
|
@ -288,7 +287,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-Function-Generic",
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
|
|
@ -333,8 +332,7 @@
|
|||
"value": "@variables('SQLStatement')",
|
||||
"type": "Expression"
|
||||
},
|
||||
"queryTimeout": "02:00:00",
|
||||
"partitionOption": "None"
|
||||
"queryTimeout": "02:00:00"
|
||||
},
|
||||
"sink": {
|
||||
"type": "ParquetSink",
|
||||
|
|
@ -354,7 +352,7 @@
|
|||
},
|
||||
"inputs": [
|
||||
{
|
||||
"referenceName": "SqlServerTable_OnPrem_SH_IR",
|
||||
"referenceName": "SqlServerTable_@GF{IR}",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"TableSchema": {
|
||||
|
|
@ -390,7 +388,7 @@
|
|||
],
|
||||
"outputs": [
|
||||
{
|
||||
"referenceName": "BlobParquet_OnPrem_SH_IR",
|
||||
"referenceName": "BlobParquet_@GF{IR}",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"RelativePath": {
|
||||
|
|
@ -420,7 +418,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-Function-Generic",
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
|
|
@ -487,7 +485,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"dataset": {
|
||||
"referenceName": "BlobParquet_SH_IR",
|
||||
"referenceName": "BlobParquet_@GF{IR}",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"RelativePath": {
|
||||
|
|
@ -531,7 +529,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-Function-Generic",
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
|
|
@ -559,7 +557,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ-Function-Generic",
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
|
|
@ -628,7 +626,7 @@
|
|||
"Name": "adsgofastdatakakeacceladf",
|
||||
"ResourceGroup": "AdsGoFastDataLakeAccel",
|
||||
"SubscriptionId": "035a1364-f00d-48e2-b582-4fe125905ee3",
|
||||
"ADFPipeline": "AZ-SQL-AZ-Storage-Parquet-SH-IR"
|
||||
"ADFPipeline": "AZ_SQL_AZ_Storage_Parquet_@GF{IR}"
|
||||
}
|
||||
}
|
||||
},
|
||||
|
|
@ -648,10 +646,9 @@
|
|||
}
|
||||
},
|
||||
"folder": {
|
||||
"name": "ADS Go Fast/Data Movement/OnPrem SH IR/Components"
|
||||
"name": "ADS Go Fast/Data Movement/@GF{IR}/Components"
|
||||
},
|
||||
"annotations": [],
|
||||
"lastPublishTime": "2020-08-05T00:59:22Z"
|
||||
"annotations": []
|
||||
},
|
||||
"type": "Microsoft.DataFactory/factories/pipelines"
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,364 @@
|
|||
{
|
||||
"name": "SH_SQL_GEN_File_Parquet_@GF{IR}",
|
||||
"properties": {
|
||||
"activities": [
|
||||
{
|
||||
"name": "Copy SQL to File",
|
||||
"type": "Copy",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "Pipeline AF Log - SQL to File Start",
|
||||
"dependencyConditions": [
|
||||
"Succeeded"
|
||||
]
|
||||
}
|
||||
],
|
||||
"policy": {
|
||||
"timeout": "7.00:00:00",
|
||||
"retry": 0,
|
||||
"retryIntervalInSeconds": 30,
|
||||
"secureOutput": false,
|
||||
"secureInput": false
|
||||
},
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"source": {
|
||||
"type": "SqlServerSource",
|
||||
"queryTimeout": "02:00:00",
|
||||
"partitionOption": "None"
|
||||
},
|
||||
"sink": {
|
||||
"type": "ParquetSink",
|
||||
"storeSettings": {
|
||||
"type": "FileServerWriteSettings"
|
||||
}
|
||||
},
|
||||
"enableStaging": false,
|
||||
"parallelCopies": {
|
||||
"value": "@pipeline().parameters.TaskObject.DegreeOfCopyParallelism",
|
||||
"type": "Expression"
|
||||
},
|
||||
"translator": {
|
||||
"value": "@activity('AF Persist Metadata and Get Mapping').output.value",
|
||||
"type": "Expression"
|
||||
}
|
||||
},
|
||||
"inputs": [
|
||||
{
|
||||
"referenceName": "SqlServerTable_@GF{IR}",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"TableSchema": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.Extraction.TableSchema",
|
||||
"type": "Expression"
|
||||
},
|
||||
"TableName": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.Extraction.TableName",
|
||||
"type": "Expression"
|
||||
},
|
||||
"KeyVaultBaseUrl": {
|
||||
"value": "@pipeline().parameters.TaskObject.KeyVaultBaseUrl",
|
||||
"type": "Expression"
|
||||
},
|
||||
"PasswordSecret": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.Database.PasswordKeyVaultSecretName",
|
||||
"type": "Expression"
|
||||
},
|
||||
"Server": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.Database.SystemName",
|
||||
"type": "Expression"
|
||||
},
|
||||
"Database": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.Database.Name",
|
||||
"type": "Expression"
|
||||
},
|
||||
"UserName": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.Database.Username",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"referenceName": "FileParquet_@GF{IR}",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"Host": {
|
||||
"value": "@pipeline().parameters.TaskObject.Target.Host",
|
||||
"type": "Expression"
|
||||
},
|
||||
"UserId": {
|
||||
"value": "@pipeline().parameters.TaskObject.Target.UserId",
|
||||
"type": "Expression"
|
||||
},
|
||||
"Secret": {
|
||||
"value": "@pipeline().parameters.TaskObject.Target.PasswordKeyVaultSecretName",
|
||||
"type": "Expression"
|
||||
},
|
||||
"Directory": {
|
||||
"value": "@pipeline().parameters.TaskObject.Target.Directory",
|
||||
"type": "Expression"
|
||||
},
|
||||
"File": {
|
||||
"value": "@pipeline().parameters.TaskObject.Target.File",
|
||||
"type": "Expression"
|
||||
},
|
||||
"KeyVaultBaseUrl": {
|
||||
"value": "@pipeline().parameters.TaskObject.KeyVaultBaseUrl",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "Pipeline AF Log - SQL to File Start",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "AF Persist Metadata and Get Mapping",
|
||||
"dependencyConditions": [
|
||||
"Succeeded"
|
||||
]
|
||||
}
|
||||
],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
"parameters": {
|
||||
"Body": {
|
||||
"value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":3,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy SQL Server to File\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"Status\":\"Started\"}'))",
|
||||
"type": "Expression"
|
||||
},
|
||||
"FunctionName": "Log",
|
||||
"Method": "Post"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Pipeline AF Log - SQL to File Succeed",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "Copy SQL to File",
|
||||
"dependencyConditions": [
|
||||
"Succeeded"
|
||||
]
|
||||
}
|
||||
],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
"parameters": {
|
||||
"Body": {
|
||||
"value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":1,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy SQL to File\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"EndDateTimeOffSet\":\"', string(utcnow()), '\",\"RowsInserted\":\"', string(activity('Copy SQL to File').output.rowsCopied), '\",\"Comment\":\"\",\"Status\":\"Complete\"}'))",
|
||||
"type": "Expression"
|
||||
},
|
||||
"FunctionName": "Log",
|
||||
"Method": "Post"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Pipeline AF Log - SQL to File Failed",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "Copy SQL to File",
|
||||
"dependencyConditions": [
|
||||
"Failed"
|
||||
]
|
||||
}
|
||||
],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
"parameters": {
|
||||
"Body": {
|
||||
"value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"LogTypeId\":1,\"LogSource\":\"ADF\",\"ActivityType\":\"Copy SQL Server to File Share\",\"StartDateTimeOffSet\":\"', string(pipeline().TriggerTime), '\",\"EndDateTimeOffSet\":\"', string(utcnow()), '\",\"Comment\":\"', string(activity('Copy SQL to File').error.message), '\",\"Status\":\"Failed\"}'))",
|
||||
"type": "Expression"
|
||||
},
|
||||
"FunctionName": "Log",
|
||||
"Method": "Post"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "AF Log - Get Metadata Failed",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "Lookup Get SQL Metadata",
|
||||
"dependencyConditions": [
|
||||
"Failed"
|
||||
]
|
||||
}
|
||||
],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
"parameters": {
|
||||
"Body": {
|
||||
"value": "[\n {\n \"TaskInstanceId\":@{pipeline().parameters.TaskObject.TaskInstanceId},\n \"ExecutionUid\":@{pipeline().parameters.TaskObject.ExecutionUid},\n \"RunId\":@{pipeline().RunId}, \n \"EndTime\":@{utcnow()},\n \"Status\":\"Failed\",\n \"Activity\":\"Get Metadata\",\n \"Comments\":@{activity('Lookup Get SQL Metadata').errors}\n }\n]",
|
||||
"type": "Expression"
|
||||
},
|
||||
"FunctionName": "Log",
|
||||
"Method": "Post"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "AF Get Information Schema SQL",
|
||||
"type": "AzureFunctionActivity",
|
||||
"dependsOn": [],
|
||||
"policy": {
|
||||
"timeout": "7.00:00:00",
|
||||
"retry": 0,
|
||||
"retryIntervalInSeconds": 30,
|
||||
"secureOutput": false,
|
||||
"secureInput": false
|
||||
},
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"functionName": "GetInformationSchemaSQL",
|
||||
"method": "POST",
|
||||
"body": {
|
||||
"value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"TableSchema\":\"', string(pipeline().parameters.TaskObject.Source.Extraction.TableSchema), '\",\"TableName\":\"', string(pipeline().parameters.TaskObject.Source.Extraction.TableName),'\"}'))",
|
||||
"type": "Expression"
|
||||
}
|
||||
},
|
||||
"linkedServiceName": {
|
||||
"referenceName": "AzureFunctionAdsGoFastDataLakeAccelFunApp",
|
||||
"type": "LinkedServiceReference"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "AF Persist Metadata and Get Mapping",
|
||||
"type": "AzureFunctionActivity",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "Lookup Get SQL Metadata",
|
||||
"dependencyConditions": [
|
||||
"Succeeded"
|
||||
]
|
||||
}
|
||||
],
|
||||
"policy": {
|
||||
"timeout": "7.00:00:00",
|
||||
"retry": 0,
|
||||
"retryIntervalInSeconds": 30,
|
||||
"secureOutput": false,
|
||||
"secureInput": false
|
||||
},
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"functionName": "TaskExecutionSchemaFile",
|
||||
"method": "POST",
|
||||
"body": {
|
||||
"value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"StorageAccountName\":\"', string(pipeline().parameters.TaskObject.Target.StorageAccountName), '\",\"StorageAccountContainer\":\"', string(pipeline().parameters.TaskObject.Target.StorageAccountContainer), '\",\"RelativePath\":\"', string(pipeline().parameters.TaskObject.Target.RelativePath), '\",\"SchemaFileName\":\"', string(pipeline().parameters.TaskObject.Target.SchemaFileName), '\",\"SourceType\":\"', string(pipeline().parameters.TaskObject.Source.Type), '\",\"TargetType\":\"', string(pipeline().parameters.TaskObject.Target.Type), '\",\"Data\":',string(activity('Lookup Get SQL Metadata').output),',\"MetadataType\":\"SQL\"}'))",
|
||||
"type": "Expression"
|
||||
}
|
||||
},
|
||||
"linkedServiceName": {
|
||||
"referenceName": "AzureFunctionAdsGoFastDataLakeAccelFunApp",
|
||||
"type": "LinkedServiceReference"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Lookup Get SQL Metadata",
|
||||
"type": "Lookup",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "AF Get Information Schema SQL",
|
||||
"dependencyConditions": [
|
||||
"Succeeded"
|
||||
]
|
||||
}
|
||||
],
|
||||
"policy": {
|
||||
"timeout": "7.00:00:00",
|
||||
"retry": 0,
|
||||
"retryIntervalInSeconds": 30,
|
||||
"secureOutput": false,
|
||||
"secureInput": false
|
||||
},
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"source": {
|
||||
"type": "SqlServerSource",
|
||||
"sqlReaderQuery": {
|
||||
"value": "@activity('AF Get Information Schema SQL').output.InformationSchemaSQL",
|
||||
"type": "Expression"
|
||||
},
|
||||
"queryTimeout": "02:00:00",
|
||||
"partitionOption": "None"
|
||||
},
|
||||
"dataset": {
|
||||
"referenceName": "SqlServerTable_@GF{IR}",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"TableSchema": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.Extraction.TableSchema",
|
||||
"type": "Expression"
|
||||
},
|
||||
"TableName": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.Extraction.TableName",
|
||||
"type": "Expression"
|
||||
},
|
||||
"KeyVaultBaseUrl": {
|
||||
"value": "@pipeline().parameters.TaskObject.KeyVaultBaseUrl",
|
||||
"type": "Expression"
|
||||
},
|
||||
"PasswordSecret": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.Database.PasswordKeyVaultSecretName",
|
||||
"type": "Expression"
|
||||
},
|
||||
"Server": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.Database.SystemName",
|
||||
"type": "Expression"
|
||||
},
|
||||
"Database": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.Database.Name",
|
||||
"type": "Expression"
|
||||
},
|
||||
"UserName": {
|
||||
"value": "@pipeline().parameters.TaskObject.Source.Database.Username",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
},
|
||||
"firstRowOnly": false
|
||||
}
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"TaskObject": {
|
||||
"type": "object",
|
||||
"defaultValue": "[ { \"TaskInstanceId\": 2, \"TaskMasterId\": 1, \"TaskStatus\": \"Untried\", \"TaskType\": \"RelationalDataExtractToDataLake\", \"Enabled\": true, \"ExecutionUid\": 1, \"KeyVaultBaseUrl\" : \"https://adsgofastkeyvault.vault.azure.net/\", \"Source\": { \"Type\":\"MsSqlServer\", \"Database\": { \"Name\": \"AdventureWorks2017\", \"SystemName\": \"adsgofast-onpre\", \"Type\": \"MsSqlServer\", \"AuthenticationType\": \"SQLAuth\", \"Username\": \"sqladfir\", \"PasswordKeyVaultSecretName\": \"adsgofast-onpre-sqladfir-password\" }, \"Extraction\": { \"Type\": \"Table\", \"FullOrIncremental\": true, \"IncrementalType\": null, \"TableSchema\": \"Sales\", \"TableName\": \"SalesOrderDetail\" } }, \"Target\": { \"Type\" : \"File\", \"Host\": \"D:\\\\dataingestion\\\\\", \"UserId\": \"AzureUser\", \"PasswordKeyVaultSecretName\": \"adsgofast-onpre_File_password\", \"Directory\": \"AdventureWorks2017/2020/06/15/14/22/\", \"File\": \"Sales.SalesOrderDetails.parquet\" }, \"DataFactory\": { \"Id\": 1, \"Name\": \"adsgofastdatakakeacceladf\", \"ResourceGroup\": \"AdsGoFastDataLakeAccel\", \"SubscriptionId\": \"035a1364-f00d-48e2-b582-4fe125905ee3\", } } ]"
|
||||
}
|
||||
},
|
||||
"folder": {
|
||||
"name": "ADS Go Fast/Data Movement/@GF{IR}"
|
||||
},
|
||||
"annotations": [],
|
||||
"lastPublishTime": "2020-07-27T03:24:22Z"
|
||||
},
|
||||
"type": "Microsoft.DataFactory/factories/pipelines"
|
||||
}
|
||||
|
|
@ -0,0 +1,153 @@
|
|||
{
|
||||
"name": "SH_SQL_Watermark_Chunk_@GF{IR}",
|
||||
"properties": {
|
||||
"activities": [
|
||||
{
|
||||
"name": "AF Set New Watermark",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "ForEach Chunk",
|
||||
"dependencyConditions": [
|
||||
"Succeeded"
|
||||
]
|
||||
}
|
||||
],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "AZ_Function_Generic",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
"parameters": {
|
||||
"Body": {
|
||||
"value": "@json(concat('{\"TaskInstanceId\":\"', string(pipeline().parameters.TaskObject.TaskInstanceId), '\",\"ExecutionUid\":\"', string(pipeline().parameters.TaskObject.ExecutionUid), '\",\"RunId\":\"', string(pipeline().RunId), '\",\"TaskMasterId\":\"', string(pipeline().parameters.TaskObject.TaskMasterId),'\",\"TaskMasterWaterMarkColumnType\":\"', string(pipeline().parameters.TaskObject.Source.Extraction.IncrementalColumnType),'\",\"WaterMarkValue\":\"', string(pipeline().parameters.NewWatermark), '\"}'))",
|
||||
"type": "Expression"
|
||||
},
|
||||
"FunctionName": "WaterMark",
|
||||
"Method": "Post"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "ForEach Chunk",
|
||||
"type": "ForEach",
|
||||
"dependsOn": [],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"items": {
|
||||
"value": "@range(1, pipeline().parameters.BatchCount)",
|
||||
"type": "Expression"
|
||||
},
|
||||
"isSequential": true,
|
||||
"activities": [
|
||||
{
|
||||
"name": "Execute SH_SQL_Watermark",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "SH_SQL_Watermark_@GF{IR}",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": true,
|
||||
"parameters": {
|
||||
"TaskObject": {
|
||||
"value": "@pipeline().parameters.TaskObject",
|
||||
"type": "Expression"
|
||||
},
|
||||
"Mapping": {
|
||||
"value": "@pipeline().parameters.Mapping",
|
||||
"type": "Expression"
|
||||
},
|
||||
"NewWaterMark": {
|
||||
"value": "@pipeline().parameters.NewWatermark",
|
||||
"type": "Expression"
|
||||
},
|
||||
"Item": {
|
||||
"value": "@item()",
|
||||
"type": "Expression"
|
||||
},
|
||||
"BatchCount": {
|
||||
"value": "@pipeline().parameters.BatchCount",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"TaskObject": {
|
||||
"type": "object",
|
||||
"defaultValue": {
|
||||
"TaskInstanceId": 75,
|
||||
"TaskMasterId": 12,
|
||||
"TaskStatus": "Untried",
|
||||
"TaskType": "SQL Database to Azure Storage",
|
||||
"Enabled": 1,
|
||||
"ExecutionUid": "2c5924ee-b855-4d2b-bb7e-4f5dde4c4dd3",
|
||||
"NumberOfRetries": 111,
|
||||
"DegreeOfCopyParallelism": 1,
|
||||
"KeyVaultBaseUrl": "https://adsgofastkeyvault.vault.azure.net/",
|
||||
"ScheduleMasterId": 2,
|
||||
"TaskGroupConcurrency": 10,
|
||||
"TaskGroupPriority": 0,
|
||||
"Source": {
|
||||
"Type": "Azure SQL",
|
||||
"Database": {
|
||||
"SystemName": "adsgofastdatakakeaccelsqlsvr.database.windows.net",
|
||||
"Name": "AWSample",
|
||||
"AuthenticationType": "MSI"
|
||||
},
|
||||
"Extraction": {
|
||||
"Type": "Table",
|
||||
"FullOrIncremental": "Full",
|
||||
"IncrementalType": null,
|
||||
"TableSchema": "SalesLT",
|
||||
"TableName": "SalesOrderHeader"
|
||||
}
|
||||
},
|
||||
"Target": {
|
||||
"Type": "Azure Blob",
|
||||
"StorageAccountName": "https://adsgofastdatalakeaccelst.blob.core.windows.net",
|
||||
"StorageAccountContainer": "datalakeraw",
|
||||
"StorageAccountAccessMethod": "MSI",
|
||||
"RelativePath": "/AwSample/SalesLT/SalesOrderHeader/2020/7/9/14/12/",
|
||||
"DataFileName": "SalesLT.SalesOrderHeader.parquet",
|
||||
"SchemaFileName": "SalesLT.SalesOrderHeader",
|
||||
"FirstRowAsHeader": null,
|
||||
"SheetName": null,
|
||||
"SkipLineCount": null,
|
||||
"MaxConcorrentConnections": null
|
||||
},
|
||||
"DataFactory": {
|
||||
"Id": 1,
|
||||
"Name": "adsgofastdatakakeacceladf",
|
||||
"ResourceGroup": "AdsGoFastDataLakeAccel",
|
||||
"SubscriptionId": "035a1364-f00d-48e2-b582-4fe125905ee3",
|
||||
"ADFPipeline": "AZ_SQL_AZ_Storage_Parquet_@GF{IR}"
|
||||
}
|
||||
}
|
||||
},
|
||||
"Mapping": {
|
||||
"type": "object"
|
||||
},
|
||||
"NewWatermark": {
|
||||
"type": "string"
|
||||
},
|
||||
"BatchCount": {
|
||||
"type": "int"
|
||||
}
|
||||
},
|
||||
"folder": {
|
||||
"name": "ADS Go Fast/Data Movement/@GF{IR}/Components"
|
||||
},
|
||||
"annotations": []
|
||||
},
|
||||
"type": "Microsoft.DataFactory/factories/pipelines"
|
||||
}
|
||||
Некоторые файлы не были показаны из-за слишком большого количества измененных файлов Показать больше
Загрузка…
Ссылка в новой задаче