Merge pull request #69 from Syncret/master
remove outdated templates and add new templates
This commit is contained in:
gauravmalhot
GitHub
Коммит
9308a505a7
|
|
@ -0,0 +1,223 @@
|
|||
{
|
||||
"$schema": "http://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#",
|
||||
"contentVersion": "1.0.0.0",
|
||||
"parameters": {
|
||||
"factoryName": {
|
||||
"type": "string",
|
||||
"metadata": "Data Factory Name"
|
||||
},
|
||||
"DataSourceConnection": {
|
||||
"type": "string"
|
||||
},
|
||||
"ExternalControlTableConnection": {
|
||||
"type": "string"
|
||||
},
|
||||
"DataDestinationConnection": {
|
||||
"type": "string"
|
||||
},
|
||||
"allowPolyBase_parameter": {
|
||||
"type": "bool"
|
||||
},
|
||||
"preCopyScript_parameter": {
|
||||
"type": "bool"
|
||||
},
|
||||
"writeBatchSize_parameter": {
|
||||
"type": "int"
|
||||
},
|
||||
"polyBaseSettings_parameter": {
|
||||
"type": "object"
|
||||
},
|
||||
"enableStaging_parameter": {
|
||||
"type": "bool"
|
||||
},
|
||||
"stagingSettings_parameter": {
|
||||
"type": "object"
|
||||
},
|
||||
"copyactivity_output_parameter": {
|
||||
"type": "object"
|
||||
},
|
||||
"DataDestination_parameter_paramter": {
|
||||
"type": "object"
|
||||
},
|
||||
"DataDestination_typeproperties_paramter": {
|
||||
"type": "object"
|
||||
},
|
||||
"pipeline_parameter": {
|
||||
"type": "object"
|
||||
}
|
||||
},
|
||||
"variables": {
|
||||
"factoryId": "[concat('Microsoft.DataFactory/factories/', parameters('factoryName'))]"
|
||||
},
|
||||
"resources": [
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/BulkCopyfromDB_with_ControlTable')]",
|
||||
"type": "Microsoft.DataFactory/factories/pipelines",
|
||||
"apiVersion": "2017-09-01-preview",
|
||||
"properties": {
|
||||
"description": "Copy huge amount of data in bulk from database using external control table to store source table list with partitions for each table.\n\nWhen you want to migrate data from your data warehouse like Oracle server, Netezza server, Teradata server or SQL Server to Azure, you have to load huge amount of data from multiple tables in data sources. In most cases, data has to be further partitioned in each table so that you can load rows with multiple threads in parallel from single table.",
|
||||
"activities": [
|
||||
{
|
||||
"name": "GetPartitionList",
|
||||
"description": "Lookup activity to retrieve the list of partitions stored in the external control table.",
|
||||
"type": "Lookup",
|
||||
"policy": {
|
||||
"timeout": "7.00:00:00",
|
||||
"retry": 0,
|
||||
"retryIntervalInSeconds": 30,
|
||||
"secureOutput": false,
|
||||
"secureInput": false
|
||||
},
|
||||
"typeProperties": {
|
||||
"source": {
|
||||
"type": "SqlSource",
|
||||
"sqlReaderQuery": {
|
||||
"value": "select @{pipeline().parameters.Control_Table_Schema_PatitionID} as PatitionID, @{pipeline().parameters.Control_Table_Schema_SourceTableName} as SourceTableName, @{pipeline().parameters.Control_Table_Schema_FilterQuery} as FilterQuery from @{pipeline().parameters.Control_Table_Name}",
|
||||
"type": "Expression"
|
||||
}
|
||||
},
|
||||
"dataset": {
|
||||
"referenceName": "External_ControlTable",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"TableName": {
|
||||
"value": "@pipeline().parameters.Control_Table_Name",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
},
|
||||
"firstRowOnly": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "ForEachPartition",
|
||||
"description": "ForEach activity to get the partition list from Lookup activity and then iterate each of them to Copy activity. ",
|
||||
"type": "ForEach",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "GetPartitionList",
|
||||
"dependencyConditions": [
|
||||
"Succeeded"
|
||||
]
|
||||
}
|
||||
],
|
||||
"typeProperties": {
|
||||
"items": {
|
||||
"value": "@activity('GetPartitionList').output.value",
|
||||
"type": "Expression"
|
||||
},
|
||||
"batchCount": 5,
|
||||
"activities": [
|
||||
{
|
||||
"name": "CopyOnePartition",
|
||||
"description": "Copy activity to copy each partition from source database store to destination store.",
|
||||
"type": "Copy",
|
||||
"policy": {
|
||||
"timeout": "7.00:00:00",
|
||||
"retry": 0,
|
||||
"retryIntervalInSeconds": 30,
|
||||
"secureOutput": false,
|
||||
"secureInput": false
|
||||
},
|
||||
"typeProperties": {
|
||||
"source": {
|
||||
"type": "SqlSource",
|
||||
"sqlReaderQuery": {
|
||||
"value": "@item().FilterQuery",
|
||||
"type": "Expression"
|
||||
}
|
||||
},
|
||||
"sink": {
|
||||
"type": "AzureDataLakeStoreSink",
|
||||
"allowPolyBase": "[parameters('allowPolyBase_parameter')]",
|
||||
"preCopyScript": "[parameters('preCopyScript_parameter')]",
|
||||
"writeBatchSize": "[parameters('writeBatchSize_parameter')]",
|
||||
"polyBaseSettings": "[parameters('polyBaseSettings_parameter')]"
|
||||
},
|
||||
"enableStaging": "[parameters('enableStaging_parameter')]",
|
||||
"stagingSettings": "[parameters('stagingSettings_parameter')]",
|
||||
"dataIntegrationUnits": 0
|
||||
},
|
||||
"inputs": [
|
||||
{
|
||||
"referenceName": "DataSource",
|
||||
"type": "DatasetReference"
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"referenceName": "DataDestination",
|
||||
"type": "DatasetReference",
|
||||
"parameters": "[parameters('copyactivity_output_parameter')]"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"parameters": "[parameters('pipeline_parameter')]"
|
||||
},
|
||||
"dependsOn": [
|
||||
"[concat(variables('factoryId'), '/datasets/External_ControlTable')]",
|
||||
"[concat(variables('factoryId'), '/datasets/DataSource')]",
|
||||
"[concat(variables('factoryId'), '/datasets/DataDestination')]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/External_ControlTable')]",
|
||||
"type": "Microsoft.DataFactory/factories/datasets",
|
||||
"apiVersion": "2017-09-01-preview",
|
||||
"properties": {
|
||||
"description": "The external control table is used to store the partition list of data source tables.",
|
||||
"linkedServiceName": {
|
||||
"referenceName": "[parameters('ExternalControlTableConnection')]",
|
||||
"type": "LinkedServiceReference"
|
||||
},
|
||||
"parameters": {
|
||||
"TableName": {
|
||||
"type": "String"
|
||||
}
|
||||
},
|
||||
"type": "AzureSqlTable",
|
||||
"typeProperties": {
|
||||
"tableName": {
|
||||
"value": "@dataset().TableName",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/DataSource')]",
|
||||
"type": "Microsoft.DataFactory/factories/datasets",
|
||||
"apiVersion": "2017-09-01-preview",
|
||||
"properties": {
|
||||
"description": "Connection to your data source store.",
|
||||
"linkedServiceName": {
|
||||
"referenceName": "[parameters('DataSourceConnection')]",
|
||||
"type": "LinkedServiceReference"
|
||||
},
|
||||
"type": "AzureSqlTable",
|
||||
"typeProperties": {
|
||||
"tableName": "[concat('[', 'dbo].[xuyetable]')]"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/DataDestination')]",
|
||||
"type": "Microsoft.DataFactory/factories/datasets",
|
||||
"apiVersion": "2017-09-01-preview",
|
||||
"properties": {
|
||||
"description": "Connection to your data destination store.",
|
||||
"linkedServiceName": {
|
||||
"referenceName": "[parameters('DataDestinationConnection')]",
|
||||
"type": "LinkedServiceReference"
|
||||
},
|
||||
"parameters": "[parameters('DataDestination_parameter_paramter')]",
|
||||
"type": "AzureDataLakeStoreFile",
|
||||
"typeProperties": "[parameters('DataDestination_typeproperties_paramter')]"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
Различия файлов скрыты, потому что одна или несколько строк слишком длинны
|
|
@ -0,0 +1,96 @@
|
|||
{
|
||||
"$schema": "http://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#",
|
||||
"contentVersion": "1.0.0.0",
|
||||
"parameters": {
|
||||
"factoryName": {
|
||||
"type": "string",
|
||||
"metadata": "Data Factory Name"
|
||||
},
|
||||
"DataSourceConnection": {
|
||||
"type": "string"
|
||||
},
|
||||
"DataDestinationConnection": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"variables": {
|
||||
"factoryId": "[concat('Microsoft.DataFactory/factories/', parameters('factoryName'))]"
|
||||
},
|
||||
"resources": [
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/CopyData_from_Amazon_S3_to_AzureDataLakeStorage')]",
|
||||
"type": "Microsoft.DataFactory/factories/pipelines",
|
||||
"apiVersion": "2017-09-01-preview",
|
||||
"properties": {
|
||||
"description": "Copy data from Amazon S3 to Azure Data Lake Storage.\n\nYou can also go to \"Copy Data Tool\" to get the pipeline for more connectors and scenarios.",
|
||||
"activities": [
|
||||
{
|
||||
"name": "AmazonS3_to_Azure",
|
||||
"description": "Copy data from Amazon S3 to Azure Data Lake Storage.",
|
||||
"type": "Copy",
|
||||
"policy": {
|
||||
"timeout": "7.00:00:00",
|
||||
"retry": 0,
|
||||
"retryIntervalInSeconds": 30,
|
||||
"secureOutput": false,
|
||||
"secureInput": false
|
||||
},
|
||||
"typeProperties": {
|
||||
"source": {
|
||||
"type": "FileSystemSource",
|
||||
"recursive": true
|
||||
},
|
||||
"sink": {
|
||||
"type": "AzureBlobFSSink"
|
||||
},
|
||||
"enableStaging": false,
|
||||
"dataIntegrationUnits": 0
|
||||
},
|
||||
"inputs": [
|
||||
{
|
||||
"referenceName": "DataSourceStore",
|
||||
"type": "DatasetReference"
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"referenceName": "DataDestinationStore",
|
||||
"type": "DatasetReference"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"dependsOn": [
|
||||
"[concat(variables('factoryId'), '/datasets/DataSourceStore')]",
|
||||
"[concat(variables('factoryId'), '/datasets/DataDestinationStore')]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/DataSourceStore')]",
|
||||
"type": "Microsoft.DataFactory/factories/datasets",
|
||||
"apiVersion": "2017-09-01-preview",
|
||||
"properties": {
|
||||
"description": "Connection your Amazon S3 store.",
|
||||
"linkedServiceName": {
|
||||
"referenceName": "[parameters('DataSourceConnection')]",
|
||||
"type": "LinkedServiceReference"
|
||||
},
|
||||
"type": "AmazonS3Object"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/DataDestinationStore')]",
|
||||
"type": "Microsoft.DataFactory/factories/datasets",
|
||||
"apiVersion": "2017-09-01-preview",
|
||||
"properties": {
|
||||
"description": "Connection to your Azure Data Lake Store.",
|
||||
"linkedServiceName": {
|
||||
"referenceName": "[parameters('DataDestinationConnection')]",
|
||||
"type": "LinkedServiceReference"
|
||||
},
|
||||
"type": "AzureBlobFSFile"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
Различия файлов скрыты, потому что одна или несколько строк слишком длинны
|
|
@ -0,0 +1,106 @@
|
|||
{
|
||||
"$schema": "http://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#",
|
||||
"contentVersion": "1.0.0.0",
|
||||
"parameters": {
|
||||
"factoryName": {
|
||||
"type": "string",
|
||||
"metadata": "Data Factory Name"
|
||||
},
|
||||
"DataSourceConnection": {
|
||||
"type": "string"
|
||||
},
|
||||
"DataDestinationConnection": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"variables": {
|
||||
"factoryId": "[concat('Microsoft.DataFactory/factories/', parameters('factoryName'))]"
|
||||
},
|
||||
"resources": [
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/CopyData_from_Google_BigQuery_to_AzureDataLakeStorage')]",
|
||||
"type": "Microsoft.DataFactory/factories/pipelines",
|
||||
"apiVersion": "2017-09-01-preview",
|
||||
"properties": {
|
||||
"description": "Copy data from Google BigQuery to Azure Data Lake Storage.\n\n\nYou can also go to \"Copy Data Tool\" to get the pipeline for more connectors and scenarios.",
|
||||
"activities": [
|
||||
{
|
||||
"name": "Google_BigQuery_to_AzureDataLakeStorage",
|
||||
"description": "Copy data from Google BigQuery to Azure Data Lake Storage",
|
||||
"type": "Copy",
|
||||
"policy": {
|
||||
"timeout": "7.00:00:00",
|
||||
"retry": 0,
|
||||
"retryIntervalInSeconds": 30,
|
||||
"secureOutput": false,
|
||||
"secureInput": false
|
||||
},
|
||||
"typeProperties": {
|
||||
"source": {
|
||||
"type": "GoogleBigQuerySource"
|
||||
},
|
||||
"sink": {
|
||||
"type": "AzureBlobFSSink"
|
||||
},
|
||||
"enableStaging": false,
|
||||
"dataIntegrationUnits": 0
|
||||
},
|
||||
"inputs": [
|
||||
{
|
||||
"referenceName": "DataSourceStore",
|
||||
"type": "DatasetReference"
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"referenceName": "DataDestinationStore",
|
||||
"type": "DatasetReference"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"dependsOn": [
|
||||
"[concat(variables('factoryId'), '/datasets/DataSourceStore')]",
|
||||
"[concat(variables('factoryId'), '/datasets/DataDestinationStore')]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/DataSourceStore')]",
|
||||
"type": "Microsoft.DataFactory/factories/datasets",
|
||||
"apiVersion": "2017-09-01-preview",
|
||||
"properties": {
|
||||
"description": "Connection to your Google BigQuery store.",
|
||||
"linkedServiceName": {
|
||||
"referenceName": "[parameters('DataSourceConnection')]",
|
||||
"type": "LinkedServiceReference"
|
||||
},
|
||||
"type": "GoogleBigQueryObject"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/DataDestinationStore')]",
|
||||
"type": "Microsoft.DataFactory/factories/datasets",
|
||||
"apiVersion": "2017-09-01-preview",
|
||||
"properties": {
|
||||
"description": "Connection to your Azure Data Lake Store.",
|
||||
"linkedServiceName": {
|
||||
"referenceName": "[parameters('DataDestinationConnection')]",
|
||||
"type": "LinkedServiceReference"
|
||||
},
|
||||
"type": "AzureBlobFSFile",
|
||||
"typeProperties": {
|
||||
"format": {
|
||||
"type": "TextFormat",
|
||||
"columnDelimiter": ",",
|
||||
"rowDelimiter": "",
|
||||
"nullValue": "\\N",
|
||||
"treatEmptyAsNull": true,
|
||||
"skipLineCount": 0,
|
||||
"firstRowAsHeader": false
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
Различия файлов скрыты, потому что одна или несколько строк слишком длинны
|
|
@ -0,0 +1,96 @@
|
|||
{
|
||||
"$schema": "http://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#",
|
||||
"contentVersion": "1.0.0.0",
|
||||
"parameters": {
|
||||
"factoryName": {
|
||||
"type": "string",
|
||||
"metadata": "Data Factory Name"
|
||||
},
|
||||
"DataSourceConnection": {
|
||||
"type": "string"
|
||||
},
|
||||
"DataDestinationConnection": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"variables": {
|
||||
"factoryId": "[concat('Microsoft.DataFactory/factories/', parameters('factoryName'))]"
|
||||
},
|
||||
"resources": [
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/CopyData_from_HDFS_to_AzureDataLakeStorage')]",
|
||||
"type": "Microsoft.DataFactory/factories/pipelines",
|
||||
"apiVersion": "2017-09-01-preview",
|
||||
"properties": {
|
||||
"description": "Copy data from HDFS (Hadoop Distributed File System) to Azure Data Lake Storage.\n\n\nYou can also go to \"Copy Data Tool\" to get the pipeline for more connectors and scenarios.",
|
||||
"activities": [
|
||||
{
|
||||
"name": "HDFS_to_AzureDataLakeStorage",
|
||||
"description": "Copy data from HDFS (Hadoop Distributed File System) to Azure Data Lake Storage.",
|
||||
"type": "Copy",
|
||||
"policy": {
|
||||
"timeout": "7.00:00:00",
|
||||
"retry": 0,
|
||||
"retryIntervalInSeconds": 30,
|
||||
"secureOutput": false,
|
||||
"secureInput": false
|
||||
},
|
||||
"typeProperties": {
|
||||
"source": {
|
||||
"type": "HdfsSource",
|
||||
"recursive": true
|
||||
},
|
||||
"sink": {
|
||||
"type": "AzureBlobFSSink"
|
||||
},
|
||||
"enableStaging": false,
|
||||
"dataIntegrationUnits": 0
|
||||
},
|
||||
"inputs": [
|
||||
{
|
||||
"referenceName": "DataSourceStore",
|
||||
"type": "DatasetReference"
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"referenceName": "DataDestinationStore",
|
||||
"type": "DatasetReference"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"dependsOn": [
|
||||
"[concat(variables('factoryId'), '/datasets/DataSourceStore')]",
|
||||
"[concat(variables('factoryId'), '/datasets/DataDestinationStore')]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/DataSourceStore')]",
|
||||
"type": "Microsoft.DataFactory/factories/datasets",
|
||||
"apiVersion": "2017-09-01-preview",
|
||||
"properties": {
|
||||
"description": "Connection to your HDFS.",
|
||||
"linkedServiceName": {
|
||||
"referenceName": "[parameters('DataSourceConnection')]",
|
||||
"type": "LinkedServiceReference"
|
||||
},
|
||||
"type": "FileShare"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/DataDestinationStore')]",
|
||||
"type": "Microsoft.DataFactory/factories/datasets",
|
||||
"apiVersion": "2017-09-01-preview",
|
||||
"properties": {
|
||||
"description": "Connection to your Azure Data Lake Store.",
|
||||
"linkedServiceName": {
|
||||
"referenceName": "[parameters('DataDestinationConnection')]",
|
||||
"type": "LinkedServiceReference"
|
||||
},
|
||||
"type": "AzureBlobFSFile"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
Различия файлов скрыты, потому что одна или несколько строк слишком длинны
|
|
@ -0,0 +1,106 @@
|
|||
{
|
||||
"$schema": "http://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#",
|
||||
"contentVersion": "1.0.0.0",
|
||||
"parameters": {
|
||||
"factoryName": {
|
||||
"type": "string",
|
||||
"metadata": "Data Factory Name"
|
||||
},
|
||||
"DataSourceConnection": {
|
||||
"type": "string"
|
||||
},
|
||||
"DataDestinationConnection": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"variables": {
|
||||
"factoryId": "[concat('Microsoft.DataFactory/factories/', parameters('factoryName'))]"
|
||||
},
|
||||
"resources": [
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/CopyData_from_Netezza_to_AzureDataLakeStorage')]",
|
||||
"type": "Microsoft.DataFactory/factories/pipelines",
|
||||
"apiVersion": "2017-09-01-preview",
|
||||
"properties": {
|
||||
"description": "Copy data from Netezza server to Azure Data Lake Storage.\n\nYou can also go to \"Copy Data Tool\" to get the pipeline for more connectors and scenarios.",
|
||||
"activities": [
|
||||
{
|
||||
"name": "Netezza_to_AzureDataLakeStorage",
|
||||
"description": "Copy data from Netezza server to Azure Data Lake Storage.",
|
||||
"type": "Copy",
|
||||
"policy": {
|
||||
"timeout": "7.00:00:00",
|
||||
"retry": 0,
|
||||
"retryIntervalInSeconds": 30,
|
||||
"secureOutput": false,
|
||||
"secureInput": false
|
||||
},
|
||||
"typeProperties": {
|
||||
"source": {
|
||||
"type": "NetezzaSource"
|
||||
},
|
||||
"sink": {
|
||||
"type": "AzureDataLakeStoreSink"
|
||||
},
|
||||
"enableStaging": false,
|
||||
"dataIntegrationUnits": 0
|
||||
},
|
||||
"inputs": [
|
||||
{
|
||||
"referenceName": "DataSourceStore",
|
||||
"type": "DatasetReference"
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"referenceName": "DataDestinationStore",
|
||||
"type": "DatasetReference"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"dependsOn": [
|
||||
"[concat(variables('factoryId'), '/datasets/DataSourceStore')]",
|
||||
"[concat(variables('factoryId'), '/datasets/DataDestinationStore')]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/DataSourceStore')]",
|
||||
"type": "Microsoft.DataFactory/factories/datasets",
|
||||
"apiVersion": "2017-09-01-preview",
|
||||
"properties": {
|
||||
"description": "Connection to your Netezza server.",
|
||||
"linkedServiceName": {
|
||||
"referenceName": "[parameters('DataSourceConnection')]",
|
||||
"type": "LinkedServiceReference"
|
||||
},
|
||||
"type": "NetezzaTable"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/DataDestinationStore')]",
|
||||
"type": "Microsoft.DataFactory/factories/datasets",
|
||||
"apiVersion": "2017-09-01-preview",
|
||||
"properties": {
|
||||
"description": "Connection to your Azure Data Lake Store.",
|
||||
"linkedServiceName": {
|
||||
"referenceName": "[parameters('DataDestinationConnection')]",
|
||||
"type": "LinkedServiceReference"
|
||||
},
|
||||
"type": "AzureDataLakeStoreFile",
|
||||
"typeProperties": {
|
||||
"format": {
|
||||
"type": "TextFormat",
|
||||
"columnDelimiter": ",",
|
||||
"rowDelimiter": "",
|
||||
"nullValue": "\\N",
|
||||
"treatEmptyAsNull": true,
|
||||
"skipLineCount": 0,
|
||||
"firstRowAsHeader": false
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
Различия файлов скрыты, потому что одна или несколько строк слишком длинны
|
|
@ -0,0 +1,103 @@
|
|||
{
|
||||
"$schema": "http://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#",
|
||||
"contentVersion": "1.0.0.0",
|
||||
"parameters": {
|
||||
"factoryName": {
|
||||
"type": "string",
|
||||
"metadata": "Data Factory Name"
|
||||
},
|
||||
"DataSourceConnection": {
|
||||
"type": "string"
|
||||
},
|
||||
"DataDestinationConnection": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"variables": {
|
||||
"factoryId": "[concat('Microsoft.DataFactory/factories/', parameters('factoryName'))]"
|
||||
},
|
||||
"resources": [
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/CopyData_from_Oracle_to_SQLDataWarehouse')]",
|
||||
"type": "Microsoft.DataFactory/factories/pipelines",
|
||||
"apiVersion": "2017-09-01-preview",
|
||||
"properties": {
|
||||
"description": "Copy data from Oracle server to SQL Data Warehouse.\n\n\nYou can also go to \"Copy Data Tool\" to get the pipeline for more connectors and scenarios.",
|
||||
"activities": [
|
||||
{
|
||||
"name": "Oracle_to_SQLDataWarehouse",
|
||||
"description": "Copy data from Oracle server to SQL Data Warehouse",
|
||||
"type": "Copy",
|
||||
"policy": {
|
||||
"timeout": "7.00:00:00",
|
||||
"retry": 0,
|
||||
"retryIntervalInSeconds": 30,
|
||||
"secureOutput": false,
|
||||
"secureInput": false
|
||||
},
|
||||
"typeProperties": {
|
||||
"source": {
|
||||
"type": "OracleSource"
|
||||
},
|
||||
"sink": {
|
||||
"type": "SqlDWSink",
|
||||
"allowPolyBase": true,
|
||||
"writeBatchSize": 10000,
|
||||
"polyBaseSettings": {
|
||||
"rejectValue": 0,
|
||||
"rejectType": "value",
|
||||
"useTypeDefault": true
|
||||
}
|
||||
},
|
||||
"enableStaging": true,
|
||||
"enableSkipIncompatibleRow": false,
|
||||
"dataIntegrationUnits": 0
|
||||
},
|
||||
"inputs": [
|
||||
{
|
||||
"referenceName": "DataSourceStore",
|
||||
"type": "DatasetReference"
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"referenceName": "DataDestinationStore",
|
||||
"type": "DatasetReference"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"dependsOn": [
|
||||
"[concat(variables('factoryId'), '/datasets/DataSourceStore')]",
|
||||
"[concat(variables('factoryId'), '/datasets/DataDestinationStore')]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/DataSourceStore')]",
|
||||
"type": "Microsoft.DataFactory/factories/datasets",
|
||||
"apiVersion": "2017-09-01-preview",
|
||||
"properties": {
|
||||
"description": "Connection to your Oracle server.",
|
||||
"linkedServiceName": {
|
||||
"referenceName": "[parameters('DataSourceConnection')]",
|
||||
"type": "LinkedServiceReference"
|
||||
},
|
||||
"type": "OracleTable"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/DataDestinationStore')]",
|
||||
"type": "Microsoft.DataFactory/factories/datasets",
|
||||
"apiVersion": "2017-09-01-preview",
|
||||
"properties": {
|
||||
"description": "Connection to your SQL Data Warehouse.",
|
||||
"linkedServiceName": {
|
||||
"referenceName": "[parameters('DataDestinationConnection')]",
|
||||
"type": "LinkedServiceReference"
|
||||
},
|
||||
"type": "AzureSqlDWTable"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
Различия файлов скрыты, потому что одна или несколько строк слишком длинны
|
|
@ -0,0 +1,96 @@
|
|||
{
|
||||
"$schema": "http://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#",
|
||||
"contentVersion": "1.0.0.0",
|
||||
"parameters": {
|
||||
"factoryName": {
|
||||
"type": "string",
|
||||
"metadata": "Data Factory Name"
|
||||
},
|
||||
"DataSourceConnection": {
|
||||
"type": "string"
|
||||
},
|
||||
"DataDestinationConnection": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"variables": {
|
||||
"factoryId": "[concat('Microsoft.DataFactory/factories/', parameters('factoryName'))]"
|
||||
},
|
||||
"resources": [
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/Copy data from on premise SQL Server to SQL Azure')]",
|
||||
"type": "Microsoft.DataFactory/factories/pipelines",
|
||||
"apiVersion": "2017-09-01-preview",
|
||||
"properties": {
|
||||
"description": "Copy data from on premise SQL Server to SQL Azure.\n\nYou can also go to \"Copy Data Tool\" to get the pipeline for more connectors and scenarios.",
|
||||
"activities": [
|
||||
{
|
||||
"name": "SQL_to_SQLAzure",
|
||||
"description": "Copy data from SQL server to SQL Azure",
|
||||
"type": "Copy",
|
||||
"policy": {
|
||||
"timeout": "7.00:00:00",
|
||||
"retry": 0,
|
||||
"retryIntervalInSeconds": 30,
|
||||
"secureOutput": false,
|
||||
"secureInput": false
|
||||
},
|
||||
"typeProperties": {
|
||||
"source": {
|
||||
"type": "SqlSource"
|
||||
},
|
||||
"sink": {
|
||||
"type": "SqlSink",
|
||||
"writeBatchSize": 10000
|
||||
},
|
||||
"enableStaging": false,
|
||||
"dataIntegrationUnits": 0
|
||||
},
|
||||
"inputs": [
|
||||
{
|
||||
"referenceName": "DataSource",
|
||||
"type": "DatasetReference"
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"referenceName": "DataDestination",
|
||||
"type": "DatasetReference"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"dependsOn": [
|
||||
"[concat(variables('factoryId'), '/datasets/DataSource')]",
|
||||
"[concat(variables('factoryId'), '/datasets/DataDestination')]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/DataSource')]",
|
||||
"type": "Microsoft.DataFactory/factories/datasets",
|
||||
"apiVersion": "2017-09-01-preview",
|
||||
"properties": {
|
||||
"description": "Connect to your data source store.",
|
||||
"linkedServiceName": {
|
||||
"referenceName": "[parameters('DataSourceConnection')]",
|
||||
"type": "LinkedServiceReference"
|
||||
},
|
||||
"type": "SqlServerTable"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/DataDestination')]",
|
||||
"type": "Microsoft.DataFactory/factories/datasets",
|
||||
"apiVersion": "2017-09-01-preview",
|
||||
"properties": {
|
||||
"description": "Connect to your data destination store.",
|
||||
"linkedServiceName": {
|
||||
"referenceName": "[parameters('DataDestinationConnection')]",
|
||||
"type": "LinkedServiceReference"
|
||||
},
|
||||
"type": "AzureSqlTable"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
Различия файлов скрыты, потому что одна или несколько строк слишком длинны
|
|
@ -0,0 +1,102 @@
|
|||
{
|
||||
"$schema": "http://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#",
|
||||
"contentVersion": "1.0.0.0",
|
||||
"parameters": {
|
||||
"factoryName": {
|
||||
"type": "string",
|
||||
"metadata": "Data Factory Name"
|
||||
},
|
||||
"DataSourceConnection": {
|
||||
"type": "string"
|
||||
},
|
||||
"DataDestinationConnection": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"variables": {
|
||||
"factoryId": "[concat('Microsoft.DataFactory/factories/', parameters('factoryName'))]"
|
||||
},
|
||||
"resources": [
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/Copy data from on premise SQL Server to SQL Datawarehouse')]",
|
||||
"type": "Microsoft.DataFactory/factories/pipelines",
|
||||
"apiVersion": "2017-09-01-preview",
|
||||
"properties": {
|
||||
"description": "Copy data from on premise SQL Server to SQL Data Warehouse.\n\nYou can also go to \"Copy Data Tool\" to get the pipeline for more connectors and scenarios.",
|
||||
"activities": [
|
||||
{
|
||||
"name": "SQL_to_SQLDataWarehouse",
|
||||
"description": "Copy data from SQL server to SQL Data Warehouse",
|
||||
"type": "Copy",
|
||||
"policy": {
|
||||
"timeout": "7.00:00:00",
|
||||
"retry": 0,
|
||||
"retryIntervalInSeconds": 30,
|
||||
"secureOutput": false,
|
||||
"secureInput": false
|
||||
},
|
||||
"typeProperties": {
|
||||
"source": {
|
||||
"type": "SqlSource"
|
||||
},
|
||||
"sink": {
|
||||
"type": "SqlDWSink",
|
||||
"allowPolyBase": true,
|
||||
"writeBatchSize": 10000,
|
||||
"polyBaseSettings": {
|
||||
"rejectValue": 0,
|
||||
"rejectType": "value",
|
||||
"useTypeDefault": true
|
||||
}
|
||||
},
|
||||
"enableStaging": true,
|
||||
"dataIntegrationUnits": 0
|
||||
},
|
||||
"inputs": [
|
||||
{
|
||||
"referenceName": "DataSource",
|
||||
"type": "DatasetReference"
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"referenceName": "DataDestination",
|
||||
"type": "DatasetReference"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"dependsOn": [
|
||||
"[concat(variables('factoryId'), '/datasets/DataSource')]",
|
||||
"[concat(variables('factoryId'), '/datasets/DataDestination')]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/DataSource')]",
|
||||
"type": "Microsoft.DataFactory/factories/datasets",
|
||||
"apiVersion": "2017-09-01-preview",
|
||||
"properties": {
|
||||
"description": "Connect to your data source store.",
|
||||
"linkedServiceName": {
|
||||
"referenceName": "[parameters('DataSourceConnection')]",
|
||||
"type": "LinkedServiceReference"
|
||||
},
|
||||
"type": "SqlServerTable"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/DataDestination')]",
|
||||
"type": "Microsoft.DataFactory/factories/datasets",
|
||||
"apiVersion": "2017-09-01-preview",
|
||||
"properties": {
|
||||
"description": "Connect to your data destination store.",
|
||||
"linkedServiceName": {
|
||||
"referenceName": "[parameters('DataDestinationConnection')]",
|
||||
"type": "LinkedServiceReference"
|
||||
},
|
||||
"type": "AzureSqlDWTable"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
Различия файлов скрыты, потому что одна или несколько строк слишком длинны
|
|
@ -4,52 +4,41 @@
|
|||
"parameters": {
|
||||
"factoryName": {
|
||||
"type": "string",
|
||||
"metadata": "Data Factory Name",
|
||||
"defaultValue": "liliuMadridTestGit"
|
||||
"metadata": "Data Factory Name"
|
||||
},
|
||||
"AzureBlobSource_properties_typeProperties_fileName": {
|
||||
"type": "string",
|
||||
"defaultValue": "@dataset().file"
|
||||
"Source Data Connection": {
|
||||
"type": "string"
|
||||
},
|
||||
"AzureBlobSource_properties_typeProperties_folderPath": {
|
||||
"type": "string",
|
||||
"defaultValue": "@dataset().folder"
|
||||
"Destination Data Connection": {
|
||||
"type": "string"
|
||||
},
|
||||
"AzureBlobSink_properties_typeProperties_fileName": {
|
||||
"type": "string",
|
||||
"defaultValue": ""
|
||||
},
|
||||
"AzureBlobSink_properties_typeProperties_folderPath": {
|
||||
"type": "string",
|
||||
"defaultValue": "@dataset().folder"
|
||||
},
|
||||
"AzureBlobSource_properties_linkedServiceName": {
|
||||
"type": "string",
|
||||
"metadata": {
|
||||
"linkedServiceType": "AzureBlobStorage"
|
||||
}
|
||||
},
|
||||
"AzureBlobSink_properties_linkedServiceName": {
|
||||
"type": "string",
|
||||
"metadata": {
|
||||
"linkedServiceType": "AzureBlobStorage"
|
||||
}
|
||||
}
|
||||
"sourceFolderPath_parameter": {
|
||||
"type": "object"
|
||||
},
|
||||
"sourceFileName_parameter": {
|
||||
"type": "object"
|
||||
},
|
||||
"sourceS3BucketName_parameter": {
|
||||
"type": "object"
|
||||
},
|
||||
"sourceS3Key_parameter": {
|
||||
"type": "object"
|
||||
}
|
||||
},
|
||||
"variables": {
|
||||
"factoryId": "[concat('Microsoft.DataFactory/factories/', parameters('factoryName'))]"
|
||||
},
|
||||
"resources": [
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/ForeachFilePartitionCopy')]",
|
||||
"name": "[concat(parameters('factoryName'), '/CopyFilesfromMultipleContainers')]",
|
||||
"type": "Microsoft.DataFactory/factories/pipelines",
|
||||
"apiVersion": "2018-06-01",
|
||||
"properties": {
|
||||
"description": "Copying a large number of files between file-based storages template",
|
||||
"description": "Copy all files across multiple containers or buckets from one file-based storage store to another",
|
||||
"activities": [
|
||||
{
|
||||
"name": "GetPartitionList",
|
||||
"description": "get item list",
|
||||
"name": "EnumerateContainersList",
|
||||
"description": "Scan the file path on your source storage store and get the container list.",
|
||||
"type": "GetMetadata",
|
||||
"policy": {
|
||||
"timeout": "7.00:00:00",
|
||||
|
|
@ -60,11 +49,11 @@
|
|||
},
|
||||
"typeProperties": {
|
||||
"dataset": {
|
||||
"referenceName": "AzureBlobSource",
|
||||
"referenceName": "SourceFileStore",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"folder": {
|
||||
"value": "@pipeline().parameters.sourceFolder",
|
||||
"value": "@pipeline().parameters.SourceFilePath",
|
||||
"type": "Expression"
|
||||
},
|
||||
"file": " "
|
||||
|
|
@ -76,12 +65,12 @@
|
|||
}
|
||||
},
|
||||
{
|
||||
"name": "ForEachPartition",
|
||||
"description": "copy for each item",
|
||||
"name": "ForEachContainer",
|
||||
"description": "Get the container list from Metadata activity and then iterates each of them to Copy activity.",
|
||||
"type": "ForEach",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "GetPartitionList",
|
||||
"activity": "EnumerateContainersList",
|
||||
"dependencyConditions": [
|
||||
"Succeeded"
|
||||
]
|
||||
|
|
@ -89,14 +78,14 @@
|
|||
],
|
||||
"typeProperties": {
|
||||
"items": {
|
||||
"value": "@activity('GetPartitionList').output.childItems",
|
||||
"value": "@activity('EnumerateContainersList').output.childItems",
|
||||
"type": "Expression"
|
||||
},
|
||||
"batchCount": 10,
|
||||
"batchCount": 4,
|
||||
"activities": [
|
||||
{
|
||||
"name": "CopyOnePartition",
|
||||
"description": "Do the copy",
|
||||
"name": "CopyOneContainer",
|
||||
"description": "Copy each container from source storage store to destination store.",
|
||||
"type": "Copy",
|
||||
"policy": {
|
||||
"timeout": "7.00:00:00",
|
||||
|
|
@ -108,11 +97,11 @@
|
|||
"userProperties": [
|
||||
{
|
||||
"name": "Source",
|
||||
"value": "@{item().name}/"
|
||||
"value": "@concat(pipeline().parameters.SourceFilePath,'/',item().name)"
|
||||
},
|
||||
{
|
||||
"name": "Destination",
|
||||
"value": "@{item().name}/"
|
||||
"value": "@concat(pipeline().parameters.DestinationFilePath,'/',item().name)"
|
||||
}
|
||||
],
|
||||
"typeProperties": {
|
||||
|
|
@ -121,22 +110,19 @@
|
|||
"recursive": true
|
||||
},
|
||||
"sink": {
|
||||
"type": "BlobSink"
|
||||
"type": "BlobSink",
|
||||
"copyBehavior": "PreserveHierarchy"
|
||||
},
|
||||
"enableStaging": false,
|
||||
"dataIntegrationUnits": 0
|
||||
},
|
||||
"inputs": [
|
||||
{
|
||||
"referenceName": "AzureBlobSource",
|
||||
"referenceName": "SourceFileStore",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"folder": {
|
||||
"value": "@pipeline().parameters.sourceFolder",
|
||||
"type": "Expression"
|
||||
},
|
||||
"file": {
|
||||
"value": "@item().name",
|
||||
"value": "@concat(pipeline().parameters.SourceFilePath,'/',item().name)",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
|
|
@ -144,11 +130,11 @@
|
|||
],
|
||||
"outputs": [
|
||||
{
|
||||
"referenceName": "AzureBlobSink",
|
||||
"referenceName": "SinkFileStore",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"folder": {
|
||||
"value": "@pipeline().parameters.sinkFolder",
|
||||
"value": "@concat(pipeline().parameters.DestinationFilePath,'/',item().name)",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
|
|
@ -160,56 +146,53 @@
|
|||
}
|
||||
],
|
||||
"parameters": {
|
||||
"sourceFolder": {
|
||||
"type": "String"
|
||||
"SourceFilePath": {
|
||||
"type": "String",
|
||||
"defaultValue": "/"
|
||||
},
|
||||
"sinkFolder": {
|
||||
"type": "String"
|
||||
"DestinationFilePath": {
|
||||
"type": "String",
|
||||
"defaultValue": "/"
|
||||
}
|
||||
}
|
||||
},
|
||||
"dependsOn": [
|
||||
"[concat(variables('factoryId'), '/datasets/AzureBlobSource')]",
|
||||
"[concat(variables('factoryId'), '/datasets/AzureBlobSink')]"
|
||||
"[concat(variables('factoryId'), '/datasets/SourceFileStore')]",
|
||||
"[concat(variables('factoryId'), '/datasets/SinkFileStore')]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/AzureBlobSource')]",
|
||||
"name": "[concat(parameters('factoryName'), '/SourceFileStore')]",
|
||||
"type": "Microsoft.DataFactory/factories/datasets",
|
||||
"apiVersion": "2018-06-01",
|
||||
"properties": {
|
||||
"description": "Connection to your source data store.",
|
||||
"linkedServiceName": {
|
||||
"referenceName": "[parameters('AzureBlobSource_properties_linkedServiceName')]",
|
||||
"referenceName": "[parameters('Source Data Connection')]",
|
||||
"type": "LinkedServiceReference"
|
||||
},
|
||||
"parameters": {
|
||||
"folder": {
|
||||
"type": "String"
|
||||
},
|
||||
"file": {
|
||||
"type": "String"
|
||||
}
|
||||
},
|
||||
"type": "AzureBlob",
|
||||
"typeProperties": {
|
||||
"fileName": {
|
||||
"value": "[parameters('AzureBlobSource_properties_typeProperties_fileName')]",
|
||||
"type": "Expression"
|
||||
},
|
||||
"folderPath": {
|
||||
"value": "[parameters('AzureBlobSource_properties_typeProperties_folderPath')]",
|
||||
"type": "Expression"
|
||||
}
|
||||
"key": "[parameters('sourceS3Key_parameter')]",
|
||||
"bucketName": "[parameters('sourceS3BucketName_parameter')]",
|
||||
"fileName": "[parameters('sourceFileName_parameter')]",
|
||||
"folderPath": "[parameters('sourceFolderPath_parameter')]"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/AzureBlobSink')]",
|
||||
"name": "[concat(parameters('factoryName'), '/SinkFileStore')]",
|
||||
"type": "Microsoft.DataFactory/factories/datasets",
|
||||
"apiVersion": "2018-06-01",
|
||||
"properties": {
|
||||
"description": "Connection to your destination data store.",
|
||||
"linkedServiceName": {
|
||||
"referenceName": "[parameters('AzureBlobSink_properties_linkedServiceName')]",
|
||||
"referenceName": "[parameters('Destination Data Connection')]",
|
||||
"type": "LinkedServiceReference"
|
||||
},
|
||||
"parameters": {
|
||||
|
|
@ -219,9 +202,9 @@
|
|||
},
|
||||
"type": "AzureBlob",
|
||||
"typeProperties": {
|
||||
"fileName": "[parameters('AzureBlobSink_properties_typeProperties_fileName')]",
|
||||
"fileName": "",
|
||||
"folderPath": {
|
||||
"value": "[parameters('AzureBlobSink_properties_typeProperties_folderPath')]",
|
||||
"value": "@dataset().folder",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
Различия файлов скрыты, потому что одна или несколько строк слишком длинны
|
|
@ -0,0 +1,256 @@
|
|||
{
|
||||
"$schema": "http://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#",
|
||||
"contentVersion": "1.0.0.0",
|
||||
"parameters": {
|
||||
"factoryName": {
|
||||
"type": "string",
|
||||
"metadata": "Data Factory Name"
|
||||
},
|
||||
"ExternalControlTableConnection": {
|
||||
"type": "string"
|
||||
},
|
||||
"DataSourceConnection": {
|
||||
"type": "string"
|
||||
},
|
||||
"DataDestinationConnection": {
|
||||
"type": "string"
|
||||
},
|
||||
"allowPolyBase_parameter": {
|
||||
"type": "bool"
|
||||
},
|
||||
"writeBatchSize_parameter": {
|
||||
"type": "int"
|
||||
},
|
||||
"polyBaseSettings_parameter": {
|
||||
"type": "object"
|
||||
},
|
||||
"enableStaging_parameter": {
|
||||
"type": "bool"
|
||||
},
|
||||
"stagingSettings_parameter": {
|
||||
"type": "object"
|
||||
},
|
||||
"copyactivity_output_parameter": {
|
||||
"type": "object"
|
||||
},
|
||||
"DataDestination_parameter_paramter": {
|
||||
"type": "object"
|
||||
},
|
||||
"DataDestination_typeproperties_paramter": {
|
||||
"type": "object"
|
||||
},
|
||||
"pipeline_parameter": {
|
||||
"type": "object"
|
||||
},
|
||||
"sqlReaderQuery_parameter": {
|
||||
"type": "object"
|
||||
},
|
||||
"OdbcReaderQuery_parameter": {
|
||||
"type": "object"
|
||||
},
|
||||
"OracleReaderQuery_parameter": {
|
||||
"type": "object"
|
||||
}
|
||||
},
|
||||
"variables": {
|
||||
"factoryId": "[concat('Microsoft.DataFactory/factories/', parameters('factoryName'))]"
|
||||
},
|
||||
"resources": [
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/DeltaCopyfromDB_using_ControlTable')]",
|
||||
"type": "Microsoft.DataFactory/factories/pipelines",
|
||||
"apiVersion": "2017-09-01-preview",
|
||||
"properties": {
|
||||
"description": "Copy new or updated rows only from database using external control table",
|
||||
"activities": [
|
||||
{
|
||||
"name": "LookupLastWaterMark",
|
||||
"description": "Retrieve the last high-watermark value stored in external control table",
|
||||
"type": "Lookup",
|
||||
"policy": {
|
||||
"timeout": "7.00:00:00",
|
||||
"retry": 0,
|
||||
"retryIntervalInSeconds": 30
|
||||
},
|
||||
"typeProperties": {
|
||||
"source": {
|
||||
"type": "SqlSource",
|
||||
"sqlReaderQuery": {
|
||||
"value": "select @{pipeline().parameters.Control_Table_Column_Name} as WatermarkValue from @{pipeline().parameters.Control_Table_Table_Name}",
|
||||
"type": "Expression"
|
||||
}
|
||||
},
|
||||
"dataset": {
|
||||
"referenceName": "External_ControlTable",
|
||||
"type": "DatasetReference"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "LookupCurrentWaterMark",
|
||||
"description": "Retrieve the current maximum value in watermark column of source data store",
|
||||
"type": "Lookup",
|
||||
"policy": {
|
||||
"timeout": "7.00:00:00",
|
||||
"retry": 0,
|
||||
"retryIntervalInSeconds": 30
|
||||
},
|
||||
"typeProperties": {
|
||||
"source": {
|
||||
"type": "SqlSource",
|
||||
"sqlReaderQuery": {
|
||||
"value": "@concat('select MAX(',pipeline().parameters.Data_Source_WaterMarkColumn, ') as NewWatermarkValue from ',pipeline().parameters.Data_Source_Table_Name)",
|
||||
"type": "Expression"
|
||||
}
|
||||
},
|
||||
"dataset": {
|
||||
"referenceName": "DataSource",
|
||||
"type": "DatasetReference"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "DeltaCopyfromDB",
|
||||
"description": "Copy activity to use query to filter the delta data by > last high-watermark and <= current high-watermark from source database, and then only copy the changes to the destination store.",
|
||||
"type": "Copy",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "LookupCurrentWaterMark",
|
||||
"dependencyConditions": [
|
||||
"Succeeded"
|
||||
]
|
||||
},
|
||||
{
|
||||
"activity": "LookupLastWaterMark",
|
||||
"dependencyConditions": [
|
||||
"Succeeded"
|
||||
]
|
||||
}
|
||||
],
|
||||
"policy": {
|
||||
"timeout": "7.00:00:00",
|
||||
"retry": 0,
|
||||
"retryIntervalInSeconds": 30
|
||||
},
|
||||
"typeProperties": {
|
||||
"source": {
|
||||
"type": "SqlSource",
|
||||
"sqlReaderQuery": "[parameters('sqlReaderQuery_parameter')]",
|
||||
"query":"[parameters('OdbcReaderQuery_parameter')]",
|
||||
"oracleReaderQuery":"[parameters('OracleReaderQuery_parameter')]"
|
||||
},
|
||||
"sink": {
|
||||
"type": "BlobSink",
|
||||
"allowPolyBase": "[parameters('allowPolyBase_parameter')]",
|
||||
"writeBatchSize": "[parameters('writeBatchSize_parameter')]",
|
||||
"polyBaseSettings": "[parameters('polyBaseSettings_parameter')]"
|
||||
},
|
||||
"enableStaging": "[parameters('enableStaging_parameter')]",
|
||||
"stagingSettings": "[parameters('stagingSettings_parameter')]",
|
||||
"dataIntegrationUnits": 0
|
||||
},
|
||||
"inputs": [
|
||||
{
|
||||
"referenceName": "DataSource",
|
||||
"type": "DatasetReference"
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"referenceName": "DataDestination",
|
||||
"type": "DatasetReference",
|
||||
"parameters": "[parameters('copyactivity_output_parameter')]"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "UpdateWaterMark",
|
||||
"description": "Stored procedure activity to store the new high-watermark value into external control table for delta data loading. ",
|
||||
"type": "SqlServerStoredProcedure",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "DeltaCopyfromDB",
|
||||
"dependencyConditions": [
|
||||
"Succeeded"
|
||||
]
|
||||
}
|
||||
],
|
||||
"policy": {
|
||||
"timeout": "7.00:00:00",
|
||||
"retry": 0,
|
||||
"retryIntervalInSeconds": 30
|
||||
},
|
||||
"typeProperties": {
|
||||
"storedProcedureParameters": {
|
||||
"LastModifyDate": {
|
||||
"value": {
|
||||
"value": "@activity('LookupCurrentWaterMark').output.firstRow.NewWatermarkValue",
|
||||
"type": "Expression"
|
||||
},
|
||||
"type": "datetime"
|
||||
}
|
||||
}
|
||||
},
|
||||
"linkedServiceName": {
|
||||
"referenceName": "[parameters('ExternalControlTableConnection')]",
|
||||
"type": "LinkedServiceReference"
|
||||
}
|
||||
}
|
||||
],
|
||||
"parameters": "[parameters('pipeline_parameter')]"
|
||||
},
|
||||
"dependsOn": [
|
||||
"[concat(variables('factoryId'), '/datasets/External_ControlTable')]",
|
||||
"[concat(variables('factoryId'), '/datasets/DataSource')]",
|
||||
"[concat(variables('factoryId'), '/datasets/DataDestination')]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/DataSource')]",
|
||||
"type": "Microsoft.DataFactory/factories/datasets",
|
||||
"apiVersion": "2017-09-01-preview",
|
||||
"properties": {
|
||||
"description": "Connection to your source data store.",
|
||||
"linkedServiceName": {
|
||||
"referenceName": "[parameters('DataSourceConnection')]",
|
||||
"type": "LinkedServiceReference"
|
||||
},
|
||||
"type": "SqlServerTable",
|
||||
"typeProperties": {
|
||||
"tableName": "DataSourceTable"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/DataDestination')]",
|
||||
"type": "Microsoft.DataFactory/factories/datasets",
|
||||
"apiVersion": "2017-09-01-preview",
|
||||
"properties": {
|
||||
"description": "Connection to your destination data store.",
|
||||
"linkedServiceName": {
|
||||
"referenceName": "[parameters('DataDestinationConnection')]",
|
||||
"type": "LinkedServiceReference"
|
||||
},
|
||||
"parameters": "[parameters('DataDestination_parameter_paramter')]",
|
||||
"type": "AzureBlob",
|
||||
"typeProperties": "[parameters('DataDestination_typeproperties_paramter')]"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/External_ControlTable')]",
|
||||
"type": "Microsoft.DataFactory/factories/datasets",
|
||||
"apiVersion": "2017-09-01-preview",
|
||||
"properties": {
|
||||
"description": "External control table to store high water mark value.",
|
||||
"linkedServiceName": {
|
||||
"referenceName": "[parameters('ExternalControlTableConnection')]",
|
||||
"type": "LinkedServiceReference"
|
||||
},
|
||||
"type": "SqlServerTable",
|
||||
"typeProperties": {
|
||||
"tableName": "watermarktable"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
Различия файлов скрыты, потому что одна или несколько строк слишком длинны
|
|
@ -0,0 +1,223 @@
|
|||
{
|
||||
"$schema": "http://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#",
|
||||
"contentVersion": "1.0.0.0",
|
||||
"parameters": {
|
||||
"factoryName": {
|
||||
"type": "string",
|
||||
"metadata": "Data Factory Name"
|
||||
},
|
||||
"Azure Databricks Connection": {
|
||||
"type": "string"
|
||||
},
|
||||
"Source Blob Connection": {
|
||||
"type": "string"
|
||||
},
|
||||
"Destination Blob Connection": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"variables": {
|
||||
"factoryId": "[concat('Microsoft.DataFactory/factories/', parameters('factoryName'))]"
|
||||
},
|
||||
"resources": [
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/ETL with Azure Databricks')]",
|
||||
"type": "Microsoft.DataFactory/factories/pipelines",
|
||||
"apiVersion": "2018-06-01",
|
||||
"properties": {
|
||||
"description": "Run a simple ETL job using Azure Databricks, with a single pane of glass monitoring from ADF.\n\nIn the template, we check for source dataset availability. Once it is available we copy it into a blob storage for staging using a Copy activity. The same storage is accessed from Databricks clusters while processing the data (ETL). The output is stored in the same storage under 'output' folder. Various notebook properties are referenced as expressions using pipeline parameters, which lets you configure more generic and reusable pipelines. \n \nFor steps on setting up storage and databricks notebook refer https://aka.ms/databricks-instructions. ",
|
||||
"activities": [
|
||||
{
|
||||
"name": "Availability flag",
|
||||
"description": "Lookup (or Get Metadata) activity is used to get information about the source files if they are available for processing. \nIn this template, only when the '_success' flag/ file is available at source, would the downstream activities be triggered. ",
|
||||
"type": "Lookup",
|
||||
"policy": {
|
||||
"timeout": "7.00:00:00",
|
||||
"retry": 0,
|
||||
"retryIntervalInSeconds": 30,
|
||||
"secureOutput": false,
|
||||
"secureInput": false
|
||||
},
|
||||
"typeProperties": {
|
||||
"source": {
|
||||
"type": "BlobSource",
|
||||
"recursive": true
|
||||
},
|
||||
"dataset": {
|
||||
"referenceName": "SourceAvailabilityBlobStore",
|
||||
"type": "DatasetReference"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "file-to-blob",
|
||||
"description": "Copy activity copies the actual files/ dataset to be processed by Databricks into a staging store. This storage should be accessible by the Azure Databricks cluster referenced in the next activity. ",
|
||||
"type": "Copy",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "Availability flag",
|
||||
"dependencyConditions": [
|
||||
"Succeeded"
|
||||
]
|
||||
}
|
||||
],
|
||||
"policy": {
|
||||
"timeout": "7.00:00:00",
|
||||
"retry": 0,
|
||||
"retryIntervalInSeconds": 30,
|
||||
"secureOutput": false,
|
||||
"secureInput": false
|
||||
},
|
||||
"typeProperties": {
|
||||
"source": {
|
||||
"type": "BlobSource",
|
||||
"recursive": true
|
||||
},
|
||||
"sink": {
|
||||
"type": "BlobSink"
|
||||
},
|
||||
"enableStaging": false,
|
||||
"dataIntegrationUnits": 0
|
||||
},
|
||||
"inputs": [
|
||||
{
|
||||
"referenceName": "SourceFilesBlobStore",
|
||||
"type": "DatasetReference"
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"referenceName": "DestinationFilesBlobStore",
|
||||
"type": "DatasetReference"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "ETL",
|
||||
"description": "Databricks Notebook activity does the processing of the data copied in the previous step (copy activity). Please ensure you have added the databricks notebook (<a href='https://adflabstaging1.blob.core.windows.net/share/Transformations.html' target='_blank'>https://adflabstaging1.blob.core.windows.net/share/Transformations.html</a>) in the databricks work-space and referenced it in the notebook activity in ADF.",
|
||||
"type": "DatabricksNotebook",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "file-to-blob",
|
||||
"dependencyConditions": [
|
||||
"Succeeded"
|
||||
]
|
||||
}
|
||||
],
|
||||
"policy": {
|
||||
"timeout": "7.00:00:00",
|
||||
"retry": 0,
|
||||
"retryIntervalInSeconds": 30,
|
||||
"secureOutput": false,
|
||||
"secureInput": false
|
||||
},
|
||||
"typeProperties": {
|
||||
"notebookPath": "/Shared/Transformations",
|
||||
"baseParameters": {
|
||||
"input": {
|
||||
"value": "@pipeline().parameters.inputPath",
|
||||
"type": "Expression"
|
||||
},
|
||||
"output": {
|
||||
"value": "@pipeline().parameters.outputPath",
|
||||
"type": "Expression"
|
||||
},
|
||||
"filename": {
|
||||
"value": "@pipeline().parameters.fileName",
|
||||
"type": "Expression"
|
||||
},
|
||||
"pipelineRunId": {
|
||||
"value": "@pipeline().RunId",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
},
|
||||
"linkedServiceName": {
|
||||
"referenceName": "[parameters('Azure Databricks Connection')]",
|
||||
"type": "LinkedServiceReference"
|
||||
}
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"inputPath": {
|
||||
"type": "String",
|
||||
"defaultValue": "/staged_sink"
|
||||
},
|
||||
"outputPath": {
|
||||
"type": "String",
|
||||
"defaultValue": "/processed_sink"
|
||||
},
|
||||
"fileName": {
|
||||
"type": "String",
|
||||
"defaultValue": "Product.csv"
|
||||
}
|
||||
}
|
||||
},
|
||||
"dependsOn": [
|
||||
"[concat(variables('factoryId'), '/datasets/SourceAvailabilityBlobStore')]",
|
||||
"[concat(variables('factoryId'), '/datasets/SourceFilesBlobStore')]",
|
||||
"[concat(variables('factoryId'), '/datasets/DestinationFilesBlobStore')]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/SourceAvailabilityBlobStore')]",
|
||||
"type": "Microsoft.DataFactory/factories/datasets",
|
||||
"apiVersion": "2018-06-01",
|
||||
"properties": {
|
||||
"description": "Dataset to check if '_success' flag/ file is available in source. If not, then the activity fails, letting you know that source dataset is not ready.\nContext: In Spark/ Hadoop environments, generally users leverage flags to identify if the data has been correctly written through the distributed compute engines. E.g. '_success' defines all nodes have successfully written the data.",
|
||||
"linkedServiceName": {
|
||||
"referenceName": "[parameters('Source Blob Connection')]",
|
||||
"type": "LinkedServiceReference"
|
||||
},
|
||||
"type": "AzureBlob",
|
||||
"typeProperties": {
|
||||
"format": {
|
||||
"type": "TextFormat",
|
||||
"columnDelimiter": ",",
|
||||
"rowDelimiter": "",
|
||||
"nullValue": "\\N",
|
||||
"treatEmptyAsNull": true,
|
||||
"skipLineCount": 0,
|
||||
"firstRowAsHeader": false
|
||||
},
|
||||
"fileName": "_success",
|
||||
"folderPath": "data/source"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/SourceFilesBlobStore')]",
|
||||
"type": "Microsoft.DataFactory/factories/datasets",
|
||||
"apiVersion": "2018-06-01",
|
||||
"properties": {
|
||||
"description": "Connection to your source files in Azure Blob Store.",
|
||||
"linkedServiceName": {
|
||||
"referenceName": "[parameters('Source Blob Connection')]",
|
||||
"type": "LinkedServiceReference"
|
||||
},
|
||||
"type": "AzureBlob",
|
||||
"typeProperties": {
|
||||
"fileName": "Product.csv",
|
||||
"folderPath": "data/source"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/DestinationFilesBlobStore')]",
|
||||
"type": "Microsoft.DataFactory/factories/datasets",
|
||||
"apiVersion": "2018-06-01",
|
||||
"properties": {
|
||||
"description": "Connection to your destination Azure Blob Store.",
|
||||
"linkedServiceName": {
|
||||
"referenceName": "[parameters('Destination Blob Connection')]",
|
||||
"type": "LinkedServiceReference"
|
||||
},
|
||||
"type": "AzureBlob",
|
||||
"typeProperties": {
|
||||
"fileName": "",
|
||||
"folderPath": "sinkdata/staged_sink"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
Различия файлов скрыты, потому что одна или несколько строк слишком длинны
Различия файлов скрыты, потому что одна или несколько строк слишком длинны
|
|
@ -1,215 +0,0 @@
|
|||
{
|
||||
"$schema": "http://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#",
|
||||
"contentVersion": "1.0.0.0",
|
||||
"parameters": {
|
||||
"factoryName": {
|
||||
"type": "string",
|
||||
"metadata": "Data Factory Name",
|
||||
"defaultValue": "liliuMadridTestGit"
|
||||
},
|
||||
"AzureBlobDestination_properties_typeProperties_folderPath": {
|
||||
"type": "string",
|
||||
"defaultValue": "@dataset().folder"
|
||||
},
|
||||
"AzureSqlTableLookup_properties_linkedServiceName": {
|
||||
"type": "string",
|
||||
"metadata": {
|
||||
"linkedServiceType": "AzureSqlDatabase"
|
||||
}
|
||||
},
|
||||
"AzureSqlTableSource_properties_linkedServiceName": {
|
||||
"type": "string",
|
||||
"metadata": {
|
||||
"linkedServiceType": "AzureSqlDatabase"
|
||||
}
|
||||
},
|
||||
"AzureBlobDestination_properties_linkedServiceName": {
|
||||
"type": "string",
|
||||
"metadata": {
|
||||
"linkedServiceType": "AzureBlobStorage"
|
||||
}
|
||||
}
|
||||
},
|
||||
"variables": {
|
||||
"factoryId": "[concat('Microsoft.DataFactory/factories/', parameters('factoryName'))]"
|
||||
},
|
||||
"resources": [
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/ForeachTablePartitionCopy')]",
|
||||
"type": "Microsoft.DataFactory/factories/pipelines",
|
||||
"apiVersion": "2018-06-01",
|
||||
"properties": {
|
||||
"activities": [
|
||||
{
|
||||
"name": "ForEachPartition",
|
||||
"type": "ForEach",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "GetPartitionList",
|
||||
"dependencyConditions": [
|
||||
"Succeeded"
|
||||
]
|
||||
}
|
||||
],
|
||||
"typeProperties": {
|
||||
"items": {
|
||||
"value": "@activity('GetPartitionList').output.value",
|
||||
"type": "Expression"
|
||||
},
|
||||
"batchCount": 10,
|
||||
"activities": [
|
||||
{
|
||||
"name": "CopyOnePartition",
|
||||
"description": "Do the real copy",
|
||||
"type": "Copy",
|
||||
"policy": {
|
||||
"timeout": "7.00:00:00",
|
||||
"retry": 0,
|
||||
"retryIntervalInSeconds": 30,
|
||||
"secureOutput": false,
|
||||
"secureInput": false
|
||||
},
|
||||
"typeProperties": {
|
||||
"source": {
|
||||
"type": "SqlSource"
|
||||
},
|
||||
"sink": {
|
||||
"type": "BlobSink"
|
||||
},
|
||||
"enableStaging": false,
|
||||
"dataIntegrationUnits": 0
|
||||
},
|
||||
"inputs": [
|
||||
{
|
||||
"referenceName": "AzureSqlTableSource",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"tableName": {
|
||||
"value": "[concat('[', '@{item().TABLE_SCHEMA}].[@{item().TABLE_NAME}]')]",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"referenceName": "AzureBlobDestination",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"folder": {
|
||||
"value": "@pipeline().parameters.sinkFolder",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "GetPartitionList",
|
||||
"type": "Lookup",
|
||||
"policy": {
|
||||
"timeout": "7.00:00:00",
|
||||
"retry": 0,
|
||||
"retryIntervalInSeconds": 30,
|
||||
"secureOutput": false,
|
||||
"secureInput": false
|
||||
},
|
||||
"typeProperties": {
|
||||
"source": {
|
||||
"type": "SqlSource",
|
||||
"sqlReaderQuery": "SELECT TABLE_SCHEMA, TABLE_NAME FROM information_schema.TABLES"
|
||||
},
|
||||
"dataset": {
|
||||
"referenceName": "AzureSqlTableLookup",
|
||||
"type": "DatasetReference"
|
||||
},
|
||||
"firstRowOnly": false
|
||||
}
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"sinkFolder": {
|
||||
"type": "String"
|
||||
}
|
||||
}
|
||||
},
|
||||
"dependsOn": [
|
||||
"[concat(variables('factoryId'), '/datasets/AzureSqlTableLookup')]",
|
||||
"[concat(variables('factoryId'), '/datasets/AzureSqlTableSource')]",
|
||||
"[concat(variables('factoryId'), '/datasets/AzureBlobDestination')]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/AzureSqlTableLookup')]",
|
||||
"type": "Microsoft.DataFactory/factories/datasets",
|
||||
"apiVersion": "2018-06-01",
|
||||
"properties": {
|
||||
"description": "Data source for lookup activity, should has the same linked service with AzureSqlTableSource",
|
||||
"linkedServiceName": {
|
||||
"referenceName": "[parameters('AzureSqlTableLookup_properties_linkedServiceName')]",
|
||||
"type": "LinkedServiceReference"
|
||||
},
|
||||
"type": "AzureSqlTable"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/AzureSqlTableSource')]",
|
||||
"type": "Microsoft.DataFactory/factories/datasets",
|
||||
"apiVersion": "2018-06-01",
|
||||
"properties": {
|
||||
"description": "Data source for copy activity, should has the same linked service with AzureSqlTableLookup",
|
||||
"linkedServiceName": {
|
||||
"referenceName": "[parameters('AzureSqlTableSource_properties_linkedServiceName')]",
|
||||
"type": "LinkedServiceReference"
|
||||
},
|
||||
"parameters": {
|
||||
"tableName": {
|
||||
"type": "String"
|
||||
}
|
||||
},
|
||||
"type": "AzureSqlTable",
|
||||
"typeProperties": {
|
||||
"tableName": {
|
||||
"value": "@dataset().tableName",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/AzureBlobDestination')]",
|
||||
"type": "Microsoft.DataFactory/factories/datasets",
|
||||
"apiVersion": "2018-06-01",
|
||||
"properties": {
|
||||
"linkedServiceName": {
|
||||
"referenceName": "[parameters('AzureBlobDestination_properties_linkedServiceName')]",
|
||||
"type": "LinkedServiceReference"
|
||||
},
|
||||
"parameters": {
|
||||
"folder": {
|
||||
"type": "String"
|
||||
}
|
||||
},
|
||||
"type": "AzureBlob",
|
||||
"typeProperties": {
|
||||
"format": {
|
||||
"type": "TextFormat",
|
||||
"columnDelimiter": ",",
|
||||
"rowDelimiter": "",
|
||||
"nullValue": "\\N",
|
||||
"treatEmptyAsNull": true,
|
||||
"skipLineCount": 0,
|
||||
"firstRowAsHeader": false
|
||||
},
|
||||
"folderPath": {
|
||||
"value": "[parameters('AzureBlobDestination_properties_typeProperties_folderPath')]",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
Различия файлов скрыты, потому что одна или несколько строк слишком длинны
|
|
@ -0,0 +1,87 @@
|
|||
{
|
||||
"$schema": "http://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#",
|
||||
"contentVersion": "1.0.0.0",
|
||||
"parameters": {
|
||||
"factoryName": {
|
||||
"type": "string",
|
||||
"metadata": "Data Factory Name"
|
||||
}
|
||||
},
|
||||
"variables": {
|
||||
"factoryId": "[concat('Microsoft.DataFactory/factories/', parameters('factoryName'))]"
|
||||
},
|
||||
"resources": [
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/ScheduleAzure-SSISIntegrationRuntime')]",
|
||||
"type": "Microsoft.DataFactory/factories/pipelines",
|
||||
"apiVersion": "2017-09-01-preview",
|
||||
"properties": {
|
||||
"activities": [
|
||||
{
|
||||
"name": "Start Azure-SSIS IR",
|
||||
"description": "Invoke the webhook created in Azure Automation runbook to start Azure-SSIS IR.",
|
||||
"type": "WebActivity",
|
||||
"policy": {
|
||||
"timeout": "7.00:00:00",
|
||||
"retry": 0,
|
||||
"retryIntervalInSeconds": 30,
|
||||
"secureOutput": false,
|
||||
"secureInput": false
|
||||
},
|
||||
"typeProperties": {
|
||||
"url": "",
|
||||
"method": "POST"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Execute SSIS Package",
|
||||
"description": "Run a SSIS package by Execute SSIS Package Activity.",
|
||||
"type": "ExecuteSSISPackage",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "Start Azure-SSIS IR",
|
||||
"dependencyConditions": [
|
||||
"Succeeded"
|
||||
]
|
||||
}
|
||||
],
|
||||
"policy": {
|
||||
"timeout": "7.00:00:00",
|
||||
"retry": 0,
|
||||
"retryIntervalInSeconds": 30,
|
||||
"secureOutput": false,
|
||||
"secureInput": false
|
||||
},
|
||||
"typeProperties": {
|
||||
"loggingLevel": "Basic"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Stop Azure-SSIS IR",
|
||||
"description": "Invoke the webhook created in Azure Automation runbook to stop Azure-SSIS IR.",
|
||||
"type": "WebActivity",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "Execute SSIS Package",
|
||||
"dependencyConditions": [
|
||||
"Succeeded"
|
||||
]
|
||||
}
|
||||
],
|
||||
"policy": {
|
||||
"timeout": "7.00:00:00",
|
||||
"retry": 0,
|
||||
"retryIntervalInSeconds": 30,
|
||||
"secureOutput": false,
|
||||
"secureInput": false
|
||||
},
|
||||
"typeProperties": {
|
||||
"url": "",
|
||||
"method": "POST"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
Различия файлов скрыты, потому что одна или несколько строк слишком длинны
Загрузка…
Ссылка в новой задаче