microsoft
/
azure-data-services-go-fast-codebase
зеркало из https://github.com/microsoft/azure-data-services-go-fast-codebase.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

Updated Delimited Text connections to allow for dynamic delimiter

This commit is contained in:
John Rampono 2022-11-20 22:01:18 +08:00
Родитель 4772bc8d33
Коммит 5b0bbb33ae
28 изменённых файлов: 423 добавлений и 8037 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

8
.devcontainer/Dockerfile
Просмотреть файл

@ -38,8 +38,12 @@ RUN bash /tmp/library-scripts/common-debian.sh "${INSTALL_ZSH}" "${USERNAME}" "$
&& sudo apt-get install -y lolcat \
&& sudo gem install lolcat \
&& sudo apt-get install -y boxes
# DBT
# https://learn.microsoft.com/nl-be/sql/connect/odbc/linux-mac/installing-the-microsoft-odbc-driver-for-sql-server?view=sql-server-ver16#ubuntu18
#&& sudo apt install python3.10-venv \
#&& sudo apt-get install -y pip \
#&& pip install dbt-core \
#&& pip install dbt-sqlserver
# [Optional] Uncomment this section to install additional OS packages.
# RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
# && apt-get -y install --no-install-recommends <your-package-list-here>

3
solution/DataFactory/Patterns/SqlTests_Generate.ps1
Просмотреть файл

@ -1,7 +1,7 @@
$sql = @"
delete from [dbo].[TaskGroup] where taskgroupid <=0;
delete from [dbo].[TaskGroup] where taskgroupid <=0 and taskgroupid > -8;
SET IDENTITY_INSERT [dbo].[TaskGroup] ON
INSERT INTO [dbo].[TaskGroup] ([TaskGroupId],[TaskGroupName],[SubjectAreaId], [TaskGroupPriority],[TaskGroupConcurrency],[TaskGroupJSON],[ActiveYN])
@ -25,6 +25,7 @@ Values (-6,'DependencyChainL2',1, 0,10,null,1)
INSERT INTO [dbo].[TaskGroup] ([TaskGroupId],[TaskGroupName],[SubjectAreaId], [TaskGroupPriority],[TaskGroupConcurrency],[TaskGroupJSON],[ActiveYN])
Values (-7,'DependencyChainL3',1, 0,10,null,1)
SET IDENTITY_INSERT [dbo].[TaskGroup] OFF
delete from [dbo].[TaskMaster] where taskmasterid <=0 and taskmasterid > -2000;

7662
solution/DataFactory/Patterns/TaskTypeMapping.sql
Просмотреть файл

Разница между файлами не показана из-за своего большого размера Загрузить разницу

2
solution/DataFactory/Patterns/UploadGeneratedPatternsToADF.ps1
Просмотреть файл

@ -75,7 +75,7 @@ function UploadADFItem ($items) {
}
$UploadGDS = $false
$UploadGDS = $true
$UploadGLS = $false
if($UploadGLS -eq $true)

11
solution/DataFactory/Patterns/dataset/GDS_AzureBlobFS_DelimitedText_(IRName).libsonnet
Просмотреть файл

@ -27,7 +27,11 @@ function(shortIRName = "", fullIRName = "")
},
"FirstRowAsHeader": {
"type": "bool"
}
},
"Delimiter": {
"type": "string",
"defaultValue": ","
}
},
"folder": {
"name": "ADS Go Fast/Generic/" + fullIRName
@ -50,7 +54,10 @@ function(shortIRName = "", fullIRName = "")
"type": "Expression"
}
},
"columnDelimiter": ",",
"columnDelimiter": {
"value": "@dataset().Delimiter",
"type": "Expression"
},
"escapeChar": "\\",
"firstRowAsHeader": {
"value": "@dataset().FirstRowAsHeader",

11
solution/DataFactory/Patterns/dataset/GDS_AzureBlobStorage_DelimitedText_(IRName).libsonnet
Просмотреть файл

@ -26,7 +26,11 @@ function(shortIRName = "", fullIRName = "") {
},
"FirstRowAsHeader": {
"type": "bool"
}
},
"Delimiter": {
"type": "string",
"defaultValue": ","
}
},
"folder": {
"name": "ADS Go Fast/Generic/" + fullIRName
@ -49,7 +53,10 @@ function(shortIRName = "", fullIRName = "") {
"type": "Expression"
}
},
"columnDelimiter": ",",
"columnDelimiter": {
"value": "@dataset().Delimiter",
"type": "Expression"
},
"escapeChar": "\\",
"firstRowAsHeader": {
"value": "@dataset().FirstRowAsHeader",

4
solution/DataFactory/Patterns/pipeline/Azure-Storage-to-Azure-Storage/functionapptests/partials/functionapptest.libsonnet
Просмотреть файл

@ -46,7 +46,7 @@ function(
then {"SkipLineCount": SourceSkipLineCount, "FirstRowAsHeader":SourceFirstRowAsHeader, "SheetName":SourceSheetName}
else {}
+ if (SourceFormat == "Csv" || SourceFormat == "DelimitedText")
then {"SkipLineCount": SourceSkipLineCount, "FirstRowAsHeader":SourceFirstRowAsHeader}
then {"SkipLineCount": SourceSkipLineCount, "FirstRowAsHeader":SourceFirstRowAsHeader, "Delimiter":","}
else {},
"Target":{
@ -62,7 +62,7 @@ function(
then {"SkipLineCount": 0, "FirstRowAsHeader":TargetFirstRowAsHeader, "SheetName":TargetSheetName}
else {}
+ if (TargetFormat == "Csv" || TargetFormat == "DelimitedText")
then {"SkipLineCount": 0, "FirstRowAsHeader":TargetFirstRowAsHeader}
then {"SkipLineCount": 0, "FirstRowAsHeader":TargetFirstRowAsHeader, "Delimiter":","}
else {},
},

7
solution/DataFactory/Patterns/pipeline/Azure-Storage-to-Azure-Storage/jsonschema/Partial_DelimitedText.libsonnet
Просмотреть файл

@ -56,6 +56,13 @@ function()
"infoText": "(required) Set to true if you want the first row of data to be used as column names."
}
},
"Delimiter": {
"type": "string",
"default": ",",
"options": {
"infoText": "The delimiter character or string."
}
},
"MaxConcurrentConnections": {
"type": "integer",
"default": 100,

4
solution/DataFactory/Patterns/pipeline/Azure-Storage-to-Azure-Storage/partials/datasetReferences/DelimitedText.libsonnet
Просмотреть файл

@ -25,6 +25,10 @@ function (GenerateArm=false, Type = "AzureBlobStorage", GFPIR = "{IRA}", SourceO
"FirstRowAsHeader": {
"value": "@pipeline().parameters.TaskObject.%(SourceOrTarget)s.FirstRowAsHeader" % { SourceOrTarget : SourceOrTarget},
"type": "Expression"
},
"Delimiter": {
"value": "@pipeline().parameters.TaskObject.%(SourceOrTarget)s.Delimiter" % { SourceOrTarget : SourceOrTarget},
"type": "Expression"
}
}
}

2
solution/DataFactory/Patterns/pipeline/Azure-Storage-to-SQL-Database/functionapptests/partials/functionapptest.libsonnet
Просмотреть файл

@ -49,7 +49,7 @@ function(
then {"FirstRowAsHeader":FirstRowAsHeader,"SkipLineCount": SkipLineCount, "SheetName":SheetName}
else {}
+ if (SourceFormat == "Csv" || SourceFormat == "DelimitedText")
then {"SkipLineCount": SkipLineCount, "FirstRowAsHeader":FirstRowAsHeader}
then {"SkipLineCount": SkipLineCount, "FirstRowAsHeader":FirstRowAsHeader, "Delimiter":","}
else {},
"Target":{
"Type":TargetFormat,

7
solution/DataFactory/Patterns/pipeline/Azure-Storage-to-SQL-Database/jsonschema/Partial_LoadFromDelimitedText.libsonnet
Просмотреть файл

@ -55,6 +55,13 @@ function()
"options": {
"infoText": "(required) Set to true if you want the first row of data to be used as column names."
}
},
"Delimiter": {
"type": "string",
"default": ",",
"options": {
"infoText": "The delimiter character or string."
}
},
"MaxConcurrentConnections": {
"type": "integer",

6
solution/DataFactory/Patterns/pipeline/Azure-Storage-to-SQL-Database/partials/MainCopyActivity/datasetReferences/DelimitedText.libsonnet
Просмотреть файл

@ -25,6 +25,10 @@ function (GenerateArm=false, Type = "AzureBlobStorage", GFPIR = "{IRA}", SourceO
"FirstRowAsHeader": {
"value": "@pipeline().parameters.TaskObject.%(SourceOrTarget)s.FirstRowAsHeader" % { SourceOrTarget : SourceOrTarget},
"type": "Expression"
}
},
"Delimiter": {
"value": "@pipeline().parameters.TaskObject.%(SourceOrTarget)s.Delimiter" % { SourceOrTarget : SourceOrTarget},
"type": "Expression"
},
}
}

4
solution/DataFactory/Patterns/pipeline/Azure-Storage-to-SQL-Database/partials/oldtoberemoved/Main_CopyActivity_AzureBlobFS_DelimitedText_Inputs.libsonnet
Просмотреть файл

@ -28,6 +28,10 @@ local referenceName = "GDS_AzureBlobFS_DelimitedText_",
"FirstRowAsHeader": {
"value": "@pipeline().parameters.TaskObject.Source.FirstRowAsHeader",
"type": "Expression"
},
"Delimiter": {
"value": "@pipeline().parameters.TaskObject.Source.Delimiter",
"type": "Expression"
}
}
}

6
solution/DataFactory/Patterns/pipeline/Azure-Storage-to-SQL-Database/partials/oldtoberemoved/Main_CopyActivity_AzureBlobStorage_DelimitedText_Inputs.libsonnet
Просмотреть файл

@ -28,7 +28,11 @@ local referenceName = "GDS_AzureBlobStorage_DelimitedText_",
"FirstRowAsHeader": {
"value": "@pipeline().parameters.TaskObject.Source.FirstRowAsHeader",
"type": "Expression"
}
},
"Delimiter": {
"value": "@pipeline().parameters.TaskObject.Source.Delimiter",
"type": "Expression"
},
}
}
]

4
solution/Dbt/dbt_init.sh
Просмотреть файл

@ -1,8 +1,8 @@
python3 -m venv .dbtenv
source .dbtenv/bin/activate
source ./.dbtenv/bin/activate
pip install dbt-core --quiet
pip install dbt-sqlserver --quiet
echo '.dbtenv/' > .gitignore
#echo '.dbtenv/' > .gitignore
export DBT_PROFILES_DIR=./.dbt/

1
solution/DeploymentV2/environments/vars/common_vars_template.jsonnet
Просмотреть файл

@ -408,3 +408,4 @@ local SecretFileSensitiveVars = { // Object comprehension.
"ForSecretFile": SecretFileVars,
"ForSecretFileSensitive": SecretFileSensitiveVars
}

11
solution/DeploymentV2/terraform_layer2/modules/data_factory_datasets/arm/GDS_AzureBlobFS_DelimitedText.json
Просмотреть файл

@ -51,7 +51,11 @@
"type": "String"
},
"FirstRowAsHeader": {
"type": "bool"
"type": "Bool"
},
"Delimiter": {
"type": "String",
"defaultValue": ","
}
},
"folder": {
@ -75,7 +79,10 @@
"type": "Expression"
}
},
"columnDelimiter": ",",
"columnDelimiter": {
"value": "@dataset().Delimiter",
"type": "Expression"
},
"escapeChar": "\\",
"firstRowAsHeader": {
"value": "@dataset().FirstRowAsHeader",

9
solution/DeploymentV2/terraform_layer2/modules/data_factory_datasets/arm/GDS_AzureBlobStorage_DelimitedText.json
Просмотреть файл

@ -51,6 +51,10 @@
},
"FirstRowAsHeader": {
"type": "bool"
},
"Delimiter": {
"type": "String",
"defaultValue": ","
}
},
"folder": {
@ -74,7 +78,10 @@
"type": "Expression"
}
},
"columnDelimiter": ",",
"columnDelimiter": {
"value": "@dataset().Delimiter",
"type": "Expression"
},
"escapeChar": "\\",
"firstRowAsHeader": {
"value": "@dataset().FirstRowAsHeader",

132
solution/FunctionApp/FunctionApp/Models/GetTaskInstanceJSON/ADFJsonBaseTask_ProcessTaskMaster.cs
Просмотреть файл

@ -386,57 +386,82 @@ namespace FunctionApp.Models.GetTaskInstanceJSON
sqlParams["tableSchema"] = sqlParams["tableSchema"].ToUpper();
}
//If Extraction SQL Explicitly set then overide _SQLStatement with that explicit value
if (!string.IsNullOrWhiteSpace(extractionSql))
{
sqlStatement = extractionSql;
sqlStatement = extractionSql;
if (incrementalType == "full_chunk" || incrementalType == "watermark_chunk")
{
if (incrementalType == "full_chunk")
{
sqlStatement = sqlStatement.Replace("{chunkField}", chunkField.ToString());
}
if (incrementalType == "watermark_chunk")
{
sqlStatement = sqlStatement.Replace("{chunkField}", chunkField.ToString());
sqlStatement = sqlStatement.Replace("{incrementalField}", incrementalField.ToString());
if (incrementalColumnType.ToString() == "DateTime")
{
DateTime incrementalValueDateTime = (DateTime)_taskInstanceJson["IncrementalValue"];
if (sourceType == "Oracle Server")
{
sqlStatement = sqlStatement.Replace("{incrementalValueDateTime}",incrementalValueDateTime.ToString("yyyy-MM-dd HH:mm:ss.ff"));
}
else
{
sqlStatement = sqlStatement.Replace("{incrementalValueDateTime}",incrementalValueDateTime.ToString("yyyy-MM-dd HH:mm:ss.fff"));
}
}
else if (incrementalColumnType.ToString() == "BigInt")
{
int incrementalValueBigInt = (int)_taskInstanceJson["IncrementalValue"];
sqlStatement = sqlStatement.Replace("{incrementalValueBigInt}",incrementalValueBigInt.ToString());
}
}
}
else //Non Chunk
{
if (incrementalType == "watermark")
{
sqlStatement = sqlStatement.Replace("{incrementalField}", incrementalField.ToString());
if (incrementalColumnType.ToString() == "DateTime")
{
DateTime incrementalValueDateTime = (DateTime)_taskInstanceJson["IncrementalValue"];
if (sourceType == "Oracle Server")
{
sqlStatement = sqlStatement.Replace("{incrementalValueDateTime}",incrementalValueDateTime.ToString("yyyy-MM-dd HH:mm:ss.ff"));
}
else
{
sqlStatement = sqlStatement.Replace("{incrementalValueDateTime}",incrementalValueDateTime.ToString("yyyy-MM-dd HH:mm:ss.fff"));
}
}
else if (incrementalColumnType.ToString() == "BigInt")
{
int incrementalValueBigInt = (int)_taskInstanceJson["IncrementalValue"];
sqlStatement = sqlStatement.Replace("{incrementalValueBigInt}",incrementalValueBigInt.ToString());
}
}
}
goto EndOfSQLStatementSet;
}
//Chunk branch
if (incrementalType == "full_chunk" || incrementalType == "watermark_chunk")
{
{
if (incrementalType == "full_chunk")
{
templateFile = "Full";
//sqlStatement = $"SELECT * FROM {tableSchema}.{tableName}";
}
else if (incrementalType == "Full")
{
templateFile = "Full_Chunk";
sqlParams.Add("chunkField", chunkField.ToString());
//sqlStatement = $"SELECT * FROM {tableSchema}.{tableName} WHERE CAST({chunkField} AS BIGINT) % <batchcount> = <item> -1. ";
}
else if (incrementalType == "Watermark")
{
if (incrementalColumnType.ToString() == "DateTime")
{
DateTime incrementalValueDateTime = (DateTime)_taskInstanceJson["IncrementalValue"];
templateFile = "WatermarkDateTime";
if (sourceType == "Oracle Server")
{
sqlParams.Add("incrementalValueDateTime", incrementalValueDateTime.ToString("yyyy-MM-dd HH:mm:ss.ff"));
}
else
{
sqlParams.Add("incrementalValueDateTime", incrementalValueDateTime.ToString("yyyy-MM-dd HH:mm:ss.fff"));
}
//sqlStatement = string.Format("SELECT * FROM {0}.{1} WHERE {2} > Cast('{3}' as datetime) AND {2} <= Cast('<newWatermark>' as datetime)", tableSchema, tableName, incrementalField, incrementalValueDateTime.ToString("yyyy-MM-dd HH:mm:ss.fff"));
}
else if (incrementalColumnType.ToString() == "BigInt")
{
int incrementalValueBigInt = (int)_taskInstanceJson["IncrementalValue"];
templateFile = "WatermarkBigInt";
sqlParams.Add("incrementalValueBigInt", incrementalValueBigInt.ToString());
//sqlStatement = string.Format("SELECT * FROM {0}.{1} WHERE {2} > Cast('{3}' as bigint) AND {2} <= cast('<newWatermark>' as bigint)", tableSchema, tableName, incrementalField, incrementalValueBigInt);
}
}
else if (incrementalType == "Watermark" && !string.IsNullOrWhiteSpace(_taskMasterJsonSource["Source"]["ChunkSize"].ToString()))
}
else if (incrementalType == "watermark_chunk" && !string.IsNullOrWhiteSpace(_taskMasterJsonSource["Source"]["ChunkSize"].ToString()))
{
if (incrementalColumnType.ToString() == "DateTime")
{
@ -449,17 +474,13 @@ namespace FunctionApp.Models.GetTaskInstanceJSON
else
{
sqlParams.Add("incrementalValueDateTime", incrementalValueDateTime.ToString("yyyy-MM-dd HH:mm:ss.fff"));
}
//sqlStatement = string.Format("SELECT * FROM {0}.{1} WHERE {2} > Cast('{3}' as datetime) AND {2} <= Cast('<newWatermark>' as datetime) AND CAST({4} AS BIGINT) % <batchcount> = <item> -1.", tableSchema, tableName, incrementalField, incrementalValueDateTime.ToString("yyyy-MM-dd HH:mm:ss.fff"), chunkField);
}
}
else if (incrementalColumnType.ToString() == "BigInt")
{
int incrementalValueBigInt = (int)_taskInstanceJson["IncrementalValue"];
templateFile = "WatermarkBigInt_Chunk";
sqlParams.Add("incrementalValueBigInt", incrementalValueBigInt.ToString());
//sqlStatement = string.Format("SELECT * FROM {0}.{1} WHERE {2} > Cast('{3}' as bigint) AND {2} <= Cast('<newWatermark>' as bigint) AND CAST({4} AS BIGINT) % <batchcount> = <item> -1.", tableSchema, tableName, incrementalField, incrementalValueBigInt, chunkField);
}
sqlParams.Add("incrementalValueBigInt", incrementalValueBigInt.ToString()); }
sqlParams.Add("chunkField", chunkField.ToString());
}
@ -469,10 +490,8 @@ namespace FunctionApp.Models.GetTaskInstanceJSON
{
if (incrementalType == "full")
{
templateFile = "Full";
//sqlStatement = string.Format("SELECT * FROM {0}.{1}", tableSchema, tableName);
}
templateFile = "Full";
}
else if (incrementalType == "watermark")
{
if (incrementalColumnType.ToString() == "DateTime")
@ -487,23 +506,26 @@ namespace FunctionApp.Models.GetTaskInstanceJSON
{
sqlParams.Add("incrementalValueDateTime", incrementalValueDateTime.ToString("yyyy-MM-dd HH:mm:ss.fff"));
}
//sqlStatement = string.Format("SELECT * FROM {0}.{1} WHERE {2} > Cast('{3}' as datetime) AND {2} <= Cast('<newWatermark>' as datetime)", tableSchema, tableName, incrementalField, incrementalValueDateTime.ToString("yyyy-MM-dd HH:mm:ss.fff"));
}
else if (incrementalColumnType.ToString() == "BigInt")
{
int incrementalValueBigInt = (int)_taskInstanceJson["IncrementalValue"];
templateFile = "WatermarkBigInt";
sqlParams.Add("incrementalValueBigInt", incrementalValueBigInt.ToString());
//sqlStatement = string.Format("SELECT * FROM {0}.{1} WHERE {2} > Cast('{3}' as bigint) AND {2} <= cast('<newWatermark>' as bigint)", tableSchema, tableName, incrementalField, incrementalValueBigInt);
}
}
}
sqlStatement = GenerateSqlStatementTemplates.GetSql(System.IO.Path.Combine(EnvironmentHelper.GetWorkingFolder(), _appOptions.LocalPaths.SQLTemplateLocation), "CreateSQLStatement_" + templateFile + "_" + SqlTemplateLanguage, sqlParams);
EndOfSQLStatementSet:
if (string.IsNullOrWhiteSpace(sqlStatement))
{
Exception e = new Exception("SqlStatement for Extraction has not been set. IncrementalType = " + incrementalType);
_logging.LogErrors(e);
throw e;
}
return sqlStatement;
}

160
solution/Synapse/Patterns/pipeline/Azure-Storage-to-Azure-Storage/functionapptests/tests/tests.json
Просмотреть файл

@ -3,27 +3,27 @@
"DegreeOfCopyParallelism": 1,
"DependencyChainTag": "DeltaLoad",
"EngineId": -2,
"EngineJson": "{\"endpoint\": \"https://adsstgsynwadsr6p7.dev.azuresynapse.net\", \"DeltaProcessingNotebook\": \"DeltaProcessingNotebook\"}",
"EngineName": "ads-stg-adf-ads-r6p7",
"EngineResourceGroup": "gfuat2",
"EngineJson": "{\"endpoint\": \"https://adsstgsynwadsgo2n.dev.azuresynapse.net\", \"DeltaProcessingNotebook\": \"DeltaProcessingNotebook\"}",
"EngineName": "ads-stg-adf-ads-go2n",
"EngineResourceGroup": "gfloc1",
"EngineSubscriptionId": "035a1364-f00d-48e2-b582-4fe125905ee3",
"NumberOfRetries": 3,
"ScheduleMasterId": "-2",
"SourceKeyVaultBaseUrl": "https://ads-stg-kv-ads-r6p7.vault.azure.net",
"SourceKeyVaultBaseUrl": "https://ads-stg-kv-ads-go2n.vault.azure.net",
"SourceSystemAuthType": "MSI",
"SourceSystemId": -4,
"SourceSystemJSON": "{\n \"Container\": \"datalakeraw\"\n}",
"SourceSystemSecretName": "",
"SourceSystemServer": "https://adsstgdlsadsr6p7adsl.dfs.core.windows.net",
"SourceSystemServer": "https://adsstgdlsadsgo2nadsl.dfs.core.windows.net",
"SourceSystemType": "ADLS",
"SourceSystemUserName": "",
"SynapsePipeline": "GPL_SparkNotebookExecution_Primary_Azure",
"TargetKeyVaultBaseUrl": "https://ads-stg-kv-ads-r6p7.vault.azure.net",
"TargetKeyVaultBaseUrl": "https://ads-stg-kv-ads-go2n.vault.azure.net",
"TargetSystemAuthType": "MSI",
"TargetSystemId": -4,
"TargetSystemJSON": "{\n \"Container\": \"datalakeraw\"\n}",
"TargetSystemSecretName": "",
"TargetSystemServer": "https://adsstgdlsadsr6p7adsl.dfs.core.windows.net",
"TargetSystemServer": "https://adsstgdlsadsgo2nadsl.dfs.core.windows.net",
"TargetSystemType": "ADLS",
"TargetSystemUserName": "",
"TaskDatafactoryIR": "Azure",
@ -33,7 +33,7 @@
"TaskInstanceId": -1,
"TaskInstanceJson": "{\n \"SourceRelativePath\": \"samples/\",\n \"TargetRelativePath\": \"/Tests/Azure Storage to Azure Storage/\"\n}",
"TaskMasterId": -1000,
"TaskMasterJson": "{\n \"CDCSource\": \"Disabled\",\n \"ModifiedDate\": \"Disabled\",\n \"Purview\": \"Disabled\",\n \"QualifiedIDAssociation\": \"TaskMasterId\",\n \"Source\": {\n \"DataFileName\": \"SalesLT.Customer*.parquet\",\n \"DeleteAfterCompletion\": \"false\",\n \"MaxConcurrentConnections\": 0,\n \"Recursively\": \"false\",\n \"RelativePath\": \"samples/SalesLT_Customer_CDC/\",\n \"SchemaFileName\": \"SalesLT.Customer*.json\",\n \"Type\": \"Parquet\",\n \"WriteSchemaToPurview\": \"Disabled\"\n },\n \"SparkTableCreate\": \"Disabled\",\n \"SparkTableDBName\": \"\",\n \"SparkTableName\": \"\",\n \"Target\": {\n \"DataFileName\": \"SalesLT.Customer\",\n \"DeleteAfterCompletion\": \"false\",\n \"MaxConcurrentConnections\": 0,\n \"Recursively\": \"false\",\n \"RelativePath\": \"/Tests/Azure Storage to Azure Storage/-1000/\",\n \"SchemaFileName\": \"SalesLT.Customer.json\",\n \"Type\": \"Delta\",\n \"WriteSchemaToPurview\": \"Disabled\"\n },\n \"UseNotebookActivity\": \"Disabled\"\n}",
"TaskMasterJson": "{\n \"CDCSource\": \"Disabled\",\n \"ModifiedDate\": \"Disabled\",\n \"PrimaryKeyOverride\": \"\",\n \"Purview\": \"Disabled\",\n \"QualifiedIDAssociation\": \"TaskMasterId\",\n \"Source\": {\n \"DataFileName\": \"SalesLT.Customer*.parquet\",\n \"DeleteAfterCompletion\": \"false\",\n \"MaxConcurrentConnections\": 0,\n \"Recursively\": \"false\",\n \"RelativePath\": \"samples/SalesLT_Customer_CDC/\",\n \"SchemaFileName\": \"SalesLT.Customer*.json\",\n \"Type\": \"Parquet\",\n \"WriteSchemaToPurview\": \"Disabled\"\n },\n \"SparkTableCreate\": \"Disabled\",\n \"SparkTableDBName\": \"\",\n \"SparkTableName\": \"\",\n \"Target\": {\n \"DataFileName\": \"SalesLT.Customer\",\n \"DeleteAfterCompletion\": \"false\",\n \"MaxConcurrentConnections\": 0,\n \"Recursively\": \"false\",\n \"RelativePath\": \"/Tests/Azure Storage to Azure Storage/-1000/\",\n \"SchemaFileName\": \"SalesLT.Customer.json\",\n \"Type\": \"Delta\",\n \"WriteSchemaToPurview\": \"Disabled\"\n },\n \"UseNotebookActivity\": \"Disabled\"\n}",
"TaskStatus": "Untried",
"TaskType": "Azure Storage to Azure Storage",
"TaskTypeId": -2,
@ -43,27 +43,27 @@
"DegreeOfCopyParallelism": 1,
"DependencyChainTag": "DeltaLoad",
"EngineId": -2,
"EngineJson": "{\"endpoint\": \"https://adsstgsynwadsr6p7.dev.azuresynapse.net\", \"DeltaProcessingNotebook\": \"DeltaProcessingNotebook\"}",
"EngineName": "ads-stg-adf-ads-r6p7",
"EngineResourceGroup": "gfuat2",
"EngineJson": "{\"endpoint\": \"https://adsstgsynwadsgo2n.dev.azuresynapse.net\", \"DeltaProcessingNotebook\": \"DeltaProcessingNotebook\"}",
"EngineName": "ads-stg-adf-ads-go2n",
"EngineResourceGroup": "gfloc1",
"EngineSubscriptionId": "035a1364-f00d-48e2-b582-4fe125905ee3",
"NumberOfRetries": 3,
"ScheduleMasterId": "-2",
"SourceKeyVaultBaseUrl": "https://ads-stg-kv-ads-r6p7.vault.azure.net",
"SourceKeyVaultBaseUrl": "https://ads-stg-kv-ads-go2n.vault.azure.net",
"SourceSystemAuthType": "MSI",
"SourceSystemId": -4,
"SourceSystemJSON": "{\n \"Container\": \"datalakeraw\"\n}",
"SourceSystemSecretName": "",
"SourceSystemServer": "https://adsstgdlsadsr6p7adsl.dfs.core.windows.net",
"SourceSystemServer": "https://adsstgdlsadsgo2nadsl.dfs.core.windows.net",
"SourceSystemType": "ADLS",
"SourceSystemUserName": "",
"SynapsePipeline": "GPL_SparkNotebookExecution_Primary_Azure",
"TargetKeyVaultBaseUrl": "https://ads-stg-kv-ads-r6p7.vault.azure.net",
"TargetKeyVaultBaseUrl": "https://ads-stg-kv-ads-go2n.vault.azure.net",
"TargetSystemAuthType": "MSI",
"TargetSystemId": -4,
"TargetSystemJSON": "{\n \"Container\": \"datalakeraw\"\n}",
"TargetSystemSecretName": "",
"TargetSystemServer": "https://adsstgdlsadsr6p7adsl.dfs.core.windows.net",
"TargetSystemServer": "https://adsstgdlsadsgo2nadsl.dfs.core.windows.net",
"TargetSystemType": "ADLS",
"TargetSystemUserName": "",
"TaskDatafactoryIR": "Azure",
@ -73,7 +73,7 @@
"TaskInstanceId": -1,
"TaskInstanceJson": "{\n \"SourceRelativePath\": \"samples/\",\n \"TargetRelativePath\": \"/Tests/Azure Storage to Azure Storage/\"\n}",
"TaskMasterId": -1001,
"TaskMasterJson": "{\n \"CDCSource\": \"Disabled\",\n \"ModifiedDate\": \"Disabled\",\n \"Purview\": \"Disabled\",\n \"QualifiedIDAssociation\": \"TaskMasterId\",\n \"Source\": {\n \"DataFileName\": \"SalesLT.Customer*.parquet\",\n \"DeleteAfterCompletion\": \"false\",\n \"MaxConcurrentConnections\": 0,\n \"Recursively\": \"false\",\n \"RelativePath\": \"samples/SalesLT_Customer_CDC/\",\n \"SchemaFileName\": \"SalesLT.Customer*.json\",\n \"Type\": \"Parquet\",\n \"WriteSchemaToPurview\": \"Disabled\"\n },\n \"SparkTableCreate\": \"Enabled\",\n \"SparkTableDBName\": \"TestTable\",\n \"SparkTableName\": \"TestDB\",\n \"Target\": {\n \"DataFileName\": \"SalesLT.Customer\",\n \"DeleteAfterCompletion\": \"false\",\n \"MaxConcurrentConnections\": 0,\n \"Recursively\": \"false\",\n \"RelativePath\": \"/Tests/Azure Storage to Azure Storage/-1001/\",\n \"SchemaFileName\": \"SalesLT.Customer.json\",\n \"Type\": \"Delta\",\n \"WriteSchemaToPurview\": \"Disabled\"\n },\n \"UseNotebookActivity\": \"Disabled\"\n}",
"TaskMasterJson": "{\n \"CDCSource\": \"Disabled\",\n \"ModifiedDate\": \"Disabled\",\n \"PrimaryKeyOverride\": \"\",\n \"Purview\": \"Disabled\",\n \"QualifiedIDAssociation\": \"TaskMasterId\",\n \"Source\": {\n \"DataFileName\": \"SalesLT.Customer*.parquet\",\n \"DeleteAfterCompletion\": \"false\",\n \"MaxConcurrentConnections\": 0,\n \"Recursively\": \"false\",\n \"RelativePath\": \"samples/SalesLT_Customer_CDC/\",\n \"SchemaFileName\": \"SalesLT.Customer*.json\",\n \"Type\": \"Parquet\",\n \"WriteSchemaToPurview\": \"Disabled\"\n },\n \"SparkTableCreate\": \"Enabled\",\n \"SparkTableDBName\": \"TestTable\",\n \"SparkTableName\": \"TestDB\",\n \"Target\": {\n \"DataFileName\": \"SalesLT.Customer\",\n \"DeleteAfterCompletion\": \"false\",\n \"MaxConcurrentConnections\": 0,\n \"Recursively\": \"false\",\n \"RelativePath\": \"/Tests/Azure Storage to Azure Storage/-1001/\",\n \"SchemaFileName\": \"SalesLT.Customer.json\",\n \"Type\": \"Delta\",\n \"WriteSchemaToPurview\": \"Disabled\"\n },\n \"UseNotebookActivity\": \"Disabled\"\n}",
"TaskStatus": "Untried",
"TaskType": "Azure Storage to Azure Storage",
"TaskTypeId": -2,
@ -83,27 +83,27 @@
"DegreeOfCopyParallelism": 1,
"DependencyChainTag": "DeltaLoad",
"EngineId": -2,
"EngineJson": "{\"endpoint\": \"https://adsstgsynwadsr6p7.dev.azuresynapse.net\", \"DeltaProcessingNotebook\": \"DeltaProcessingNotebook\"}",
"EngineName": "ads-stg-adf-ads-r6p7",
"EngineResourceGroup": "gfuat2",
"EngineJson": "{\"endpoint\": \"https://adsstgsynwadsgo2n.dev.azuresynapse.net\", \"DeltaProcessingNotebook\": \"DeltaProcessingNotebook\"}",
"EngineName": "ads-stg-adf-ads-go2n",
"EngineResourceGroup": "gfloc1",
"EngineSubscriptionId": "035a1364-f00d-48e2-b582-4fe125905ee3",
"NumberOfRetries": 3,
"ScheduleMasterId": "-2",
"SourceKeyVaultBaseUrl": "https://ads-stg-kv-ads-r6p7.vault.azure.net",
"SourceKeyVaultBaseUrl": "https://ads-stg-kv-ads-go2n.vault.azure.net",
"SourceSystemAuthType": "MSI",
"SourceSystemId": -4,
"SourceSystemJSON": "{\n \"Container\": \"datalakeraw\"\n}",
"SourceSystemSecretName": "",
"SourceSystemServer": "https://adsstgdlsadsr6p7adsl.dfs.core.windows.net",
"SourceSystemServer": "https://adsstgdlsadsgo2nadsl.dfs.core.windows.net",
"SourceSystemType": "ADLS",
"SourceSystemUserName": "",
"SynapsePipeline": "GPL_SparkNotebookExecution_Primary_Azure",
"TargetKeyVaultBaseUrl": "https://ads-stg-kv-ads-r6p7.vault.azure.net",
"TargetKeyVaultBaseUrl": "https://ads-stg-kv-ads-go2n.vault.azure.net",
"TargetSystemAuthType": "MSI",
"TargetSystemId": -4,
"TargetSystemJSON": "{\n \"Container\": \"datalakeraw\"\n}",
"TargetSystemSecretName": "",
"TargetSystemServer": "https://adsstgdlsadsr6p7adsl.dfs.core.windows.net",
"TargetSystemServer": "https://adsstgdlsadsgo2nadsl.dfs.core.windows.net",
"TargetSystemType": "ADLS",
"TargetSystemUserName": "",
"TaskDatafactoryIR": "Azure",
@ -113,7 +113,7 @@
"TaskInstanceId": -1,
"TaskInstanceJson": "{\n \"SourceRelativePath\": \"samples/\",\n \"TargetRelativePath\": \"/Tests/Azure Storage to Azure Storage/\"\n}",
"TaskMasterId": -1002,
"TaskMasterJson": "{\n \"CDCSource\": \"Disabled\",\n \"ModifiedDate\": \"Disabled\",\n \"Purview\": \"Enabled\",\n \"QualifiedIDAssociation\": \"TaskMasterId\",\n \"Source\": {\n \"DataFileName\": \"SalesLT.Customer*.parquet\",\n \"DeleteAfterCompletion\": \"false\",\n \"MaxConcurrentConnections\": 0,\n \"Recursively\": \"false\",\n \"RelativePath\": \"samples/SalesLT_Customer_CDC/\",\n \"SchemaFileName\": \"SalesLT.Customer*.json\",\n \"Type\": \"Parquet\",\n \"WriteSchemaToPurview\": \"Disabled\"\n },\n \"SparkTableCreate\": \"Disabled\",\n \"SparkTableDBName\": \"\",\n \"SparkTableName\": \"\",\n \"Target\": {\n \"DataFileName\": \"SalesLT.Customer\",\n \"DeleteAfterCompletion\": \"false\",\n \"MaxConcurrentConnections\": 0,\n \"Recursively\": \"false\",\n \"RelativePath\": \"/Tests/Azure Storage to Azure Storage/-1002/\",\n \"SchemaFileName\": \"SalesLT.Customer.json\",\n \"Type\": \"Delta\",\n \"WriteSchemaToPurview\": \"Disabled\"\n },\n \"UseNotebookActivity\": \"Disabled\"\n}",
"TaskMasterJson": "{\n \"CDCSource\": \"Disabled\",\n \"ModifiedDate\": \"Disabled\",\n \"PrimaryKeyOverride\": \"\",\n \"Purview\": \"Enabled\",\n \"QualifiedIDAssociation\": \"TaskMasterId\",\n \"Source\": {\n \"DataFileName\": \"SalesLT.Customer*.parquet\",\n \"DeleteAfterCompletion\": \"false\",\n \"MaxConcurrentConnections\": 0,\n \"Recursively\": \"false\",\n \"RelativePath\": \"samples/SalesLT_Customer_CDC/\",\n \"SchemaFileName\": \"SalesLT.Customer*.json\",\n \"Type\": \"Parquet\",\n \"WriteSchemaToPurview\": \"Disabled\"\n },\n \"SparkTableCreate\": \"Disabled\",\n \"SparkTableDBName\": \"\",\n \"SparkTableName\": \"\",\n \"Target\": {\n \"DataFileName\": \"SalesLT.Customer\",\n \"DeleteAfterCompletion\": \"false\",\n \"MaxConcurrentConnections\": 0,\n \"Recursively\": \"false\",\n \"RelativePath\": \"/Tests/Azure Storage to Azure Storage/-1002/\",\n \"SchemaFileName\": \"SalesLT.Customer.json\",\n \"Type\": \"Delta\",\n \"WriteSchemaToPurview\": \"Disabled\"\n },\n \"UseNotebookActivity\": \"Disabled\"\n}",
"TaskStatus": "Untried",
"TaskType": "Azure Storage to Azure Storage",
"TaskTypeId": -2,
@ -123,27 +123,27 @@
"DegreeOfCopyParallelism": 1,
"DependencyChainTag": "DeltaLoad",
"EngineId": -2,
"EngineJson": "{\"endpoint\": \"https://adsstgsynwadsr6p7.dev.azuresynapse.net\", \"DeltaProcessingNotebook\": \"DeltaProcessingNotebook\"}",
"EngineName": "ads-stg-adf-ads-r6p7",
"EngineResourceGroup": "gfuat2",
"EngineJson": "{\"endpoint\": \"https://adsstgsynwadsgo2n.dev.azuresynapse.net\", \"DeltaProcessingNotebook\": \"DeltaProcessingNotebook\"}",
"EngineName": "ads-stg-adf-ads-go2n",
"EngineResourceGroup": "gfloc1",
"EngineSubscriptionId": "035a1364-f00d-48e2-b582-4fe125905ee3",
"NumberOfRetries": 3,
"ScheduleMasterId": "-2",
"SourceKeyVaultBaseUrl": "https://ads-stg-kv-ads-r6p7.vault.azure.net",
"SourceKeyVaultBaseUrl": "https://ads-stg-kv-ads-go2n.vault.azure.net",
"SourceSystemAuthType": "MSI",
"SourceSystemId": -4,
"SourceSystemJSON": "{\n \"Container\": \"datalakeraw\"\n}",
"SourceSystemSecretName": "",
"SourceSystemServer": "https://adsstgdlsadsr6p7adsl.dfs.core.windows.net",
"SourceSystemServer": "https://adsstgdlsadsgo2nadsl.dfs.core.windows.net",
"SourceSystemType": "ADLS",
"SourceSystemUserName": "",
"SynapsePipeline": "GPL_SparkNotebookExecution_Primary_Azure",
"TargetKeyVaultBaseUrl": "https://ads-stg-kv-ads-r6p7.vault.azure.net",
"TargetKeyVaultBaseUrl": "https://ads-stg-kv-ads-go2n.vault.azure.net",
"TargetSystemAuthType": "MSI",
"TargetSystemId": -4,
"TargetSystemJSON": "{\n \"Container\": \"datalakeraw\"\n}",
"TargetSystemSecretName": "",
"TargetSystemServer": "https://adsstgdlsadsr6p7adsl.dfs.core.windows.net",
"TargetSystemServer": "https://adsstgdlsadsgo2nadsl.dfs.core.windows.net",
"TargetSystemType": "ADLS",
"TargetSystemUserName": "",
"TaskDatafactoryIR": "Azure",
@ -153,7 +153,7 @@
"TaskInstanceId": -1,
"TaskInstanceJson": "{\n \"SourceRelativePath\": \"samples/\",\n \"TargetRelativePath\": \"/Tests/Azure Storage to Azure Storage/\"\n}",
"TaskMasterId": -1003,
"TaskMasterJson": "{\n \"CDCSource\": \"Enabled\",\n \"ModifiedDate\": \"Disabled\",\n \"Purview\": \"Disabled\",\n \"QualifiedIDAssociation\": \"TaskMasterId\",\n \"Source\": {\n \"DataFileName\": \"SalesLT.CustomerCDCInit.parquet\",\n \"DeleteAfterCompletion\": \"false\",\n \"MaxConcurrentConnections\": 0,\n \"Recursively\": \"false\",\n \"RelativePath\": \"samples/SalesLT_Customer_CDC/\",\n \"SchemaFileName\": \"SalesLT.CustomerCDCInit.json\",\n \"Type\": \"Parquet\",\n \"WriteSchemaToPurview\": \"Disabled\"\n },\n \"SparkTableCreate\": \"Disabled\",\n \"SparkTableDBName\": \"\",\n \"SparkTableName\": \"\",\n \"Target\": {\n \"DataFileName\": \"CustomerCDCDeltaTest\",\n \"DeleteAfterCompletion\": \"false\",\n \"MaxConcurrentConnections\": 0,\n \"Recursively\": \"false\",\n \"RelativePath\": \"/Tests/Azure Storage to Azure Storage/-1003/\",\n \"SchemaFileName\": \"CustomerCDCDeltaTest.json\",\n \"Type\": \"Delta\",\n \"WriteSchemaToPurview\": \"Disabled\"\n },\n \"UseNotebookActivity\": \"Disabled\"\n}",
"TaskMasterJson": "{\n \"CDCSource\": \"Enabled\",\n \"ModifiedDate\": \"Disabled\",\n \"PrimaryKeyOverride\": \"\",\n \"Purview\": \"Disabled\",\n \"QualifiedIDAssociation\": \"TaskMasterId\",\n \"Source\": {\n \"DataFileName\": \"SalesLT.CustomerCDCInit.parquet\",\n \"DeleteAfterCompletion\": \"false\",\n \"MaxConcurrentConnections\": 0,\n \"Recursively\": \"false\",\n \"RelativePath\": \"samples/SalesLT_Customer_CDC/\",\n \"SchemaFileName\": \"SalesLT.CustomerCDCInit.json\",\n \"Type\": \"Parquet\",\n \"WriteSchemaToPurview\": \"Disabled\"\n },\n \"SparkTableCreate\": \"Disabled\",\n \"SparkTableDBName\": \"\",\n \"SparkTableName\": \"\",\n \"Target\": {\n \"DataFileName\": \"CustomerCDCDeltaTest\",\n \"DeleteAfterCompletion\": \"false\",\n \"MaxConcurrentConnections\": 0,\n \"Recursively\": \"false\",\n \"RelativePath\": \"/Tests/Azure Storage to Azure Storage/-1003/\",\n \"SchemaFileName\": \"CustomerCDCDeltaTest.json\",\n \"Type\": \"Delta\",\n \"WriteSchemaToPurview\": \"Disabled\"\n },\n \"UseNotebookActivity\": \"Disabled\"\n}",
"TaskStatus": "Untried",
"TaskType": "Azure Storage to Azure Storage",
"TaskTypeId": -2,
@ -163,27 +163,27 @@
"DegreeOfCopyParallelism": 1,
"DependencyChainTag": "DeltaLoad",
"EngineId": -2,
"EngineJson": "{\"endpoint\": \"https://adsstgsynwadsr6p7.dev.azuresynapse.net\", \"DeltaProcessingNotebook\": \"DeltaProcessingNotebook\"}",
"EngineName": "ads-stg-adf-ads-r6p7",
"EngineResourceGroup": "gfuat2",
"EngineJson": "{\"endpoint\": \"https://adsstgsynwadsgo2n.dev.azuresynapse.net\", \"DeltaProcessingNotebook\": \"DeltaProcessingNotebook\"}",
"EngineName": "ads-stg-adf-ads-go2n",
"EngineResourceGroup": "gfloc1",
"EngineSubscriptionId": "035a1364-f00d-48e2-b582-4fe125905ee3",
"NumberOfRetries": 3,
"ScheduleMasterId": "-2",
"SourceKeyVaultBaseUrl": "https://ads-stg-kv-ads-r6p7.vault.azure.net",
"SourceKeyVaultBaseUrl": "https://ads-stg-kv-ads-go2n.vault.azure.net",
"SourceSystemAuthType": "MSI",
"SourceSystemId": -4,
"SourceSystemJSON": "{\n \"Container\": \"datalakeraw\"\n}",
"SourceSystemSecretName": "",
"SourceSystemServer": "https://adsstgdlsadsr6p7adsl.dfs.core.windows.net",
"SourceSystemServer": "https://adsstgdlsadsgo2nadsl.dfs.core.windows.net",
"SourceSystemType": "ADLS",
"SourceSystemUserName": "",
"SynapsePipeline": "GPL_SparkNotebookExecution_Primary_Azure",
"TargetKeyVaultBaseUrl": "https://ads-stg-kv-ads-r6p7.vault.azure.net",
"TargetKeyVaultBaseUrl": "https://ads-stg-kv-ads-go2n.vault.azure.net",
"TargetSystemAuthType": "MSI",
"TargetSystemId": -4,
"TargetSystemJSON": "{\n \"Container\": \"datalakeraw\"\n}",
"TargetSystemSecretName": "",
"TargetSystemServer": "https://adsstgdlsadsr6p7adsl.dfs.core.windows.net",
"TargetSystemServer": "https://adsstgdlsadsgo2nadsl.dfs.core.windows.net",
"TargetSystemType": "ADLS",
"TargetSystemUserName": "",
"TaskDatafactoryIR": "Azure",
@ -193,7 +193,7 @@
"TaskInstanceId": -1,
"TaskInstanceJson": "{\n \"SourceRelativePath\": \"samples/\",\n \"TargetRelativePath\": \"/Tests/Azure Storage to Azure Storage/\"\n}",
"TaskMasterId": -1004,
"TaskMasterJson": "{\n \"CDCSource\": \"Disabled\",\n \"ModifiedDate\": \"Disabled\",\n \"Purview\": \"Enabled\",\n \"QualifiedIDAssociation\": \"TaskMasterId\",\n \"Source\": {\n \"DataFileName\": \"SalesLT.Customer*.parquet\",\n \"DeleteAfterCompletion\": \"false\",\n \"MaxConcurrentConnections\": 0,\n \"Recursively\": \"false\",\n \"RelativePath\": \"samples/SalesLT_Customer_CDC/\",\n \"SchemaFileName\": \"SalesLT.Customer*.json\",\n \"Type\": \"Parquet\",\n \"WriteSchemaToPurview\": \"Disabled\"\n },\n \"SparkTableCreate\": \"Enabled\",\n \"SparkTableDBName\": \"TestTable2\",\n \"SparkTableName\": \"TestDB\",\n \"Target\": {\n \"DataFileName\": \"SalesLT.Customer\",\n \"DeleteAfterCompletion\": \"false\",\n \"MaxConcurrentConnections\": 0,\n \"Recursively\": \"false\",\n \"RelativePath\": \"/Tests/Azure Storage to Azure Storage/-1004/\",\n \"SchemaFileName\": \"SalesLT.Customer.json\",\n \"Type\": \"Delta\",\n \"WriteSchemaToPurview\": \"Disabled\"\n },\n \"UseNotebookActivity\": \"Disabled\"\n}",
"TaskMasterJson": "{\n \"CDCSource\": \"Disabled\",\n \"ModifiedDate\": \"Disabled\",\n \"PrimaryKeyOverride\": \"\",\n \"Purview\": \"Enabled\",\n \"QualifiedIDAssociation\": \"TaskMasterId\",\n \"Source\": {\n \"DataFileName\": \"SalesLT.Customer*.parquet\",\n \"DeleteAfterCompletion\": \"false\",\n \"MaxConcurrentConnections\": 0,\n \"Recursively\": \"false\",\n \"RelativePath\": \"samples/SalesLT_Customer_CDC/\",\n \"SchemaFileName\": \"SalesLT.Customer*.json\",\n \"Type\": \"Parquet\",\n \"WriteSchemaToPurview\": \"Disabled\"\n },\n \"SparkTableCreate\": \"Enabled\",\n \"SparkTableDBName\": \"TestTable2\",\n \"SparkTableName\": \"TestDB\",\n \"Target\": {\n \"DataFileName\": \"SalesLT.Customer\",\n \"DeleteAfterCompletion\": \"false\",\n \"MaxConcurrentConnections\": 0,\n \"Recursively\": \"false\",\n \"RelativePath\": \"/Tests/Azure Storage to Azure Storage/-1004/\",\n \"SchemaFileName\": \"SalesLT.Customer.json\",\n \"Type\": \"Delta\",\n \"WriteSchemaToPurview\": \"Disabled\"\n },\n \"UseNotebookActivity\": \"Disabled\"\n}",
"TaskStatus": "Untried",
"TaskType": "Azure Storage to Azure Storage",
"TaskTypeId": -2,
@ -203,27 +203,27 @@
"DegreeOfCopyParallelism": 1,
"DependencyChainTag": "DeltaLoad",
"EngineId": -2,
"EngineJson": "{\"endpoint\": \"https://adsstgsynwadsr6p7.dev.azuresynapse.net\", \"DeltaProcessingNotebook\": \"DeltaProcessingNotebook\"}",
"EngineName": "ads-stg-adf-ads-r6p7",
"EngineResourceGroup": "gfuat2",
"EngineJson": "{\"endpoint\": \"https://adsstgsynwadsgo2n.dev.azuresynapse.net\", \"DeltaProcessingNotebook\": \"DeltaProcessingNotebook\"}",
"EngineName": "ads-stg-adf-ads-go2n",
"EngineResourceGroup": "gfloc1",
"EngineSubscriptionId": "035a1364-f00d-48e2-b582-4fe125905ee3",
"NumberOfRetries": 3,
"ScheduleMasterId": "-2",
"SourceKeyVaultBaseUrl": "https://ads-stg-kv-ads-r6p7.vault.azure.net",
"SourceKeyVaultBaseUrl": "https://ads-stg-kv-ads-go2n.vault.azure.net",
"SourceSystemAuthType": "MSI",
"SourceSystemId": -4,
"SourceSystemJSON": "{\n \"Container\": \"datalakeraw\"\n}",
"SourceSystemSecretName": "",
"SourceSystemServer": "https://adsstgdlsadsr6p7adsl.dfs.core.windows.net",
"SourceSystemServer": "https://adsstgdlsadsgo2nadsl.dfs.core.windows.net",
"SourceSystemType": "ADLS",
"SourceSystemUserName": "",
"SynapsePipeline": "GPL_SparkNotebookExecution_Primary_Azure",
"TargetKeyVaultBaseUrl": "https://ads-stg-kv-ads-r6p7.vault.azure.net",
"TargetKeyVaultBaseUrl": "https://ads-stg-kv-ads-go2n.vault.azure.net",
"TargetSystemAuthType": "MSI",
"TargetSystemId": -4,
"TargetSystemJSON": "{\n \"Container\": \"datalakeraw\"\n}",
"TargetSystemSecretName": "",
"TargetSystemServer": "https://adsstgdlsadsr6p7adsl.dfs.core.windows.net",
"TargetSystemServer": "https://adsstgdlsadsgo2nadsl.dfs.core.windows.net",
"TargetSystemType": "ADLS",
"TargetSystemUserName": "",
"TaskDatafactoryIR": "Azure",
@ -233,7 +233,7 @@
"TaskInstanceId": -1,
"TaskInstanceJson": "{\n \"SourceRelativePath\": \"samples/\",\n \"TargetRelativePath\": \"/Tests/Azure Storage to Azure Storage/\"\n}",
"TaskMasterId": -1005,
"TaskMasterJson": "{\n \"CDCSource\": \"Enabled\",\n \"ModifiedDate\": \"Disabled\",\n \"Purview\": \"Disabled\",\n \"QualifiedIDAssociation\": \"TaskMasterId\",\n \"Source\": {\n \"DataFileName\": \"SalesLT.CustomerCDCEdit1.parquet\",\n \"DeleteAfterCompletion\": \"false\",\n \"MaxConcurrentConnections\": 0,\n \"Recursively\": \"false\",\n \"RelativePath\": \"samples/SalesLT_Customer_CDC/\",\n \"SchemaFileName\": \"SalesLT.CustomerCDCEdit1.json\",\n \"Type\": \"Parquet\",\n \"WriteSchemaToPurview\": \"Disabled\"\n },\n \"SparkTableCreate\": \"Enabled\",\n \"SparkTableDBName\": \"TestTable3\",\n \"SparkTableName\": \"TestDB\",\n \"Target\": {\n \"DataFileName\": \"CustomerCDCDeltaTest\",\n \"DeleteAfterCompletion\": \"false\",\n \"MaxConcurrentConnections\": 0,\n \"Recursively\": \"false\",\n \"RelativePath\": \"/Tests/Azure Storage to Azure Storage/-1005/\",\n \"SchemaFileName\": \"CustomerCDCDeltaTest.json\",\n \"Type\": \"Delta\",\n \"WriteSchemaToPurview\": \"Disabled\"\n },\n \"UseNotebookActivity\": \"Disabled\"\n}",
"TaskMasterJson": "{\n \"CDCSource\": \"Enabled\",\n \"ModifiedDate\": \"Disabled\",\n \"PrimaryKeyOverride\": \"\",\n \"Purview\": \"Disabled\",\n \"QualifiedIDAssociation\": \"TaskMasterId\",\n \"Source\": {\n \"DataFileName\": \"SalesLT.CustomerCDCEdit1.parquet\",\n \"DeleteAfterCompletion\": \"false\",\n \"MaxConcurrentConnections\": 0,\n \"Recursively\": \"false\",\n \"RelativePath\": \"samples/SalesLT_Customer_CDC/\",\n \"SchemaFileName\": \"SalesLT.CustomerCDCEdit1.json\",\n \"Type\": \"Parquet\",\n \"WriteSchemaToPurview\": \"Disabled\"\n },\n \"SparkTableCreate\": \"Enabled\",\n \"SparkTableDBName\": \"TestTable3\",\n \"SparkTableName\": \"TestDB\",\n \"Target\": {\n \"DataFileName\": \"CustomerCDCDeltaTest\",\n \"DeleteAfterCompletion\": \"false\",\n \"MaxConcurrentConnections\": 0,\n \"Recursively\": \"false\",\n \"RelativePath\": \"/Tests/Azure Storage to Azure Storage/-1005/\",\n \"SchemaFileName\": \"CustomerCDCDeltaTest.json\",\n \"Type\": \"Delta\",\n \"WriteSchemaToPurview\": \"Disabled\"\n },\n \"UseNotebookActivity\": \"Disabled\"\n}",
"TaskStatus": "Untried",
"TaskType": "Azure Storage to Azure Storage",
"TaskTypeId": -2,
@ -243,27 +243,27 @@
"DegreeOfCopyParallelism": 1,
"DependencyChainTag": "DeltaLoad",
"EngineId": -2,
"EngineJson": "{\"endpoint\": \"https://adsstgsynwadsr6p7.dev.azuresynapse.net\", \"DeltaProcessingNotebook\": \"DeltaProcessingNotebook\"}",
"EngineName": "ads-stg-adf-ads-r6p7",
"EngineResourceGroup": "gfuat2",
"EngineJson": "{\"endpoint\": \"https://adsstgsynwadsgo2n.dev.azuresynapse.net\", \"DeltaProcessingNotebook\": \"DeltaProcessingNotebook\"}",
"EngineName": "ads-stg-adf-ads-go2n",
"EngineResourceGroup": "gfloc1",
"EngineSubscriptionId": "035a1364-f00d-48e2-b582-4fe125905ee3",
"NumberOfRetries": 3,
"ScheduleMasterId": "-2",
"SourceKeyVaultBaseUrl": "https://ads-stg-kv-ads-r6p7.vault.azure.net",
"SourceKeyVaultBaseUrl": "https://ads-stg-kv-ads-go2n.vault.azure.net",
"SourceSystemAuthType": "MSI",
"SourceSystemId": -4,
"SourceSystemJSON": "{\n \"Container\": \"datalakeraw\"\n}",
"SourceSystemSecretName": "",
"SourceSystemServer": "https://adsstgdlsadsr6p7adsl.dfs.core.windows.net",
"SourceSystemServer": "https://adsstgdlsadsgo2nadsl.dfs.core.windows.net",
"SourceSystemType": "ADLS",
"SourceSystemUserName": "",
"SynapsePipeline": "GPL_SparkNotebookExecution_Primary_Azure",
"TargetKeyVaultBaseUrl": "https://ads-stg-kv-ads-r6p7.vault.azure.net",
"TargetKeyVaultBaseUrl": "https://ads-stg-kv-ads-go2n.vault.azure.net",
"TargetSystemAuthType": "MSI",
"TargetSystemId": -4,
"TargetSystemJSON": "{\n \"Container\": \"datalakeraw\"\n}",
"TargetSystemSecretName": "",
"TargetSystemServer": "https://adsstgdlsadsr6p7adsl.dfs.core.windows.net",
"TargetSystemServer": "https://adsstgdlsadsgo2nadsl.dfs.core.windows.net",
"TargetSystemType": "ADLS",
"TargetSystemUserName": "",
"TaskDatafactoryIR": "Azure",
@ -273,7 +273,7 @@
"TaskInstanceId": -1,
"TaskInstanceJson": "{\n \"SourceRelativePath\": \"samples/\",\n \"TargetRelativePath\": \"/Tests/Azure Storage to Azure Storage/\"\n}",
"TaskMasterId": -1006,
"TaskMasterJson": "{\n \"CDCSource\": \"Enabled\",\n \"ModifiedDate\": \"Disabled\",\n \"Purview\": \"Enabled\",\n \"QualifiedIDAssociation\": \"TaskMasterId\",\n \"Source\": {\n \"DataFileName\": \"SalesLT.CustomerCDCEdit2.parquet\",\n \"DeleteAfterCompletion\": \"false\",\n \"MaxConcurrentConnections\": 0,\n \"Recursively\": \"false\",\n \"RelativePath\": \"samples/SalesLT_Customer_CDC/\",\n \"SchemaFileName\": \"SalesLT.CustomerCDCEdit2.json\",\n \"Type\": \"Parquet\",\n \"WriteSchemaToPurview\": \"Disabled\"\n },\n \"SparkTableCreate\": \"Disabled\",\n \"SparkTableDBName\": \"\",\n \"SparkTableName\": \"\",\n \"Target\": {\n \"DataFileName\": \"CustomerCDCDeltaTest\",\n \"DeleteAfterCompletion\": \"false\",\n \"MaxConcurrentConnections\": 0,\n \"Recursively\": \"false\",\n \"RelativePath\": \"/Tests/Azure Storage to Azure Storage/-1006/\",\n \"SchemaFileName\": \"CustomerCDCDeltaTest.json\",\n \"Type\": \"Delta\",\n \"WriteSchemaToPurview\": \"Disabled\"\n },\n \"UseNotebookActivity\": \"Disabled\"\n}",
"TaskMasterJson": "{\n \"CDCSource\": \"Enabled\",\n \"ModifiedDate\": \"Disabled\",\n \"PrimaryKeyOverride\": \"\",\n \"Purview\": \"Enabled\",\n \"QualifiedIDAssociation\": \"TaskMasterId\",\n \"Source\": {\n \"DataFileName\": \"SalesLT.CustomerCDCEdit2.parquet\",\n \"DeleteAfterCompletion\": \"false\",\n \"MaxConcurrentConnections\": 0,\n \"Recursively\": \"false\",\n \"RelativePath\": \"samples/SalesLT_Customer_CDC/\",\n \"SchemaFileName\": \"SalesLT.CustomerCDCEdit2.json\",\n \"Type\": \"Parquet\",\n \"WriteSchemaToPurview\": \"Disabled\"\n },\n \"SparkTableCreate\": \"Disabled\",\n \"SparkTableDBName\": \"\",\n \"SparkTableName\": \"\",\n \"Target\": {\n \"DataFileName\": \"CustomerCDCDeltaTest\",\n \"DeleteAfterCompletion\": \"false\",\n \"MaxConcurrentConnections\": 0,\n \"Recursively\": \"false\",\n \"RelativePath\": \"/Tests/Azure Storage to Azure Storage/-1006/\",\n \"SchemaFileName\": \"CustomerCDCDeltaTest.json\",\n \"Type\": \"Delta\",\n \"WriteSchemaToPurview\": \"Disabled\"\n },\n \"UseNotebookActivity\": \"Disabled\"\n}",
"TaskStatus": "Untried",
"TaskType": "Azure Storage to Azure Storage",
"TaskTypeId": -2,
@ -283,27 +283,27 @@
"DegreeOfCopyParallelism": 1,
"DependencyChainTag": "DeltaLoad",
"EngineId": -2,
"EngineJson": "{\"endpoint\": \"https://adsstgsynwadsr6p7.dev.azuresynapse.net\", \"DeltaProcessingNotebook\": \"DeltaProcessingNotebook\"}",
"EngineName": "ads-stg-adf-ads-r6p7",
"EngineResourceGroup": "gfuat2",
"EngineJson": "{\"endpoint\": \"https://adsstgsynwadsgo2n.dev.azuresynapse.net\", \"DeltaProcessingNotebook\": \"DeltaProcessingNotebook\"}",
"EngineName": "ads-stg-adf-ads-go2n",
"EngineResourceGroup": "gfloc1",
"EngineSubscriptionId": "035a1364-f00d-48e2-b582-4fe125905ee3",
"NumberOfRetries": 3,
"ScheduleMasterId": "-2",
"SourceKeyVaultBaseUrl": "https://ads-stg-kv-ads-r6p7.vault.azure.net",
"SourceKeyVaultBaseUrl": "https://ads-stg-kv-ads-go2n.vault.azure.net",
"SourceSystemAuthType": "MSI",
"SourceSystemId": -4,
"SourceSystemJSON": "{\n \"Container\": \"datalakeraw\"\n}",
"SourceSystemSecretName": "",
"SourceSystemServer": "https://adsstgdlsadsr6p7adsl.dfs.core.windows.net",
"SourceSystemServer": "https://adsstgdlsadsgo2nadsl.dfs.core.windows.net",
"SourceSystemType": "ADLS",
"SourceSystemUserName": "",
"SynapsePipeline": "GPL_SparkNotebookExecution_Primary_Azure",
"TargetKeyVaultBaseUrl": "https://ads-stg-kv-ads-r6p7.vault.azure.net",
"TargetKeyVaultBaseUrl": "https://ads-stg-kv-ads-go2n.vault.azure.net",
"TargetSystemAuthType": "MSI",
"TargetSystemId": -4,
"TargetSystemJSON": "{\n \"Container\": \"datalakeraw\"\n}",
"TargetSystemSecretName": "",
"TargetSystemServer": "https://adsstgdlsadsr6p7adsl.dfs.core.windows.net",
"TargetSystemServer": "https://adsstgdlsadsgo2nadsl.dfs.core.windows.net",
"TargetSystemType": "ADLS",
"TargetSystemUserName": "",
"TaskDatafactoryIR": "Azure",
@ -313,7 +313,7 @@
"TaskInstanceId": -1,
"TaskInstanceJson": "{\n \"SourceRelativePath\": \"samples/\",\n \"TargetRelativePath\": \"/Tests/Azure Storage to Azure Storage/\"\n}",
"TaskMasterId": -1007,
"TaskMasterJson": "{\n \"CDCSource\": \"Enabled\",\n \"ModifiedDate\": \"Disabled\",\n \"Purview\": \"Enabled\",\n \"QualifiedIDAssociation\": \"TaskMasterId\",\n \"Source\": {\n \"DataFileName\": \"SalesLT.CustomerCDCEdit3.parquet\",\n \"DeleteAfterCompletion\": \"false\",\n \"MaxConcurrentConnections\": 0,\n \"Recursively\": \"false\",\n \"RelativePath\": \"samples/SalesLT_Customer_CDC/\",\n \"SchemaFileName\": \"SalesLT.CustomerCDCEdit3.json\",\n \"Type\": \"Parquet\",\n \"WriteSchemaToPurview\": \"Disabled\"\n },\n \"SparkTableCreate\": \"Enabled\",\n \"SparkTableDBName\": \"TestTable4\",\n \"SparkTableName\": \"TestDB\",\n \"Target\": {\n \"DataFileName\": \"CustomerCDCDeltaTest\",\n \"DeleteAfterCompletion\": \"false\",\n \"MaxConcurrentConnections\": 0,\n \"Recursively\": \"false\",\n \"RelativePath\": \"/Tests/Azure Storage to Azure Storage/-1007/\",\n \"SchemaFileName\": \"CustomerCDCDeltaTest.json\",\n \"Type\": \"Delta\",\n \"WriteSchemaToPurview\": \"Disabled\"\n },\n \"UseNotebookActivity\": \"Disabled\"\n}",
"TaskMasterJson": "{\n \"CDCSource\": \"Enabled\",\n \"ModifiedDate\": \"Disabled\",\n \"PrimaryKeyOverride\": \"\",\n \"Purview\": \"Enabled\",\n \"QualifiedIDAssociation\": \"TaskMasterId\",\n \"Source\": {\n \"DataFileName\": \"SalesLT.CustomerCDCEdit3.parquet\",\n \"DeleteAfterCompletion\": \"false\",\n \"MaxConcurrentConnections\": 0,\n \"Recursively\": \"false\",\n \"RelativePath\": \"samples/SalesLT_Customer_CDC/\",\n \"SchemaFileName\": \"SalesLT.CustomerCDCEdit3.json\",\n \"Type\": \"Parquet\",\n \"WriteSchemaToPurview\": \"Disabled\"\n },\n \"SparkTableCreate\": \"Enabled\",\n \"SparkTableDBName\": \"TestTable4\",\n \"SparkTableName\": \"TestDB\",\n \"Target\": {\n \"DataFileName\": \"CustomerCDCDeltaTest\",\n \"DeleteAfterCompletion\": \"false\",\n \"MaxConcurrentConnections\": 0,\n \"Recursively\": \"false\",\n \"RelativePath\": \"/Tests/Azure Storage to Azure Storage/-1007/\",\n \"SchemaFileName\": \"CustomerCDCDeltaTest.json\",\n \"Type\": \"Delta\",\n \"WriteSchemaToPurview\": \"Disabled\"\n },\n \"UseNotebookActivity\": \"Disabled\"\n}",
"TaskStatus": "Untried",
"TaskType": "Azure Storage to Azure Storage",
"TaskTypeId": -2,
@ -323,27 +323,27 @@
"DegreeOfCopyParallelism": 1,
"DependencyChainTag": "DeltaLoad",
"EngineId": -2,
"EngineJson": "{\"endpoint\": \"https://adsstgsynwadsr6p7.dev.azuresynapse.net\", \"DeltaProcessingNotebook\": \"DeltaProcessingNotebook\"}",
"EngineName": "ads-stg-adf-ads-r6p7",
"EngineResourceGroup": "gfuat2",
"EngineJson": "{\"endpoint\": \"https://adsstgsynwadsgo2n.dev.azuresynapse.net\", \"DeltaProcessingNotebook\": \"DeltaProcessingNotebook\"}",
"EngineName": "ads-stg-adf-ads-go2n",
"EngineResourceGroup": "gfloc1",
"EngineSubscriptionId": "035a1364-f00d-48e2-b582-4fe125905ee3",
"NumberOfRetries": 3,
"ScheduleMasterId": "-2",
"SourceKeyVaultBaseUrl": "https://ads-stg-kv-ads-r6p7.vault.azure.net",
"SourceKeyVaultBaseUrl": "https://ads-stg-kv-ads-go2n.vault.azure.net",
"SourceSystemAuthType": "MSI",
"SourceSystemId": -4,
"SourceSystemJSON": "{\n \"Container\": \"datalakeraw\"\n}",
"SourceSystemSecretName": "",
"SourceSystemServer": "https://adsstgdlsadsr6p7adsl.dfs.core.windows.net",
"SourceSystemServer": "https://adsstgdlsadsgo2nadsl.dfs.core.windows.net",
"SourceSystemType": "ADLS",
"SourceSystemUserName": "",
"SynapsePipeline": "GPL_SparkNotebookExecution_Primary_Azure",
"TargetKeyVaultBaseUrl": "https://ads-stg-kv-ads-r6p7.vault.azure.net",
"TargetKeyVaultBaseUrl": "https://ads-stg-kv-ads-go2n.vault.azure.net",
"TargetSystemAuthType": "MSI",
"TargetSystemId": -4,
"TargetSystemJSON": "{\n \"Container\": \"datalakeraw\"\n}",
"TargetSystemSecretName": "",
"TargetSystemServer": "https://adsstgdlsadsr6p7adsl.dfs.core.windows.net",
"TargetSystemServer": "https://adsstgdlsadsgo2nadsl.dfs.core.windows.net",
"TargetSystemType": "ADLS",
"TargetSystemUserName": "",
"TaskDatafactoryIR": "Azure",
@ -353,7 +353,7 @@
"TaskInstanceId": -1,
"TaskInstanceJson": "{\n \"SourceRelativePath\": \"samples/\",\n \"TargetRelativePath\": \"/Tests/Azure Storage to Azure Storage/\"\n}",
"TaskMasterId": -1008,
"TaskMasterJson": "{\n \"CDCSource\": \"Disabled\",\n \"ModifiedDate\": \"Disabled\",\n \"Purview\": \"Disabled\",\n \"QualifiedIDAssociation\": \"TaskMasterId\",\n \"Source\": {\n \"DataFileName\": \"SalesLT_Customer_Delta/SalesLT.Customer/\",\n \"DeleteAfterCompletion\": \"false\",\n \"MaxConcurrentConnections\": 0,\n \"Recursively\": \"false\",\n \"RelativePath\": \"samples/SalesLT_Customer_CDC/\",\n \"SchemaFileName\": \"SalesLT.Customer*.json\",\n \"Type\": \"Delta\",\n \"WriteSchemaToPurview\": \"Disabled\"\n },\n \"SparkTableCreate\": \"Disabled\",\n \"SparkTableDBName\": \"\",\n \"SparkTableName\": \"\",\n \"Target\": {\n \"DataFileName\": \"DeltaTable/SalesLT.Customer\",\n \"DeleteAfterCompletion\": \"false\",\n \"MaxConcurrentConnections\": 0,\n \"Recursively\": \"false\",\n \"RelativePath\": \"/Tests/Azure Storage to Azure Storage/-1008/\",\n \"SchemaFileName\": \"SalesLT.Customer.json\",\n \"Type\": \"Delta\",\n \"WriteSchemaToPurview\": \"Disabled\"\n },\n \"UseNotebookActivity\": \"Disabled\"\n}",
"TaskMasterJson": "{\n \"CDCSource\": \"Disabled\",\n \"ModifiedDate\": \"Disabled\",\n \"PrimaryKeyOverride\": \"\",\n \"Purview\": \"Disabled\",\n \"QualifiedIDAssociation\": \"TaskMasterId\",\n \"Source\": {\n \"DataFileName\": \"SalesLT_Customer_Delta/SalesLT.Customer/\",\n \"DeleteAfterCompletion\": \"false\",\n \"MaxConcurrentConnections\": 0,\n \"Recursively\": \"false\",\n \"RelativePath\": \"samples/SalesLT_Customer_CDC/\",\n \"SchemaFileName\": \"SalesLT.Customer*.json\",\n \"Type\": \"Delta\",\n \"WriteSchemaToPurview\": \"Disabled\"\n },\n \"SparkTableCreate\": \"Disabled\",\n \"SparkTableDBName\": \"\",\n \"SparkTableName\": \"\",\n \"Target\": {\n \"DataFileName\": \"DeltaTable/SalesLT.Customer\",\n \"DeleteAfterCompletion\": \"false\",\n \"MaxConcurrentConnections\": 0,\n \"Recursively\": \"false\",\n \"RelativePath\": \"/Tests/Azure Storage to Azure Storage/-1008/\",\n \"SchemaFileName\": \"SalesLT.Customer.json\",\n \"Type\": \"Delta\",\n \"WriteSchemaToPurview\": \"Disabled\"\n },\n \"UseNotebookActivity\": \"Disabled\"\n}",
"TaskStatus": "Untried",
"TaskType": "Azure Storage to Azure Storage",
"TaskTypeId": -2,
@ -363,27 +363,27 @@
"DegreeOfCopyParallelism": 1,
"DependencyChainTag": "DeltaLoad",
"EngineId": -2,
"EngineJson": "{\"endpoint\": \"https://adsstgsynwadsr6p7.dev.azuresynapse.net\", \"DeltaProcessingNotebook\": \"DeltaProcessingNotebook\"}",
"EngineName": "ads-stg-adf-ads-r6p7",
"EngineResourceGroup": "gfuat2",
"EngineJson": "{\"endpoint\": \"https://adsstgsynwadsgo2n.dev.azuresynapse.net\", \"DeltaProcessingNotebook\": \"DeltaProcessingNotebook\"}",
"EngineName": "ads-stg-adf-ads-go2n",
"EngineResourceGroup": "gfloc1",
"EngineSubscriptionId": "035a1364-f00d-48e2-b582-4fe125905ee3",
"NumberOfRetries": 3,
"ScheduleMasterId": "-2",
"SourceKeyVaultBaseUrl": "https://ads-stg-kv-ads-r6p7.vault.azure.net",
"SourceKeyVaultBaseUrl": "https://ads-stg-kv-ads-go2n.vault.azure.net",
"SourceSystemAuthType": "MSI",
"SourceSystemId": -4,
"SourceSystemJSON": "{\n \"Container\": \"datalakeraw\"\n}",
"SourceSystemSecretName": "",
"SourceSystemServer": "https://adsstgdlsadsr6p7adsl.dfs.core.windows.net",
"SourceSystemServer": "https://adsstgdlsadsgo2nadsl.dfs.core.windows.net",
"SourceSystemType": "ADLS",
"SourceSystemUserName": "",
"SynapsePipeline": "GPL_SparkNotebookExecution_Primary_Azure",
"TargetKeyVaultBaseUrl": "https://ads-stg-kv-ads-r6p7.vault.azure.net",
"TargetKeyVaultBaseUrl": "https://ads-stg-kv-ads-go2n.vault.azure.net",
"TargetSystemAuthType": "MSI",
"TargetSystemId": -4,
"TargetSystemJSON": "{\n \"Container\": \"datalakeraw\"\n}",
"TargetSystemSecretName": "",
"TargetSystemServer": "https://adsstgdlsadsr6p7adsl.dfs.core.windows.net",
"TargetSystemServer": "https://adsstgdlsadsgo2nadsl.dfs.core.windows.net",
"TargetSystemType": "ADLS",
"TargetSystemUserName": "",
"TaskDatafactoryIR": "Azure",
@ -393,7 +393,7 @@
"TaskInstanceId": -1,
"TaskInstanceJson": "{\n \"SourceRelativePath\": \"samples/\",\n \"TargetRelativePath\": \"/Tests/Azure Storage to Azure Storage/\"\n}",
"TaskMasterId": -1009,
"TaskMasterJson": "{\n \"CDCSource\": \"Disabled\",\n \"ModifiedDate\": \"Disabled\",\n \"Purview\": \"Disabled\",\n \"QualifiedIDAssociation\": \"TaskMasterId\",\n \"Source\": {\n \"DataFileName\": \"SalesLT_Customer_Delta/SalesLT.Customer/\",\n \"DeleteAfterCompletion\": \"false\",\n \"MaxConcurrentConnections\": 0,\n \"Recursively\": \"false\",\n \"RelativePath\": \"samples/SalesLT_Customer_CDC/\",\n \"SchemaFileName\": \"SalesLT.Customer*.json\",\n \"Type\": \"Delta\",\n \"WriteSchemaToPurview\": \"Disabled\"\n },\n \"SparkTableCreate\": \"Disabled\",\n \"SparkTableDBName\": \"\",\n \"SparkTableName\": \"\",\n \"Target\": {\n \"DataFileName\": \"SalesLT.Customer*.parquet\",\n \"DeleteAfterCompletion\": \"false\",\n \"MaxConcurrentConnections\": 0,\n \"Recursively\": \"false\",\n \"RelativePath\": \"/Tests/Azure Storage to Azure Storage/-1009/\",\n \"SchemaFileName\": \"SalesLT.Customer.json\",\n \"Type\": \"Parquet\",\n \"WriteSchemaToPurview\": \"Disabled\"\n },\n \"UseNotebookActivity\": \"Disabled\"\n}",
"TaskMasterJson": "{\n \"CDCSource\": \"Disabled\",\n \"ModifiedDate\": \"Disabled\",\n \"PrimaryKeyOverride\": \"\",\n \"Purview\": \"Disabled\",\n \"QualifiedIDAssociation\": \"TaskMasterId\",\n \"Source\": {\n \"DataFileName\": \"SalesLT_Customer_Delta/SalesLT.Customer/\",\n \"DeleteAfterCompletion\": \"false\",\n \"MaxConcurrentConnections\": 0,\n \"Recursively\": \"false\",\n \"RelativePath\": \"samples/SalesLT_Customer_CDC/\",\n \"SchemaFileName\": \"SalesLT.Customer*.json\",\n \"Type\": \"Delta\",\n \"WriteSchemaToPurview\": \"Disabled\"\n },\n \"SparkTableCreate\": \"Disabled\",\n \"SparkTableDBName\": \"\",\n \"SparkTableName\": \"\",\n \"Target\": {\n \"DataFileName\": \"SalesLT.Customer*.parquet\",\n \"DeleteAfterCompletion\": \"false\",\n \"MaxConcurrentConnections\": 0,\n \"Recursively\": \"false\",\n \"RelativePath\": \"/Tests/Azure Storage to Azure Storage/-1009/\",\n \"SchemaFileName\": \"SalesLT.Customer.json\",\n \"Type\": \"Parquet\",\n \"WriteSchemaToPurview\": \"Disabled\"\n },\n \"UseNotebookActivity\": \"Disabled\"\n}",
"TaskStatus": "Untried",
"TaskType": "Azure Storage to Azure Storage",
"TaskTypeId": -2,

12
solution/Synapse/Patterns/pipeline/Execute-Notebook/functionapptests/tests/tests.json
Просмотреть файл

@ -3,13 +3,13 @@
"DegreeOfCopyParallelism": 1,
"DependencyChainTag": "",
"EngineId": -2,
"EngineJson": "{\"endpoint\": \"https://arkstgsynwadsf829.dev.azuresynapse.net\", \"DeltaProcessingNotebook\": \"DeltaProcessingNotebook\"}",
"EngineName": "ark-stg-adf-ads-f829",
"EngineResourceGroup": "lockboxdevtemp",
"EngineSubscriptionId": "687fe1ae-a520-4f86-b921-a80664c40f9b",
"EngineJson": "{\"endpoint\": \"https://adsstgsynwadsgo2n.dev.azuresynapse.net\", \"DeltaProcessingNotebook\": \"DeltaProcessingNotebook\"}",
"EngineName": "ads-stg-adf-ads-go2n",
"EngineResourceGroup": "gfloc1",
"EngineSubscriptionId": "035a1364-f00d-48e2-b582-4fe125905ee3",
"NumberOfRetries": 3,
"ScheduleMasterId": "-2",
"SourceKeyVaultBaseUrl": "https://ark-stg-kv-ads-f829.vault.azure.net",
"SourceKeyVaultBaseUrl": "https://ads-stg-kv-ads-go2n.vault.azure.net",
"SourceSystemAuthType": "MSI",
"SourceSystemId": -16,
"SourceSystemJSON": "{\n\n}",
@ -18,7 +18,7 @@
"SourceSystemType": "N/A",
"SourceSystemUserName": "",
"SynapsePipeline": "GPL_SparkNotebookExecution_Primary_Azure",
"TargetKeyVaultBaseUrl": "https://ads-stg-kv-ads-r6p7.vault.azure.net",
"TargetKeyVaultBaseUrl": "https://ads-stg-kv-ads-go2n.vault.azure.net",
"TargetSystemAuthType": "MSI",
"TargetSystemId": -16,
"TargetSystemJSON": "{\n\n}",

2
solution/Synapse/Patterns/pipeline/PowerBI-Dataflow-Refresh/functionapptests/partials/functionapptest.libsonnet
Просмотреть файл

@ -73,7 +73,7 @@ function(
},
"TaskGroupId": TaskGroupId,
"TaskInstanceJson":std.manifestJson(TaskInstanceJson),
"TaskTypeId":-6,
"TaskTypeId":-11,
"TaskType":Pattern,
"EngineName":vars.datafactory_name,
"EngineResourceGroup":vars.resource_group_name,

82
solution/Synapse/Patterns/pipeline/Rest-API-to-Azure-Storage/functionapptests/tests/tests.json
Просмотреть файл

@ -3,13 +3,13 @@
"DegreeOfCopyParallelism": 1,
"DependencyChainTag": "",
"EngineId": -2,
"EngineJson": "{\"endpoint\": \"https://adsstgsynwadsr6p7.dev.azuresynapse.net\", \"DeltaProcessingNotebook\": \"DeltaProcessingNotebook\"}",
"EngineName": "ads-stg-adf-ads-r6p7",
"EngineResourceGroup": "gfuat2",
"EngineJson": "{\"endpoint\": \"https://adsstgsynwadsgo2n.dev.azuresynapse.net\", \"DeltaProcessingNotebook\": \"DeltaProcessingNotebook\"}",
"EngineName": "ads-stg-adf-ads-go2n",
"EngineResourceGroup": "gfloc1",
"EngineSubscriptionId": "035a1364-f00d-48e2-b582-4fe125905ee3",
"NumberOfRetries": 3,
"ScheduleMasterId": "-2",
"SourceKeyVaultBaseUrl": "https://ads-stg-kv-ads-r6p7.vault.azure.net",
"SourceKeyVaultBaseUrl": "https://ads-stg-kv-ads-go2n.vault.azure.net",
"SourceSystemAuthType": "MSI",
"SourceSystemId": -17,
"SourceSystemJSON": "{\n\n}",
@ -18,47 +18,7 @@
"SourceSystemType": "Rest",
"SourceSystemUserName": "",
"SynapsePipeline": "GPL_SparkNotebookExecution_Primary_Azure",
"TargetKeyVaultBaseUrl": "https://ads-stg-kv-ads-r6p7.vault.azure.net",
"TargetSystemAuthType": "MSI",
"TargetSystemId": -8,
"TargetSystemJSON": "{\n\n}",
"TargetSystemSecretName": "",
"TargetSystemServer": "N/A",
"TargetSystemType": "ADLS",
"TargetSystemUserName": "",
"TaskDatafactoryIR": "Azure",
"TaskExecutionType": "Synapse",
"TaskGroupConcurrency": "10",
"TaskGroupPriority": 0,
"TaskInstanceId": -1,
"TaskInstanceJson": "{\n\n}",
"TaskMasterId": -1011,
"TaskMasterJson": "{\n \"ExecuteNotebook\": \"RestAPINotebook\",\n \"PartitionSize\": \"Single\",\n \"Purview\": \"Disabled\",\n \"QualifiedIDAssociation\": \"TaskMasterId\",\n \"Source\": {\n \"Pagination\": \"Disabled\",\n \"RelativeUrl\": \"/facts?limit=50\",\n \"RequestBody\": \"\",\n \"RequestMethod\": \"GET\",\n \"Type\": \"Rest\"\n },\n \"Target\": {\n \"DataFileName\": \"TestNoAuthGet.json\",\n \"RelativePath\": \"\",\n \"Type\": \"Json\"\n },\n \"UseNotebookActivity\": \"Enabled\"\n}",
"TaskStatus": "Untried",
"TaskType": "REST API to Azure Storage",
"TaskTypeId": -9,
"TestDescription": "[-1011] Rest to Storage - No Auth GET no Headers"
},
{
"DegreeOfCopyParallelism": 1,
"DependencyChainTag": "",
"EngineId": -2,
"EngineJson": "{\"endpoint\": \"https://adsstgsynwadsr6p7.dev.azuresynapse.net\", \"DeltaProcessingNotebook\": \"DeltaProcessingNotebook\"}",
"EngineName": "ads-stg-adf-ads-r6p7",
"EngineResourceGroup": "gfuat2",
"EngineSubscriptionId": "035a1364-f00d-48e2-b582-4fe125905ee3",
"NumberOfRetries": 3,
"ScheduleMasterId": "-2",
"SourceKeyVaultBaseUrl": "https://ads-stg-kv-ads-r6p7.vault.azure.net",
"SourceSystemAuthType": "MSI",
"SourceSystemId": -17,
"SourceSystemJSON": "{\n\n}",
"SourceSystemSecretName": "",
"SourceSystemServer": "N/A",
"SourceSystemType": "Rest",
"SourceSystemUserName": "",
"SynapsePipeline": "GPL_SparkNotebookExecution_Primary_Azure",
"TargetKeyVaultBaseUrl": "https://ads-stg-kv-ads-r6p7.vault.azure.net",
"TargetKeyVaultBaseUrl": "https://ads-stg-kv-ads-go2n.vault.azure.net",
"TargetSystemAuthType": "MSI",
"TargetSystemId": -8,
"TargetSystemJSON": "{\n\n}",
@ -83,13 +43,13 @@
"DegreeOfCopyParallelism": 1,
"DependencyChainTag": "",
"EngineId": -2,
"EngineJson": "{\"endpoint\": \"https://adsstgsynwadsr6p7.dev.azuresynapse.net\", \"DeltaProcessingNotebook\": \"DeltaProcessingNotebook\"}",
"EngineName": "ads-stg-adf-ads-r6p7",
"EngineResourceGroup": "gfuat2",
"EngineJson": "{\"endpoint\": \"https://adsstgsynwadsgo2n.dev.azuresynapse.net\", \"DeltaProcessingNotebook\": \"DeltaProcessingNotebook\"}",
"EngineName": "ads-stg-adf-ads-go2n",
"EngineResourceGroup": "gfloc1",
"EngineSubscriptionId": "035a1364-f00d-48e2-b582-4fe125905ee3",
"NumberOfRetries": 3,
"ScheduleMasterId": "-2",
"SourceKeyVaultBaseUrl": "https://ads-stg-kv-ads-r6p7.vault.azure.net",
"SourceKeyVaultBaseUrl": "https://ads-stg-kv-ads-go2n.vault.azure.net",
"SourceSystemAuthType": "MSI",
"SourceSystemId": -17,
"SourceSystemJSON": "{\n\n}",
@ -98,7 +58,7 @@
"SourceSystemType": "Rest",
"SourceSystemUserName": "",
"SynapsePipeline": "GPL_SparkNotebookExecution_Primary_Azure",
"TargetKeyVaultBaseUrl": "https://ads-stg-kv-ads-r6p7.vault.azure.net",
"TargetKeyVaultBaseUrl": "https://ads-stg-kv-ads-go2n.vault.azure.net",
"TargetSystemAuthType": "MSI",
"TargetSystemId": -8,
"TargetSystemJSON": "{\n\n}",
@ -123,13 +83,13 @@
"DegreeOfCopyParallelism": 1,
"DependencyChainTag": "",
"EngineId": -2,
"EngineJson": "{\"endpoint\": \"https://adsstgsynwadsr6p7.dev.azuresynapse.net\", \"DeltaProcessingNotebook\": \"DeltaProcessingNotebook\"}",
"EngineName": "ads-stg-adf-ads-r6p7",
"EngineResourceGroup": "gfuat2",
"EngineJson": "{\"endpoint\": \"https://adsstgsynwadsgo2n.dev.azuresynapse.net\", \"DeltaProcessingNotebook\": \"DeltaProcessingNotebook\"}",
"EngineName": "ads-stg-adf-ads-go2n",
"EngineResourceGroup": "gfloc1",
"EngineSubscriptionId": "035a1364-f00d-48e2-b582-4fe125905ee3",
"NumberOfRetries": 3,
"ScheduleMasterId": "-2",
"SourceKeyVaultBaseUrl": "https://ads-stg-kv-ads-r6p7.vault.azure.net",
"SourceKeyVaultBaseUrl": "https://ads-stg-kv-ads-go2n.vault.azure.net",
"SourceSystemAuthType": "MSI",
"SourceSystemId": -17,
"SourceSystemJSON": "{\n\n}",
@ -138,7 +98,7 @@
"SourceSystemType": "Rest",
"SourceSystemUserName": "",
"SynapsePipeline": "GPL_SparkNotebookExecution_Primary_Azure",
"TargetKeyVaultBaseUrl": "https://ads-stg-kv-ads-r6p7.vault.azure.net",
"TargetKeyVaultBaseUrl": "https://ads-stg-kv-ads-go2n.vault.azure.net",
"TargetSystemAuthType": "MSI",
"TargetSystemId": -8,
"TargetSystemJSON": "{\n\n}",
@ -163,13 +123,13 @@
"DegreeOfCopyParallelism": 1,
"DependencyChainTag": "",
"EngineId": -2,
"EngineJson": "{\"endpoint\": \"https://arkstgsynwadsf829.dev.azuresynapse.net\", \"DeltaProcessingNotebook\": \"DeltaProcessingNotebook\"}",
"EngineName": "ark-stg-adf-ads-f829",
"EngineResourceGroup": "lockboxdevtemp",
"EngineSubscriptionId": "687fe1ae-a520-4f86-b921-a80664c40f9b",
"EngineJson": "{\"endpoint\": \"https://adsstgsynwadsgo2n.dev.azuresynapse.net\", \"DeltaProcessingNotebook\": \"DeltaProcessingNotebook\"}",
"EngineName": "ads-stg-adf-ads-go2n",
"EngineResourceGroup": "gfloc1",
"EngineSubscriptionId": "035a1364-f00d-48e2-b582-4fe125905ee3",
"NumberOfRetries": 3,
"ScheduleMasterId": "-2",
"SourceKeyVaultBaseUrl": "https://ark-stg-kv-ads-f829.vault.azure.net",
"SourceKeyVaultBaseUrl": "https://ads-stg-kv-ads-go2n.vault.azure.net",
"SourceSystemAuthType": "MSI",
"SourceSystemId": -17,
"SourceSystemJSON": "{\n\n}",
@ -178,7 +138,7 @@
"SourceSystemType": "Rest",
"SourceSystemUserName": "",
"SynapsePipeline": "GPL_SparkNotebookExecution_Primary_Azure",
"TargetKeyVaultBaseUrl": "https://ark-stg-kv-ads-f829.vault.azure.net",
"TargetKeyVaultBaseUrl": "https://ads-stg-kv-ads-go2n.vault.azure.net",
"TargetSystemAuthType": "MSI",
"TargetSystemId": -8,
"TargetSystemJSON": "{\n\n}",

24
solution/Synapse/Patterns/pipeline/Synapse-SQLPool-Start-Stop/functionapptests/tests/tests.json
Просмотреть файл

@ -3,13 +3,13 @@
"DegreeOfCopyParallelism": 1,
"DependencyChainTag": "",
"EngineId": -2,
"EngineJson": "{\"endpoint\": \"https://adsstgsynwadsr6p7.dev.azuresynapse.net\", \"DeltaProcessingNotebook\": \"DeltaProcessingNotebook\"}",
"EngineName": "ads-stg-adf-ads-r6p7",
"EngineResourceGroup": "gfuat2",
"EngineJson": "{\"endpoint\": \"https://adsstgsynwadsgo2n.dev.azuresynapse.net\", \"DeltaProcessingNotebook\": \"DeltaProcessingNotebook\"}",
"EngineName": "ads-stg-adf-ads-go2n",
"EngineResourceGroup": "gfloc1",
"EngineSubscriptionId": "035a1364-f00d-48e2-b582-4fe125905ee3",
"NumberOfRetries": 3,
"ScheduleMasterId": "-2",
"SourceKeyVaultBaseUrl": "https://ads-stg-kv-ads-r6p7.vault.azure.net",
"SourceKeyVaultBaseUrl": "https://ads-stg-kv-ads-go2n.vault.azure.net",
"SourceSystemAuthType": "MSI",
"SourceSystemId": -16,
"SourceSystemJSON": "{\n\n}",
@ -18,7 +18,7 @@
"SourceSystemType": "N/A",
"SourceSystemUserName": "",
"SynapsePipeline": "Not-Applicable",
"TargetKeyVaultBaseUrl": "https://ads-stg-kv-ads-r6p7.vault.azure.net",
"TargetKeyVaultBaseUrl": "https://ads-stg-kv-ads-go2n.vault.azure.net",
"TargetSystemAuthType": "MSI",
"TargetSystemId": -10,
"TargetSystemJSON": "{\n\n}",
@ -34,7 +34,7 @@
"TaskInstanceId": -1,
"TaskInstanceJson": "{\n\n}",
"TaskMasterId": -1016,
"TaskMasterJson": "{\n \"SQLPoolName\": \"arkstgsyndpads\",\n \"SQLPoolOperation\": \"start\",\n \"Source\": {\n \"DataFileName\": \"\",\n \"DeleteAfterCompletion\": \"false\",\n \"MaxConcurrentConnections\": 0,\n \"Recursively\": \"false\",\n \"RelativePath\": \"\",\n \"SchemaFileName\": \"\",\n \"Type\": \"Not-Applicable\"\n },\n \"Target\": {\n \"DataFileName\": \"\",\n \"DeleteAfterCompletion\": \"false\",\n \"MaxConcurrentConnections\": 0,\n \"Recursively\": \"false\",\n \"RelativePath\": \"\",\n \"SchemaFileName\": \"\",\n \"Type\": \"Not-Applicable\"\n }\n}",
"TaskMasterJson": "{\n \"SQLPoolName\": \"adsstgsyndpads\",\n \"SQLPoolOperation\": \"start\",\n \"Source\": {\n \"DataFileName\": \"\",\n \"DeleteAfterCompletion\": \"false\",\n \"MaxConcurrentConnections\": 0,\n \"Recursively\": \"false\",\n \"RelativePath\": \"\",\n \"SchemaFileName\": \"\",\n \"Type\": \"Not-Applicable\"\n },\n \"Target\": {\n \"DataFileName\": \"\",\n \"DeleteAfterCompletion\": \"false\",\n \"MaxConcurrentConnections\": 0,\n \"Recursively\": \"false\",\n \"RelativePath\": \"\",\n \"SchemaFileName\": \"\",\n \"Type\": \"Not-Applicable\"\n }\n}",
"TaskStatus": "Untried",
"TaskType": "Synapse-SQLPool-Start-Stop",
"TaskTypeId": -6,
@ -44,13 +44,13 @@
"DegreeOfCopyParallelism": 1,
"DependencyChainTag": "",
"EngineId": -2,
"EngineJson": "{\"endpoint\": \"https://adsstgsynwadsr6p7.dev.azuresynapse.net\", \"DeltaProcessingNotebook\": \"DeltaProcessingNotebook\"}",
"EngineName": "ads-stg-adf-ads-r6p7",
"EngineResourceGroup": "gfuat2",
"EngineJson": "{\"endpoint\": \"https://adsstgsynwadsgo2n.dev.azuresynapse.net\", \"DeltaProcessingNotebook\": \"DeltaProcessingNotebook\"}",
"EngineName": "ads-stg-adf-ads-go2n",
"EngineResourceGroup": "gfloc1",
"EngineSubscriptionId": "035a1364-f00d-48e2-b582-4fe125905ee3",
"NumberOfRetries": 3,
"ScheduleMasterId": "-2",
"SourceKeyVaultBaseUrl": "https://ads-stg-kv-ads-r6p7.vault.azure.net",
"SourceKeyVaultBaseUrl": "https://ads-stg-kv-ads-go2n.vault.azure.net",
"SourceSystemAuthType": "MSI",
"SourceSystemId": -16,
"SourceSystemJSON": "{\n\n}",
@ -59,7 +59,7 @@
"SourceSystemType": "N/A",
"SourceSystemUserName": "",
"SynapsePipeline": "Not-Applicable",
"TargetKeyVaultBaseUrl": "https://ads-stg-kv-ads-r6p7.vault.azure.net",
"TargetKeyVaultBaseUrl": "https://ads-stg-kv-ads-go2n.vault.azure.net",
"TargetSystemAuthType": "MSI",
"TargetSystemId": -10,
"TargetSystemJSON": "{\n\n}",
@ -75,7 +75,7 @@
"TaskInstanceId": -1,
"TaskInstanceJson": "{\n\n}",
"TaskMasterId": -1017,
"TaskMasterJson": "{\n \"SQLPoolName\": \"arkstgsyndpads\",\n \"SQLPoolOperation\": \"pause\",\n \"Source\": {\n \"DataFileName\": \"\",\n \"DeleteAfterCompletion\": \"false\",\n \"MaxConcurrentConnections\": 0,\n \"Recursively\": \"false\",\n \"RelativePath\": \"\",\n \"SchemaFileName\": \"\",\n \"Type\": \"Not-Applicable\"\n },\n \"Target\": {\n \"DataFileName\": \"\",\n \"DeleteAfterCompletion\": \"false\",\n \"MaxConcurrentConnections\": 0,\n \"Recursively\": \"false\",\n \"RelativePath\": \"\",\n \"SchemaFileName\": \"\",\n \"Type\": \"Not-Applicable\"\n }\n}",
"TaskMasterJson": "{\n \"SQLPoolName\": \"adsstgsyndpads\",\n \"SQLPoolOperation\": \"pause\",\n \"Source\": {\n \"DataFileName\": \"\",\n \"DeleteAfterCompletion\": \"false\",\n \"MaxConcurrentConnections\": 0,\n \"Recursively\": \"false\",\n \"RelativePath\": \"\",\n \"SchemaFileName\": \"\",\n \"Type\": \"Not-Applicable\"\n },\n \"Target\": {\n \"DataFileName\": \"\",\n \"DeleteAfterCompletion\": \"false\",\n \"MaxConcurrentConnections\": 0,\n \"Recursively\": \"false\",\n \"RelativePath\": \"\",\n \"SchemaFileName\": \"\",\n \"Type\": \"Not-Applicable\"\n }\n}",
"TaskStatus": "Untried",
"TaskType": "Synapse-SQLPool-Start-Stop",
"TaskTypeId": -6,

6
solution/WebApplication/WebApplication/Controllers/TaskMasterController.cs
Просмотреть файл

@ -398,9 +398,9 @@ namespace WebApplication.Controllers
recordsTotal = await modelDataAll.CountAsync();
//Paging
var data = await modelDataAll.Skip(skip).Take(pageSize).ToListAsync();
//Paging
var data = await modelDataAll.Skip(skip).Take(pageSize).ToListAsync();

278
solution/WebApplication/WebApplication/Services/EntityRoleProvider.cs
Просмотреть файл

@ -1,80 +1,80 @@
using Microsoft.EntityFrameworkCore;
using System;
using System.Linq;
using System.Threading.Tasks;
using WebApplication.Models;
namespace WebApplication.Services
{
/// <summary>
/// Centralised goo that provides the lokup form entity to subject area role map (or other future role maps)
/// </summary>
public class EntityRoleProvider : IEntityRoleProvider
{
private readonly AdsGoFastContext _context;
public EntityRoleProvider(AdsGoFastContext context)
{
_context = context;
}
public async Task<string[]> GetUserRoles(object entity, params Guid[] groups)
{
switch (entity)
{
case FrameworkTaskRunner x:
return await LoadFrameworkTaskRunnerRoles(x.TaskRunnerId, groups);
case SubjectArea x:
return await LoadSubjectAreaRoles(x.SubjectAreaId, groups);
case SubjectAreaForm x:
return await LoadSubjectAreaFormRoles(x.SubjectAreaFormId, groups);
case TaskGroup x:
return await LoadSubjectAreaRoles(x.SubjectAreaId, groups);
case TaskInstance x:
return await LoadTaskInstanceRoles(x.TaskInstanceId, groups);
case TaskInstanceExecution x:
return await LoadTaskInstanceRoles(x.TaskInstanceId, groups);
case TaskMaster x:
return await LoadTaskMasterRoles(x.TaskMasterId, groups);
case TaskMasterWaterMark x:
return await LoadTaskMasterRoles(x.TaskMasterId, groups);
//these don't link back to subject area.
case SourceAndTargetSystems x:
return await LoadSourceAndTargetSystemRoles(x.SystemId, groups);
case ScheduleInstance x:
if (!x.ScheduleMasterId.HasValue)
{
return new string[0];
}
return await LoadScheduleMasterRoles(x.ScheduleMasterId.Value, groups);
case ScheduleMaster x:
return await LoadScheduleMasterRoles(x.ScheduleMasterId, groups);
case ExecutionEngine x:
return await LoadExecutionEngineRoles(x.EngineId, groups);
default:
return new string[0];
}
}
private Task<string[]> LoadExecutionEngineRoles(long xEngineId, Guid[] groups) =>
(
from tr in _context.ExecutionEngine
join rm in ActiveRoleMaps(groups)
on tr.EngineId equals rm.EntityId
where rm.EntityTypeName == EntityRoleMap.ExecutionEngineTypeName
&& rm.EntityId == xEngineId
select rm.ApplicationRoleName
).ToArrayAsync();
using Microsoft.EntityFrameworkCore;
using System;
using System.Linq;
using System.Threading.Tasks;
using WebApplication.Models;
namespace WebApplication.Services
{
/// <summary>
/// Centralised goo that provides the lokup form entity to subject area role map (or other future role maps)
/// </summary>
public class EntityRoleProvider : IEntityRoleProvider
{
private readonly AdsGoFastContext _context;
public EntityRoleProvider(AdsGoFastContext context)
{
_context = context;
}
public async Task<string[]> GetUserRoles(object entity, params Guid[] groups)
{
switch (entity)
{
case FrameworkTaskRunner x:
return await LoadFrameworkTaskRunnerRoles(x.TaskRunnerId, groups);
case SubjectArea x:
return await LoadSubjectAreaRoles(x.SubjectAreaId, groups);
case SubjectAreaForm x:
return await LoadSubjectAreaFormRoles(x.SubjectAreaFormId, groups);
case TaskGroup x:
return await LoadSubjectAreaRoles(x.SubjectAreaId, groups);
case TaskInstance x:
return await LoadTaskInstanceRoles(x.TaskInstanceId, groups);
case TaskInstanceExecution x:
return await LoadTaskInstanceRoles(x.TaskInstanceId, groups);
case TaskMaster x:
return await LoadTaskMasterRoles(x.TaskMasterId, groups);
case TaskMasterWaterMark x:
return await LoadTaskMasterRoles(x.TaskMasterId, groups);
//these don't link back to subject area.
case SourceAndTargetSystems x:
return await LoadSourceAndTargetSystemRoles(x.SystemId, groups);
case ScheduleInstance x:
if (!x.ScheduleMasterId.HasValue)
{
return new string[0];
}
return await LoadScheduleMasterRoles(x.ScheduleMasterId.Value, groups);
case ScheduleMaster x:
return await LoadScheduleMasterRoles(x.ScheduleMasterId, groups);
case ExecutionEngine x:
return await LoadExecutionEngineRoles(x.EngineId, groups);
default:
return new string[0];
}
}
private Task<string[]> LoadExecutionEngineRoles(long xEngineId, Guid[] groups) =>
(
from tr in _context.ExecutionEngine
join rm in ActiveRoleMaps(groups)
on tr.EngineId equals rm.EntityId
where rm.EntityTypeName == EntityRoleMap.ExecutionEngineTypeName
&& rm.EntityId == xEngineId
select rm.ApplicationRoleName
).ToArrayAsync();
private Task<string[]> LoadScheduleMasterRoles(long systemId, Guid[] groups)
{
{
return (from tr in _context.ScheduleMaster
join rm in ActiveRoleMaps(groups)
on tr.ScheduleMasterId equals rm.EntityId
where rm.EntityTypeName == EntityRoleMap.ScheduleMasterTypeName
&& rm.EntityId == systemId
select rm.ApplicationRoleName
).ToArrayAsync();
select rm.ApplicationRoleName
).ToArrayAsync();
}
private Task<string[]> LoadSourceAndTargetSystemRoles(long systemId, Guid[] groups) =>
@ -85,71 +85,71 @@ namespace WebApplication.Services
where rm.EntityTypeName == EntityRoleMap.SourceAndTargetTypeName
&& rm.EntityId == systemId
select rm.ApplicationRoleName
).ToArrayAsync();
private Task<string[]> LoadFrameworkTaskRunnerRoles(long taskRunnerId, Guid[] groups) =>
(
from tr in _context.FrameworkTaskRunner
join ti in _context.TaskInstance
on tr.TaskRunnerId equals ti.TaskRunnerId
join tm in _context.TaskMaster
on ti.TaskMasterId equals tm.TaskMasterId
join tg in _context.TaskGroup
on tm.TaskGroupId equals tg.TaskGroupId
join rm in ActiveRoleMaps(groups)
on tg.SubjectAreaId equals rm.EntityId
where tr.TaskRunnerId == taskRunnerId
&& rm.EntityTypeName == EntityRoleMap.SubjectAreaTypeName
select rm.ApplicationRoleName
).ToArrayAsync();
private Task<string[]> LoadSubjectAreaRoles(long subjectAreaId, Guid[] groups) =>
(
from rm in ActiveRoleMaps(groups)
where rm.EntityId == subjectAreaId
&& rm.EntityTypeName == EntityRoleMap.SubjectAreaTypeName
select rm.ApplicationRoleName
).ToArrayAsync();
private Task<string[]> LoadSubjectAreaFormRoles(long subjectAreaFormId, Guid[] groups) =>
(
from rm in ActiveRoleMaps(groups)
join sa in _context.SubjectArea
on rm.EntityId equals sa.SubjectAreaId
where sa.SubjectAreaFormId == subjectAreaFormId
&& rm.EntityTypeName == EntityRoleMap.SubjectAreaTypeName
select rm.ApplicationRoleName
).ToArrayAsync();
private Task<string[]> LoadTaskInstanceRoles(long taskInstanceId, Guid[] groups) =>
(
from ti in _context.TaskInstance
join tm in _context.TaskMaster
on ti.TaskMasterId equals tm.TaskMasterId
join tg in _context.TaskGroup
on tm.TaskGroupId equals tg.TaskGroupId
join rm in ActiveRoleMaps(groups)
on tg.SubjectAreaId equals rm.EntityId
where ti.TaskInstanceId == taskInstanceId
&& rm.EntityTypeName == EntityRoleMap.SubjectAreaTypeName
select rm.ApplicationRoleName
).ToArrayAsync();
private Task<string[]> LoadTaskMasterRoles(long taskMasterId, Guid[] groups) =>
(
from tm in _context.TaskMaster
join tg in _context.TaskGroup
on tm.TaskGroupId equals tg.TaskGroupId
join rm in ActiveRoleMaps(groups)
on tg.SubjectAreaId equals rm.EntityId
where tm.TaskMasterId == taskMasterId
&& rm.EntityTypeName == EntityRoleMap.SubjectAreaTypeName
select rm.ApplicationRoleName
).ToArrayAsync();
IQueryable<EntityRoleMap> ActiveRoleMaps(Guid[] groups) =>
_context.EntityRoleMap.Where(rm => groups.Contains(rm.AadGroupUid) && rm.ActiveYN && rm.ExpiryDate > DateTimeOffset.Now);
}
}
).ToArrayAsync();
private Task<string[]> LoadFrameworkTaskRunnerRoles(long taskRunnerId, Guid[] groups) =>
(
from tr in _context.FrameworkTaskRunner
join ti in _context.TaskInstance
on tr.TaskRunnerId equals ti.TaskRunnerId
join tm in _context.TaskMaster
on ti.TaskMasterId equals tm.TaskMasterId
join tg in _context.TaskGroup
on tm.TaskGroupId equals tg.TaskGroupId
join rm in ActiveRoleMaps(groups)
on tg.SubjectAreaId equals rm.EntityId
where tr.TaskRunnerId == taskRunnerId
&& rm.EntityTypeName == EntityRoleMap.SubjectAreaTypeName
select rm.ApplicationRoleName
).ToArrayAsync();
private Task<string[]> LoadSubjectAreaRoles(long subjectAreaId, Guid[] groups) =>
(
from rm in ActiveRoleMaps(groups)
where rm.EntityId == subjectAreaId
&& rm.EntityTypeName == EntityRoleMap.SubjectAreaTypeName
select rm.ApplicationRoleName
).ToArrayAsync();
private Task<string[]> LoadSubjectAreaFormRoles(long subjectAreaFormId, Guid[] groups) =>
(
from rm in ActiveRoleMaps(groups)
join sa in _context.SubjectArea
on rm.EntityId equals sa.SubjectAreaId
where sa.SubjectAreaFormId == subjectAreaFormId
&& rm.EntityTypeName == EntityRoleMap.SubjectAreaTypeName
select rm.ApplicationRoleName
).ToArrayAsync();
private Task<string[]> LoadTaskInstanceRoles(long taskInstanceId, Guid[] groups) =>
(
from ti in _context.TaskInstance
join tm in _context.TaskMaster
on ti.TaskMasterId equals tm.TaskMasterId
join tg in _context.TaskGroup
on tm.TaskGroupId equals tg.TaskGroupId
join rm in ActiveRoleMaps(groups)
on tg.SubjectAreaId equals rm.EntityId
where ti.TaskInstanceId == taskInstanceId
&& rm.EntityTypeName == EntityRoleMap.SubjectAreaTypeName
select rm.ApplicationRoleName
).ToArrayAsync();
private Task<string[]> LoadTaskMasterRoles(long taskMasterId, Guid[] groups) =>
(
from tm in _context.TaskMaster
join tg in _context.TaskGroup
on tm.TaskGroupId equals tg.TaskGroupId
join rm in ActiveRoleMaps(groups)
on tg.SubjectAreaId equals rm.EntityId
where tm.TaskMasterId == taskMasterId
&& rm.EntityTypeName == EntityRoleMap.SubjectAreaTypeName
select rm.ApplicationRoleName
).ToArrayAsync();
IQueryable<EntityRoleMap> ActiveRoleMaps(Guid[] groups) =>
_context.EntityRoleMap.Where(rm => groups.Contains(rm.AadGroupUid) && rm.ActiveYN && rm.ExpiryDate > DateTimeOffset.Now);
}
}

1
solution/WebApplication/WebApplication/Startup.cs
Просмотреть файл

@ -59,6 +59,7 @@ namespace WebApplication
DataSource = appOptions.Value.AdsGoFastTaskMetaDataDatabaseServer,
InitialCatalog = appOptions.Value.AdsGoFastTaskMetaDataDatabaseName
};
//options.UseSqlServer(scsb.ConnectionString).EnableSensitiveDataLogging();
options.UseSqlServer(scsb.ConnectionString);
options.AddInterceptors(provider.GetRequiredService<AadAuthenticationDbConnectionInterceptor>());
}

1
solution/WebApplication/WebApplication/appsettings.json
Просмотреть файл

@ -5,6 +5,7 @@
"Microsoft.Aspnetcore": "Warning",
"Microsoft": "Warning",
"Microsoft.Hosting.Lifetime": "Information"
//"Microsoft.EntityFrameworkCore.Database.Command": "Information"
}
},
"ApplicationOptions": {