microsoft
/
data-accelerator
зеркало из https://github.com/microsoft/data-accelerator.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

1. DataProcessing – (#187) 

a. Rev packages flagged by component governance

2. ARM Template –
a. Change SF durability to silver.
b. Set min TLS for HDInsight to 1.2 .
c. Add zookeeper node config for HDInsight

3. Services –
a. Add API for start and stop all spark jobs

4. Website –
a. Accessibility: Tweak borderline color for tabs on flow definition page, charts on metric page and flows list table on flows page to achieve minimum 3:1 luminosity contrast
b. Rev packages flagged by component governance
c. Add button to ‘start all jobs’ and ‘stop all jobs’ on the jobs page
This commit is contained in:
Rohit Agrawal - MSFT 2021-09-24 15:56:53 -07:00 коммит произвёл GitHub
Родитель 605c7457bf
Коммит d368b8d95b
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: 4AEE18F83AFDEB23
34 изменённых файлов: 259 добавлений и 131 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

7
DataProcessing/Spark.nuspec
Просмотреть файл

@ -19,7 +19,7 @@
<file src="**\azure-documentdb-2.4.7.jar" target="lib" />
<file src="**\azure-eventhubs-1.2.1.jar" target="lib" />
<file src="**\azure-eventhubs-spark_2.11-2.3.6.jar" target="lib" />
<file src="**\azure-keyvault-webkey-1.2.2.jar" target="lib" />
<file src="**\azure-keyvault-webkey-1.2.4.jar" target="lib" />
<file src="**\datax-core*.jar" target="lib" />
<file src="**\datax-host*.jar" target="lib" />
<file src="**\datax-utility*.jar" target="lib" />
@ -31,11 +31,10 @@
<file src="**\kafka-clients-2.4.1.jar" target="lib" />
<file src="**\spark-streaming-kafka-0-10_2.11-2.4.5.jar" target="lib" />
<file src="**\azure-sqldb-spark-1.0.2.jar" target="lib" />
<file src="**\hadoop-azure-3.2.1.jar" target="lib" />
<file src="**\hadoop-azure-3.3.1.jar" target="lib" />
<file src="**\jetty-util-6.1.25.jar" target="lib" />
<file src="**\json-20180813.jar" target="lib" />
<file src="**\json4s-jackson_2.11-3.2.11.jar" target="lib" />
<file src="**\azure-storage-3.1.0.jar" target="lib" />
<file src="**\azure-storage-8.6.6.jar" target="lib" />
<file src="NOTICE.txt" target="" />
</files>
</package>

2
DataProcessing/datax-core/pom.xml
Просмотреть файл

@ -51,7 +51,7 @@ SOFTWARE
<groupId>com.microsoft.datax</groupId>
<artifactId>datax-core_2.4_2.11</artifactId>
<version>1.2.2-SNAPSHOT</version>
<version>1.2.3-SNAPSHOT</version>
<name>Data Accelerator Core</name>
<description>This package contains the core module of Data Accelerator functionality.</description>
<url>https://github.com/Microsoft/data-accelerator</url>

12
DataProcessing/datax-host/pom.xml
Просмотреть файл

@ -51,7 +51,7 @@ SOFTWARE
<groupId>com.microsoft.datax</groupId>
<artifactId>datax-host_2.4_2.11</artifactId>
<version>1.2.2-SNAPSHOT</version>
<version>1.2.3-SNAPSHOT</version>
<properties>
<spark.version>2.4.5</spark.version>
@ -118,12 +118,12 @@ SOFTWARE
<dependency>
<groupId>com.microsoft.datax</groupId>
<artifactId>datax-core_2.4_2.11</artifactId>
<version>1.2.2-SNAPSHOT</version>
<version>1.2.3-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>com.microsoft.datax</groupId>
<artifactId>datax-utility_2.4_2.11</artifactId>
<version>1.2.2-SNAPSHOT</version>
<version>1.2.3-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>com.microsoft.azure</groupId>
@ -133,12 +133,12 @@ SOFTWARE
<dependency>
<groupId>io.lettuce</groupId>
<artifactId>lettuce-core</artifactId>
<version>5.2.2.RELEASE</version>
<version>6.1.5.RELEASE</version>
</dependency>
<dependency>
<groupId>com.microsoft.azure</groupId>
<artifactId>azure-storage</artifactId>
<version>3.1.0</version>
<version>8.6.6</version>
</dependency>
<dependency>
<groupId>com.microsoft.azure</groupId>
@ -148,7 +148,7 @@ SOFTWARE
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-azure</artifactId>
<version>3.2.1</version>
<version>3.3.1</version>
</dependency>
<dependency>
<groupId>org.mortbay.jetty</groupId>

14
DataProcessing/datax-keyvault/pom.xml
Просмотреть файл

@ -51,7 +51,7 @@ SOFTWARE
<groupId>com.microsoft.datax</groupId>
<artifactId>datax-keyvault_2.4_2.11</artifactId>
<version>1.2.2-SNAPSHOT</version>
<version>1.2.3-SNAPSHOT</version>
<properties>
<spark.version>2.4.5</spark.version>
@ -73,7 +73,7 @@ SOFTWARE
<dependency>
<groupId>com.microsoft.azure</groupId>
<artifactId>adal4j</artifactId>
<version>1.6.5</version>
<version>1.6.7</version>
</dependency>
<dependency>
<groupId>com.microsoft.azure</groupId>
@ -84,23 +84,23 @@ SOFTWARE
<dependency>
<groupId>com.microsoft.azure</groupId>
<artifactId>azure</artifactId>
<version>1.11.0</version>
<version>1.41.2</version>
</dependency>
<!--http-->
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
<version>4.5.3</version>
<version>4.5.13</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-core</artifactId>
<version>2.10.0</version>
<version>2.12.5</version>
</dependency>
<dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
<version>2.10.0</version>
<version>2.12.5</version>
</dependency>
<!--Testing-->
<dependency>

4
DataProcessing/datax-udf-samples/pom.xml
Просмотреть файл

@ -51,7 +51,7 @@ SOFTWARE
<groupId>com.microsoft.datax</groupId>
<artifactId>datax-udf-samples_2.4_2.11</artifactId>
<version>1.2.2-SNAPSHOT</version>
<version>1.2.3-SNAPSHOT</version>
<properties>
<spark.version>2.4.5</spark.version>
@ -113,7 +113,7 @@ SOFTWARE
<dependency>
<groupId>com.microsoft.datax</groupId>
<artifactId>datax-core_2.4_2.11</artifactId>
<version>1.2.2-SNAPSHOT</version>
<version>1.2.3-SNAPSHOT</version>
</dependency>
</dependencies>

16
DataProcessing/datax-utility/pom.xml
Просмотреть файл

@ -51,7 +51,7 @@ SOFTWARE
<groupId>com.microsoft.datax</groupId>
<artifactId>datax-utility_2.4_2.11</artifactId>
<version>1.2.2-SNAPSHOT</version>
<version>1.2.3-SNAPSHOT</version>
<properties>
<spark.version>2.4.5</spark.version>
@ -100,9 +100,9 @@ SOFTWARE
<scope>provided</scope>
</dependency>
<dependency>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
<version>1.2.17</version>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-core</artifactId>
<version>2.14.1</version>
<scope>provided</scope>
</dependency>
<dependency>
@ -120,7 +120,7 @@ SOFTWARE
<dependency>
<groupId>com.microsoft.datax</groupId>
<artifactId>datax-core_2.4_2.11</artifactId>
<version>1.2.2-SNAPSHOT</version>
<version>1.2.3-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>com.microsoft.azure</groupId>
@ -130,12 +130,12 @@ SOFTWARE
<dependency>
<groupId>com.microsoft.datax</groupId>
<artifactId>datax-keyvault_2.4_2.11</artifactId>
<version>1.2.2-SNAPSHOT</version>
<version>1.2.3-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>com.microsoft.azure</groupId>
<artifactId>azure-keyvault-webkey</artifactId>
<version>1.2.2</version>
<version>1.2.4</version>
</dependency>
<dependency>
<groupId>com.microsoft.azure</groupId>
@ -165,7 +165,7 @@ SOFTWARE
<dependency>
<groupId>io.lettuce</groupId>
<artifactId>lettuce-core</artifactId>
<version>5.2.2.RELEASE</version>
<version>6.1.5.RELEASE</version>
</dependency>
</dependencies>

45
DeploymentCloud/Deployment.Common/CosmosDB/commons.json
Просмотреть файл

@ -130,26 +130,25 @@
"cluster": "$sparkName",
"options": {
"name": "${sparkJobName}",
"file" : "wasbs:///datax/bin/datax-host_2.3_2.11-1.1.0.jar",
"file" : "wasbs:///datax/bin/datax-host_2.4_2.11-1.2.0.jar",
"className" : "datax.app.DirectStreamingApp",
"args": [
"conf=${sparkJobConfigFilePath}",
"driverLogLevel=${sparkJobDriverLogLevel}"
],
"jars": [
"wasbs:///datax/bin/datax-core_2.3_2.11-1.1.0.jar",
"wasbs:///datax/bin/datax-utility_2.3_2.11-1.1.0.jar",
"wasbs:///datax/bin/datax-core_2.4_2.11-1.2.0.jar",
"wasbs:///datax/bin/datax-utility_2.4_2.11-1.2.0.jar",
"wasbs:///datax/bin/applicationinsights-core-2.2.1.jar",
"wasbs:///datax/bin/azure-documentdb-2.4.7.jar",
"wasbs:///datax/bin/azure-eventhubs-1.2.1.jar",
"wasbs:///datax/bin/azure-eventhubs-spark_2.11-2.3.6.jar",
"wasbs:///datax/bin/azure-keyvault-webkey-1.2.2.jar",
"wasbs:///datax/bin/datax-keyvault_2.3_2.11-1.1.0-with-dependencies.jar",
"wasbs:///datax/bin/azure-keyvault-webkey-1.2.4.jar",
"wasbs:///datax/bin/datax-keyvault_2.4_2.11-1.2.0-with-dependencies.jar",
"wasbs:///datax/bin/java-uuid-generator-3.1.5.jar",
"wasbs:///datax/bin/proton-j-0.31.0.jar",
"wasbs:///datax/bin/scala-java8-compat_2.11-0.9.0.jar",
"wasbs:///datax/bin/azure-sqldb-spark-1.0.2.jar",
"wasbs:///datax/bin/json4s-jackson_2.11-3.2.11.jar"
"wasbs:///datax/bin/azure-sqldb-spark-1.0.2.jar"
],
"driverMemory": "1024m",
"executorCores": 2,
@ -231,7 +230,7 @@
"jar" : "dbfs:/datax/azure-eventhubs-spark_2.11-2.3.6.jar"
},
{
"jar" : "dbfs:/datax/azure-keyvault-webkey-1.2.2.jar"
"jar" : "dbfs:/datax/azure-keyvault-webkey-1.2.4.jar"
},
{
"jar" : "dbfs:/datax/datax-core_2.4_2.11-1.2.0.jar"
@ -655,28 +654,27 @@
"cluster": "$sparkName",
"options" : {
"name" : "${sparkJobName}",
"file" : "wasbs:///datax/bin/datax-host_2.3_2.11-1.1.0.jar",
"file" : "wasbs:///datax/bin/datax-host_2.4_2.11-1.2.0.jar",
"className" : "datax.app.DirectKafkaStreamingApp",
"args" : [
"conf=${sparkJobConfigFilePath}",
"driverLogLevel=${sparkJobDriverLogLevel}"
],
"jars" : [
"wasbs:///datax/bin/datax-core_2.3_2.11-1.1.0.jar",
"wasbs:///datax/bin/datax-utility_2.3_2.11-1.1.0.jar",
"wasbs:///datax/bin/datax-core_2.4_2.11-1.2.0.jar",
"wasbs:///datax/bin/datax-utility_2.4_2.11-1.2.0.jar",
"wasbs:///datax/bin/applicationinsights-core-2.2.1.jar",
"wasbs:///datax/bin/azure-documentdb-2.4.7.jar",
"wasbs:///datax/bin/azure-eventhubs-1.2.1.jar",
"wasbs:///datax/bin/azure-eventhubs-spark_2.11-2.3.6.jar",
"wasbs:///datax/bin/azure-keyvault-webkey-1.2.2.jar",
"wasbs:///datax/bin/datax-keyvault_2.3_2.11-1.1.0-with-dependencies.jar",
"wasbs:///datax/bin/azure-keyvault-webkey-1.2.4.jar",
"wasbs:///datax/bin/datax-keyvault_2.4_2.11-1.2.0-with-dependencies.jar",
"wasbs:///datax/bin/java-uuid-generator-3.1.5.jar",
"wasbs:///datax/bin/proton-j-0.31.0.jar",
"wasbs:///datax/bin/scala-java8-compat_2.11-0.9.0.jar",
"wasbs:///datax/bin/spark-streaming-kafka-0-10_2.11-2.4.5.jar",
"wasbs:///datax/bin/kafka-clients-2.4.1.jar",
"wasbs:///datax/bin/azure-sqldb-spark-1.0.2.jar",
"wasbs:///datax/bin/json4s-jackson_2.11-3.2.11.jar"
"wasbs:///datax/bin/azure-sqldb-spark-1.0.2.jar"
],
"driverMemory" : "1024m",
"executorCores" : 2,
@ -758,7 +756,7 @@
"jar" : "dbfs:/datax/azure-eventhubs-spark_2.11-2.3.6.jar"
},
{
"jar" : "dbfs:/datax/azure-keyvault-webkey-1.2.2.jar"
"jar" : "dbfs:/datax/azure-keyvault-webkey-1.2.4.jar"
},
{
"jar" : "dbfs:/datax/datax-core_2.4_2.11-1.2.0.jar"
@ -1062,26 +1060,25 @@
"cluster": "$sparkName",
"options" : {
"name" : "${sparkJobName}",
"file" : "wasbs:///datax/bin/datax-host_2.3_2.11-1.1.0.jar",
"file" : "wasbs:///datax/bin/datax-host_2.4_2.11-1.2.0.jar",
"className" : "datax.app.BatchApp",
"args" : [
"conf=${sparkJobConfigFilePath}",
"driverLogLevel=${sparkJobDriverLogLevel}"
],
"jars" : [
"wasbs:///datax/bin/datax-core_2.3_2.11-1.1.0.jar",
"wasbs:///datax/bin/datax-utility_2.3_2.11-1.1.0.jar",
"wasbs:///datax/bin/datax-core_2.4_2.11-1.2.0.jar",
"wasbs:///datax/bin/datax-utility_2.4_2.11-1.2.0.jar",
"wasbs:///datax/bin/applicationinsights-core-2.2.1.jar",
"wasbs:///datax/bin/azure-documentdb-2.4.7.jar",
"wasbs:///datax/bin/azure-eventhubs-1.2.1.jar",
"wasbs:///datax/bin/azure-eventhubs-spark_2.11-2.3.6.jar",
"wasbs:///datax/bin/azure-keyvault-webkey-1.2.2.jar",
"wasbs:///datax/bin/datax-keyvault_2.3_2.11-1.1.0-with-dependencies.jar",
"wasbs:///datax/bin/azure-keyvault-webkey-1.2.4.jar",
"wasbs:///datax/bin/datax-keyvault_2.4_2.11-1.2.0-with-dependencies.jar",
"wasbs:///datax/bin/java-uuid-generator-3.1.5.jar",
"wasbs:///datax/bin/proton-j-0.31.0.jar",
"wasbs:///datax/bin/scala-java8-compat_2.11-0.9.0.jar",
"wasbs:///datax/bin/azure-sqldb-spark-1.0.2.jar",
"wasbs:///datax/bin/json4s-jackson_2.11-3.2.11.jar"
"wasbs:///datax/bin/azure-sqldb-spark-1.0.2.jar"
],
"driverMemory" : "1024m",
"executorCores" : 2,
@ -1163,7 +1160,7 @@
"jar" : "dbfs:/datax/azure-eventhubs-spark_2.11-2.3.6.jar"
},
{
"jar" : "dbfs:/datax/azure-keyvault-webkey-1.2.2.jar"
"jar" : "dbfs:/datax/azure-keyvault-webkey-1.2.4.jar"
},
{
"jar" : "dbfs:/datax/datax-core_2.4_2.11-1.2.0.jar"

8
DeploymentCloud/Deployment.Common/CosmosDB/configgenConfigs.json
Просмотреть файл

@ -20,10 +20,10 @@
"serviceKeyVaultName" : "$servicesKVName",
"sparkKeyVaultName" : "$sparkKVName",
"jobURLBase": "$sparkName",
"binaryName" : [
"datax/bin/datax-utility_2.3_2.11-1.1.0.jar",
"datax/bin/datax-host_2.3_2.11-1.1.0.jar",
"datax/bin/datax-core_2.3_2.11-1.1.0.jar"
"binaryName": [
"datax/bin/datax-utility_2.4_2.11-1.2.0.jar",
"datax/bin/datax-host_2.4_2.11-1.2.0.jar",
"datax/bin/datax-core_2.4_2.11-1.2.0.jar"
],
"configgenClientId" : "$configgenClientId",
"configgenClientSecret" : "keyvault://$servicesKVName/$serviceSecretPrefix-clientsecret",

2
DeploymentCloud/Deployment.Common/Resources/Parameters/SF-Parameter.json
Просмотреть файл

@ -127,7 +127,7 @@
"value": "Standard_LRS"
},
"nt0InstanceCount": {
"value": $vmNodeinstanceCount,
"value": $vmNodeinstanceCount
},
"vmNodeType0Name": {
"value": "$vmNodeTypeName"

68
DeploymentCloud/Deployment.Common/Resources/Templates/SF-Template.json
Просмотреть файл

@ -125,10 +125,10 @@
}
},
"reverseProxyCertificateThumbprint": {
"type": "string",
"type": "string"
},
"reverseProxyCertificateUrlValue": {
"type": "string",
"type": "string"
},
"storageAccountType": {
"type": "string",
@ -377,22 +377,22 @@
"StorageAccountKey2": "[listKeys(resourceId('Microsoft.Storage/storageAccounts', variables('supportLogStorageAccountName')),'2015-05-01-preview').key2]"
},
"publisher": "Microsoft.Azure.ServiceFabric",
"settings": {
"clusterEndpoint": "[reference(parameters('clusterName')).clusterEndpoint]",
"nodeTypeRef": "[parameters('vmNodeType0Name')]",
"dataPath": "D:\\\\SvcFab",
"durabilityLevel": "Bronze",
"enableParallelJobs": true,
"nicPrefixOverride": "[parameters('subnet0Prefix')]",
"reverseProxyCertificate": {
"thumbprint": "[parameters('reverseProxyCertificateThumbprint')]",
"x509StoreName": "[parameters('reverseProxyCertificateStoreValue')]"
},
"certificate": {
"thumbprint": "[parameters('certificateThumbprint')]",
"x509StoreName": "[parameters('certificateStoreValue')]"
}
"settings": {
"clusterEndpoint": "[reference(parameters('clusterName')).clusterEndpoint]",
"nodeTypeRef": "[parameters('vmNodeType0Name')]",
"dataPath": "D:\\\\SvcFab",
"durabilityLevel": "Silver",
"enableParallelJobs": true,
"nicPrefixOverride": "[parameters('subnet0Prefix')]",
"reverseProxyCertificate": {
"thumbprint": "[parameters('reverseProxyCertificateThumbprint')]",
"x509StoreName": "[parameters('reverseProxyCertificateStoreValue')]"
},
"certificate": {
"thumbprint": "[parameters('certificateThumbprint')]",
"x509StoreName": "[parameters('certificateStoreValue')]"
}
},
"typeHandlerVersion": "1.0"
}
},
@ -568,23 +568,23 @@
],
"managementEndpoint": "[concat('https://',reference(concat(parameters('lbIPName'),'-','0')).dnsSettings.fqdn,':',parameters('nt0fabricHttpGatewayPort'))]",
"nodeTypes": [
{
"name": "[parameters('vmNodeType0Name')]",
"applicationPorts": {
"endPort": "[parameters('nt0applicationEndPort')]",
"startPort": "[parameters('nt0applicationStartPort')]"
},
"clientConnectionEndpointPort": "[parameters('nt0fabricTcpGatewayPort')]",
"durabilityLevel": "Bronze",
"ephemeralPorts": {
"endPort": "[parameters('nt0ephemeralEndPort')]",
"startPort": "[parameters('nt0ephemeralStartPort')]"
},
"httpGatewayEndpointPort": "[parameters('nt0fabricHttpGatewayPort')]",
"isPrimary": true,
"reverseProxyEndpointPort": "[parameters('nt0reverseProxyEndpointPort')]",
"vmInstanceCount": "[parameters('nt0InstanceCount')]"
}
{
"name": "[parameters('vmNodeType0Name')]",
"applicationPorts": {
"endPort": "[parameters('nt0applicationEndPort')]",
"startPort": "[parameters('nt0applicationStartPort')]"
},
"clientConnectionEndpointPort": "[parameters('nt0fabricTcpGatewayPort')]",
"durabilityLevel": "Silver",
"ephemeralPorts": {
"endPort": "[parameters('nt0ephemeralEndPort')]",
"startPort": "[parameters('nt0ephemeralStartPort')]"
},
"httpGatewayEndpointPort": "[parameters('nt0fabricHttpGatewayPort')]",
"isPrimary": true,
"reverseProxyEndpointPort": "[parameters('nt0reverseProxyEndpointPort')]",
"vmInstanceCount": "[parameters('nt0InstanceCount')]"
}
],
"provisioningState": "Default",
"reliabilityLevel": "Silver",

24
DeploymentCloud/Deployment.Common/Resources/Templates/Spark-Template.json
Просмотреть файл

@ -1,5 +1,5 @@
{
"$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#",
"$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#",
"contentVersion": "1.0.0.0",
"parameters": {
"default_resource_location": {
@ -105,7 +105,7 @@
"properties": {}
},
{
"apiVersion": "2015-03-01-preview",
"apiVersion": "2018-06-01-preview",
"name": "[parameters('sparkClusterName')]",
"type": "Microsoft.HDInsight/clusters",
"location": "[parameters('default_resource_location')]",
@ -139,6 +139,7 @@
}
]
},
"minSupportedTlsVersion": "1.2",
"computeProfile": {
"roles": [
{
@ -179,7 +180,24 @@
"subnet": "[variables('subnet0Ref')]"
},
"scriptActions": []
}
},
{
"name": "zookeepernode",
"targetInstanceCount": 3,
"hardwareProfile": {
"vmSize": "Standard_A8_V2"
},
"osProfile": {
"linuxOperatingSystemProfile": {
"username": "[parameters('sparkSshUserName')]",
"password": "[parameters('sparkSshPassword')]"
}
},
"virtualNetworkProfile": {
"id": "[variables('vnetID')]",
"subnet": "[variables('subnet0Ref')]"
}
}
]
}
},

28
Services/DataX.Config/DataX.Config/InternalService/SparkJobOperation.cs
Просмотреть файл

@ -219,6 +219,34 @@ namespace DataX.Config
return await SyncJobStateByNames(jobs.Select(job => job.Name).ToArray());
}
/// <summary>
/// Stop all jobs
/// </summary>
/// <returns></returns>
public async Task<SparkJobConfig[]> StopAllJobs()
{
var jobs = await JobData.GetAll();
string[] jobNames = jobs.Select(job => job.Name).ToArray();
await Task.WhenAll(jobNames.Select(name => StopJob(name)));
Logger.LogInformation($"sync'ing job states for all jobs'");
return await SyncJobStateByNames(jobs.Select(job => job.Name).ToArray());
}
/// <summary>
/// Start all jobs
/// </summary>
/// <returns></returns>
public async Task<SparkJobConfig[]> StartAllJobs()
{
var jobs = await JobData.GetAll();
string[] jobNames = jobs.Select(job => job.Name).ToArray();
await Task.WhenAll(jobNames.Select(name => StartJob(name)));
Logger.LogInformation($"sync'ing job states for all jobs'");
return await SyncJobStateByNames(jobs.Select(job => job.Name).ToArray());
}
/// <summary>
/// Make sure a job is in the specified state within
/// </summary>

10
Services/DataX.Config/DataX.Config/PublicService/JobOperation.cs
Просмотреть файл

@ -99,6 +99,16 @@ namespace DataX.Config.PublicService
return ConvertToFrontEnd(SparkJobOperation.SyncAllJobState());
}
public Task<SparkJobFrontEnd[]> StopAllJobs()
{
return ConvertToFrontEnd(SparkJobOperation.StopAllJobs());
}
public Task<SparkJobFrontEnd[]> StartAllJobs()
{
return ConvertToFrontEnd(SparkJobOperation.StartAllJobs());
}
public async Task<Result> DeleteJob(string jobName)
{
return await this.SparkJobData.DeleteByName(jobName);

40
Services/DataX.Flow/Flow.ManagementService/Controllers/FlowManagementController.cs
Просмотреть файл

@ -490,6 +490,46 @@ namespace Flow.Management.Controllers
}
}
[HttpGet]
[Route("job/stopall")]
[DataXWriter]
public async Task<ApiResult> StopAllJobs()
{
try
{
RolesCheck.EnsureWriter(Request, _isLocal);
// Sync all jobs
var jobOpResult = await _jobOperation.StopAllJobs();
return ApiResult.CreateSuccess(JToken.FromObject(jobOpResult));
}
catch (Exception e)
{
_logger.LogError(e, e.Message);
return ApiResult.CreateError(e.Message);
}
}
[HttpGet]
[Route("job/startall")]
[DataXWriter]
public async Task<ApiResult> StartAllJobs()
{
try
{
RolesCheck.EnsureWriter(Request, _isLocal);
// Sync all jobs
var jobOpResult = await _jobOperation.StartAllJobs();
return ApiResult.CreateSuccess(JToken.FromObject(jobOpResult));
}
catch (Exception e)
{
_logger.LogError(e, e.Message);
return ApiResult.CreateError(e.Message);
}
}
[HttpPost]
[Route("job/syncbynames")]
public async Task<ApiResult> SyncJobsByNames([FromBody] string[] jobNames)

2
Website/Packages/datax-common/package.json
Просмотреть файл

@ -1,6 +1,6 @@
{
"name": "datax-common",
"version": "1.4.0-SNAPSHOT-7",
"version": "1.4.0-SNAPSHOT-8",
"description": "Common UX, styles, utilities, and modules",
"author": "Microsoft",
"license": "MIT",

4
Website/Packages/datax-common/src/styles/styles.css
Просмотреть файл

@ -74,8 +74,8 @@
border-left: 10px solid orange;
color: #40474b;
background-color: #dadada;
border-top: 1px solid #dadada;
border-bottom: 1px solid #dadada;
border-top: 1px solid #474747;
border-bottom: 1px solid #474747;
}
.vertical-tabs-item.selected {

4
Website/Packages/datax-home/package.json
Просмотреть файл

@ -1,6 +1,6 @@
{
"name": "datax-home",
"version": "1.4.0-SNAPSHOT-7",
"version": "1.4.0-SNAPSHOT-8",
"description": "Home page",
"author": "Microsoft",
"license": "MIT",
@ -30,7 +30,7 @@
"css-loader": "1.0.1",
"file-loader": "2.0.0",
"mini-css-extract-plugin": "0.4.4",
"datax-common": "1.4.0-SNAPSHOT-7",
"datax-common": "1.4.0-SNAPSHOT-8",
"office-ui-fabric-react": "6.111.2",
"q": "1.5.1",
"react": "16.6.3",

4
Website/Packages/datax-jobs/package.json
Просмотреть файл

@ -1,6 +1,6 @@
{
"name": "datax-jobs",
"version": "1.4.0-SNAPSHOT-7",
"version": "1.4.0-SNAPSHOT-8",
"description": "Job features",
"author": "Microsoft",
"license": "MIT",
@ -30,7 +30,7 @@
"css-loader": "1.0.1",
"file-loader": "2.0.0",
"mini-css-extract-plugin": "0.4.4",
"datax-common": "1.4.0-SNAPSHOT-7",
"datax-common": "1.4.0-SNAPSHOT-8",
"office-ui-fabric-react": "6.111.2",
"q": "1.5.1",
"prop-types": "15.6.2",

6
Website/Packages/datax-jobs/src/modules/jobs/api.js
Просмотреть файл

@ -11,6 +11,12 @@ export const listSparkJobs = () =>
export const syncSparkJobs = () =>
serviceGetApi(Constants.serviceRouteApi, Constants.serviceApplication, Constants.services.flow, 'job/syncall');
export const stopAllSparkJobs = () =>
serviceGetApi(Constants.serviceRouteApi, Constants.serviceApplication, Constants.services.flow, 'job/stopall');
export const startAllSparkJobs = () =>
serviceGetApi(Constants.serviceRouteApi, Constants.serviceApplication, Constants.services.flow, 'job/startall');
export const startSparkJob = name =>
servicePostApi(Constants.serviceRouteApi, Constants.serviceApplication, Constants.services.flow, 'job/start', name);

30
Website/Packages/datax-jobs/src/modules/jobs/components/sparkJobs.jsx
Просмотреть файл

@ -71,6 +71,12 @@ class SparkJobs extends React.Component {
<div style={buttonStyle}>
<PrimaryButton text="Refresh Jobs Status" onClick={() => this.syncSparkItems()} />
</div>
<div style={buttonStyle}>
<PrimaryButton text="Stop All Jobs" onClick={() => this.stopSparkItems()} />
</div>
<div style={buttonStyle}>
<PrimaryButton text="Start All Jobs" onClick={() => this.startSparkItems()} />
</div>
</div>
<div style={detailsContainerStyle}>{details}</div>
@ -124,6 +130,30 @@ class SparkJobs extends React.Component {
});
}
stopSparkItems() {
Api.stopAllSparkJobs()
.then(jobs => this.setJobs(jobs))
.catch(error => {
const message = getApiErrorMessage(error);
this.setState({
errorMessage: message,
showMessageBar: true
});
});
}
startSparkItems() {
Api.startAllSparkJobs()
.then(jobs => this.setJobs(jobs))
.catch(error => {
const message = getApiErrorMessage(error);
this.setState({
errorMessage: message,
showMessageBar: true
});
});
}
updateFilter(text) {
this.setState({ filter: text });
}

6
Website/Packages/datax-metrics/package.json
Просмотреть файл

@ -1,6 +1,6 @@
{
"name": "datax-metrics",
"version": "1.4.0-SNAPSHOT-7",
"version": "1.4.0-SNAPSHOT-8",
"description": "Metric features",
"author": "Microsoft",
"license": "MIT",
@ -22,7 +22,7 @@
"devpatch": "rimraf dist && webpack --mode development --config webpack.config.js --bail --progress --display-modules --display-error-details --colors --display-chunks --display-reasons && localdevpatch.bat"
},
"dependencies": {
"moment": "2.22.2"
"moment": "2.29.1"
},
"devDependencies": {
"@babel/core": "7.1.6",
@ -33,7 +33,7 @@
"d3": "4.10.2",
"file-loader": "2.0.0",
"mini-css-extract-plugin": "0.4.4",
"datax-common": "1.4.0-SNAPSHOT-7",
"datax-common": "1.4.0-SNAPSHOT-8",
"office-ui-fabric-react": "6.111.2",
"q": "1.5.1",
"plotly.js": "1.44.4",

2
Website/Packages/datax-metrics/src/modules/metrics/components/metricWidgetDetailsList.jsx
Просмотреть файл

@ -58,7 +58,7 @@ export default class MetricWidgetDetailsList extends React.Component {
return (
<div>
<h3 style={{ fontWeight: 'normal', color: `${Colors.themeDarker}` }} tabindex="0">{this.props.displayName}</h3>
<div style={{ width: 'auto', border: `1px solid ${Colors.customGray}` }}>
<div style={{ width: 'auto', border: `1px solid ${Colors.customNeutralDarkGray}` }}>
<DetailsList
ariaLabelForGrid={this.props.displayName}
items={items}

2
Website/Packages/datax-metrics/src/modules/metrics/components/metricWidgetGauge.jsx
Просмотреть файл

@ -95,7 +95,7 @@ class MetricWidgetGauge extends React.Component {
width: 300,
height: 120,
padding: 10,
borderColor: Colors.customGray,
borderColor: Colors.customNeutralDarkGray,
borderWidth: 1,
borderStyle: 'solid',
backgroundColor: Colors.white,

2
Website/Packages/datax-metrics/src/modules/metrics/components/metricWidgetLineChart.jsx
Просмотреть файл

@ -82,7 +82,7 @@ class MetricWidgetLineChart extends React.Component {
ref={div => {
this.graphDiv = div;
}}
style={{ width: 'auto', height: 300, paddingTop: 25, backgroundColor: Colors.white, border: `1px solid ${Colors.customGray}` }}
style={{ width: 'auto', height: 300, paddingTop: 25, backgroundColor: Colors.white, border: `1px solid ${Colors.customNeutralDarkGray}` }}
/>
</div>
);

2
Website/Packages/datax-metrics/src/modules/metrics/components/metricWidgetMultiLineChart.jsx
Просмотреть файл

@ -149,7 +149,7 @@ class MetricWidgetMultiLineChart extends React.Component {
ref={div => {
this.graphDiv = div;
}}
style={{ width: 'auto', height: 300, paddingTop: 25, backgroundColor: Colors.white, border: `1px solid ${Colors.customGray}` }}
style={{ width: 'auto', height: 300, paddingTop: 25, backgroundColor: Colors.white, border: `1px solid ${Colors.customNeutralDarkGray}` }}
/>
</div>
);

2
Website/Packages/datax-metrics/src/modules/metrics/components/metricWidgetNumberBox.jsx
Просмотреть файл

@ -63,7 +63,7 @@ class MetricWidgetNumberBox extends React.Component {
width: 300,
height: 120,
padding: 10,
borderColor: Colors.customGray,
borderColor: Colors.customNeutralDarkGray,
borderWidth: 1,
borderStyle: 'solid',
backgroundColor: Colors.white,

2
Website/Packages/datax-metrics/src/modules/metrics/components/metricWidgetPercentageBox.jsx
Просмотреть файл

@ -23,7 +23,7 @@ class MetricWidgetPercentageBox extends React.Component {
width: 300,
height: 120,
padding: 10,
borderColor: Colors.customGray,
borderColor: Colors.customNeutralDarkGray,
borderWidth: 1,
borderStyle: 'solid',
backgroundColor: Colors.white,

2
Website/Packages/datax-metrics/src/modules/metrics/components/metricWidgetSimpleBox.jsx
Просмотреть файл

@ -18,7 +18,7 @@ class MetricWidgetSimpleBox extends React.Component {
width: 300,
height: 120,
padding: 10,
borderColor: Colors.customGray,
borderColor: Colors.customNeutralDarkGray,
borderWidth: 1,
borderStyle: 'solid',
backgroundColor: Colors.white,

2
Website/Packages/datax-metrics/src/modules/metrics/components/metricWidgetStackedAreaChart.jsx
Просмотреть файл

@ -124,7 +124,7 @@ class MetricWidgetStackedAreaChart extends React.Component {
ref={div => {
this.graphDiv = div;
}}
style={{ width: 'auto', height: 300, paddingTop: 25, backgroundColor: Colors.white, border: `1px solid ${Colors.customGray}` }}
style={{ width: 'auto', height: 300, paddingTop: 25, backgroundColor: Colors.white, border: `1px solid ${Colors.customNeutralDarkGray}` }}
/>
</div>
);

6
Website/Packages/datax-pipeline/package.json
Просмотреть файл

@ -1,6 +1,6 @@
{
"name": "datax-pipeline",
"version": "1.4.0-SNAPSHOT-7",
"version": "1.4.0-SNAPSHOT-8",
"description": "Pipeline features",
"author": "Microsoft",
"license": "MIT",
@ -34,8 +34,8 @@
"css-loader": "1.0.1",
"file-loader": "2.0.0",
"mini-css-extract-plugin": "0.4.4",
"datax-common": "1.4.0-SNAPSHOT-7",
"datax-query": "1.4.0-SNAPSHOT-7",
"datax-common": "1.4.0-SNAPSHOT-8",
"datax-query": "1.4.0-SNAPSHOT-8",
"office-ui-fabric-react": "6.111.2",
"q": "1.5.1",
"promise-polyfill": "8.1.0",

2
Website/Packages/datax-pipeline/src/modules/flowList/components/flowListPanel.jsx
Просмотреть файл

@ -198,7 +198,7 @@ const filterContainerStyle = {
const listContainerStyle = {
backgroundColor: Colors.white,
border: `1px solid ${Colors.neutralTertiaryAlt}`,
border: `1px solid ${Colors.neutralPrimaryAlt}`,
flex: 1,
overflowY: 'auto',
minHeight: '100px'

4
Website/Packages/datax-query/package.json
Просмотреть файл

@ -1,6 +1,6 @@
{
"name": "datax-query",
"version": "1.4.0-SNAPSHOT-7",
"version": "1.4.0-SNAPSHOT-8",
"description": "Query features",
"author": "Microsoft",
"license": "MIT",
@ -32,7 +32,7 @@
"css-loader": "1.0.1",
"file-loader": "2.0.0",
"mini-css-extract-plugin": "0.4.4",
"datax-common": "1.4.0-SNAPSHOT-7",
"datax-common": "1.4.0-SNAPSHOT-8",
"office-ui-fabric-react": "6.111.2",
"q": "1.5.1",
"promise-polyfill": "8.1.0",

24
Website/Website/package.json
Просмотреть файл

@ -23,30 +23,28 @@
"author": "Microsoft",
"license": "MIT",
"dependencies": {
"adal-node": "0.1.28",
"adal-node": "0.2.3",
"applicationinsights": "1.0.6",
"archiver": "3.0.0",
"azure-keyvault": "3.0.4",
"body-parser": "1.18.3",
"csurf": "1.9.0",
"diskdb": "0.1.17",
"express": "4.16.4",
"express-session": "1.15.6",
"fast-csv": "2.4.1",
"generate-schema": "2.6.0",
"helmet": "3.22.0",
"moment": "2.22.2",
"moment": "2.29.1",
"mongodb": "3.2.7",
"ms-rest-azure": "2.5.9",
"ms-rest-azure": "2.6.0",
"mssql": "4.2.3",
"passport": "0.4.0",
"passport-azure-ad": "4.0.0",
"passport-azure-ad": "4.3.1",
"q": "1.5.1",
"react": "16.6.3",
"react-ace": "6.2.0",
"react-dom": "16.6.3",
"react-loadable": "5.5.0",
"redis": "2.8.0",
"redis": "3.1.2",
"request": "2.88.0",
"request-promise": "4.2.2",
"tree-kill": "1.2.2",
@ -63,11 +61,11 @@
"css-loader": "1.0.1",
"file-loader": "2.0.0",
"mini-css-extract-plugin": "0.4.4",
"datax-common": "1.4.0-SNAPSHOT-7",
"datax-home": "1.4.0-SNAPSHOT-7",
"datax-jobs": "1.4.0-SNAPSHOT-7",
"datax-metrics": "1.4.0-SNAPSHOT-7",
"datax-pipeline": "1.4.0-SNAPSHOT-7",
"datax-common": "1.4.0-SNAPSHOT-8",
"datax-home": "1.4.0-SNAPSHOT-8",
"datax-jobs": "1.4.0-SNAPSHOT-8",
"datax-metrics": "1.4.0-SNAPSHOT-8",
"datax-pipeline": "1.4.0-SNAPSHOT-8",
"office-ui-fabric-react": "6.111.2",
"promise-polyfill": "8.1.0",
"prop-types": "15.6.2",
@ -83,7 +81,7 @@
"style-loader": "0.23.1",
"terser-webpack-plugin": "1.1.0",
"url-loader": "1.1.2",
"webpack": "4.26.0",
"webpack": "4.46.0",
"webpack-cli": "3.1.2",
"whatwg-fetch": "2.0.3"
}

2
Website/Website/web.composition.json
Просмотреть файл

@ -121,6 +121,8 @@
"^DataX.Flow\/Flow.ManagementService\/flow\/stopjobs$",
"^DataX.Flow\/Flow.ManagementService\/flow\/restartjobs$",
"^DataX.Flow\/Flow.ManagementService\/job\/syncall$",
"^DataX.Flow\/Flow.ManagementService\/job\/stopall$",
"^DataX.Flow\/Flow.ManagementService\/job\/startall$",
"^DataX.Flow\/Flow.ManagementService\/job\/syncbynames$",
"^DataX.Flow\/Flow.ManagementService\/job\/start$",
"^DataX.Flow\/Flow.ManagementService\/job\/stop$",