microsoft
/
data-accelerator
зеркало из https://github.com/microsoft/data-accelerator.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

Docker: Fix debian security updates (#175) 

* update datax package version

* Add null check

* Fix file path incase of local scenario

* Include debian security updates
This commit is contained in:
Rohit Agrawal - MSFT 2020-08-24 11:54:09 -07:00 коммит произвёл GitHub
Родитель b67c607e04
Коммит ccfccdc1eb
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: 4AEE18F83AFDEB23
7 изменённых файлов: 52 добавлений и 25 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

2
DataProcessing/datax-host/src/main/scala/datax/host/StreamingHost.scala
Просмотреть файл

@ -65,7 +65,7 @@ import scala.concurrent.duration._
createStreamContextLogger.warn(s"Create streaming context checkpoints folder=${streamingConf.checkpointDir}, internalInSeconds=${streamingConf.intervalInSeconds}")
val streamingContext = createStreamingContext(spark, streamingConf.intervalInSeconds)
val batchInterval = streamingConf.intervalInSeconds.seconds
val repartitionNumber = inputConf.repartition.getOrElse(0)
val repartitionNumber = if (inputConf.repartition != null) inputConf.repartition.getOrElse(0) else 0
val repartition = if(repartitionNumber==0) (r:RDD[T])=>r else (r:RDD[T])=>r.repartition(repartitionNumber)
streamingFactory.getStream(streamingContext, inputConf, (rdd, time) => {
val streamingLogger = LogManager.getLogger("StreamingLoop")

11
DeploymentLocal/Dockerfile
Просмотреть файл

@ -1,10 +1,14 @@
FROM microsoft/dotnet:2.2-sdk AS build-env
WORKDIR /app/aspnetcore
FROM microsoft/dotnet:2.2-aspnetcore-runtime
WORKDIR /app/aspnetcore
COPY "finalrun.sh" "devices.csv" "./"
ENV NODE_VERSION="8.9.4" NODE_DOWNLOAD_SHA="21fb4690e349f82d708ae766def01d7fec1b085ce1f5ab30d9bda8ee126ca8fc" \
ENV NODE_VERSION="14.3.0" NODE_DOWNLOAD_SHA="ea08acdf403d13ab67303d503eab080a5285c005c75c0710551d928f3cc74057" \
DATAX_ENABLE_ONEBOX="true" DATAX_ONEBOX_ROOT="./datax" NPM_CONFIG_LOGLEVEL="info" \
SPARK_VERSION="2.3.0" ASPNETCORE_URLS="http://+:5000" port="2020" \
DATAXDEV_FLOW_MANAGEMENT_LOCAL_SERVICE="http://localhost:5000" NODE_ENV="production"
@ -38,6 +42,11 @@ RUN curl -sL --retry 3 \
| tar vxz -C /usr/ \
&& mv /usr/$SPARK_PACKAGE $SPARK_HOME \
&& chown -R root:root "$SPARK_HOME"
# Debian security updates
RUN apt-get update || true && apt-get -y install apt-transport-https
RUN echo "deb http://security.debian.org/debian-security buster/updates main contrib non-free" >> /etc/apt/sources.list
RUN apt-get update && apt-get upgrade -y
ADD "FlowManagement.tar" "deployment.tar" "./"
ADD "datax.tar" "./datax/bin"

2
DeploymentLocal/sample/HomeAutomationLocal.json
Просмотреть файл

@ -316,7 +316,7 @@
"properties":
{
"name": null,
"path": "/app/aspnetcore/datax/bin/datax-udf-samples_2.3_2.11-1.1.0-SNAPSHOT.jar",
"path": "/app/aspnetcore/datax/bin/datax-udf-samples_2.4_2.11-1.2.2-SNAPSHOT.jar",
"class": "datax.sample.udf.UdfHelloWorld",
"libs": []
},

8
Services/DataX.Config/DataX.Config.Local/Resources/localSparkJobTemplate.json
Просмотреть файл

@ -3,19 +3,19 @@
"cluster": "${sparkClusterName}",
"options": {
"name": "${sparkJobName}",
"file": "${localRoot}/bin/datax-host_2.3_2.11-1.1.0-SNAPSHOT.jar",
"file": "${localRoot}/bin/datax-host_2.4_2.11-1.2.2-SNAPSHOT.jar",
"className": "datax.app.DirectLocalStreamingApp",
"args": [
"conf=${sparkJobConfigFilePath}",
"driverLogLevel=${sparkJobDriverLogLevel}"
],
"jars": [
"${localRoot}/bin/datax-core_2.3_2.11-1.1.0-SNAPSHOT.jar",
"${localRoot}/bin/datax-utility_2.3_2.11-1.1.0-SNAPSHOT.jar",
"${localRoot}/bin/datax-core_2.4_2.11-1.2.2-SNAPSHOT.jar",
"${localRoot}/bin/datax-utility_2.4_2.11-1.2.2-SNAPSHOT.jar",
"${localRoot}/bin/applicationinsights-core-2.2.1.jar",
"${localRoot}/bin/azure-eventhubs-1.2.1.jar",
"${localRoot}/bin/azure-eventhubs-spark_2.11-2.3.6.jar",
"${localRoot}/bin/datax-keyvault_2.3_2.11-1.1.0-SNAPSHOT-with-dependencies.jar",
"${localRoot}/bin/datax-keyvault_2.4_2.11-1.2.2-SNAPSHOT-with-dependencies.jar",
"${localRoot}/bin/proton-j-0.31.0.jar"
],

18
Services/DataX.Config/DataX.Config/ConfigGeneration/Processor/S450_PrepareTransformFile.cs
Просмотреть файл

@ -65,12 +65,18 @@ namespace DataX.Config.ConfigGeneration.Processor
var runtimeKeyVaultName = flowToDeploy.GetTokenString(PortConfigurationSettings.TokenName_RuntimeKeyVaultName);
Ensure.NotNull(runtimeKeyVaultName, "runtimeKeyVaultName");
var secretName = $"{config.Name}-transform";
Configuration.TryGet(Constants.ConfigSettingName_SparkType, out string sparkType);
var uriPrefix = KeyVaultClient.GetUriPrefix(sparkType);
var transformFileSecret = SecretUriParser.ComposeUri(runtimeKeyVaultName, secretName, uriPrefix);
flowToDeploy.SetStringToken(TokenName_TransformFile, transformFileSecret);
if (runtimeKeyVaultName != "local")
{
var secretName = $"{config.Name}-transform";
Configuration.TryGet(Constants.ConfigSettingName_SparkType, out string sparkType);
var uriPrefix = KeyVaultClient.GetUriPrefix(sparkType);
var transformFileSecret = SecretUriParser.ComposeUri(runtimeKeyVaultName, secretName, uriPrefix);
flowToDeploy.SetStringToken(TokenName_TransformFile, transformFileSecret);
}
else
{
flowToDeploy.SetStringToken(TokenName_TransformFile, flowToDeploy.ResultProperties[PrepareJobConfigVariables.ResultPropertyName_RuntimeConfigFolder].ToString() + $"/{config.Name}-combined.txt");
}
await Task.CompletedTask;
return "done";

18
Services/DataX.Config/DataX.Config/ConfigGeneration/Processor/S500_PrepareProjectionFile.cs
Просмотреть файл

@ -47,12 +47,18 @@ namespace DataX.Config.ConfigGeneration.Processor
var runtimeKeyVaultName = flowToDeploy.GetTokenString(PortConfigurationSettings.TokenName_RuntimeKeyVaultName);
Ensure.NotNull(runtimeKeyVaultName, "runtimeKeyVaultName");
var secretName = $"{config.Name}-projectionfile";
Configuration.TryGet(Constants.ConfigSettingName_SparkType, out string sparkType);
var uriPrefix = KeyVaultClient.GetUriPrefix(sparkType);
var projectionFileSecret = SecretUriParser.ComposeUri(runtimeKeyVaultName, secretName, uriPrefix);
flowToDeploy.SetObjectToken(TokenName_ProjectionFiles, new string[] { projectionFileSecret });
if (runtimeKeyVaultName != "local")
{
var secretName = $"{config.Name}-projectionfile";
Configuration.TryGet(Constants.ConfigSettingName_SparkType, out string sparkType);
var uriPrefix = KeyVaultClient.GetUriPrefix(sparkType);
var projectionFileSecret = SecretUriParser.ComposeUri(runtimeKeyVaultName, secretName, uriPrefix);
flowToDeploy.SetObjectToken(TokenName_ProjectionFiles, new string[] { projectionFileSecret });
}
else
{
flowToDeploy.SetObjectToken(TokenName_ProjectionFiles, new string[] { flowToDeploy.ResultProperties[PrepareJobConfigVariables.ResultPropertyName_RuntimeConfigFolder].ToString() + "/projection.txt" });
}
await Task.CompletedTask;
return "done";

18
Services/DataX.Config/DataX.Config/ConfigGeneration/Processor/S500_PrepareSchemaFile.cs
Просмотреть файл

@ -45,12 +45,18 @@ namespace DataX.Config.ConfigGeneration.Processor
var config = flowToDeploy.Config;
var runtimeKeyVaultName = flowToDeploy.GetTokenString(PortConfigurationSettings.TokenName_RuntimeKeyVaultName);
Ensure.NotNull(runtimeKeyVaultName, "runtimeKeyVaultName");
var secretName = $"{config.Name}-inputschemafile";
Configuration.TryGet(Constants.ConfigSettingName_SparkType, out string sparkType);
var uriPrefix = KeyVaultClient.GetUriPrefix(sparkType);
var schemaFileSecret = SecretUriParser.ComposeUri(runtimeKeyVaultName, secretName, uriPrefix);
flowToDeploy.SetStringToken(TokenName_InputSchemaFilePath, schemaFileSecret);
if(runtimeKeyVaultName != "local")
{
var secretName = $"{config.Name}-inputschemafile";
Configuration.TryGet(Constants.ConfigSettingName_SparkType, out string sparkType);
var uriPrefix = KeyVaultClient.GetUriPrefix(sparkType);
var schemaFileSecret = SecretUriParser.ComposeUri(runtimeKeyVaultName, secretName, uriPrefix);
flowToDeploy.SetStringToken(TokenName_InputSchemaFilePath, schemaFileSecret);
}
else
{
flowToDeploy.SetStringToken(TokenName_InputSchemaFilePath, flowToDeploy.ResultProperties[PrepareJobConfigVariables.ResultPropertyName_RuntimeConfigFolder].ToString()+ "/inputschema.json");
}
await Task.CompletedTask;
return "done";