diff --git a/DataProcessing/DataX.Utilities/.gitignore b/DataProcessing/DataX.Utilities/.gitignore new file mode 100644 index 00000000..28a11147 --- /dev/null +++ b/DataProcessing/DataX.Utilities/.gitignore @@ -0,0 +1 @@ +*/obj \ No newline at end of file diff --git a/Services/.gitignore b/Services/.gitignore index 5f74ee22..27f1c8bf 100644 --- a/Services/.gitignore +++ b/Services/.gitignore @@ -128,4 +128,7 @@ BundleArtifacts/ /DeploymentCloud/Deployment.DataX/cachedVariables /DeploymentCloud/Deployment.DataX/Temp /DeploymentCloud/Deployment.DataX/Outputs -/DeploymentCloud/Deployment.DataX/Packages \ No newline at end of file +/DeploymentCloud/Deployment.DataX/Packages + +# Development settings +appsettings.Development.json \ No newline at end of file diff --git a/Services/AspnetCore/DataX.FlowManagement/DataX.FlowManagement/Startup.cs b/Services/AspnetCore/DataX.FlowManagement/DataX.FlowManagement/Startup.cs index e0bf3c25..301a1c0f 100644 --- a/Services/AspnetCore/DataX.FlowManagement/DataX.FlowManagement/Startup.cs +++ b/Services/AspnetCore/DataX.FlowManagement/DataX.FlowManagement/Startup.cs @@ -27,8 +27,9 @@ namespace DataX.FlowManagement { private const string _MetricsHttpEndpointRelativeUri = "/api/data/upload"; + private readonly ILoggerFactory _loggerFactory; - public Startup(IHostingEnvironment env) + public Startup(IHostingEnvironment env, ILoggerFactory loggerFactory) { var builder = new ConfigurationBuilder() .SetBasePath(env.ContentRootPath) @@ -37,6 +38,8 @@ namespace DataX.FlowManagement .AddEnvironmentVariables(); Configuration = builder.Build(); + + _loggerFactory = loggerFactory; } public IConfiguration Configuration { get; } @@ -51,7 +54,7 @@ namespace DataX.FlowManagement // Export the Config dependencies Type[] exportTypes = new Type[] { typeof(FlowOperation), typeof(RuntimeConfigGeneration), typeof(JobOperation) }; - services.AddMefExportsFromAssemblies(ServiceLifetime.Scoped, GetOneBoxModeDependencyAssemblies(), exportTypes, new object[] { }); + services.AddMefExportsFromAssemblies(ServiceLifetime.Scoped, GetOneBoxModeDependencyAssemblies(), exportTypes, null, _loggerFactory, true); var result = InitTemplatesForLocal(services); Ensure.IsSuccessResult(result); } diff --git a/Services/DataX.Config/DataX.Config.Local/LocalSparkClientFactory.cs b/Services/DataX.Config/DataX.Config.Local/LocalSparkClientFactory.cs index 9b071e4b..67be2104 100644 --- a/Services/DataX.Config/DataX.Config.Local/LocalSparkClientFactory.cs +++ b/Services/DataX.Config/DataX.Config.Local/LocalSparkClientFactory.cs @@ -17,7 +17,7 @@ namespace DataX.Config.Local private ILogger _logger { get; } [ImportingConstructor] - public LocalSparkClientFactory(ILogger logger) + public LocalSparkClientFactory(ILogger logger) { this._logger = logger; } diff --git a/Services/DataX.Config/DataX.Config/InternalService/SparkJobOperation.cs b/Services/DataX.Config/DataX.Config/InternalService/SparkJobOperation.cs index e093e8f7..e7c4ee45 100644 --- a/Services/DataX.Config/DataX.Config/InternalService/SparkJobOperation.cs +++ b/Services/DataX.Config/DataX.Config/InternalService/SparkJobOperation.cs @@ -21,7 +21,7 @@ namespace DataX.Config public class SparkJobOperation { [ImportingConstructor] - public SparkJobOperation(SparkJobData jobData, SparkClusterManager clusterManager, ILogger logger) + public SparkJobOperation(SparkJobData jobData, SparkClusterManager clusterManager, ILogger logger) { this.JobData= jobData; this.ClusterManager = clusterManager; diff --git a/Services/DataX.Config/DataX.Config/PublicService/RuntimeConfigGeneration.cs b/Services/DataX.Config/DataX.Config/PublicService/RuntimeConfigGeneration.cs index 1f912eb8..2432e0f4 100644 --- a/Services/DataX.Config/DataX.Config/PublicService/RuntimeConfigGeneration.cs +++ b/Services/DataX.Config/DataX.Config/PublicService/RuntimeConfigGeneration.cs @@ -25,7 +25,7 @@ namespace DataX.Config /// [ImportingConstructor] public RuntimeConfigGeneration( - ILogger logger, + ILogger logger, FlowDataManager flows, JobDataManager jobData, [ImportMany] IEnumerable flowProcessors diff --git a/Services/DataX.Config/DataX.Config/Utility/LoggerExportDescriptorProvider.cs b/Services/DataX.Config/DataX.Config/Utility/LoggerExportDescriptorProvider.cs new file mode 100644 index 00000000..bc8558d9 --- /dev/null +++ b/Services/DataX.Config/DataX.Config/Utility/LoggerExportDescriptorProvider.cs @@ -0,0 +1,76 @@ +// ********************************************************************* +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License +// ********************************************************************* +using Microsoft.Extensions.Logging; +using System; +using System.Collections.Generic; +using System.Composition.Hosting.Core; +using System.Linq; +using System.Reflection; + +namespace DataX.Config.Utility +{ + /// + /// Using a LoggerFactory, dynamically creates Logger to resolve for MEF + /// Falls back to InstanceExportDescriptorProvider implementation if not a Logger + /// + public class LoggerAndInstanceExportDescriptorProvider : InstanceExportDescriptorProvider + { + private static readonly Type _ILoggerType = typeof(ILogger); + + private static readonly MethodInfo _CreateLogger = + typeof(Microsoft.Extensions.Logging.LoggerFactoryExtensions) + .GetMethods() + .Where(m => m.Name == nameof(LoggerFactoryExtensions.CreateLogger) && m.IsGenericMethod) + .FirstOrDefault(); + + private readonly ILoggerFactory _loggerFactory; + private readonly bool _hasInstances; + + public LoggerAndInstanceExportDescriptorProvider(object[] instances, ILoggerFactory loggerFactory) + : base(instances) + { + _hasInstances = instances?.Length > 0; + _loggerFactory = loggerFactory; + } + + /// + public override IEnumerable GetExportDescriptors(CompositionContract contract, DependencyAccessor descriptorAccessor) + { + if (_loggerFactory != null && _ILoggerType.IsAssignableFrom(contract.ContractType)) + { + ILogger logger; + + if (contract.ContractType.GenericTypeArguments.Length > 0) + { + logger = CreateLogger(contract.ContractType.GenericTypeArguments.FirstOrDefault()); + } + else + { + logger = _loggerFactory.CreateLogger(contract.ContractType); + } + + yield return new ExportDescriptorPromise( + contract, + contract.ContractType.FullName, + true, + NoDependencies, + dependencies => ExportDescriptor.Create((context, operation) => logger, NoMetadata)); + } + else if(_hasInstances) + { + foreach (var descriptor in base.GetExportDescriptors(contract, descriptorAccessor)) + { + yield return descriptor; + } + } + } + + private ILogger CreateLogger(Type t) + { + var genericMethod = _CreateLogger.MakeGenericMethod(t); + return genericMethod.Invoke(null, new object[] { _loggerFactory }) as ILogger; + } + } +} diff --git a/Services/DataX.Flow/DataX.Flow/ApplicationPackageRoot/ApplicationManifest.xml b/Services/DataX.Flow/DataX.Flow/ApplicationPackageRoot/ApplicationManifest.xml index 28feb120..bcbe0d6a 100644 --- a/Services/DataX.Flow/DataX.Flow/ApplicationPackageRoot/ApplicationManifest.xml +++ b/Services/DataX.Flow/DataX.Flow/ApplicationPackageRoot/ApplicationManifest.xml @@ -22,6 +22,7 @@ Licensed under the MIT License +