diff --git a/Core/Microsoft.DataTransfer.Core/Microsoft.DataTransfer.Core.csproj b/Core/Microsoft.DataTransfer.Core/Microsoft.DataTransfer.Core.csproj
index 8056ee9..733ebd9 100644
--- a/Core/Microsoft.DataTransfer.Core/Microsoft.DataTransfer.Core.csproj
+++ b/Core/Microsoft.DataTransfer.Core/Microsoft.DataTransfer.Core.csproj
@@ -13,6 +13,7 @@
+
diff --git a/Core/Microsoft.DataTransfer.Core/Program.cs b/Core/Microsoft.DataTransfer.Core/Program.cs
index 516d37f..a9ba0b2 100644
--- a/Core/Microsoft.DataTransfer.Core/Program.cs
+++ b/Core/Microsoft.DataTransfer.Core/Program.cs
@@ -3,6 +3,7 @@ using Microsoft.DataTransfer.Interfaces;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
+using Microsoft.Extensions.Logging;
namespace Microsoft.DataTransfer.Core;
@@ -18,6 +19,8 @@ class Program
.Build();
IConfiguration configuration = host.Services.GetRequiredService();
+ var loggerFactory = host.Services.GetRequiredService();
+ var log = loggerFactory.CreateLogger();
var options = configuration.Get();
@@ -34,7 +37,7 @@ class Program
var sources = LoadExtensions(container);
var sinks = LoadExtensions(container);
- Console.WriteLine($"{sources.Count + sinks.Count} Extensions Loaded");
+ log.LogInformation("{TotalExtensionCount} Extensions Loaded", sources.Count + sinks.Count);
var source = GetExtensionSelection(options.Source, sources, "Source");
var sourceConfig = BuildSettingsConfiguration(configuration, options.SourceSettingsPath, $"{source.DisplayName}SourceSettings", options.Source == null);
@@ -42,10 +45,10 @@ class Program
var sink = GetExtensionSelection(options.Sink, sinks, "Sink");
var sinkConfig = BuildSettingsConfiguration(configuration, options.SinkSettingsPath, $"{sink.DisplayName}SinkSettings", options.Sink == null);
- var data = source.ReadAsync(sourceConfig);
- await sink.WriteAsync(data, sinkConfig);
+ var data = source.ReadAsync(sourceConfig, loggerFactory.CreateLogger(source.GetType().Name));
+ await sink.WriteAsync(data, sinkConfig, source, loggerFactory.CreateLogger(sink.GetType().Name));
- Console.WriteLine("Done");
+ log.LogInformation("Done");
Console.WriteLine("Enter to Quit...");
Console.ReadLine();
diff --git a/Core/Microsoft.DataTransfer.Core/appsettings.json b/Core/Microsoft.DataTransfer.Core/appsettings.json
index b9b5535..04c781d 100644
--- a/Core/Microsoft.DataTransfer.Core/appsettings.json
+++ b/Core/Microsoft.DataTransfer.Core/appsettings.json
@@ -1,4 +1,9 @@
{
+ "Logging": {
+ "LogLevel": {
+ "Default": "Information"
+ }
+ },
"JsonSourceSettings": {
},
"JsonSinkSettings": {
diff --git a/Extensions/AzureTableAPI/Microsoft.DataTransfer.AzureTableAPIExtension/AzureTableAPIDataSinkExtension.cs b/Extensions/AzureTableAPI/Microsoft.DataTransfer.AzureTableAPIExtension/AzureTableAPIDataSinkExtension.cs
index 6d52e14..7811f03 100644
--- a/Extensions/AzureTableAPI/Microsoft.DataTransfer.AzureTableAPIExtension/AzureTableAPIDataSinkExtension.cs
+++ b/Extensions/AzureTableAPI/Microsoft.DataTransfer.AzureTableAPIExtension/AzureTableAPIDataSinkExtension.cs
@@ -3,6 +3,7 @@ using Microsoft.DataTransfer.AzureTableAPIExtension.Data;
using Microsoft.DataTransfer.AzureTableAPIExtension.Settings;
using Microsoft.DataTransfer.Interfaces;
using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.Logging;
using System.ComponentModel.Composition;
namespace Microsoft.DataTransfer.AzureTableAPIExtension
@@ -12,7 +13,7 @@ namespace Microsoft.DataTransfer.AzureTableAPIExtension
{
public string DisplayName => "AzureTableAPI";
- public async Task WriteAsync(IAsyncEnumerable dataItems, IConfiguration config, CancellationToken cancellationToken = default)
+ public async Task WriteAsync(IAsyncEnumerable dataItems, IConfiguration config, IDataSourceExtension dataSource, ILogger logger, CancellationToken cancellationToken = default)
{
var settings = config.Get();
settings.Validate();
diff --git a/Extensions/AzureTableAPI/Microsoft.DataTransfer.AzureTableAPIExtension/AzureTableAPIDataSourceExtension.cs b/Extensions/AzureTableAPI/Microsoft.DataTransfer.AzureTableAPIExtension/AzureTableAPIDataSourceExtension.cs
index 1fb8c20..2d546ab 100644
--- a/Extensions/AzureTableAPI/Microsoft.DataTransfer.AzureTableAPIExtension/AzureTableAPIDataSourceExtension.cs
+++ b/Extensions/AzureTableAPI/Microsoft.DataTransfer.AzureTableAPIExtension/AzureTableAPIDataSourceExtension.cs
@@ -4,6 +4,7 @@ using Microsoft.DataTransfer.AzureTableAPIExtension.Data;
using Microsoft.DataTransfer.AzureTableAPIExtension.Settings;
using Microsoft.DataTransfer.Interfaces;
using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.Logging;
using System.ComponentModel.Composition;
using System.Runtime.CompilerServices;
@@ -14,7 +15,7 @@ namespace Microsoft.DataTransfer.AzureTableAPIExtension
{
public string DisplayName => "AzureTableAPI";
- public async IAsyncEnumerable ReadAsync(IConfiguration config, [EnumeratorCancellation] CancellationToken cancellationToken = default)
+ public async IAsyncEnumerable ReadAsync(IConfiguration config, ILogger logger, [EnumeratorCancellation] CancellationToken cancellationToken = default)
{
var settings = config.Get();
settings.Validate();
diff --git a/Extensions/Cosmos/Microsoft.DataTransfer.CosmosExtension/CosmosDataSinkExtension.cs b/Extensions/Cosmos/Microsoft.DataTransfer.CosmosExtension/CosmosDataSinkExtension.cs
index deadc90..5624b50 100644
--- a/Extensions/Cosmos/Microsoft.DataTransfer.CosmosExtension/CosmosDataSinkExtension.cs
+++ b/Extensions/Cosmos/Microsoft.DataTransfer.CosmosExtension/CosmosDataSinkExtension.cs
@@ -8,6 +8,7 @@ using System.Text;
using Microsoft.Azure.Cosmos;
using Microsoft.DataTransfer.Interfaces;
using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.Logging;
using Newtonsoft.Json;
using Polly;
using Polly.Retry;
@@ -19,7 +20,7 @@ namespace Microsoft.DataTransfer.CosmosExtension
{
public string DisplayName => "Cosmos-nosql";
- public async Task WriteAsync(IAsyncEnumerable dataItems, IConfiguration config, CancellationToken cancellationToken = default)
+ public async Task WriteAsync(IAsyncEnumerable dataItems, IConfiguration config, IDataSourceExtension dataSource, ILogger logger, CancellationToken cancellationToken = default)
{
var settings = config.Get();
settings.Validate();
@@ -33,7 +34,7 @@ namespace Microsoft.DataTransfer.CosmosExtension
var entryAssembly = Assembly.GetEntryAssembly();
bool isShardedImport = false;
- string sourceName = "Unknown"; // TODO: add source as parameter
+ string sourceName = dataSource.DisplayName;
string sinkName = DisplayName;
string userAgentString = string.Format(CultureInfo.InvariantCulture, "{0}-{1}-{2}-{3}{4}",
entryAssembly == null ? "dtr" : entryAssembly.GetName().Name,
@@ -82,7 +83,7 @@ namespace Microsoft.DataTransfer.CosmosExtension
insertCount += i;
if (insertCount % 500 == 0)
{
- Console.WriteLine($"{insertCount} records added after {timer.ElapsedMilliseconds / 1000.0:F2}s");
+ logger.LogInformation("{InsertCount} records added after {TotalSeconds}s", insertCount, $"{timer.ElapsedMilliseconds / 1000.0:F2}");
}
}
@@ -92,14 +93,14 @@ namespace Microsoft.DataTransfer.CosmosExtension
await foreach (var batch in batches.WithCancellation(cancellationToken))
{
var insertTasks = settings.InsertStreams
- ? batch.Select(item => InsertItemStreamAsync(container, item, settings.PartitionKeyPath, retry, cancellationToken)).ToList()
- : batch.Select(item => InsertItemAsync(container, item, retry, cancellationToken)).ToList();
+ ? batch.Select(item => InsertItemStreamAsync(container, item, settings.PartitionKeyPath, retry, logger, cancellationToken)).ToList()
+ : batch.Select(item => InsertItemAsync(container, item, retry, logger, cancellationToken)).ToList();
var results = await Task.WhenAll(insertTasks);
ReportCount(results.Sum());
}
- Console.WriteLine($"Added {insertCount} total records in {timer.ElapsedMilliseconds / 1000.0:F2}s");
+ logger.LogInformation("Added {InsertCount} total records in {TotalSeconds}s", insertCount, $"{timer.ElapsedMilliseconds / 1000.0:F2}");
}
private static AsyncRetryPolicy GetRetryPolicy(int maxRetryCount, int initialRetryDuration)
@@ -115,8 +116,9 @@ namespace Microsoft.DataTransfer.CosmosExtension
return retryPolicy;
}
- private static Task InsertItemAsync(Container container, ExpandoObject item, AsyncRetryPolicy retryPolicy, CancellationToken cancellationToken)
+ private static Task InsertItemAsync(Container container, ExpandoObject item, AsyncRetryPolicy retryPolicy, ILogger logger, CancellationToken cancellationToken)
{
+ logger.LogTrace("Inserting item {Id}", GetPropertyValue(item, "id"));
var task = retryPolicy.ExecuteAsync(() => container.CreateItemAsync(item, cancellationToken: cancellationToken))
.ContinueWith(t =>
{
@@ -127,7 +129,7 @@ namespace Microsoft.DataTransfer.CosmosExtension
if (t.IsFaulted)
{
- Console.WriteLine($"Error adding record: {t.Exception?.Message}");
+ logger.LogWarning(t.Exception, "Error adding record: {ErrorMessage}", t.Exception?.Message);
}
return 0;
@@ -135,12 +137,16 @@ namespace Microsoft.DataTransfer.CosmosExtension
return task;
}
- private static Task InsertItemStreamAsync(Container container, ExpandoObject item, string partitionKeyPath, AsyncRetryPolicy retryPolicy, CancellationToken cancellationToken)
+ private static Task InsertItemStreamAsync(Container container, ExpandoObject item, string? partitionKeyPath, AsyncRetryPolicy retryPolicy, ILogger logger, CancellationToken cancellationToken)
{
+ if (partitionKeyPath == null)
+ throw new ArgumentNullException(nameof(partitionKeyPath));
+
+ logger.LogTrace("Inserting item {Id}", GetPropertyValue(item, "id"));
var json = JsonConvert.SerializeObject(item);
-
+
var ms = new MemoryStream(Encoding.UTF8.GetBytes(json));
- var task = retryPolicy.ExecuteAsync(() => container.CreateItemStreamAsync(ms, new PartitionKey(((IDictionary)item)[partitionKeyPath.TrimStart('/')]?.ToString()), cancellationToken: cancellationToken))
+ var task = retryPolicy.ExecuteAsync(() => container.CreateItemStreamAsync(ms, new PartitionKey(GetPropertyValue(item, partitionKeyPath.TrimStart('/'))), cancellationToken: cancellationToken))
.ContinueWith(t =>
{
if (t.IsCompletedSuccessfully)
@@ -150,7 +156,7 @@ namespace Microsoft.DataTransfer.CosmosExtension
if (t.IsFaulted)
{
- Console.WriteLine($"Error adding record: {t.Exception?.Message}");
+ logger.LogWarning(t.Exception, "Error adding record: {ErrorMessage}", t.Exception?.Message);
}
return 0;
@@ -158,6 +164,11 @@ namespace Microsoft.DataTransfer.CosmosExtension
return task;
}
+ private static string? GetPropertyValue(ExpandoObject item, string propertyName)
+ {
+ return ((IDictionary)item)[propertyName]?.ToString();
+ }
+
private static ExpandoObject? BuildObject(IDataItem? source, bool requireStringId = false)
{
if (source == null)
diff --git a/Extensions/Cosmos/Microsoft.DataTransfer.CosmosExtension/CosmosDataSourceExtension.cs b/Extensions/Cosmos/Microsoft.DataTransfer.CosmosExtension/CosmosDataSourceExtension.cs
index cf4621a..faf43dc 100644
--- a/Extensions/Cosmos/Microsoft.DataTransfer.CosmosExtension/CosmosDataSourceExtension.cs
+++ b/Extensions/Cosmos/Microsoft.DataTransfer.CosmosExtension/CosmosDataSourceExtension.cs
@@ -4,6 +4,7 @@ using Microsoft.Azure.Cosmos;
using Microsoft.DataTransfer.Interfaces;
using Microsoft.Extensions.Configuration;
using System;
+using Microsoft.Extensions.Logging;
namespace Microsoft.DataTransfer.CosmosExtension
{
@@ -12,7 +13,7 @@ namespace Microsoft.DataTransfer.CosmosExtension
{
public string DisplayName => "Cosmos-nosql";
- public async IAsyncEnumerable ReadAsync(IConfiguration config, [EnumeratorCancellation] CancellationToken cancellationToken = default)
+ public async IAsyncEnumerable ReadAsync(IConfiguration config, ILogger logger, [EnumeratorCancellation] CancellationToken cancellationToken = default)
{
var settings = config.Get();
settings.Validate();
@@ -31,7 +32,7 @@ namespace Microsoft.DataTransfer.CosmosExtension
requestOptions.PartitionKey = new PartitionKey(settings.PartitionKey);
}
- Console.WriteLine($"Reading from {settings.Database}.{settings.Container}");
+ logger.LogInformation("Reading from {Database}.{Container}", settings.Database, settings.Container);
using FeedIterator> feedIterator = GetFeedIterator>(settings, container, requestOptions);
while (feedIterator.HasMoreResults)
{
diff --git a/Extensions/Json/Microsoft.DataTransfer.JsonExtension.UnitTests/JsonRoundTripTests.cs b/Extensions/Json/Microsoft.DataTransfer.JsonExtension.UnitTests/JsonRoundTripTests.cs
index e6352e3..aaa71d4 100644
--- a/Extensions/Json/Microsoft.DataTransfer.JsonExtension.UnitTests/JsonRoundTripTests.cs
+++ b/Extensions/Json/Microsoft.DataTransfer.JsonExtension.UnitTests/JsonRoundTripTests.cs
@@ -1,3 +1,4 @@
+using Microsoft.Extensions.Logging.Abstractions;
using Newtonsoft.Json.Linq;
namespace Microsoft.DataTransfer.JsonExtension.UnitTests
@@ -24,7 +25,7 @@ namespace Microsoft.DataTransfer.JsonExtension.UnitTests
{ "Indented", "true" },
});
- await output.WriteAsync(input.ReadAsync(sourceConfig), sinkConfig);
+ await output.WriteAsync(input.ReadAsync(sourceConfig, NullLogger.Instance), sinkConfig, input, NullLogger.Instance);
bool areEqual = JToken.DeepEquals(JToken.Parse(await File.ReadAllTextAsync(fileIn)), JToken.Parse(await File.ReadAllTextAsync(fileOut)));
Assert.IsTrue(areEqual);
@@ -50,7 +51,7 @@ namespace Microsoft.DataTransfer.JsonExtension.UnitTests
{ "Indented", "true" },
});
- await output.WriteAsync(input.ReadAsync(sourceConfig), sinkConfig);
+ await output.WriteAsync(input.ReadAsync(sourceConfig, NullLogger.Instance), sinkConfig, input, NullLogger.Instance);
bool areEqual = JToken.DeepEquals(JToken.Parse(await File.ReadAllTextAsync(fileCompare)), JToken.Parse(await File.ReadAllTextAsync(fileOut)));
Assert.IsTrue(areEqual);
diff --git a/Extensions/Json/Microsoft.DataTransfer.JsonExtension.UnitTests/JsonSinkTests.cs b/Extensions/Json/Microsoft.DataTransfer.JsonExtension.UnitTests/JsonSinkTests.cs
index 16cf0a8..017f121 100644
--- a/Extensions/Json/Microsoft.DataTransfer.JsonExtension.UnitTests/JsonSinkTests.cs
+++ b/Extensions/Json/Microsoft.DataTransfer.JsonExtension.UnitTests/JsonSinkTests.cs
@@ -1,4 +1,5 @@
using Microsoft.DataTransfer.Interfaces;
+using Microsoft.Extensions.Logging.Abstractions;
using Newtonsoft.Json;
namespace Microsoft.DataTransfer.JsonExtension.UnitTests
@@ -35,7 +36,7 @@ namespace Microsoft.DataTransfer.JsonExtension.UnitTests
{ "FilePath", outputFile }
});
- await sink.WriteAsync(data.ToAsyncEnumerable(), config);
+ await sink.WriteAsync(data.ToAsyncEnumerable(), config, new JsonDataSourceExtension(), NullLogger.Instance);
var outputData = JsonConvert.DeserializeObject>(await File.ReadAllTextAsync(outputFile));
diff --git a/Extensions/Json/Microsoft.DataTransfer.JsonExtension.UnitTests/JsonSourceTests.cs b/Extensions/Json/Microsoft.DataTransfer.JsonExtension.UnitTests/JsonSourceTests.cs
index 9bd9390..b54d3f8 100644
--- a/Extensions/Json/Microsoft.DataTransfer.JsonExtension.UnitTests/JsonSourceTests.cs
+++ b/Extensions/Json/Microsoft.DataTransfer.JsonExtension.UnitTests/JsonSourceTests.cs
@@ -1,4 +1,5 @@
using Microsoft.DataTransfer.Interfaces;
+using Microsoft.Extensions.Logging.Abstractions;
namespace Microsoft.DataTransfer.JsonExtension.UnitTests
{
@@ -14,7 +15,7 @@ namespace Microsoft.DataTransfer.JsonExtension.UnitTests
{ "FilePath", "Data/SimpleIdName.json" }
});
- await foreach (var dataItem in extension.ReadAsync(config))
+ await foreach (var dataItem in extension.ReadAsync(config, NullLogger.Instance))
{
CollectionAssert.AreEquivalent(new[] { "id", "name" }, dataItem.GetFieldNames().ToArray());
Assert.IsNotNull(dataItem.GetValue("id"));
@@ -31,7 +32,7 @@ namespace Microsoft.DataTransfer.JsonExtension.UnitTests
{ "FilePath", "Data/NestedObjects.json" }
});
- await foreach (var dataItem in extension.ReadAsync(config))
+ await foreach (var dataItem in extension.ReadAsync(config, NullLogger.Instance))
{
if (dataItem.GetValue("child") is IDataItem child)
{
@@ -56,7 +57,7 @@ namespace Microsoft.DataTransfer.JsonExtension.UnitTests
});
int counter = 0;
- await foreach (var dataItem in extension.ReadAsync(config))
+ await foreach (var dataItem in extension.ReadAsync(config, NullLogger.Instance))
{
counter++;
CollectionAssert.AreEquivalent(new[] { "id", "name" }, dataItem.GetFieldNames().ToArray());
@@ -79,7 +80,7 @@ namespace Microsoft.DataTransfer.JsonExtension.UnitTests
int counter = 0;
double lastId = -1;
- await foreach (var dataItem in extension.ReadAsync(config))
+ await foreach (var dataItem in extension.ReadAsync(config, NullLogger.Instance))
{
counter++;
CollectionAssert.AreEquivalent(new[] { "id", "name" }, dataItem.GetFieldNames().ToArray());
@@ -104,7 +105,7 @@ namespace Microsoft.DataTransfer.JsonExtension.UnitTests
});
int counter = 0;
- await foreach (var dataItem in extension.ReadAsync(config))
+ await foreach (var dataItem in extension.ReadAsync(config, NullLogger.Instance))
{
counter++;
CollectionAssert.AreEquivalent(new[] { "id", "name" }, dataItem.GetFieldNames().ToArray());
@@ -125,7 +126,7 @@ namespace Microsoft.DataTransfer.JsonExtension.UnitTests
});
int counter = 0;
- await foreach (var dataItem in extension.ReadAsync(config))
+ await foreach (var dataItem in extension.ReadAsync(config, NullLogger.Instance))
{
counter++;
CollectionAssert.AreEquivalent(new[] { "id", "name" }, dataItem.GetFieldNames().ToArray());
diff --git a/Extensions/Json/Microsoft.DataTransfer.JsonExtension/JsonDataSinkExtension.cs b/Extensions/Json/Microsoft.DataTransfer.JsonExtension/JsonDataSinkExtension.cs
index 5c73001..51be285 100644
--- a/Extensions/Json/Microsoft.DataTransfer.JsonExtension/JsonDataSinkExtension.cs
+++ b/Extensions/Json/Microsoft.DataTransfer.JsonExtension/JsonDataSinkExtension.cs
@@ -4,6 +4,7 @@ using System.Text.Json;
using Microsoft.Extensions.Configuration;
using System;
using Microsoft.DataTransfer.JsonExtension.Settings;
+using Microsoft.Extensions.Logging;
namespace Microsoft.DataTransfer.JsonExtension
{
@@ -12,14 +13,14 @@ namespace Microsoft.DataTransfer.JsonExtension
{
public string DisplayName => "JSON";
- public async Task WriteAsync(IAsyncEnumerable dataItems, IConfiguration config, CancellationToken cancellationToken = default)
+ public async Task WriteAsync(IAsyncEnumerable dataItems, IConfiguration config, IDataSourceExtension dataSource, ILogger logger, CancellationToken cancellationToken = default)
{
var settings = config.Get();
settings.Validate();
if (settings.FilePath != null)
{
- Console.WriteLine($"Writing to file '{settings.FilePath}'");
+ logger.LogInformation("Writing to file '{FilePath}'", settings.FilePath);
await using var stream = File.Create(settings.FilePath);
await using var writer = new Utf8JsonWriter(stream, new JsonWriterOptions
{
@@ -33,7 +34,7 @@ namespace Microsoft.DataTransfer.JsonExtension
}
writer.WriteEndArray();
- Console.WriteLine($"Completed writing data to file '{settings.FilePath}'");
+ logger.LogInformation("Completed writing data to file '{FilePath}'", settings.FilePath);
}
}
}
diff --git a/Extensions/Json/Microsoft.DataTransfer.JsonExtension/JsonDataSourceExtension.cs b/Extensions/Json/Microsoft.DataTransfer.JsonExtension/JsonDataSourceExtension.cs
index 0aa5808..ace401e 100644
--- a/Extensions/Json/Microsoft.DataTransfer.JsonExtension/JsonDataSourceExtension.cs
+++ b/Extensions/Json/Microsoft.DataTransfer.JsonExtension/JsonDataSourceExtension.cs
@@ -6,6 +6,7 @@ using Microsoft.DataTransfer.Interfaces;
using Microsoft.DataTransfer.JsonExtension.Settings;
using Microsoft.Extensions.Configuration;
using System;
+using Microsoft.Extensions.Logging;
namespace Microsoft.DataTransfer.JsonExtension
{
@@ -13,7 +14,7 @@ namespace Microsoft.DataTransfer.JsonExtension
public class JsonDataSourceExtension : IDataSourceExtension
{
public string DisplayName => "JSON";
- public async IAsyncEnumerable ReadAsync(IConfiguration config, [EnumeratorCancellation] CancellationToken cancellationToken = default)
+ public async IAsyncEnumerable ReadAsync(IConfiguration config, ILogger logger, [EnumeratorCancellation] CancellationToken cancellationToken = default)
{
var settings = config.Get();
settings.Validate();
@@ -22,8 +23,8 @@ namespace Microsoft.DataTransfer.JsonExtension
{
if (File.Exists(settings.FilePath))
{
- Console.WriteLine($"Reading file '{settings.FilePath}'");
- var list = await ReadFileAsync(cancellationToken, settings.FilePath);
+ logger.LogInformation("Reading file '{FilePath}'", settings.FilePath);
+ var list = await ReadFileAsync(cancellationToken, settings.FilePath, logger);
if (list != null)
{
@@ -36,11 +37,11 @@ namespace Microsoft.DataTransfer.JsonExtension
else if (Directory.Exists(settings.FilePath))
{
string[] files = Directory.GetFiles(settings.FilePath, "*.json", SearchOption.AllDirectories);
- Console.WriteLine($"Reading {files.Length} files from '{settings.FilePath}'");
+ logger.LogInformation("Reading {FileCount} files from '{Folder}'", files.Length, settings.FilePath);
foreach (string filePath in files.OrderBy(f => f))
{
- Console.WriteLine($"Reading file '{filePath}'");
- var list = await ReadFileAsync(cancellationToken, filePath);
+ logger.LogInformation("Reading file '{FilePath}'", filePath);
+ var list = await ReadFileAsync(cancellationToken, filePath, logger);
if (list != null)
{
@@ -51,11 +52,11 @@ namespace Microsoft.DataTransfer.JsonExtension
}
}
}
- Console.WriteLine($"Completed reading '{settings.FilePath}'");
+ logger.LogInformation("Completed reading '{FilePath}'", settings.FilePath);
}
}
- private static async Task>?> ReadFileAsync(CancellationToken cancellationToken, string filePath)
+ private static async Task>?> ReadFileAsync(CancellationToken cancellationToken, string filePath, ILogger logger)
{
var file = await File.ReadAllTextAsync(filePath, cancellationToken);
try
@@ -85,7 +86,7 @@ namespace Microsoft.DataTransfer.JsonExtension
if (!list.Any())
{
- Console.WriteLine($"No records read from '{filePath}'");
+ logger.LogWarning("No records read from '{FilePath}'", filePath);
}
return list;
diff --git a/Extensions/Mongo/Microsoft.DataTransfer.MongoExtension/MongoDataSinkExtension.cs b/Extensions/Mongo/Microsoft.DataTransfer.MongoExtension/MongoDataSinkExtension.cs
index 9e5fa58..aa950bc 100644
--- a/Extensions/Mongo/Microsoft.DataTransfer.MongoExtension/MongoDataSinkExtension.cs
+++ b/Extensions/Mongo/Microsoft.DataTransfer.MongoExtension/MongoDataSinkExtension.cs
@@ -3,6 +3,7 @@ using Microsoft.DataTransfer.MongoExtension.Settings;
using Microsoft.Extensions.Configuration;
using MongoDB.Bson;
using System.ComponentModel.Composition;
+using Microsoft.Extensions.Logging;
namespace Microsoft.DataTransfer.MongoExtension;
[Export(typeof(IDataSinkExtension))]
@@ -10,7 +11,7 @@ public class MongoDataSinkExtension : IDataSinkExtension
{
public string DisplayName => "Mongo";
- public async Task WriteAsync(IAsyncEnumerable dataItems, IConfiguration config, CancellationToken cancellationToken = default)
+ public async Task WriteAsync(IAsyncEnumerable dataItems, IConfiguration config, IDataSourceExtension dataSource, ILogger logger, CancellationToken cancellationToken = default)
{
var settings = config.Get();
settings.Validate();
diff --git a/Extensions/Mongo/Microsoft.DataTransfer.MongoExtension/MongoDataSourceExtension.cs b/Extensions/Mongo/Microsoft.DataTransfer.MongoExtension/MongoDataSourceExtension.cs
index 870674a..d315a5d 100644
--- a/Extensions/Mongo/Microsoft.DataTransfer.MongoExtension/MongoDataSourceExtension.cs
+++ b/Extensions/Mongo/Microsoft.DataTransfer.MongoExtension/MongoDataSourceExtension.cs
@@ -4,6 +4,7 @@ using Microsoft.Extensions.Configuration;
using Microsoft.DataTransfer.Interfaces;
using MongoDB.Bson;
using Microsoft.DataTransfer.MongoExtension.Settings;
+using Microsoft.Extensions.Logging;
namespace Microsoft.DataTransfer.MongoExtension;
[Export(typeof(IDataSourceExtension))]
@@ -11,7 +12,7 @@ internal class MongoDataSourceExtension : IDataSourceExtension
{
public string DisplayName => "Mongo";
- public async IAsyncEnumerable ReadAsync(IConfiguration config, [EnumeratorCancellation] CancellationToken cancellationToken = default)
+ public async IAsyncEnumerable ReadAsync(IConfiguration config, ILogger logger, [EnumeratorCancellation] CancellationToken cancellationToken = default)
{
var settings = config.Get();
settings.Validate();
diff --git a/Extensions/SqlServer/Microsoft.DataTransfer.SqlServerExtension/SqlServerDataSinkExtension.cs b/Extensions/SqlServer/Microsoft.DataTransfer.SqlServerExtension/SqlServerDataSinkExtension.cs
index 27d7934..7cbd68d 100644
--- a/Extensions/SqlServer/Microsoft.DataTransfer.SqlServerExtension/SqlServerDataSinkExtension.cs
+++ b/Extensions/SqlServer/Microsoft.DataTransfer.SqlServerExtension/SqlServerDataSinkExtension.cs
@@ -4,6 +4,7 @@ using System.Data;
using Microsoft.Data.SqlClient;
using Microsoft.DataTransfer.Interfaces;
using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.Logging;
namespace Microsoft.DataTransfer.SqlServerExtension
{
@@ -12,7 +13,7 @@ namespace Microsoft.DataTransfer.SqlServerExtension
{
public string DisplayName => "SqlServer";
- public async Task WriteAsync(IAsyncEnumerable dataItems, IConfiguration config, CancellationToken cancellationToken = default)
+ public async Task WriteAsync(IAsyncEnumerable dataItems, IConfiguration config, IDataSourceExtension dataSource, ILogger logger, CancellationToken cancellationToken = default)
{
var settings = config.Get();
settings.Validate();
diff --git a/Extensions/SqlServer/Microsoft.DataTransfer.SqlServerExtension/SqlServerDataSourceExtension.cs b/Extensions/SqlServer/Microsoft.DataTransfer.SqlServerExtension/SqlServerDataSourceExtension.cs
index 1078013..412ee24 100644
--- a/Extensions/SqlServer/Microsoft.DataTransfer.SqlServerExtension/SqlServerDataSourceExtension.cs
+++ b/Extensions/SqlServer/Microsoft.DataTransfer.SqlServerExtension/SqlServerDataSourceExtension.cs
@@ -3,6 +3,7 @@ using System.Runtime.CompilerServices;
using Microsoft.Data.SqlClient;
using Microsoft.DataTransfer.Interfaces;
using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.Logging;
namespace Microsoft.DataTransfer.SqlServerExtension
{
@@ -11,7 +12,7 @@ namespace Microsoft.DataTransfer.SqlServerExtension
{
public string DisplayName => "SqlServer";
- public async IAsyncEnumerable ReadAsync(IConfiguration config, [EnumeratorCancellation] CancellationToken cancellationToken = default)
+ public async IAsyncEnumerable ReadAsync(IConfiguration config, ILogger logger, [EnumeratorCancellation] CancellationToken cancellationToken = default)
{
var settings = config.Get();
settings.Validate();
diff --git a/Interfaces/Microsoft.DataTransfer.Interfaces/IDataSinkExtension.cs b/Interfaces/Microsoft.DataTransfer.Interfaces/IDataSinkExtension.cs
index 27691ff..78e7bdf 100644
--- a/Interfaces/Microsoft.DataTransfer.Interfaces/IDataSinkExtension.cs
+++ b/Interfaces/Microsoft.DataTransfer.Interfaces/IDataSinkExtension.cs
@@ -1,9 +1,10 @@
using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.Logging;
namespace Microsoft.DataTransfer.Interfaces
{
public interface IDataSinkExtension : IDataTransferExtension
{
- Task WriteAsync(IAsyncEnumerable dataItems, IConfiguration config, CancellationToken cancellationToken = default);
+ Task WriteAsync(IAsyncEnumerable dataItems, IConfiguration config, IDataSourceExtension dataSource, ILogger logger, CancellationToken cancellationToken = default);
}
}
diff --git a/Interfaces/Microsoft.DataTransfer.Interfaces/IDataSourceExtension.cs b/Interfaces/Microsoft.DataTransfer.Interfaces/IDataSourceExtension.cs
index 6ee13e8..ba964a6 100644
--- a/Interfaces/Microsoft.DataTransfer.Interfaces/IDataSourceExtension.cs
+++ b/Interfaces/Microsoft.DataTransfer.Interfaces/IDataSourceExtension.cs
@@ -1,9 +1,10 @@
using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.Logging;
namespace Microsoft.DataTransfer.Interfaces
{
public interface IDataSourceExtension : IDataTransferExtension
{
- IAsyncEnumerable ReadAsync(IConfiguration config, CancellationToken cancellationToken = default);
+ IAsyncEnumerable ReadAsync(IConfiguration config, ILogger logger, CancellationToken cancellationToken = default);
}
}
diff --git a/Interfaces/Microsoft.DataTransfer.Interfaces/Microsoft.DataTransfer.Interfaces.csproj b/Interfaces/Microsoft.DataTransfer.Interfaces/Microsoft.DataTransfer.Interfaces.csproj
index 46bed10..a6a4684 100644
--- a/Interfaces/Microsoft.DataTransfer.Interfaces/Microsoft.DataTransfer.Interfaces.csproj
+++ b/Interfaces/Microsoft.DataTransfer.Interfaces/Microsoft.DataTransfer.Interfaces.csproj
@@ -8,6 +8,7 @@
+