Merged PR 695843: Update kusto packages

This is the second PR that updates the dependencies in order to gert rid the dependencies to the stale keyvault nuget packages.

Related work items: #2011942

Public/Src/Cache/ContentStore/App/Application.cs

@@ -410,11 +410,11 @@ namespace BuildXL.Cache.ContentStore.App
                 maxFileSize: _csvLogMaxFileSize
                 );
 
-            var indexedColumns = _csvFileLog.FileSchema.Select((col, idx) => new CsvColumnMapping { ColumnName = col.ToString(), Ordinal = idx });
-            var constColumns = _csvFileLog.ConstSchema.Select(col => new CsvColumnMapping { ColumnName = col.ToString(), ConstValue = _csvFileLog.RenderConstColumn(col) });
+            var indexedColumns = _csvFileLog.FileSchema.Select((col, idx) => CreateOrdinalColumn(col.ToString(), idx));
+            var constColumns = _csvFileLog.ConstSchema.Select(col => CreateConstColumn(col.ToString(), _csvFileLog.RenderConstColumn(col)));
             var csvMapping = indexedColumns.Concat(constColumns).ToArray();
 
-            var csvMappingStr = string.Join("", csvMapping.Select(col => $"{Environment.NewLine}  Name: '{col.ColumnName}', ConstValue: '{col.ConstValue}', Ordinal: {col.Ordinal}"));
+            var csvMappingStr = string.Join("", csvMapping.Select(col => $"{Environment.NewLine}  Name: '{col.ColumnName}', ConstValue: '{GetConstValueOrDefault(col) ?? "null"}', Ordinal: {GetOrdinalOrDefault(col) ?? "N/A"}"));
             _logger.Always("Using csv mapping:{0}", csvMappingStr);
 
             _kustoUploader = new KustoUploader
@@ -442,6 +442,32 @@ namespace BuildXL.Cache.ContentStore.App
             _logger.Always("Remote telemetry enabled");
         }
 
+        private static ColumnMapping CreateOrdinalColumn(string columnName, int ordinal)
+        {
+            var result = new ColumnMapping() {ColumnName = columnName};
+            result.Properties["Ordinal"] = ordinal.ToString();
+            return result;
+        }
+
+        private static ColumnMapping CreateConstColumn(string columnName, string constValue)
+        {
+            var result = new ColumnMapping() {ColumnName = columnName};
+            result.Properties["ConstValue"] = constValue;
+            return result;
+        }
+
+        private static string GetConstValueOrDefault(ColumnMapping mapping)
+        {
+            mapping.Properties.TryGetValue("ConstValue", out var result);
+            return result;
+        }
+
+        private static string GetOrdinalOrDefault(ColumnMapping mapping)
+        {
+            mapping.Properties.TryGetValue("Ordinal", out var result);
+            return result;
+        }
+
         private void RunFileSystemContentStoreInternal(AbsolutePath rootPath, System.Func<Context, FileSystemContentStoreInternal, Task> funcAsync)
         {
             Initialize();

Public/Src/Cache/ContentStore/App/KustoUploader.cs

@@ -4,15 +4,26 @@
 using System;
 using System.Collections.Generic;
 using System.Globalization;
+using System.IO;
 using System.Linq;
+using BuildXL.Cache.ContentStore.FileSystem;
+using BuildXL.Cache.ContentStore.Interfaces.FileSystem;
 using BuildXL.Cache.ContentStore.Interfaces.Logging;
 using BuildXL.Cache.ContentStore.Logging;
 using BuildXL.Utilities.ParallelAlgorithms;
 using Kusto.Data.Common;
 using Kusto.Ingest;
 
+#nullable enable
+
 namespace BuildXL.Cache.ContentStore.App
 {
+    public record struct FileDescription
+    {
+        public required string FilePath { get; init; }
+        public required Guid SourceId { get; init; }
+    }
+
     /// <summary>
     ///     Responsible for pumping provided log files to Kusto.
     ///
@@ -34,12 +45,13 @@ namespace BuildXL.Cache.ContentStore.App
     /// </summary>
     public sealed class KustoUploader : IDisposable
     {
-        private readonly ILog _log;
+        private readonly ILog? _log;
         private readonly bool _deleteFilesOnSuccess;
         private readonly bool _checkForIngestionErrors;
         private readonly IKustoQueuedIngestClient _client;
         private readonly KustoQueuedIngestionProperties _ingestionProperties;
         private readonly ActionBlockSlim<FileDescription> _block;
+        private readonly IAbsFileSystem _fileSystem;
 
         private bool _hasUploadErrors = false;
 
@@ -56,29 +68,33 @@ namespace BuildXL.Cache.ContentStore.App
         ///     Note that at this time not all uploaded files have necessarily been ingested; this class
         ///     does not wait for ingestions to complete, it only checks for failures of those that have completed.
         /// </param>
-        /// <param name="log">Optional log to which to write some debug information.</param>
+        /// <param name="log">An optional log to which to write some debug information.</param>
+        /// <param name="fileSystem">An option file system abstraction layer.</param>
         public KustoUploader
             (
             string connectionString,
             string database,
             string table,
-            IEnumerable<CsvColumnMapping> csvMapping,
+            IEnumerable<ColumnMapping> csvMapping,
             bool deleteFilesOnSuccess,
             bool checkForIngestionErrors,
-            ILog log = null
+            ILog? log = null,
+            IAbsFileSystem? fileSystem = null
             )
         {
             _log = log;
+            _fileSystem = fileSystem ?? new PassThroughFileSystem();
             _deleteFilesOnSuccess = deleteFilesOnSuccess;
             _checkForIngestionErrors = checkForIngestionErrors;
             _client = KustoIngestFactory.CreateQueuedIngestClient(connectionString);
             _hasUploadErrors = false;
             _ingestionProperties = new KustoQueuedIngestionProperties(database, table)
             {
-                CSVMapping   = csvMapping,
                 ReportLevel  = IngestionReportLevel.FailuresOnly,
                 ReportMethod = IngestionReportMethod.Queue,
             };
+            _ingestionProperties.IngestionMapping.IngestionMappings = csvMapping;
+
             _block = ActionBlockSlim.Create<FileDescription>(
                 degreeOfParallelism: 1,
                 UploadSingleCsvFile);
@@ -134,7 +150,8 @@ namespace BuildXL.Cache.ContentStore.App
             try
             {
                 var start = DateTime.UtcNow;
-                var result = _client.IngestFromSingleFile(fileDesc, _deleteFilesOnSuccess, _ingestionProperties);
+                using var file = _fileSystem.Open(new AbsolutePath(fileDesc.FilePath), FileAccess.Read, FileMode.Open, FileShare.None, FileOptions.DeleteOnClose);
+                var result = _client.IngestFromStream(file, _ingestionProperties);
                 var duration = DateTime.UtcNow.Subtract(start);
 
                 Always("Uploading file '{0}' took {1} ms", fileDesc.FilePath, duration.TotalMilliseconds);
@@ -154,7 +171,7 @@ namespace BuildXL.Cache.ContentStore.App
             }
 
             var start = DateTime.UtcNow;
-            var ingestionFailures = _client.GetAndDiscardTopIngestionFailures().GetAwaiter().GetResult().ToList();
+            var ingestionFailures = _client.GetAndDiscardTopIngestionFailuresAsync().GetAwaiter().GetResult().ToList();
             var duration = DateTime.UtcNow.Subtract(start);
             Always("Checking for ingestion failures took {0} ms", duration.TotalMilliseconds);
 

Public/Src/Cache/ContentStore/BuildXL.Cache.ContentStore.dsc

@@ -15,17 +15,10 @@ export const NetFx = BuildXLSdk.NetFx;
 
 @@public
 export const kustoPackages = [
-    ...(BuildXLSdk.isDotNetCoreOrStandard ? [
-        importFrom("Microsoft.Azure.Kusto.Data.NETStandard").pkg,
-        importFrom("Microsoft.Azure.Kusto.Ingest.NETStandard").pkg,
-        importFrom("Microsoft.Azure.Kusto.Cloud.Platform.Azure.NETStandard").pkg,
-        importFrom("Microsoft.Azure.Kusto.Cloud.Platform.NETStandard").pkg,
-        importFrom("Microsoft.Extensions.PlatformAbstractions").pkg,
-        importFrom("Microsoft.IO.RecyclableMemoryStream").pkg,
-    ] : [
-        importFrom("Microsoft.Azure.Kusto.Ingest").pkg,
-    ]),
-    importFrom("Microsoft.Azure.Management.Kusto").pkg,
+    importFrom("Microsoft.Azure.Kusto.Data").pkg,
+    importFrom("Microsoft.Azure.Kusto.Cloud.Platform").pkg,
+    importFrom("Microsoft.Azure.Kusto.Ingest").pkg,
+    importFrom("Azure.ResourceManager.Kusto").pkg,
     importFrom("Microsoft.IdentityModel.Clients.ActiveDirectory").pkg,
     ...getAzureBlobStorageSdkPackages(true),
 ];

Public/Src/Cache/Monitor/Library/Az/KustoClientExtensions.cs

@@ -1,24 +1,25 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT License.
 
+using System.Collections.Generic;
 using System.Diagnostics.ContractsLight;
 using System.Threading.Tasks;
-using Kusto.Cloud.Platform.Data;
 using Kusto.Data.Common;
+using Kusto.Cloud.Platform.Data;
 
 namespace BuildXL.Cache.Monitor.App.Az
 {
     internal static class KustoClientExtensions
     {
-        public static async Task<ObjectReader<T>> QuerySingleResultSetAsync<T>(this ICslQueryProvider cslQueryProvider, string query, string database, ClientRequestProperties? requestProperties = null)
+        public static async Task<IEnumerable<T>> QuerySingleResultSetAsync<T>(this ICslQueryProvider cslQueryProvider, string query, string database, ClientRequestProperties? requestProperties = null)
         {
+            
             Contract.Requires(string.IsNullOrEmpty(query));
             Contract.Requires(string.IsNullOrEmpty(database));
 
             requestProperties ??= new ClientRequestProperties();
 
-            var dataReader = await cslQueryProvider.ExecuteQueryAsync(database, query, requestProperties);
-            return new ObjectReader<T>(dataReader, disposeReader: true, nameBasedColumnMapping: true);
+            return (await cslQueryProvider.ExecuteQueryAsync(database, query, requestProperties)).ToEnumerable<T>();
         }
     }
 }

Public/Src/Cache/Monitor/Library/Monitor.cs

@@ -20,7 +20,6 @@ using BuildXL.Cache.Monitor.Library.IcM;
 using BuildXL.Cache.Monitor.Library.Notifications;
 using BuildXL.Cache.Monitor.Library.Rules.Kusto;
 using BuildXL.Cache.Monitor.Library.Scheduling;
-using Kusto.Cloud.Platform.Utils;
 using Kusto.Data.Common;
 using Kusto.Ingest;
 

Public/Src/Cache/Monitor/Library/Notifications/KustoNotifier.cs

@@ -9,8 +9,6 @@ using System.Linq;
 using System.Text;
 using System.Threading.Tasks;
 using BuildXL.Cache.ContentStore.Interfaces.Logging;
-using BuildXL.Cache.ContentStore.Utils;
-using BuildXL.Utilities.Collections;
 using BuildXL.Utilities.ParallelAlgorithms;
 using Kusto.Data.Common;
 using Kusto.Ingest;
@@ -36,7 +34,6 @@ namespace BuildXL.Cache.Monitor.App.Notifications
         }
 
         private readonly ILogger _logger;
-        private readonly Configuration _configuration;
         private readonly IKustoIngestClient _kustoIngestClient;
 
         private readonly KustoIngestionProperties _kustoIngestionProperties;
@@ -45,23 +42,22 @@ namespace BuildXL.Cache.Monitor.App.Notifications
 
         public KustoNotifier(Configuration configuration, ILogger logger, IKustoIngestClient kustoIngestClient)
         {
-            _configuration = configuration;
             _logger = logger;
             _kustoIngestClient = kustoIngestClient;
 
-            _kustoIngestionProperties = new KustoIngestionProperties(_configuration.KustoDatabaseName, _configuration.KustoTableName)
+            _kustoIngestionProperties = new KustoIngestionProperties(configuration.KustoDatabaseName, configuration.KustoTableName)
             {
                 Format = DataSourceFormat.json,
             };
 
-            Contract.RequiresNotNullOrEmpty(_configuration.KustoTableIngestionMappingName,
+            Contract.Assert(!string.IsNullOrEmpty(configuration.KustoTableIngestionMappingName),
                 "Kusto ingestion will fail to authenticate without a proper ingestion mapping.");
-            _kustoIngestionProperties.JSONMappingReference = _configuration.KustoTableIngestionMappingName;
+            _kustoIngestionProperties.IngestionMapping.IngestionMappingReference = configuration.KustoTableIngestionMappingName;
 
             _queue = NagleQueue<T>.Create(FlushAsync,
-                _configuration.MaxDegreeOfParallelism,
-                _configuration.FlushInterval,
-                _configuration.BatchSize);
+                configuration.MaxDegreeOfParallelism,
+                configuration.FlushInterval,
+                configuration.BatchSize);
         }
 
         public void Emit(T row)

cg/nuget/cgmanifest.json

@@ -64,6 +64,15 @@
         }
       }
     },
+    {
+      "Component": {
+        "Type": "NuGet",
+        "NuGet": {
+          "Name": "Azure.Data.Tables",
+          "Version": "12.6.1"
+        }
+      }
+    },
     {
       "Component": {
         "Type": "NuGet",
@@ -73,6 +82,24 @@
         }
       }
     },
+    {
+      "Component": {
+        "Type": "NuGet",
+        "NuGet": {
+          "Name": "Azure.ResourceManager",
+          "Version": "1.3.2"
+        }
+      }
+    },
+    {
+      "Component": {
+        "Type": "NuGet",
+        "NuGet": {
+          "Name": "Azure.ResourceManager.Kusto",
+          "Version": "1.1.0"
+        }
+      }
+    },
     {
       "Component": {
         "Type": "NuGet",
@@ -118,6 +145,15 @@
         }
       }
     },
+    {
+      "Component": {
+        "Type": "NuGet",
+        "NuGet": {
+          "Name": "Azure.Storage.Queues",
+          "Version": "12.11.0"
+        }
+      }
+    },
     {
       "Component": {
         "Type": "NuGet",
@@ -734,8 +770,8 @@
       "Component": {
         "Type": "NuGet",
         "NuGet": {
-          "Name": "Microsoft.Azure.Kusto.Cloud.Platform.Azure.NETStandard",
-          "Version": "6.1.8"
+          "Name": "Microsoft.Azure.Kusto.Cloud.Platform",
+          "Version": "11.2.1"
         }
       }
     },
@@ -743,8 +779,8 @@
       "Component": {
         "Type": "NuGet",
         "NuGet": {
-          "Name": "Microsoft.Azure.Kusto.Cloud.Platform.NETStandard",
-          "Version": "6.1.8"
+          "Name": "Microsoft.Azure.Kusto.Cloud.Platform.Aad",
+          "Version": "11.2.1"
         }
       }
     },
@@ -753,16 +789,7 @@
         "Type": "NuGet",
         "NuGet": {
           "Name": "Microsoft.Azure.Kusto.Data",
-          "Version": "6.1.8"
-        }
-      }
-    },
-    {
-      "Component": {
-        "Type": "NuGet",
-        "NuGet": {
-          "Name": "Microsoft.Azure.Kusto.Data.NETStandard",
-          "Version": "6.1.8"
+          "Version": "11.2.1"
         }
       }
     },
@@ -771,16 +798,7 @@
         "Type": "NuGet",
         "NuGet": {
           "Name": "Microsoft.Azure.Kusto.Ingest",
-          "Version": "6.1.8"
-        }
-      }
-    },
-    {
-      "Component": {
-        "Type": "NuGet",
-        "NuGet": {
-          "Name": "Microsoft.Azure.Kusto.Ingest.NETStandard",
-          "Version": "6.1.8"
+          "Version": "11.2.1"
         }
       }
     },
@@ -789,16 +807,7 @@
         "Type": "NuGet",
         "NuGet": {
           "Name": "Microsoft.Azure.Kusto.Tools",
-          "Version": "2.2.2"
-        }
-      }
-    },
-    {
-      "Component": {
-        "Type": "NuGet",
-        "NuGet": {
-          "Name": "Microsoft.Azure.Management.Kusto",
-          "Version": "1.0.0"
+          "Version": "7.2.1"
         }
       }
     },
@@ -1338,7 +1347,7 @@
         "Type": "NuGet",
         "NuGet": {
           "Name": "Microsoft.IO.RecyclableMemoryStream",
-          "Version": "1.2.2"
+          "Version": "2.2.0"
         }
       }
     },
@@ -1770,7 +1779,7 @@
         "Type": "NuGet",
         "NuGet": {
           "Name": "Microsoft.Rest.ClientRuntime",
-          "Version": "2.3.21"
+          "Version": "2.3.24"
         }
       }
     },
@@ -1779,7 +1788,7 @@
         "Type": "NuGet",
         "NuGet": {
           "Name": "Microsoft.Rest.ClientRuntime.Azure",
-          "Version": "3.3.18"
+          "Version": "3.3.19"
         }
       }
     },
@@ -1788,7 +1797,7 @@
         "Type": "NuGet",
         "NuGet": {
           "Name": "Microsoft.Rest.ClientRuntime.Azure.Authentication",
-          "Version": "2.4.0"
+          "Version": "2.4.1"
         }
       }
     },

config.dsc

@@ -358,32 +358,33 @@ config({
                 // Ninja JSON Linux Text
                 { id: "BuildXL.Tools.Ninjson.linux-x64", version: "1.11.2", osSkip: [ "macOS" ] },
 
-                // Kusto SDK (for netstandard)
-                { id: "Microsoft.Azure.Kusto.Cloud.Platform.Azure.NETStandard", version: "6.1.8",
-                    dependentPackageIdsToSkip: ["Microsoft.Extensions.PlatformAbstractions"] },
-                { id: "Microsoft.Azure.Kusto.Cloud.Platform.NETStandard", version: "6.1.8",
-                    dependentPackageIdsToSkip: ["Microsoft.Extensions.PlatformAbstractions"] },
-                { id: "Microsoft.Azure.Kusto.Data.NETStandard", version: "6.1.8",
-                    dependentPackageIdsToSkip: ["Microsoft.Extensions.PlatformAbstractions"] },
-                { id: "Microsoft.Azure.Kusto.Ingest.NETStandard", version: "6.1.8",
-                    dependentPackageIdsToSkip: ["Microsoft.Extensions.PlatformAbstractions"] },
-                { id: "Microsoft.IO.RecyclableMemoryStream", version: "1.2.2" },
                 { id: "Microsoft.Azure.KeyVault", version: "3.0.1"},
                 { id: "Microsoft.Azure.KeyVault.WebKey", version: "3.0.1"},
 
-                // Kusto SDK (for full framework)
-                { id: "Microsoft.Azure.Kusto.Data", version: "6.1.8" },
-                { id: "Microsoft.Azure.Kusto.Ingest", version: "6.1.8" },
-                { id: "Microsoft.Azure.Kusto.Tools", version: "2.2.2" },
-                { id: "Microsoft.Azure.Management.Kusto", version: "1.0.0" },
+                // Kusto SDK
+                { id: "Microsoft.Azure.Kusto.Data", version: "11.2.1" },
+                { id: "Microsoft.Azure.Kusto.Ingest", version: "11.2.1" },
+                { id: "Microsoft.Azure.Kusto.Tools", version: "7.2.1" },
+                { id: "Azure.ResourceManager.Kusto", version: "1.1.0" },
+
+                { id: "Microsoft.Azure.Kusto.Cloud.Platform", version: "11.2.1" },
+                { id: "Microsoft.Azure.Kusto.Cloud.Platform.Aad", version: "11.2.1" },
+
+                { id: "Azure.ResourceManager", version: "1.3.2" },
+
+                { id: "Microsoft.IO.RecyclableMemoryStream", version: "2.2.0",
+                    dependentPackageIdsToSkip: ["System.Buffers", "System.Memory"] }, // Used by Microsoft.Azure.Kusto.Cloud.Platform
 
                 // Azure Communication
-                { id: "Microsoft.Rest.ClientRuntime", version: "2.3.21",
+                { id: "Microsoft.Rest.ClientRuntime", version: "2.3.24",
                     dependentPackageIdsToSkip: ["Microsoft.NETCore.Runtime"],
                     dependentPackageIdsToIgnore: ["Microsoft.NETCore.Runtime"],
                 },
-                { id: "Microsoft.Rest.ClientRuntime.Azure", version: "3.3.18" },
-                { id: "Microsoft.Rest.ClientRuntime.Azure.Authentication", version: "2.4.0" },
+                { id: "Microsoft.Rest.ClientRuntime.Azure", version: "3.3.19" },
+                { id: "Microsoft.Rest.ClientRuntime.Azure.Authentication", version: "2.4.1" },
+
+                { id: "Azure.Data.Tables", version: "12.6.1" },
+                { id: "Azure.Storage.Queues", version: "12.11.0" },
 
                 // FsCheck
                 { id: "FsCheck", version: "2.14.3" },