Switch to using System.Text.Json instead of Newtonsoft to improve performance. (#778)

Switch to using System.Text.Json

src/Diagnostics.DataProviders/DataProviders/KustoDataProvider.cs

@@ -14,10 +14,10 @@ namespace Diagnostics.DataProviders
 {
     public class KustoQuery
     {
-        public string Text;
-        public string Url;
-        public string KustoDesktopUrl;
-        public string OperationName;
+        public string Text { get; set; }
+        public string Url { get; set; }
+        public string KustoDesktopUrl { get; set; }
+        public string OperationName { get; set; }
     }
 
     public class KustoDataProvider : DiagnosticDataProvider, IKustoDataProvider

src/Diagnostics.DataProviders/HealthCheckResult.cs

@@ -1,7 +1,6 @@
 using System;
 using System.Collections.Generic;
-using Newtonsoft.Json;
-using Newtonsoft.Json.Converters;
+using System.Text.Json.Serialization;
 
 namespace Diagnostics.DataProviders
 {
@@ -15,7 +14,7 @@ namespace Diagnostics.DataProviders
     /// </summary>
     public sealed class HealthCheckResult
     {
-        [JsonConverter(typeof(StringEnumConverter))]
+        [JsonConverter(typeof(JsonStringEnumConverter))]
         public HealthStatus Status { get; private set; }
         public string Name { get; private set; }
         public string Description { get; private set; }

src/Diagnostics.DataProviders/KustoSDKClient.cs

@@ -308,9 +308,9 @@ namespace Diagnostics.DataProviders
             if (dataSet != null && dataSet.Tables != null && dataSet.Tables.Count >= 4)
             {
                 var statisticsTable = dataSet.Tables[dataSet.Tables.Count - 2].ToDataTableResponseObject();
-                if (statisticsTable.Rows.GetLength(0) >= 2 && statisticsTable.Rows.GetLength(1) >= 5)
+                if (statisticsTable.Rows.Length >= 2 && statisticsTable.Rows[1].Length >= 5)
                 {
-                    stats = statisticsTable.Rows[1, 4];
+                    stats = statisticsTable.Rows[1][4];
                 }
             }
 

src/Diagnostics.ModelsAndUtils/Attributes/Definition.cs

@@ -2,12 +2,13 @@
 // Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT License. See LICENSE in the project root for license information.
 // </copyright>
+
 using System;
 using System.Collections.Generic;
 using System.Linq;
 using System.Runtime.Serialization;
+using System.Text.Json.Serialization;
 using Diagnostics.ModelsAndUtils.Models;
-using Newtonsoft.Json;
 
 namespace Diagnostics.ModelsAndUtils.Attributes
 {
@@ -56,17 +57,38 @@ namespace Diagnostics.ModelsAndUtils.Attributes
 
         /// <summary>
         /// List of Support Topics for which this detector is enabled.
+        /// Mark it as JsonIgnore because the SupportTopic class is deriving
+        /// from Attribute and attributes are not serialized by System.Text.Json
         /// </summary>
-        [DataMember]
+        [JsonIgnore]
         public IEnumerable<SupportTopic> SupportTopicList { get; set; }
 
-        [JsonIgnore]
+        /// <summary>
+        /// Property created only for Json Serialization as Attributes
+        /// are not serialized today properly by System.Text.Json
+        /// https://github.com/dotnet/runtime/issues/58947
+        /// </summary>
+        [DataMember]
+        [JsonPropertyName("supportTopicList")]
+        public IEnumerable<SupportTopicSTJCompat> SupportTopicListSTJCompat
+        {
+            get
+            {
+                if (SupportTopicList == null)
+                {
+                    return null;
+                }
+                
+                return SupportTopicList
+                    .Where(st => st != null)
+                    .Select(x => new SupportTopicSTJCompat(x));
+            }
+        }
+
         public string AnalysisType { get; set; } = string.Empty;
 
-        [JsonIgnore]
         private Guid instanceGUID;
 
-        [JsonIgnore]
         public override object TypeId { get { return (object)instanceGUID; } }
 
         /// <summary>

src/Diagnostics.ModelsAndUtils/Attributes/SupportTopic.cs

@@ -24,4 +24,34 @@ namespace Diagnostics.ModelsAndUtils.Attributes
             return (this.Id == other.Id && this.PesId == other.PesId);
         }
     }
+
+    /// <summary>
+    /// Class created just for Json Serialization as Attributes 
+    /// are not serialized properly to Json using System.Text.Json
+    /// https://github.com/dotnet/runtime/issues/58947
+    /// </summary>
+    public class SupportTopicSTJCompat
+    {
+        /// <summary>
+        /// Support Topic Id
+        /// </summary>
+        /// See <see href="http://aka.ms/selfhelppreview"/>
+        public string Id { get; set; }
+
+        /// <summary>
+        /// Unique resource Id.
+        /// </summary>
+        public string PesId { get; set; }
+
+        public SupportTopicSTJCompat(SupportTopic st)
+        {
+            if (st == null)
+            {
+                throw new ArgumentNullException(nameof(st));
+            }
+
+            this.Id = st.Id;
+            this.PesId = st.Id;
+        }
+    }
 }

src/Diagnostics.ModelsAndUtils/Models/DataProviderMetadata.cs

@@ -4,7 +4,7 @@ namespace Diagnostics.ModelsAndUtils.Models
 {
     public class DataProviderMetadata
     {
-        public string ProviderName;
+        public string ProviderName { get; set; }
         public List<KeyValuePair<string, object>> PropertyBag { get; }
 
         public DataProviderMetadata()

src/Diagnostics.ModelsAndUtils/Models/DataTableResponseObject.cs

@@ -22,7 +22,7 @@ namespace Diagnostics.ModelsAndUtils.Models
 
         public IEnumerable<DataTableResponseColumn> Columns { get; set; }
 
-        public dynamic[,] Rows { get; set; }
+        public dynamic[][] Rows { get; set; }
     }
 
     public class DataTableResponseColumn
@@ -41,7 +41,7 @@ namespace Diagnostics.ModelsAndUtils.Models
     {
         public string Name { get; set; }
         public IEnumerable<AppInsightsDataTableResponseColumn> Columns { get; set; }
-        public dynamic[,] Rows { get; set; }
+        public dynamic[][] Rows { get; set; }
     }
 
     public class AppInsightsDataTableResponseColumn
@@ -68,7 +68,7 @@ namespace Diagnostics.ModelsAndUtils.Models
                 var row = dataTable.NewRow();
                 for (int j = 0; j < dataTable.Columns.Count; j++)
                 {
-                    row[j] = dataTableResponse.Rows[i, j] ?? DBNull.Value;
+                    row[j] = dataTableResponse.Rows[i][j] ?? DBNull.Value;
                 }
 
                 dataTable.Rows.Add(row);
@@ -92,7 +92,7 @@ namespace Diagnostics.ModelsAndUtils.Models
                 var row = dataTable.NewRow();
                 for (int j = 0; j < dataTable.Columns.Count; j++)
                 {
-                    row[j] = MaskPII(appInsightsDataTableResponse.Rows[i, j]) ?? DBNull.Value;
+                    row[j] = MaskPII(appInsightsDataTableResponse.Rows[i][j]) ?? DBNull.Value;
                 }
 
                 dataTable.Rows.Add(row);
@@ -125,14 +125,11 @@ namespace Diagnostics.ModelsAndUtils.Models
                 columns.Add(new DataTableResponseColumn() { ColumnName = col.ColumnName, DataType = col.DataType.ToString().Replace("System.", "") });
             }
 
-            var rows = new dynamic[table.Rows.Count, table.Columns.Count];
+            var rows = new dynamic[table.Rows.Count][];
 
             for (int i = 0; i < table.Rows.Count; i++)
             {
-                for (int j = 0; j < table.Columns.Count; j++)
-                {
-                    rows[i, j] = table.Rows[i][j] == DBNull.Value ? null : table.Rows[i][j];
-                }
+                rows[i] = table.Rows[i].ItemArray;
             }
 
             dataTableResponseObject.Columns = columns;

src/Diagnostics.ModelsAndUtils/Models/DetectorType.cs

@@ -3,15 +3,15 @@
 // Licensed under the MIT License. See LICENSE in the project root for license information.
 // </copyright>
 
-using Newtonsoft.Json;
-using Newtonsoft.Json.Converters;
+
+using System.Text.Json.Serialization;
 
 namespace Diagnostics.ModelsAndUtils.Models
 {
     /// <summary>
     /// Defines whether the Detector is of type Analysis or not.
     /// </summary>
-    [JsonConverter(typeof(StringEnumConverter))]
+    [JsonConverter(typeof(JsonStringEnumConverter))]
     public enum DetectorType
     {
         /// <summary>

src/Diagnostics.ModelsAndUtils/Models/Response.cs

@@ -1,6 +1,7 @@
 using System.Collections.Generic;
 using System.Data;
 using System.Linq;
+using System.Text.Json.Serialization;
 using Diagnostics.ModelsAndUtils.Attributes;
 using Diagnostics.ModelsAndUtils.Models.ResponseExtensions;
 

src/Diagnostics.ModelsAndUtils/Models/ResponseExtensions/Card.cs

@@ -1,7 +1,7 @@
 using System.Collections.Generic;
 using System.Data;
-using Newtonsoft.Json;
-using Newtonsoft.Json.Converters;
+using System.Text.Json;
+using System.Text.Json.Serialization;
 
 namespace Diagnostics.ModelsAndUtils.Models.ResponseExtensions
 {
@@ -16,12 +16,12 @@ namespace Diagnostics.ModelsAndUtils.Models.ResponseExtensions
         /// <summary>
         /// Title of the Card
         /// </summary>
-        public string Title;
+        public string Title { get; set; }
 
         /// <summary>
         /// A list of descriptions for this card
         /// </summary>
-        public List<string> Descriptions;
+        public List<string> Descriptions { get; set; }
 
         /// <summary>
         /// Specify and icon from the font-awesome collection (for e.g. fa-circle)
@@ -31,13 +31,13 @@ namespace Diagnostics.ModelsAndUtils.Models.ResponseExtensions
         /// <summary>
         /// Specify the action type for this card
         /// </summary>
-        [JsonConverter(typeof(StringEnumConverter))]
-        public CardActionType ActionType;
+        [JsonConverter(typeof(JsonStringEnumConverter))]
+        public CardActionType ActionType { get; set; }
 
         /// <summary>
         /// Specify the action value for the card (will be detectorId for detectors)
         /// </summary>
-        public string ActionValue;
+        public string ActionValue { get; set; }
 
         /// <summary>
         /// Creates an instance of Card class.
@@ -105,8 +105,8 @@ namespace Diagnostics.ModelsAndUtils.Models.ResponseExtensions
                 table.Rows.Add(new object[] {
                     card.Title,
                     card.Icon,
-                    JsonConvert.SerializeObject(card.Descriptions),
-                    JsonConvert.SerializeObject(card.ActionType),
+                    JsonSerializer.Serialize(card.Descriptions),
+                    JsonSerializer.Serialize(card.ActionType),
                     card.ActionValue
               });
             });

src/Diagnostics.ModelsAndUtils/Models/ResponseExtensions/Downtime.cs

@@ -10,12 +10,12 @@ namespace Diagnostics.ModelsAndUtils.Models.ResponseExtensions
         /// <summary>
         /// Represents the start time for the downtime period
         /// </summary>
-        public DateTime StartTime { get; set; } = DateTime.MinValue;
+        public DateTime StartTime { get; set; } = DateTime.SpecifyKind(DateTime.MinValue, DateTimeKind.Utc);
 
         /// <summary>
         /// The end time for the downtime period
         /// </summary>
-        public DateTime EndTime { get; set; } = DateTime.MinValue;
+        public DateTime EndTime { get; set; } = DateTime.SpecifyKind(DateTime.MinValue, DateTimeKind.Utc);
 
         /// <summary>
         /// A optional label that if specified can be used to render a label or span in downtime analysis

src/Diagnostics.ModelsAndUtils/Models/ResponseExtensions/Guage.cs

@@ -2,8 +2,8 @@ using System;
 using System.Collections.Generic;
 using System.Data;
 using System.Linq;
-using Newtonsoft.Json;
-using Newtonsoft.Json.Converters;
+using System.Text.Json;
+using System.Text.Json.Serialization;
 
 namespace Diagnostics.ModelsAndUtils.Models.ResponseExtensions
 {
@@ -25,7 +25,7 @@ namespace Diagnostics.ModelsAndUtils.Models.ResponseExtensions
         /// <summary>
         /// Insight Level for the Guage. Decides the color of the Guage. Red for Critical, Orange for Warning, Green for Success and Blue for Info & None.
         /// </summary>
-        [JsonConverter(typeof(StringEnumConverter))]
+        [JsonConverter(typeof(JsonStringEnumConverter))]
         public InsightStatus Status { get; set; }
 
         private double _percentFilled;
@@ -61,7 +61,7 @@ namespace Diagnostics.ModelsAndUtils.Models.ResponseExtensions
         /// <summary>
         /// Size of the Guage. Can be either Small, Medium or Large
         /// </summary>
-        [JsonConverter(typeof(StringEnumConverter))]
+        [JsonConverter(typeof(JsonStringEnumConverter))]
         public GuageSize Size { get; set; }
 
         /// <summary>
@@ -159,8 +159,8 @@ namespace Diagnostics.ModelsAndUtils.Models.ResponseExtensions
             foreach (Guage g in guages)
             {
                 table.Rows.Add(
-                    JsonConvert.SerializeObject(renderDirection),
-                    JsonConvert.SerializeObject(g.Size),
+                    JsonSerializer.Serialize(renderDirection),
+                    JsonSerializer.Serialize(g.Size),
                     g.Status,
                     g.PercentFilled,
                     g.DisplayValue,

src/Diagnostics.ModelsAndUtils/Models/ResponseExtensions/Solution.cs

@@ -1,7 +1,6 @@
 using System.Collections.Generic;
+using System.Text.Json.Serialization;
 using Diagnostics.ModelsAndUtils.ScriptUtilities;
-using Newtonsoft.Json;
-using Newtonsoft.Json.Converters;
 
 namespace Diagnostics.ModelsAndUtils.Models.ResponseExtensions
 {
@@ -25,7 +24,7 @@ namespace Diagnostics.ModelsAndUtils.Models.ResponseExtensions
         /// <summary>
         /// Denotes which action will be performed, such as calling an ARM API or navigating to a Portal Blade.
         /// </summary>
-        [JsonConverter(typeof(StringEnumConverter))]
+        [JsonConverter(typeof(JsonStringEnumConverter))]
         public ActionType Action { get; set; }
 
         /// <summary>
@@ -122,10 +121,10 @@ namespace Diagnostics.ModelsAndUtils.Models.ResponseExtensions
     {
         public string Label { get; set; }
 
-        [JsonConverter(typeof(StringEnumConverter))]
+        [JsonConverter(typeof(JsonStringEnumConverter))]
         public SolutionButtonType Type { get; set; } = SolutionButtonType.Button;
 
-        [JsonConverter(typeof(StringEnumConverter))]
+        [JsonConverter(typeof(JsonStringEnumConverter))]
         public SolutionButtonPosition Position { get; set; } = SolutionButtonPosition.Bottom;
 
         public SolutionButtonOption(string label, SolutionButtonType type = SolutionButtonType.Button, SolutionButtonPosition position = SolutionButtonPosition.Bottom)

src/Diagnostics.ModelsAndUtils/Models/ResponseExtensions/SummaryCard.cs

@@ -2,8 +2,7 @@ using System;
 using System.Collections.Generic;
 using System.Data;
 using System.Linq;
-using Newtonsoft.Json;
-using Newtonsoft.Json.Converters;
+using System.Text.Json.Serialization;
 
 namespace Diagnostics.ModelsAndUtils.Models.ResponseExtensions
 {
@@ -44,7 +43,7 @@ namespace Diagnostics.ModelsAndUtils.Models.ResponseExtensions
         /// <summary>
         /// Spicfy the status(Critical,Warning,Info,Success,None) shown as icon in middle left card
         /// </summary>
-        [JsonConverter(typeof(StringEnumConverter))]
+        [JsonConverter(typeof(JsonStringEnumConverter))]
         public SummaryCardStatus Status { set; get; }
 
         /// <summary>
@@ -55,7 +54,7 @@ namespace Diagnostics.ModelsAndUtils.Models.ResponseExtensions
         /// <summary>
         /// Spicfy the Action Type(Detector,Tool)
         /// </summary>
-        [JsonConverter(typeof(StringEnumConverter))]
+        [JsonConverter(typeof(JsonStringEnumConverter))]
         public SummaryCardActionType OnClickActionType { set; get; }
 
         /// <summary>

src/Diagnostics.RuntimeHost/Diagnostics.RuntimeHost.csproj

@@ -27,6 +27,7 @@
     <PackageReference Include="Microsoft.TeamFoundationServer.Client" Version="16.170.0" />
     <PackageReference Include="Microsoft.VisualStudio.Services.Client" Version="16.170.0" />
     <PackageReference Include="Octokit" Version="0.47.0" />
+    <PackageReference Include="System.Text.Json" Version="6.0.0" />
     <PackageReference Include="WindowsAzure.Storage" Version="9.3.3" />
     <PackageReference Include="Microsoft.Extensions.Logging.AzureAppServices" Version="3.1.9" />
   </ItemGroup>

src/Diagnostics.RuntimeHost/Services/CacheService/DiagEntityTableCacheService.cs

@@ -42,8 +42,8 @@ namespace Diagnostics.RuntimeHost.Services.CacheService
                 DiagnosticsETWProvider.Instance.LogAzureStorageMessage(nameof(DiagEntityTableCacheService), "Start polling Azure Storage for refreshing cache");
                 try
                 {
-                    var detectorTask = storageService.GetEntitiesByPartitionkey("Detector", startUp ? DateTime.MinValue : DateTime.UtcNow.AddMinutes(-5));
-                    var gistTask = storageService.GetEntitiesByPartitionkey("Gist", startUp ? DateTime.MinValue : DateTime.UtcNow.AddMinutes(-5));
+                    var detectorTask = storageService.GetEntitiesByPartitionkey("Detector", startUp ? DateTime.SpecifyKind(DateTime.MinValue, DateTimeKind.Utc) : DateTime.UtcNow.AddMinutes(-5));
+                    var gistTask = storageService.GetEntitiesByPartitionkey("Gist", startUp ? DateTime.SpecifyKind(DateTime.MinValue, DateTimeKind.Utc) : DateTime.UtcNow.AddMinutes(-5));
                     await Task.WhenAll(new Task[] { detectorTask, gistTask });
                     var detectorResult = await detectorTask;
                     if (startUp)

src/Diagnostics.RuntimeHost/Services/SourceWatcher/Watchers/StorageWatcher.cs

@@ -206,12 +206,12 @@ namespace Diagnostics.RuntimeHost.Services.SourceWatcher.Watchers
                 var timeRange = DateTime.UtcNow.AddMinutes(-5);
                 if(!diagEntityTableCacheService.TryGetValue("Detector", out List<DiagEntity> detectorsList) || detectorsList == null || detectorsList.Count < 1)
                 {
-                    detectorsList = await storageService.GetEntitiesByPartitionkey("Detector", startup ? DateTime.MinValue : timeRange);
+                    detectorsList = await storageService.GetEntitiesByPartitionkey("Detector", startup ? DateTime.SpecifyKind(DateTime.MinValue, DateTimeKind.Utc) : timeRange);
                 }
                 var gists = new List<DiagEntity>();
                 if (!LoadOnlyPublicDetectors && (!diagEntityTableCacheService.TryGetValue("Gist", out gists) || gists == null || gists.Count <1))
                 {
-                    gists = await storageService.GetEntitiesByPartitionkey("Gist", startup ? DateTime.MinValue : timeRange);
+                    gists = await storageService.GetEntitiesByPartitionkey("Gist", startup ? DateTime.SpecifyKind(DateTime.MinValue, DateTimeKind.Utc) : timeRange);
                 } 
                 var filteredDetectors = LoadOnlyPublicDetectors ? detectorsList.Where(row => !row.IsInternal).ToList() : detectorsList;
                 if(startup)

src/Diagnostics.RuntimeHost/Services/StorageService/StorageService.cs

@@ -99,9 +99,9 @@ namespace Diagnostics.RuntimeHost.Services.StorageService
                         partitionKey = "Detector";
                     }
                     var filterPartitionKey = TableQuery.GenerateFilterCondition(PartitionKey, QueryComparisons.Equal, partitionKey);
-                    DateTime timeFilter = startTime ?? DateTime.MinValue;
+                    DateTime timeFilter = startTime ?? DateTime.SpecifyKind(DateTime.MinValue, DateTimeKind.Utc);
                     string timestampFilter = TableQuery.GenerateFilterConditionForDate("Timestamp", QueryComparisons.GreaterThanOrEqual, new DateTimeOffset(timeFilter));
-                    string finalFilter = timeFilter.Equals(DateTime.MinValue) ? filterPartitionKey : TableQuery.CombineFilters(filterPartitionKey, TableOperators.And, timestampFilter);
+                    string finalFilter = timeFilter.Equals(DateTime.SpecifyKind(DateTime.MinValue, DateTimeKind.Utc)) ? filterPartitionKey : TableQuery.CombineFilters(filterPartitionKey, TableOperators.And, timestampFilter);
                     var tableQuery = new TableQuery<DiagEntity>();
                     tableQuery.Where(finalFilter);
                     TableContinuationToken tableContinuationToken = null;
@@ -129,7 +129,7 @@ namespace Diagnostics.RuntimeHost.Services.StorageService
                     } while (tableContinuationToken != null);
                     timeTakenStopWatch.Stop();
                     DiagnosticsETWProvider.Instance.LogAzureStorageMessage(nameof(StorageService), $"GetEntities by Partition key {partitionKey} took {timeTakenStopWatch.ElapsedMilliseconds}, Total rows = {detectorsResult.Count}, ClientRequestId = {clientRequestId} ");
-                     return startTime == DateTime.MinValue ? detectorsResult.Where(result => !result.IsDisabled).ToList() :
+                     return startTime == DateTime.SpecifyKind(DateTime.MinValue, DateTimeKind.Utc) ? detectorsResult.Where(result => !result.IsDisabled).ToList() :
                         detectorsResult.ToList();
                 }
                 catch (Exception ex)

src/Diagnostics.RuntimeHost/Startup.cs

@@ -32,6 +32,7 @@ using Diagnostics.Logger;
 using Microsoft.Extensions.Hosting;
 using Diagnostics.RuntimeHost.Services.DiagnosticsTranslator;
 using Diagnostics.RuntimeHost.Services.DevOpsClient;
+using System.Text.Json.Serialization;
 
 namespace Diagnostics.RuntimeHost
 {
@@ -145,9 +146,9 @@ namespace Diagnostics.RuntimeHost
                 services.AddControllers(options =>
                 {
                     options.Filters.Add<AllowAnonymousFilter>();
-                }).AddNewtonsoftJson(options =>
+                }).AddJsonOptions(options =>
                 {
-                    options.SerializerSettings.Formatting = Formatting.Indented;
+                    options.JsonSerializerOptions.WriteIndented = true;
                 });
             }
             else
@@ -155,10 +156,10 @@ namespace Diagnostics.RuntimeHost
                 services.AddControllers().AddJsonOptions(options =>
                 {
                     options.JsonSerializerOptions.WriteIndented = true;
-                }).AddNewtonsoftJson(options =>
+                }).AddJsonOptions(options =>
                 {
-                    options.SerializerSettings.Formatting = Formatting.Indented;
-                }); ;
+                    options.JsonSerializerOptions.WriteIndented = true;
+                });
             }
 
             services.AddSingleton<IDataSourcesConfigurationService, DataSourcesConfigurationService>();

tests/Diagnostics.Tests/UtilitiesTests/DataTableUtilityTests.cs

@@ -30,8 +30,8 @@ namespace Diagnostics.Tests
             Assert.Equal("DateTime", columns[1].DataType);
             Assert.Equal("Int32", columns[2].DataType);
 
-            Assert.Equal<int>(1, convertedTable.Rows.GetLength(0));
-            Assert.Equal<int>(3, convertedTable.Rows.GetLength(1));
+            Assert.Single(convertedTable.Rows);
+            Assert.Equal<int>(3, convertedTable.Rows[0].Length);
         }
 
         [Fact]