Disable continuous heartbeats to main kusto clusters (#629)

* Disable continuous heartbeats to main kusto clusters

* remove comments

* Fix failover unit test

* add mising files

src/Diagnostics.DataProviders/KustoHeartBeatService/KustoHeartBeatService.cs

@@ -159,7 +159,7 @@ namespace Diagnostics.DataProviders
             Exception exceptionForLog = null;
             try
             {
-                var primaryHeartBeat = await _kustoDataProvider.ExecuteQueryForHeartbeat(_configuration.HeartBeatQuery, PrimaryCluster, _configuration.HeartBeatTimeOutInSeconds, activityId);
+                var primaryHeartBeat = await _kustoDataProvider.ExecuteQueryForHeartbeat(_configuration.HeartBeatQuery, PrimaryCluster, _configuration.HeartBeatTimeOutInSeconds, activityId, "PrimaryHealthPing");
 
                 if (primaryHeartBeat.Rows.Count >= 1)
                 {
@@ -182,12 +182,20 @@ namespace Diagnostics.DataProviders
                 _ConsecutiveSuccessCount = 0;
             }
 
-            // if not in failover state
-            //  should failover?
+            /*
+             * Logic of failing over to failover cluster:
+             *  1. If currently Primary cluster is used
+             *  2. If the heartbeat failure count is more than the failure limit
+             *  3. When both #1 and #2 are TRUE, run a sample heartbeat query to failover cluster,
+             *     and if that succeeds, go ahead with the failover. 
+            */
             if (UsePrimaryCluster && _ConsecutiveFailureCount >= _configuration.HeartBeatConsecutiveFailureLimit)
             {
-                UsePrimaryCluster = false;
-            } // else should stop failover
+                // Check if heartbeat query succeeds on failover cluster
+                bool isFailoverHeartbeatSuccessful = await RunHeartBeatFailover(activityId);
+                UsePrimaryCluster = !isFailoverHeartbeatSuccessful;
+            } 
+            // else Stop the failover
             else if (!UsePrimaryCluster && _ConsecutiveSuccessCount >= _configuration.HeartBeatConsecutiveSuccessLimit)
             {
                 UsePrimaryCluster = true;
@@ -196,13 +204,13 @@ namespace Diagnostics.DataProviders
             LogHeartBeatInformation("Primary", primaryHeartBeatSuccess, PrimaryCluster, activityId, UsePrimaryCluster, exceptionForLog);
         }
 
-        private async Task RunHeartBeatFailover(string activityId)
+        private async Task<bool> RunHeartBeatFailover(string activityId)
         {
             bool failoverHeartBeatSuccess = false;
             Exception exceptionForLog = null;
             try
             {
-                var failoverHeartBeat = await _kustoDataProvider.ExecuteQueryForHeartbeat(_configuration.HeartBeatQuery, FailoverCluster, _configuration.HeartBeatTimeOutInSeconds, activityId);
+                var failoverHeartBeat = await _kustoDataProvider.ExecuteQueryForHeartbeat(_configuration.HeartBeatQuery, FailoverCluster, _configuration.HeartBeatTimeOutInSeconds, activityId, "FailoverHealthPing");
 
                 if (failoverHeartBeat.Rows.Count >= 1)
                 {
@@ -215,6 +223,7 @@ namespace Diagnostics.DataProviders
             }
 
             LogHeartBeatInformation("Failover", failoverHeartBeatSuccess, FailoverCluster, activityId, UsePrimaryCluster, exceptionForLog);
+            return failoverHeartBeatSuccess;
         }
 
         public async Task RunHeartBeatTask(CancellationToken cancellationToken)
@@ -222,17 +231,7 @@ namespace Diagnostics.DataProviders
             while (!cancellationToken.IsCancellationRequested) {
                 
                 string activityId = Guid.NewGuid().ToString();
-
-                if (string.IsNullOrWhiteSpace(FailoverCluster))
-                {
-                    await RunHeartBeatPrimary(activityId);
-                }
-                else
-                {
-                    var primaryTask = RunHeartBeatPrimary(activityId);
-                    var failoverTask = RunHeartBeatFailover(activityId);
-                    await Task.WhenAll(new Task[] { primaryTask, failoverTask });
-                }
+                await RunHeartBeatPrimary(activityId);
                 await Task.Delay(TimeSpan.FromSeconds(_configuration.HeartBeatDelayInSeconds), cancellationToken);
             }
         }

src/Diagnostics.DataProviders/MockKustoClient.cs

@@ -8,7 +8,8 @@ namespace Diagnostics.DataProviders
 {
     internal class MockKustoClient : IKustoClient
     {
-        internal static bool ShouldHeartbeatSucceed = false;
+        internal static bool ShouldPrimaryHeartbeatSucceed = false;
+        internal static bool ShouldFailoverHeartbeatSucceed = false;
         internal static int HeartBeatRuns = 0;
 
         public async Task<DataTable> ExecuteQueryAsync(string query, string cluster, string database, int timeoutSeconds, string requestId = null, string operationName = null)
@@ -28,6 +29,22 @@ namespace Diagnostics.DataProviders
                 throw new ArgumentNullException("database");
             }
 
+            switch (query)
+            {
+                case "TestA":
+                    return await GetTestA();
+                case "Heartbeat":
+                    ++HeartBeatRuns;
+                    if ((ShouldPrimaryHeartbeatSucceed && operationName.Equals("PrimaryHealthPing", StringComparison.OrdinalIgnoreCase)) || 
+                        (ShouldFailoverHeartbeatSucceed && operationName.Equals("FailoverHealthPing", StringComparison.OrdinalIgnoreCase)))
+                    //if (ShouldPrimaryHeartbeatSucceed || ShouldFailoverHeartbeatSucceed)
+                    {
+                        return await GetFakeTenantIdResults();
+                    }
+
+                    break;
+            }
+
             if (!string.IsNullOrWhiteSpace(operationName))
             {
                 switch (operationName.ToLower())
@@ -42,18 +59,6 @@ namespace Diagnostics.DataProviders
                 }
             }
 
-            switch (query)
-            {
-                case "TestA":
-                    return await GetTestA();
-                case "Heartbeat":
-                    ++HeartBeatRuns;
-                    if (ShouldHeartbeatSucceed) {
-                        return await GetFakeTenantIdResults();
-                    }
-                    break;
-            }
-
             return new DataTable();
         }
 

tests/Diagnostics.Tests/DataProviderTests/DataProviderTests.cs

@@ -236,28 +236,31 @@ namespace Diagnostics.Tests.DataProviderTests
             config.KustoConfiguration.EnableHeartBeatQuery = true;
             var kustoHeartBeatService = new KustoHeartBeatService(config.KustoConfiguration);
 
-            MockKustoClient.ShouldHeartbeatSucceed = true;
+            MockKustoClient.ShouldPrimaryHeartbeatSucceed = true;
+            MockKustoClient.ShouldFailoverHeartbeatSucceed = false;
             int startingHeartBeatRuns = MockKustoClient.HeartBeatRuns;
             do
             {
-                await Task.Delay(100);
+                await Task.Delay(1000);
             } while (startingHeartBeatRuns == MockKustoClient.HeartBeatRuns);
             Assert.Equal(config.KustoConfiguration.KustoClusterNameGroupings, await kustoHeartBeatService.GetClusterNameFromStamp("waws-prod-mockstamp"));
 
 
-            MockKustoClient.ShouldHeartbeatSucceed = false;
+            MockKustoClient.ShouldPrimaryHeartbeatSucceed = false;
+            MockKustoClient.ShouldFailoverHeartbeatSucceed = true;
             startingHeartBeatRuns = MockKustoClient.HeartBeatRuns;
             do
             {
-                await Task.Delay(100);
+                await Task.Delay(1000);
             } while (startingHeartBeatRuns == MockKustoClient.HeartBeatRuns);
             Assert.Equal(config.KustoConfiguration.KustoClusterFailoverGroupings, await kustoHeartBeatService.GetClusterNameFromStamp("waws-prod-mockstamp"));
 
-            MockKustoClient.ShouldHeartbeatSucceed = true;
+            MockKustoClient.ShouldPrimaryHeartbeatSucceed = true;
+            MockKustoClient.ShouldFailoverHeartbeatSucceed = false;
             startingHeartBeatRuns = MockKustoClient.HeartBeatRuns;
             do
             {
-                await Task.Delay(100);
+                await Task.Delay(1000);
             } while (startingHeartBeatRuns == MockKustoClient.HeartBeatRuns);
             Assert.Equal(config.KustoConfiguration.KustoClusterNameGroupings, await kustoHeartBeatService.GetClusterNameFromStamp("waws-prod-mockstamp"));