FaultInjection: Adds SendDelay ServerErrorType (#4447)

* Initial Commit

* nits

Microsoft.Azure.Cosmos.Samples/Tools/FaultInjection/src/FaultInjectionServerErrorResultBuilder.cs

@@ -42,15 +42,17 @@ namespace Microsoft.Azure.Cosmos.FaultInjection
         /// 
         /// Only used RESPONSE_DELAY and CONNECTION_DELAY.
         /// 
-        /// For RESPONSE_DELAY, it is the delay added before the response.
-        /// For CONNECTION_DELAY, it is the delay added before the connection is established.
+        /// For <see cref="FaultInjectionServerErrorType.SendDelay"/>, it is the delay added before the request is sent.
+        /// For <see cref="FaultInjectionServerErrorType.ResponseDelay"/>, it is the delay added after the response is recieved.
+        /// For <see cref="FaultInjectionServerErrorType.ConnectionDelay"/>, it is the delay added before the connection is established.
         /// 
         /// </summary>
         /// <param name="delay">The duration of the delay.</param>
         /// <returns>The current <see cref="FaultInjectionServerErrorResultBuilder"/>.</returns>
         public FaultInjectionServerErrorResultBuilder WithDelay(TimeSpan delay)
         {
-            if (this.serverErrorType == FaultInjectionServerErrorType.ResponseDelay 
+            if ( this.serverErrorType == FaultInjectionServerErrorType.SendDelay
+                || this.serverErrorType == FaultInjectionServerErrorType.ResponseDelay 
                 || this.serverErrorType == FaultInjectionServerErrorType.ConnectionDelay)
             {
                 this.delay = delay;

Microsoft.Azure.Cosmos.Samples/Tools/FaultInjection/src/FaultInjectionServerErrorType.cs

@@ -50,9 +50,16 @@ namespace Microsoft.Azure.Cosmos.FaultInjection
 
         /// <summary>
         /// Used to simulate a transient timeout/broken connection when over request timeout
+        /// In this case, the request will be sent to the server before the delay
         /// </summary>
         ResponseDelay,
 
+        /// <summary>
+        /// Used to simulate a transient timeout/broken connection when over request timeout
+        /// In this case, the delay will occur before the request is sent to the server
+        /// </summary>
+        SendDelay,
+
         /// <summary>
         ///  Used to simulate hight channel acquisiton. 
         ///  When over a connection timeouts can simulate connectionTimeoutException

Microsoft.Azure.Cosmos.Samples/Tools/FaultInjection/src/implementataion/ChaosInterceptor.cs

@@ -154,15 +154,15 @@ namespace Microsoft.Azure.Cosmos.FaultInjection
         /// <param name="args"></param>
         public async Task OnBeforeConnectionWriteAsync(ChannelCallArguments args)
         {
-            FaultInjectionServerErrorRule? serverResponseDelayRule = this.ruleStore?.FindRntbdServerResponseDelayRule(args);
+            FaultInjectionServerErrorRule? serverSendDelayRule = this.ruleStore?.FindRntbdServerSendDelayRule(args);
                        
-            if (serverResponseDelayRule != null)
+            if (serverSendDelayRule != null)
             {
-                this.applicationContext.AddRuleExecution(serverResponseDelayRule.GetId(), args.CommonArguments.ActivityId);
-                TimeSpan delay = serverResponseDelayRule.GetDelay();
+                this.applicationContext.AddRuleExecution(serverSendDelayRule.GetId(), args.CommonArguments.ActivityId);
+                TimeSpan delay = serverSendDelayRule.GetDelay();
 
                 DefaultTrace.TraceInformation("FaultInjection: FaultInjection Rule {0} Inserted {1} duration response delay for request {2}",
-                                    serverResponseDelayRule.GetId(), delay, args.CommonArguments.ActivityId);
+                                    serverSendDelayRule.GetId(), delay, args.CommonArguments.ActivityId);
 
                 await Task.Delay(delay);
             }

Microsoft.Azure.Cosmos.Samples/Tools/FaultInjection/src/implementataion/FaultInjectionRuleStore.cs

@@ -17,6 +17,7 @@ namespace Microsoft.Azure.Cosmos.FaultInjection
     internal class FaultInjectionRuleStore
     {
         private readonly ConcurrentDictionary<FaultInjectionServerErrorRule, byte> serverResponseDelayRuleSet = new ConcurrentDictionary<FaultInjectionServerErrorRule, byte>();
+        private readonly ConcurrentDictionary<FaultInjectionServerErrorRule, byte> serverSendDelayRuleSet = new ConcurrentDictionary<FaultInjectionServerErrorRule, byte>();
         private readonly ConcurrentDictionary<FaultInjectionServerErrorRule, byte> serverResponseErrorRuleSet = new ConcurrentDictionary<FaultInjectionServerErrorRule, byte>();
         private readonly ConcurrentDictionary<FaultInjectionServerErrorRule, byte> serverConnectionDelayRuleSet = new ConcurrentDictionary<FaultInjectionServerErrorRule, byte>();
         private readonly ConcurrentDictionary<FaultInjectionConnectionErrorRule, byte> connectionErrorRuleSet = new ConcurrentDictionary<FaultInjectionConnectionErrorRule, byte>();
@@ -79,6 +80,9 @@ namespace Microsoft.Azure.Cosmos.FaultInjection
                     case FaultInjectionServerErrorType.ResponseDelay:
                         this.serverResponseDelayRuleSet.TryAdd(serverErrorRule, 0);
                         break;
+                    case FaultInjectionServerErrorType.SendDelay:
+                        this.serverSendDelayRuleSet.TryAdd(serverErrorRule, 0);
+                        break;
                     case FaultInjectionServerErrorType.ConnectionDelay:
                         this.serverConnectionDelayRuleSet.TryAdd(serverErrorRule, 0);
                         break;
@@ -121,6 +125,22 @@ namespace Microsoft.Azure.Cosmos.FaultInjection
             return null;
         }
 
+        public FaultInjectionServerErrorRule? FindRntbdServerSendDelayRule(ChannelCallArguments args)
+        {
+            foreach (FaultInjectionServerErrorRule rule in this.serverSendDelayRuleSet.Keys)
+            {
+                if ((rule.GetConnectionType() == FaultInjectionConnectionType.Direct
+                    || rule.GetConnectionType() == FaultInjectionConnectionType.All)
+                    && rule.IsApplicable(args))
+                {
+                    return rule;
+                }
+            }
+
+            return null;
+        }
+
+
         public FaultInjectionServerErrorRule? FindRntbdServerConnectionDelayRule(
             Uri callUri, 
             DocumentServiceRequest request,

Microsoft.Azure.Cosmos.Samples/Tools/FaultInjection/tests/FaultInjectionDirectModeTests.cs

@@ -49,6 +49,7 @@ namespace Microsoft.Azure.Cosmos.FaultInjection.Tests
                 PartitionKeyPath = "/Pk"
             };
             this.container = await this.database.CreateContainerIfNotExistsAsync(containerProperties, 5000);
+            await Task.Delay(5000);
         }
 
         public async Task InitilizePreferredRegionsClient(FaultInjector faultInjector, List<string> preferredRegionList, bool multiRegion)
@@ -61,6 +62,7 @@ namespace Microsoft.Azure.Cosmos.FaultInjection.Tests
                 Id = "test",
                 PartitionKeyPath = "/Pk"
             };
+            await Task.Delay(5000);
             this.container = await this.database.CreateContainerIfNotExistsAsync(containerProperties, 5000);
         }
 
@@ -566,7 +568,90 @@ namespace Microsoft.Azure.Cosmos.FaultInjection.Tests
 
         [TestMethod]
         [Owner("nalutripician")]
-        [Description("Tests response delay")]
+        [Description("Tests send delay")]
+        public void FaultInjectionServerErrorRule_ServerSendDelay()
+        {
+            if (!this.Timeout_FaultInjectionServerErrorRule_ServerSendDelay().Wait(Timeout))
+            {
+                Assert.Fail("Test timed out");
+            }
+        }
+
+        private async Task Timeout_FaultInjectionServerErrorRule_ServerSendDelay()
+        {
+            string sendDelayRuleId = "sendDelayRule-" + Guid.NewGuid().ToString();
+            FaultInjectionRule delayRule = new FaultInjectionRuleBuilder(
+                id: sendDelayRuleId,
+                condition:
+                    new FaultInjectionConditionBuilder()
+                        .WithOperationType(FaultInjectionOperationType.CreateItem)
+                        .Build(),
+                result:
+                    FaultInjectionResultBuilder.GetResultBuilder(FaultInjectionServerErrorType.SendDelay)
+                        .WithDelay(TimeSpan.FromSeconds(10))
+                        .WithTimes(1)
+                        .Build())
+                .WithDuration(TimeSpan.FromMinutes(5))
+                .Build();
+
+            delayRule.Disable();
+
+            await this.Initialize(true);
+
+            try
+            {
+                FaultInjector faultInjector = new FaultInjector(new List<FaultInjectionRule> { delayRule });
+
+                CosmosClient testClient = new CosmosClient(
+                    accountEndpoint: TestCommon.EndpointMultiRegion,
+                    authKeyOrResourceToken: TestCommon.AuthKeyMultiRegion,
+                    clientOptions: faultInjector.GetFaultInjectionClientOptions(
+                        new CosmosClientOptions()
+                        {
+                            EnableContentResponseOnWrite = true,
+                            ConnectionMode = ConnectionMode.Direct,
+                            OpenTcpConnectionTimeout = TimeSpan.FromSeconds(1)
+                        }));
+
+                Container testContainer = testClient.GetContainer("testDb", "test");
+                delayRule.Enable();
+                ValueStopwatch stopwatch = ValueStopwatch.StartNew();
+                TimeSpan elapsed;
+
+                JObject createdItem = JObject.FromObject(new { id = Guid.NewGuid().ToString(), Pk = Guid.NewGuid().ToString() });
+                CosmosDiagnostics createDiagnostics = await this.PerformDocumentOperation(
+                    testContainer, 
+                    OperationType.Create, 
+                    createdItem);
+
+                elapsed = stopwatch.Elapsed;
+                stopwatch.Stop();
+                delayRule.Disable();
+
+                CosmosDiagnostics readDiagnostics = await this.PerformDocumentOperation(
+                    testContainer, 
+                    OperationType.Read, 
+                    createdItem);
+
+                Assert.IsTrue(readDiagnostics.ToString().Contains("404"));
+                Assert.IsTrue(elapsed.TotalSeconds >= 6);
+                this.ValidateHitCount(delayRule, 1);
+                this.ValidateFaultInjectionRuleApplication(
+                    createDiagnostics,
+                    (int)StatusCodes.RequestTimeout,
+                    (int)SubStatusCodes.Unknown,
+                    delayRule);
+                testClient.Dispose();
+            }
+            finally
+            {
+                delayRule.Disable();
+            }
+        }
+
+        [TestMethod]
+        [Owner("nalutripician")]
+        [Description("Tests send delay")]
         public void FaultInjectionServerErrorRule_ServerResponseDelay()
         {
             if (!this.Timeout_FaultInjectionServerErrorRule_ServerResponseDelay().Wait(Timeout))
@@ -576,6 +661,89 @@ namespace Microsoft.Azure.Cosmos.FaultInjection.Tests
         }
 
         private async Task Timeout_FaultInjectionServerErrorRule_ServerResponseDelay()
+        {
+            string responseDelayRuleId = "responseDelayRule-" + Guid.NewGuid().ToString();
+            FaultInjectionRule delayRule = new FaultInjectionRuleBuilder(
+                id: responseDelayRuleId,
+                condition:
+                    new FaultInjectionConditionBuilder()
+                        .WithOperationType(FaultInjectionOperationType.CreateItem)
+                        .Build(),
+                result:
+                    FaultInjectionResultBuilder.GetResultBuilder(FaultInjectionServerErrorType.ResponseDelay)
+                        .WithDelay(TimeSpan.FromSeconds(10))
+                        .WithTimes(1)
+                        .Build())
+                .WithDuration(TimeSpan.FromMinutes(5))
+                .Build();
+
+            delayRule.Disable();
+
+            await this.Initialize(true);
+
+            try
+            {
+                FaultInjector faultInjector = new FaultInjector(new List<FaultInjectionRule> { delayRule });
+
+                CosmosClient testClient = new CosmosClient(
+                    accountEndpoint: TestCommon.EndpointMultiRegion,
+                    authKeyOrResourceToken: TestCommon.AuthKeyMultiRegion,
+                    clientOptions: faultInjector.GetFaultInjectionClientOptions(
+                        new CosmosClientOptions()
+                        {
+                            EnableContentResponseOnWrite = true,
+                            ConnectionMode = ConnectionMode.Direct,
+                            OpenTcpConnectionTimeout = TimeSpan.FromSeconds(1)
+                        }));
+
+                Container testContainer = testClient.GetContainer("testDb", "test");
+                delayRule.Enable();
+                ValueStopwatch stopwatch = ValueStopwatch.StartNew();
+                TimeSpan elapsed;
+
+                JObject createdItem = JObject.FromObject(new { id = Guid.NewGuid().ToString(), Pk = Guid.NewGuid().ToString() });
+                CosmosDiagnostics createDiagnostics = await this.PerformDocumentOperation(
+                    testContainer,
+                    OperationType.Create,
+                    createdItem);
+
+                elapsed = stopwatch.Elapsed;
+                stopwatch.Stop();
+                delayRule.Disable();
+
+                CosmosDiagnostics readDiagnostics = await this.PerformDocumentOperation(
+                    testContainer,
+                    OperationType.Read,
+                    createdItem);
+
+                Assert.IsTrue(readDiagnostics.ToString().Contains("200"));
+                Assert.IsTrue(elapsed.TotalSeconds >= 6);
+                this.ValidateHitCount(delayRule, 1);
+                this.ValidateFaultInjectionRuleApplication(
+                    createDiagnostics,
+                    (int)StatusCodes.RequestTimeout,
+                    (int)SubStatusCodes.Unknown,
+                    delayRule);
+                testClient.Dispose();
+            }
+            finally
+            {
+                delayRule.Disable();
+            }
+        }
+
+        [TestMethod]
+        [Owner("nalutripician")]
+        [Description("Tests response delay")]
+        public void FaultInjectionServerErrorRule_ServerTimeout()
+        {
+            if (!this.Timeout_FaultInjectionServerErrorRule_ServerTimeout().Wait(Timeout))
+            {
+                Assert.Fail("Test timed out");
+            }
+        }
+
+        private async Task Timeout_FaultInjectionServerErrorRule_ServerTimeout()
         {
             string timeoutRuleId = "timeoutRule-" + Guid.NewGuid().ToString();
             FaultInjectionRule timeoutRule = new FaultInjectionRuleBuilder(
@@ -644,7 +812,6 @@ namespace Microsoft.Azure.Cosmos.FaultInjection.Tests
         }
 
         [TestMethod]
-        [Ignore("Connection Timeouts are currently broken, next Direct release will fix issue")]
         [Owner("nalutripician")]
         [Description("Tests injection a connection timeout")]
         public void FaultInjectionServerErrorRule_ConnectionTimeout()

Microsoft.Azure.Cosmos.Samples/Tools/FaultInjection/tests/Utils/TestCommon.cs

@@ -59,8 +59,13 @@
         {
             CosmosClientBuilder clientBuilder = new CosmosClientBuilder(
                 accountEndpoint: accountEndpointOverride 
-                ?? (multiRegion ? EndpointMultiRegion : Endpoint),
-                authKeyOrResourceToken: multiRegion ? AuthKeyMultiRegion : AuthKey);
+                ?? EndpointMultiRegion,
+                authKeyOrResourceToken: AuthKeyMultiRegion);
+            
+            if (!multiRegion)
+            {
+                return clientBuilder.WithApplicationPreferredRegions(new List<string> { "Central US" });
+            }
 
             return clientBuilder;
         }