Internal Gates: Fixes Tests and update pipelines (#2427)

Fixes the read many test which is only used in preview runs
Fixes the pipelines to all use the same windows version. Hoping this might be the cause the perf gates are failing for the preview build but not others.
Updated the perf tests to only update the values if it has a change greater than 2% and to list the tests that exceeded the 5% in a separate list to make it easy to see which ones changed.

Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CosmosReadManyItemsTests.cs

@@ -363,7 +363,7 @@ namespace Microsoft.Azure.Cosmos.SDK.EmulatorTests
         {
             IReadOnlyList<string> pkPaths = new List<string> { "/pk", "/description" };
             ContainerProperties containerSettings = new ContainerProperties(id: Guid.NewGuid().ToString(), partitionKeyPaths: pkPaths);
-            Container container = await this.database.CreateContainerAsync(this.containerSettings);
+            Container container = await this.database.CreateContainerAsync(containerSettings);
 
             for (int i = 0; i < 5; i++)
             {

Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Performance.Tests/Contracts/BenchmarkResults.json

@@ -19,7 +19,7 @@
   "MockedItemBenchmark.ReadFeed;[Type=Stream]": 42546.0,
   "MockedItemBenchmark.ReadItemExists;[Type=OfT]": 41814.0,
   "MockedItemBenchmark.ReadItemExists;[Type=OfTCustom]": 41806.0,
-  "MockedItemBenchmark.ReadItemExists;[Type=OfTWithDiagnosticsToString]": 70752.0,
+  "MockedItemBenchmark.ReadItemExists;[Type=OfTWithDiagnosticsToString]": 74898.0,
   "MockedItemBenchmark.ReadItemExists;[Type=Stream]": 34464.0,
   "MockedItemBenchmark.ReadItemNotExists;[Type=OfT]": 52826.0,
   "MockedItemBenchmark.ReadItemNotExists;[Type=OfTCustom]": 52864.0,

Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Performance.Tests/Contracts/PerformanceValidation.cs

@@ -72,40 +72,60 @@ namespace Microsoft.Azure.Cosmos.Performance.Tests
 
             currentDirectory += PerformanceValidation.DirectoryPath;
 
-            // Always write the updated version. This will change with each run.
-            string currentBenchmarkResults = JsonConvert.SerializeObject(operationToMemoryAllocated, Formatting.Indented);
-            File.WriteAllText(currentDirectory + PerformanceValidation.CurrentBenchmarkResultsFileName, currentBenchmarkResults);
-
             string baselineJson = File.ReadAllText(currentDirectory + PerformanceValidation.BaselineBenchmarkResultsFileName);
             Dictionary<string, double> baselineBenchmarkResults = JsonConvert.DeserializeObject<Dictionary<string, double>>(baselineJson);
 
-            if (baselineBenchmarkResults.Count != operationToMemoryAllocated.Count)
+            List<string> failures = new List<string>();
+            SortedDictionary<string, double> updatedBaseline = new SortedDictionary<string, double>();
+            foreach (KeyValuePair<string, double> currentResult in operationToMemoryAllocated)
             {
-                Console.WriteLine(PerformanceValidation.UpdateMessage + currentBenchmarkResults);
-                return 1;
-            }
-
-            foreach(KeyValuePair<string, double> currentResult in operationToMemoryAllocated)
-            {
-                double baselineResult = baselineBenchmarkResults[currentResult.Key];
+                if(!baselineBenchmarkResults.TryGetValue(
+                    currentResult.Key, 
+                    out double baselineResult))
+                {
+                    updatedBaseline.Add(currentResult.Key, currentResult.Value);
+                    continue;
+                }
 
                 // Add 5% buffer to avoid minor variation between test runs
-                double diff = currentResult.Value - baselineResult;
+                double diff = Math.Abs(currentResult.Value - baselineResult);
                 double maxAllowedDiff = baselineResult * .05;
+                double minDiffToUpdatebaseLine = baselineResult * .02;
                 if (diff > maxAllowedDiff)
                 {
-                    Console.WriteLine(PerformanceValidation.UpdateMessage + currentBenchmarkResults);
-                    return 1;
+                    updatedBaseline.Add(currentResult.Key, currentResult.Value);
+                    failures.Add($"{currentResult.Key}: {currentResult.Value}");
                 }
-                else if (-diff > maxAllowedDiff)
+                else if(diff > minDiffToUpdatebaseLine)
                 {
-                    Console.WriteLine(PerformanceValidation.UpdateMessage + currentBenchmarkResults);
-                    return 1;
+                    // Update the value if it is greater than 2% difference.
+                    // This reduces the noise and make it easier to see which values actually changed
+                    updatedBaseline.Add(currentResult.Key, currentResult.Value);
+                }
+                else
+                {
+                    // Use the baseline if the value didn't change by more than 2% to avoid updating values unnecessarily
+                    // This makes it easier to see which values actually need to be updated.
+                    updatedBaseline.Add(currentResult.Key, baselineResult);
                 }
             }
 
+            // Always write the updated version. This will change with each run.
+            string currentBenchmarkResults = JsonConvert.SerializeObject(updatedBaseline, Formatting.Indented);
+            File.WriteAllText(currentDirectory + PerformanceValidation.CurrentBenchmarkResultsFileName, currentBenchmarkResults);
             Console.WriteLine("Current benchmark results: " + currentBenchmarkResults);
 
+            if (failures.Any())
+            {
+                Console.WriteLine(PerformanceValidation.UpdateMessage);
+                foreach(string failure in failures)
+                {
+                    Console.WriteLine(failure);
+                }
+                
+                return 1;
+            }
+
             return 0;
         }
     }

azure-pipelines-functional.yml

@@ -14,7 +14,7 @@ pr:
 variables:
   DebugArguments: ' --filter "TestCategory!=Quarantine" --verbosity normal '
   ReleaseArguments: ' --filter "TestCategory!=Quarantine" --verbosity normal '
-  VmImage: vs2017-win2016 # https://docs.microsoft.com/en-us/azure/devops/pipelines/agents/hosted?view=azure-devops
+  VmImage: windows-latest # https://docs.microsoft.com/en-us/azure/devops/pipelines/agents/hosted?view=azure-devops 
 
 
 jobs: 

azure-pipelines-nightly.yml

@@ -14,7 +14,7 @@ schedules:
 
 variables:
   ReleaseArguments: ' --filter "TestCategory!=Quarantine" --verbosity normal ' 
-  VmImage: vs2017-win2016 # https://docs.microsoft.com/en-us/azure/devops/pipelines/agents/hosted?view=azure-devops 
+  VmImage: windows-latest # https://docs.microsoft.com/en-us/azure/devops/pipelines/agents/hosted?view=azure-devops 
   BuildConfiguration: Release 
   IsNightly: true 
 

azure-pipelines.yml

@@ -13,7 +13,7 @@ pr:
 variables:
   DebugArguments: ' --filter "TestCategory!=Quarantine & TestCategory!=Functional" --verbosity normal '
   ReleaseArguments: ' --filter "TestCategory!=Quarantine & TestCategory!=Functional" --verbosity normal '
-  VmImage: vs2017-win2016 # https://docs.microsoft.com/en-us/azure/devops/pipelines/agents/hosted?view=azure-devops
+  VmImage: windows-latest # https://docs.microsoft.com/en-us/azure/devops/pipelines/agents/hosted?view=azure-devops
 
 
 jobs: