Running Item benchmark in Gates (#998)

* Building perf part of gated build * Renaming to ItemBenchmark * Executing the perf tests part of gate (NOT GUARDING THEM) * fixing the perf target file * fixing the perf target file
2019-11-16 02:13:04 +05:30 · 2019-11-16 02:13:04 +05:30 · f9bbe241ff
--- a/Microsoft.Azure.Cosmos.sln
+++ b/Microsoft.Azure.Cosmos.sln
@ -9,6 +9,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Azure.Cosmos.Test
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Azure.Cosmos.EmulatorTests", "Microsoft.Azure.Cosmos\tests\Microsoft.Azure.Cosmos.EmulatorTests\Microsoft.Azure.Cosmos.EmulatorTests.csproj", "{F16D0FDA-36A1-420C-BA6B-BB30FA62B28D}"
 EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Azure.Cosmos.Performance.Tests", "Microsoft.Azure.Cosmos\tests\Microsoft.Azure.Cosmos.Performance.Tests\Microsoft.Azure.Cosmos.Performance.Tests.csproj", "{0392A590-68EF-4283-94D8-33F350BEC9BF}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Cover|Any CPU = Cover|Any CPU
@ -55,6 +57,18 @@ Global
 		{F16D0FDA-36A1-420C-BA6B-BB30FA62B28D}.Release|Any CPU.Build.0 = Release|Any CPU
 		{F16D0FDA-36A1-420C-BA6B-BB30FA62B28D}.Release|x64.ActiveCfg = Release|Any CPU
 		{F16D0FDA-36A1-420C-BA6B-BB30FA62B28D}.Release|x64.Build.0 = Release|Any CPU
+		{0392A590-68EF-4283-94D8-33F350BEC9BF}.Cover|Any CPU.ActiveCfg = Debug|Any CPU
+		{0392A590-68EF-4283-94D8-33F350BEC9BF}.Cover|Any CPU.Build.0 = Debug|Any CPU
+		{0392A590-68EF-4283-94D8-33F350BEC9BF}.Cover|x64.ActiveCfg = Debug|Any CPU
+		{0392A590-68EF-4283-94D8-33F350BEC9BF}.Cover|x64.Build.0 = Debug|Any CPU
+		{0392A590-68EF-4283-94D8-33F350BEC9BF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{0392A590-68EF-4283-94D8-33F350BEC9BF}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{0392A590-68EF-4283-94D8-33F350BEC9BF}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{0392A590-68EF-4283-94D8-33F350BEC9BF}.Debug|x64.Build.0 = Debug|Any CPU
+		{0392A590-68EF-4283-94D8-33F350BEC9BF}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{0392A590-68EF-4283-94D8-33F350BEC9BF}.Release|Any CPU.Build.0 = Release|Any CPU
+		{0392A590-68EF-4283-94D8-33F350BEC9BF}.Release|x64.ActiveCfg = Release|Any CPU
+		{0392A590-68EF-4283-94D8-33F350BEC9BF}.Release|x64.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
--- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Performance.Tests/Benchmarks/DocumentBenchmark.cs
+++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Performance.Tests/Benchmarks/DocumentBenchmark.cs
@ -9,20 +9,17 @@ namespace Microsoft.Azure.Cosmos.Performance.Tests.Benchmarks
    using System.Threading.Tasks;
    using BenchmarkDotNet.Attributes;
    using Microsoft.Azure.Cosmos;
-    using Microsoft.Azure.Cosmos.Performance.Tests.BenchmarkStrategies;
    using Newtonsoft.Json.Linq;

    /// <summary>
    /// Benchmark for Item related operations.
    /// </summary>
-    [MemoryDiagnoser]
-    [NetThroughput(new BenchmarkFrameworks[] { BenchmarkFrameworks.NetFx471, BenchmarkFrameworks.NetCore21 }, maxIterations: 50)]
    public class ItemBenchmark
    {
        private readonly CosmosClient clientForTests;
        private readonly Container container;
        private JObject baseItem;
-        private Stream baseStream;
+        private byte[] payloadBytes;

        /// <summary>
        /// Initializes a new instance of the <see cref="ItemBenchmark"/> class.
@ -32,7 +29,14 @@ namespace Microsoft.Azure.Cosmos.Performance.Tests.Benchmarks
            this.clientForTests = MockDocumentClient.CreateMockCosmosClient();
            this.container = this.clientForTests.GetDatabase("myDB").GetContainer("myColl");
            this.baseItem = JObject.Parse(File.ReadAllText("samplepayload.json"));
-            this.baseStream = File.OpenRead("samplepayload.json");
+            using (FileStream tmp = File.OpenRead("samplepayload.json"))
+            {
+                using (MemoryStream ms = new MemoryStream())
+                {
+                    tmp.CopyTo(ms);
+                    this.payloadBytes = ms.ToArray();
+                }
+            }
        }

        /// <summary>
@ -40,14 +44,19 @@ namespace Microsoft.Azure.Cosmos.Performance.Tests.Benchmarks
        /// </summary>
        /// <returns>A <see cref="Task"/> representing the asynchronous operation.</returns>
        [Benchmark]
-        public async Task InsertItem()
+        public async Task CreateItem()
        {
-            var response = await this.container.CreateItemAsync(
-                this.baseItem,
-                new Cosmos.PartitionKey(Constants.ValidOperationId));
-            if ((int)response.StatusCode > 300 || response.Resource == null)
+            using (MemoryStream ms = new MemoryStream(this.payloadBytes))
            {
-                throw new Exception();
+                using (ResponseMessage response = await this.container.CreateItemStreamAsync(
+                    ms,
+                    new Cosmos.PartitionKey(Constants.ValidOperationId)))
+                {
+                    if ((int)response.StatusCode > 300 || response.Content == null)
+                    {
+                        throw new Exception();
+                    }
+                }
            }
        }

@ -58,10 +67,10 @@ namespace Microsoft.Azure.Cosmos.Performance.Tests.Benchmarks
        [Benchmark]
        public async Task UpsertItem()
        {
-            var response = await this.container.UpsertItemAsync(
-                this.baseItem,
+            ResponseMessage response = await this.container.UpsertItemStreamAsync(
+                new MemoryStream(this.payloadBytes),
                new Cosmos.PartitionKey(Constants.ValidOperationId));
-            if ((int)response.StatusCode > 300 || response.Resource == null)
+            if ((int)response.StatusCode > 300 || response.Content == null)
            {
                throw new Exception();
            }
@ -74,9 +83,9 @@ namespace Microsoft.Azure.Cosmos.Performance.Tests.Benchmarks
        [Benchmark]
        public async Task UpsertItemStream()
        {
-            this.baseStream.Position = 0;
-            var response = await this.container.UpsertItemStreamAsync(
-                    this.baseStream,
+            
+            ResponseMessage response = await this.container.UpsertItemStreamAsync(
+                    new MemoryStream(this.payloadBytes),
                    new Cosmos.PartitionKey(Constants.ValidOperationId));
            if ((int)response.StatusCode > 300 || response.Content.Length == 0)
            {
@ -84,22 +93,6 @@ namespace Microsoft.Azure.Cosmos.Performance.Tests.Benchmarks
            }
        }

-        /// <summary>
-        /// Benchmark for ReadItemAsync.
-        /// </summary>
-        /// <returns>A <see cref="Task"/> representing the asynchronous operation.</returns>
-        [Benchmark]
-        public async Task ReadItem()
-        {
-            var response = await this.container.ReadItemAsync<JObject>(
-                Constants.ValidOperationId,
-                new Cosmos.PartitionKey(Constants.ValidOperationId));
-            if (response.StatusCode == System.Net.HttpStatusCode.NotFound || response.Resource == null)
-            {
-                throw new Exception();
-            }
-        }
-
        /// <summary>
        /// Benchmark for ReadItemAsync.
        /// </summary>
@ -107,14 +100,12 @@ namespace Microsoft.Azure.Cosmos.Performance.Tests.Benchmarks
        [Benchmark]
        public async Task ReadItemNotExists()
        {
-            try
-            {
-                var response = await this.container.ReadItemAsync<JObject>(
-                    Constants.NotFoundOperationId,
-                    new Cosmos.PartitionKey(Constants.ValidOperationId));
-            }
-            catch(CosmosException ex) when (ex.StatusCode == System.Net.HttpStatusCode.NotFound)
+            ResponseMessage response = await this.container.ReadItemStreamAsync(
+                Constants.NotFoundOperationId,
+                new Cosmos.PartitionKey(Constants.ValidOperationId));
+            if (response.StatusCode != System.Net.HttpStatusCode.NotFound)
            {
+                throw new Exception();
            }
        }

@ -125,7 +116,7 @@ namespace Microsoft.Azure.Cosmos.Performance.Tests.Benchmarks
        [Benchmark]
        public async Task ReadItemStream()
        {
-            var response = await this.container.ReadItemStreamAsync(
+            ResponseMessage response = await this.container.ReadItemStreamAsync(
                Constants.ValidOperationId,
                new Cosmos.PartitionKey(Constants.ValidOperationId));
            if (response.StatusCode == System.Net.HttpStatusCode.NotFound || response.Content == null)
@ -141,11 +132,11 @@ namespace Microsoft.Azure.Cosmos.Performance.Tests.Benchmarks
        [Benchmark]
        public async Task UpdateItem()
        {
-            var response = await this.container.ReplaceItemAsync(
-                this.baseItem,
+            ResponseMessage response = await this.container.ReplaceItemStreamAsync(
+                new MemoryStream(this.payloadBytes),
                Constants.ValidOperationId, 
                new Cosmos.PartitionKey(Constants.ValidOperationId));
-            if (response.StatusCode == System.Net.HttpStatusCode.NotFound || response.Resource == null)
+            if (response.StatusCode == System.Net.HttpStatusCode.NotFound || response.Content == null)
            {
                throw new Exception();
            }
@ -158,7 +149,7 @@ namespace Microsoft.Azure.Cosmos.Performance.Tests.Benchmarks
        [Benchmark]
        public async Task DeleteItem()
        {
-            var response = await this.container.DeleteItemAsync<JObject>(
+            ResponseMessage response = await this.container.DeleteItemStreamAsync(
                Constants.ValidOperationId,
                new Cosmos.PartitionKey(Constants.ValidOperationId));
            if (response.StatusCode == System.Net.HttpStatusCode.NotFound)
@ -174,14 +165,12 @@ namespace Microsoft.Azure.Cosmos.Performance.Tests.Benchmarks
        [Benchmark]
        public async Task DeleteItemNotExists()
        {
-            try
-            {
-                var response = await this.container.DeleteItemAsync<JObject>(
-                    Constants.NotFoundOperationId,
-                    new Cosmos.PartitionKey(Constants.ValidOperationId));
-            }
-            catch(CosmosException ex) when (ex.StatusCode == System.Net.HttpStatusCode.NotFound)
+            ResponseMessage response = await this.container.DeleteItemStreamAsync(
+                Constants.NotFoundOperationId,
+                new Cosmos.PartitionKey(Constants.ValidOperationId));
+            if (response.StatusCode != System.Net.HttpStatusCode.NotFound)
            {
+                throw new Exception();
            }
        }
    }
--- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Performance.Tests/Microsoft.Azure.Cosmos.Performance.Tests.csproj
+++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Performance.Tests/Microsoft.Azure.Cosmos.Performance.Tests.csproj
@ -2,18 +2,18 @@

  <PropertyGroup>
    <OutputType>Exe</OutputType>
-    <TargetFrameworks>netcoreapp2.1;net471</TargetFrameworks>
+    <TargetFramework>netcoreapp3.0</TargetFramework>
    <IsPackable>false</IsPackable>
    <GenerateAssemblyInfo>false</GenerateAssemblyInfo>
    <RootNamespace>Microsoft.Azure.Cosmos</RootNamespace>
  </PropertyGroup>

  <ItemGroup>
-    <PackageReference Include="BenchmarkDotNet" Version="0.11.3" />
-    <PackageReference Include="Microsoft.Extensions.Configuration.Binder" Version="2.2.0" />
-    <PackageReference Include="Microsoft.Extensions.Configuration.EnvironmentVariables" Version="2.2.0" />
-    <PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="2.2.0" />
-    <PackageReference Include="Moq" Version="4.10.1" />
+    <PackageReference Include="BenchmarkDotNet" Version="0.12.0" />
+    <PackageReference Include="Microsoft.Extensions.Configuration.Binder" Version="3.0.0" />
+    <PackageReference Include="Microsoft.Extensions.Configuration.EnvironmentVariables" Version="3.0.0" />
+    <PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="3.0.0" />
+    <PackageReference Include="Moq" Version="4.13.1" />
  </ItemGroup>

  <ItemGroup>
@ -32,6 +32,8 @@
  </ItemGroup>

  <PropertyGroup>
-    <SignAssembly>false</SignAssembly>
+    <SignAssembly>true</SignAssembly>
+    <DelaySign>true</DelaySign>
+    <AssemblyOriginatorKeyFile>..\..\..\testkey.snk</AssemblyOriginatorKeyFile>
  </PropertyGroup>
 </Project>
--- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Performance.Tests/Mocks/MockDocumentClient.cs
+++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Performance.Tests/Mocks/MockDocumentClient.cs
@ -63,14 +63,16 @@ namespace Microsoft.Azure.Cosmos.Performance.Tests
            string resourceType,
            string requestVerb,
            INameValueCollection headers,
-            AuthorizationTokenType tokenType) /* unused, use token based upon what is passed in constructor */
+            AuthorizationTokenType tokenType,
+            out string payload) /* unused, use token based upon what is passed in constructor */
        {
+            payload = null;
            return null;
        }

        private void Init()
        {
-            this.collectionCache = new Mock<ClientCollectionCache>(new ServerStoreModel(null), null, null);
+            this.collectionCache = new Mock<ClientCollectionCache>(null, new ServerStoreModel(null), null, null);
            this.collectionCache.Setup
                    (m =>
                        m.ResolveCollectionAsync(
@ -97,13 +99,13 @@ namespace Microsoft.Azure.Cosmos.Performance.Tests

        private void InitStoreModels()
        {
-            this.GatewayStoreModel = GetMockGatewayStoreModel();
+            this.GatewayStoreModel = this.GetMockGatewayStoreModel();

-            var sessionContainer = new SessionContainer("localhost");
-            this.Session = sessionContainer;
+            SessionContainer sessionContainer = new SessionContainer("localhost");
+            this.sessionContainer = sessionContainer;

-            AddressInformation[] addressInformation = GetMockAddressInformation();
-            var mockAddressCache = GetMockAddressCache(addressInformation);
+            AddressInformation[] addressInformation = this.GetMockAddressInformation();
+            Mock<IAddressResolver> mockAddressCache = this.GetMockAddressCache(addressInformation);

            ReplicationPolicy replicationPolicy = new ReplicationPolicy();
            replicationPolicy.MaxReplicaSetSize = 1;
@ -118,7 +120,7 @@ namespace Microsoft.Azure.Cosmos.Performance.Tests
                        mockServiceConfigReader.Object,
                        mockAuthorizationTokenProvider.Object,
                        Protocol.Tcp,
-                        GetMockTransportClient(addressInformation)));
+                        this.GetMockTransportClient(addressInformation)));
        }

        private Mock<IAddressResolver> GetMockAddressCache(AddressInformation[] addressInformation)
--- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Performance.Tests/Mocks/MockRequestHelper.cs
+++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Performance.Tests/Mocks/MockRequestHelper.cs
@ -12,6 +12,18 @@ namespace Microsoft.Azure.Cosmos.Performance.Tests

    internal static class MockRequestHelper
    {
+        private static readonly byte[] testPayload;
+
+        static MockRequestHelper()
+        {
+            MemoryStream ms = new MemoryStream();
+            using (FileStream fs = File.OpenRead("samplepayload.json"))
+            {
+                fs.CopyTo(ms);
+                MockRequestHelper.testPayload = ms.ToArray();
+            }
+        }
+
        /// <summary>
        /// For mocking a Gateway response
        /// </summary>
@ -25,7 +37,7 @@ namespace Microsoft.Azure.Cosmos.Performance.Tests
                if (request.ResourceAddress.EndsWith(Constants.ValidOperationId))
                {
                    response = new DocumentServiceResponse(
-                        File.OpenRead("samplepayload.json"),
+                        new MemoryStream(MockRequestHelper.testPayload),
                        new DictionaryNameValueCollection(),
                        System.Net.HttpStatusCode.OK
                    );
@ -45,7 +57,7 @@ namespace Microsoft.Azure.Cosmos.Performance.Tests
                if (request.ResourceAddress.EndsWith(Tests.Constants.ValidOperationId))
                {
                    response = new DocumentServiceResponse(
-                        File.OpenRead("samplepayload.json"),
+                        new MemoryStream(MockRequestHelper.testPayload),
                        new DictionaryNameValueCollection(),
                        System.Net.HttpStatusCode.OK
                    );
@ -66,7 +78,7 @@ namespace Microsoft.Azure.Cosmos.Performance.Tests
                || request.OperationType == OperationType.Patch)
            {
                response = new DocumentServiceResponse(
-                        File.OpenRead("samplepayload.json"),
+                        new MemoryStream(MockRequestHelper.testPayload),
                        new DictionaryNameValueCollection(),
                        System.Net.HttpStatusCode.OK
                    );
@ -89,7 +101,7 @@ namespace Microsoft.Azure.Cosmos.Performance.Tests
                {
                    response = new StoreResponse()
                    {
-                        ResponseBody = File.OpenRead("samplepayload.json"),
+                        ResponseBody = new MemoryStream(MockRequestHelper.testPayload),
                        Status = (int)System.Net.HttpStatusCode.OK,
                        ResponseHeaderNames = new string[] { WFConstants.BackendHeaders.LSN },
                        ResponseHeaderValues = new string[] { "1" }
@ -113,7 +125,7 @@ namespace Microsoft.Azure.Cosmos.Performance.Tests
                {
                    response = new StoreResponse()
                    {
-                        ResponseBody = File.OpenRead("samplepayload.json"),
+                        ResponseBody = new MemoryStream(MockRequestHelper.testPayload),
                        Status = (int)System.Net.HttpStatusCode.OK,
                        ResponseHeaderNames = Array.Empty<string>(),
                        ResponseHeaderValues = Array.Empty<string>()
@ -138,7 +150,7 @@ namespace Microsoft.Azure.Cosmos.Performance.Tests
            {
                response = new StoreResponse()
                {
-                    ResponseBody = File.OpenRead("samplepayload.json"),
+                    ResponseBody = new MemoryStream(MockRequestHelper.testPayload),
                    Status = (int)System.Net.HttpStatusCode.OK,
                    ResponseHeaderNames = Array.Empty<string>(),
                    ResponseHeaderValues = Array.Empty<string>()
--- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Performance.Tests/Program.cs
+++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Performance.Tests/Program.cs
@ -15,9 +15,13 @@ namespace Microsoft.Azure.Cosmos.Performance.Tests
    {
        static void Main(string[] args)
        {
-            CosmosDBConfiguration environmentConfiguration = ConfigurationService.Configuration;
-            Console.WriteLine($"Starting benchmark and dropping results on {environmentConfiguration.ReportsPath}.");
-            BenchmarkRunner.Run<ItemBenchmark>(new CustomBenchmarkConfiguration(environmentConfiguration));
+            //CosmosDBConfiguration environmentConfiguration = ConfigurationService.Configuration;
+            //Console.WriteLine($"Starting benchmark and dropping results on {environmentConfiguration.ReportsPath}.");
+            //BenchmarkRunner.Run<ItemBenchmark>(new CustomBenchmarkConfiguration(environmentConfiguration));
+
+            BenchmarkSwitcher
+                .FromAssembly(typeof(Program).Assembly)
+                .Run(args);
        }
    }
 }
--- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Performance.Tests/README.md
+++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Performance.Tests/README.md
@ -0,0 +1,3 @@
+Sample usage pattern 
+
+> dotnet run -c Release --framework netcoreapp3.0 -- -j short -f *ReadItemStream -m --allStats --join
--- a/templates/build-test.yml
+++ b/templates/build-test.yml
@ -26,6 +26,26 @@ jobs:
      publishTestResults: true
      nugetConfigPath: NuGet.config
      testRunTitle: Microsoft.Azure.Cosmos.Tests
+      
+- job:
+  displayName: ${{ parameters.BuildConfiguration }} - ${{ parameters.VmImage }} PerformanceTests
+  pool:
+    vmImage: ${{ parameters.VmImage }}
+
+  steps:
+  - checkout: self  # self represents the repo where the initial Pipelines YAML file was found
+    clean: true  # if true, execute `execute git clean -ffdx && git reset --hard HEAD` before fetching
+
+  - task: DotNetCoreCLI@2
+    displayName: Microsoft.Azure.Cosmos.PerformanceTests
+    condition: succeeded()
+    inputs:
+      command: run
+      projects: 'Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Performance.Tests/*.csproj'
+      arguments: --configuration ${{ parameters.BuildConfiguration }} /p:OS=${{ parameters.OS }} -c Release -- -j short -f * -m --allStats --join
+      publishTestResults: true
+      nugetConfigPath: NuGet.config
+      testRunTitle: Microsoft.Azure.Cosmos.PerformanceTests

 - job:
  displayName: ${{ parameters.BuildConfiguration }}-${{ parameters.VmImage }} Microsoft.Azure.Cosmos.EmulatorTests