diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/AzureVmBenchmark/system/cloud-init.txt b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/AzureVmBenchmark/system/cloud-init.txt index 6821dcaa1..b6a0419fd 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/AzureVmBenchmark/system/cloud-init.txt +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/AzureVmBenchmark/system/cloud-init.txt @@ -4,9 +4,6 @@ packages: - azure-cli runcmd: - - wget https://aka.ms/downloadazcopy-v10-linux - - tar -xvf downloadazcopy-v10-linux - - sudo cp ./azcopy_linux_amd64_*/azcopy /usr/bin/ - wget https://packages.microsoft.com/config/ubuntu/18.04/packages-microsoft-prod.deb - sudo dpkg -i packages-microsoft-prod.deb - sudo apt update diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/BenchmarkLatencyEventSource.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/BenchmarkLatencyEventSource.cs index 5a945d21d..6b1b6d03d 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/BenchmarkLatencyEventSource.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/BenchmarkLatencyEventSource.cs @@ -45,5 +45,17 @@ namespace CosmosBenchmark this.WriteEvent(1, dbName, containerName, durationInMs, lazyDiagnostics()); } } + + [Event(2, Level = EventLevel.Informational)] + public void OnOperationSuccess(int operationType, double durationInMs) + { + this.WriteEvent(2, operationType, durationInMs); + } + + [Event(3, Level = EventLevel.Informational)] + public void OnOperationFailure(int operationType, double durationInMs) + { + this.WriteEvent(3, operationType, durationInMs); + } } } diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/CosmosBenchmarkEventListener.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/CosmosBenchmarkEventListener.cs new file mode 100644 index 000000000..d12cb6c47 --- /dev/null +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/CosmosBenchmarkEventListener.cs @@ -0,0 +1,92 @@ +namespace CosmosBenchmark +{ + using System; + using System.Diagnostics.Tracing; + using System.Threading; + using System.Threading.Tasks; + using OpenTelemetry.Metrics; + + internal class CosmosBenchmarkEventListener : EventListener + { + static readonly string CosmosBenchmarkEventSourceName = "Azure.Cosmos.Benchmark"; + + private readonly MeterProvider meterProvider; + private readonly MetricsCollector[] metricsCollectors; + private readonly MetricCollectionWindow metricCollectionWindow; + private const int WindowCheckInterval = 10; + + public CosmosBenchmarkEventListener(MeterProvider meterProvider, BenchmarkConfig config) + { + this.meterProvider = meterProvider; + this.metricCollectionWindow ??= new MetricCollectionWindow(config.MetricsReportingIntervalInSec); + + this.metricsCollectors = new MetricsCollector[Enum.GetValues().Length]; + foreach (BenchmarkOperationType entry in Enum.GetValues()) + { + this.metricsCollectors[(int)entry] = new MetricsCollector(entry); + } + + /// + /// Flush metrics every + /// + ThreadPool.QueueUserWorkItem(async state => + { + while (true) + { + // Reset metricCollectionWindow and flush. + if (this.metricCollectionWindow.IsInvalid()) + { + this.meterProvider.ForceFlush(); + this.metricCollectionWindow.Reset(); + } + await Task.Delay(TimeSpan.FromMilliseconds(CosmosBenchmarkEventListener.WindowCheckInterval)); + } + }); + } + + + + /// + /// Override this method to get a list of all the eventSources that exist. + /// + protected override void OnEventSourceCreated(EventSource eventSource) + { + // Because we want to turn on every EventSource, we subscribe to a callback that triggers + // when new EventSources are created. It is also fired when the EventListener is created + // for all pre-existing EventSources. Thus this callback get called once for every + // EventSource regardless of the order of EventSource and EventListener creation. + + // For any EventSource we learn about, turn it on. + if (eventSource.Name == CosmosBenchmarkEventSourceName) + { + this.EnableEvents(eventSource, EventLevel.Informational, EventKeywords.All); + } + } + + /// + /// We override this method to get a callback on every event we subscribed to with EnableEvents + /// + /// + protected override void OnEventWritten(EventWrittenEventArgs eventData) + { + if (eventData.EventId == 2 // Successful + || eventData.EventId == 3) // Failure + { + int operationTypeIndex = (int)eventData.Payload[0]; + double durationInMs = (double)eventData.Payload[1]; + + switch (eventData.EventId) + { + case 2: + this.metricsCollectors[operationTypeIndex].OnOperationSuccess(durationInMs); + break; + case 3: + this.metricsCollectors[operationTypeIndex].OnOperationFailure(durationInMs); + break; + default: + break; + } + } + } + } +} diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/DiagnosticDataListener.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/DiagnosticDataListener.cs index 1171a05fa..fe7da4a2e 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/DiagnosticDataListener.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/DiagnosticDataListener.cs @@ -131,13 +131,16 @@ namespace CosmosBenchmark.Fx /// An instance of containing the request latency and diagnostics. protected override void OnEventWritten(EventWrittenEventArgs eventData) { - try + if (eventData.EventId == 1) { - this.Writer.WriteLine($"{eventData.Payload[2]} ; {eventData.Payload[3]}"); - } - catch (Exception ex) - { - Utility.TraceError("An exception ocured while writing diagnostic data to the file", ex); + try + { + this.Writer.WriteLine($"{eventData.Payload[2]} ; {eventData.Payload[3]}"); + } + catch (Exception ex) + { + Utility.TraceError("An exception ocured while writing diagnostic data to the file", ex); + } } } diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/IExecutionStrategy.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/IExecutionStrategy.cs index ed1400026..e51743941 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/IExecutionStrategy.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/IExecutionStrategy.cs @@ -20,7 +20,6 @@ namespace CosmosBenchmark BenchmarkConfig benchmarkConfig, int serialExecutorConcurrency, int serialExecutorIterationCount, - double warmupFraction, - MetricsCollectorProvider metricsCollectorProvider); + double warmupFraction); } } diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/IExecutor.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/IExecutor.cs index 235a54199..c4d1c765e 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/IExecutor.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/IExecutor.cs @@ -18,7 +18,6 @@ namespace CosmosBenchmark bool isWarmup, bool traceFailures, Action completionCallback, - BenchmarkConfig benchmarkConfig, - MetricsCollectorProvider metricsCollectorProvider); + BenchmarkConfig benchmarkConfig); } } diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/IMetricsCollector.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/IMetricsCollector.cs index 58af9d0b1..fa5558c31 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/IMetricsCollector.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/IMetricsCollector.cs @@ -12,25 +12,13 @@ namespace CosmosBenchmark public interface IMetricsCollector { /// - /// Collects the number of successful operations. + /// Successful operation with latency /// - void CollectMetricsOnSuccess(); + void OnOperationSuccess(double operationLatencyInMs); /// - /// Collects the number of failed operations. + /// Failed operation with latency /// - void CollectMetricsOnFailure(); - - /// - /// Records latency for success operations in milliseconda. - /// - /// The number of milliseconds to record. - void RecordSuccessOpLatencyAndRps(TimeSpan timeSpan); - - /// - /// Records latency for failed operations in milliseconda. - /// - /// The number of milliseconds to record. - void RecordFailedOpLatencyAndRps(TimeSpan timeSpan); + void OnOperationFailure(double operationLatencyInMs); } } \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/MetricsCollector.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/MetricsCollector.cs index 1676dd005..4f1fc6825 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/MetricsCollector.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/MetricsCollector.cs @@ -5,6 +5,7 @@ namespace CosmosBenchmark { using System; + using System.Collections.Generic; using System.Diagnostics.Metrics; /// @@ -96,45 +97,55 @@ namespace CosmosBenchmark /// Initialize new instance of . /// /// OpenTelemetry meter. - public MetricsCollector(Meter meter, string prefix) + public MetricsCollector(BenchmarkOperationType operationType) { - this.meter = meter; - this.rpsMetricNameHistogram = meter.CreateHistogram($"{prefix}OperationRpsHistogram"); - this.operationLatencyHistogram = meter.CreateHistogram($"{prefix}OperationLatencyInMsHistogram"); + this.meter = new Meter($"CosmosBenchmark{operationType}OperationMeter"); + this.rpsMetricNameHistogram = this.meter.CreateHistogram($"{operationType}OperationRpsHistogram"); + this.operationLatencyHistogram = this.meter.CreateHistogram($"{operationType}OperationLatencyInMsHistogram"); - this.rpsFailedMetricNameHistogram = meter.CreateHistogram($"{prefix}FailedOperationRpsHistogram"); - this.operationFailedLatencyHistogram = meter.CreateHistogram($"{prefix}FailedOperationLatencyInMsHistogram"); + this.rpsFailedMetricNameHistogram = this.meter.CreateHistogram($"{operationType}FailedOperationRpsHistogram"); + this.operationFailedLatencyHistogram = this.meter.CreateHistogram($"{operationType}FailedOperationLatencyInMsHistogram"); - this.successOperationCounter = meter.CreateCounter($"{prefix}OperationSuccess"); - this.failureOperationCounter = meter.CreateCounter($"{prefix}OperationFailure"); + this.successOperationCounter = this.meter.CreateCounter($"{operationType}OperationSuccess"); + this.failureOperationCounter = this.meter.CreateCounter($"{operationType}OperationFailure"); - this.latencyInMsMetricNameGauge = this.meter.CreateObservableGauge($"{prefix}OperationLatencyInMs", + this.latencyInMsMetricNameGauge = this.meter.CreateObservableGauge($"{operationType}OperationLatencyInMs", () => new Measurement(this.latencyInMs)); - this.rpsNameGauge = this.meter.CreateObservableGauge($"{prefix}OperationRps", + this.rpsNameGauge = this.meter.CreateObservableGauge($"{operationType}OperationRps", () => new Measurement(this.rps)); - this.latencyInMsFailedMetricNameGauge = this.meter.CreateObservableGauge($"{prefix}FailedOperationLatencyInMs", - () => new Measurement(this.latencyInMs)); + this.latencyInMsFailedMetricNameGauge = this.meter.CreateObservableGauge($"{operationType}FailedOperationLatencyInMs", + () => new Measurement(this.latencyFailedInMs)); - this.rpsFailedNameGauge = this.meter.CreateObservableGauge($"{prefix}FailedOperationRps", - () => new Measurement(this.rps)); + this.rpsFailedNameGauge = this.meter.CreateObservableGauge($"{operationType}FailedOperationRps", + () => new Measurement(this.rpsFailed)); + } + + internal static IEnumerable GetBenchmarkMeterNames() + { + foreach (BenchmarkOperationType entry in Enum.GetValues()) + { + yield return $"CosmosBenchmark{entry}OperationMeter"; + } } /// - /// Collects the number of successful operations. + /// Successful operation with latency /// - public void CollectMetricsOnSuccess() + public void OnOperationSuccess(double operationLatencyInMs) { this.successOperationCounter.Add(1); + this.RecordSuccessOpLatencyAndRps(operationLatencyInMs); } /// - /// Collects the number of failed operations. + /// Failed operation with latency /// - public void CollectMetricsOnFailure() + public void OnOperationFailure(double operationLatencyInMs) { this.failureOperationCounter.Add(1); + this.RecordFailedOpLatencyAndRps(operationLatencyInMs); } /// @@ -142,10 +153,10 @@ namespace CosmosBenchmark /// /// The number of milliseconds to record. public void RecordSuccessOpLatencyAndRps( - TimeSpan timeSpan) + double operationLatencyInMs) { - this.rps = timeSpan.TotalMilliseconds != 0 ? 1000 / timeSpan.TotalMilliseconds : 0; - this.latencyInMs = timeSpan.TotalMilliseconds; + this.rps = operationLatencyInMs != 0 ? 1000 / operationLatencyInMs : 0; + this.latencyInMs = operationLatencyInMs; this.rpsMetricNameHistogram.Record(this.rps); this.operationLatencyHistogram.Record(this.latencyInMs); } @@ -155,10 +166,10 @@ namespace CosmosBenchmark /// /// The number of milliseconds to record. public void RecordFailedOpLatencyAndRps( - TimeSpan timeSpan) + double operationLatencyInMs) { - this.rpsFailed = timeSpan.TotalMilliseconds != 0 ? 1000 / timeSpan.TotalMilliseconds : 0; - this.latencyFailedInMs = timeSpan.TotalMilliseconds; + this.rpsFailed = operationLatencyInMs != 0 ? 1000 / operationLatencyInMs : 0; + this.latencyFailedInMs = operationLatencyInMs; this.rpsFailedMetricNameHistogram.Record(this.rpsFailed); this.operationFailedLatencyHistogram.Record(this.latencyFailedInMs); } diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/MetricsCollectorProvider.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/MetricsCollectorProvider.cs deleted file mode 100644 index f49d9ec57..000000000 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/MetricsCollectorProvider.cs +++ /dev/null @@ -1,96 +0,0 @@ -//------------------------------------------------------------ -// Copyright (c) Microsoft Corporation. All rights reserved. -//------------------------------------------------------------ - -namespace CosmosBenchmark -{ - using System; - using System.Diagnostics.Metrics; - using System.Threading; - using System.Threading.Tasks; - using OpenTelemetry.Metrics; - - /// - /// Represents the metrics collector provider. - /// - internal class MetricsCollectorProvider - { - private const int WindowCheckInterval = 10; - private MetricCollectionWindow metricCollectionWindow; - - private static readonly object metricCollectionWindowLock = new object(); - - private readonly MetricsCollector insertOperationMetricsCollector; - - private readonly MetricsCollector queryOperationMetricsCollector; - - private readonly MetricsCollector readOperationMetricsCollector; - - private readonly Meter insertOperationMeter = new("CosmosBenchmarkInsertOperationMeter"); - - private readonly Meter queryOperationMeter = new("CosmosBenchmarkQueryOperationMeter"); - - private readonly Meter readOperationMeter = new("CosmosBenchmarkReadOperationMeter"); - - private readonly MeterProvider meterProvider; - - public MetricsCollectorProvider(BenchmarkConfig config, MeterProvider meterProvider) - { - this.meterProvider = meterProvider; - this.insertOperationMetricsCollector ??= new MetricsCollector(this.insertOperationMeter, "Insert"); - this.queryOperationMetricsCollector ??= new MetricsCollector(this.queryOperationMeter, "Query"); - this.readOperationMetricsCollector ??= new MetricsCollector(this.readOperationMeter, "Read"); - this.metricCollectionWindow ??= new MetricCollectionWindow(config); - - /// - /// Flush metrics every - /// - ThreadPool.QueueUserWorkItem(async state => - { - while (true) - { - MetricCollectionWindow metricCollectionWindow = this.GetCurrentMetricCollectionWindow(config); - - // Reset metricCollectionWindow and flush. - if (!metricCollectionWindow.IsValid) - { - this.meterProvider.ForceFlush(); - this.metricCollectionWindow.Reset(config); - } - await Task.Delay(TimeSpan.FromMilliseconds(MetricsCollectorProvider.WindowCheckInterval)); - } - }); - } - - private MetricCollectionWindow GetCurrentMetricCollectionWindow(BenchmarkConfig config) - { - if (this.metricCollectionWindow is null || !this.metricCollectionWindow.IsValid) - { - lock (metricCollectionWindowLock) - { - this.metricCollectionWindow ??= new MetricCollectionWindow(config); - } - } - - return this.metricCollectionWindow; - } - - /// - /// Gets the metric collector. - /// - /// Benchmark operation. - /// Benchmark configuration. - /// Metrics collector. - /// Thrown if provided benchmark operation is not covered supported to collect metrics. - public IMetricsCollector GetMetricsCollector(IBenchmarkOperation benchmarkOperation) - { - return benchmarkOperation.OperationType switch - { - BenchmarkOperationType.Insert => this.insertOperationMetricsCollector, - BenchmarkOperationType.Query => this.queryOperationMetricsCollector, - BenchmarkOperationType.Read => this.readOperationMetricsCollector, - _ => throw new NotSupportedException($"The type of {nameof(benchmarkOperation)} is not supported for collecting metrics."), - }; - } - } -} diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/OperationResult.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/OperationResult.cs index 0e87c33b3..a7f34b8bb 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/OperationResult.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/OperationResult.cs @@ -7,6 +7,7 @@ { public string DatabseName { get; set; } public string ContainerName { get; set; } + public BenchmarkOperationType OperationType { get; set; } public double RuCharges { get; set; } public Func LazyDiagnostics { get; set; } public CosmosDiagnostics CosmosDiagnostics { get; set; } diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/ParallelExecutionStrategy.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/ParallelExecutionStrategy.cs index d32052e3f..018f7f161 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/ParallelExecutionStrategy.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/ParallelExecutionStrategy.cs @@ -29,8 +29,7 @@ namespace CosmosBenchmark BenchmarkConfig benchmarkConfig, int serialExecutorConcurrency, int serialExecutorIterationCount, - double warmupFraction, - MetricsCollectorProvider metricsCollectorProvider) + double warmupFraction) { IExecutor warmupExecutor = new SerialOperationExecutor( executorId: "Warmup", @@ -39,9 +38,8 @@ namespace CosmosBenchmark (int)(serialExecutorIterationCount * warmupFraction), isWarmup: true, traceFailures: benchmarkConfig.TraceFailures, - completionCallback: () => { }, - benchmarkConfig, - metricsCollectorProvider); + completionCallback: () => { }, + benchmarkConfig); Utility.TeePrint("Starting execution {0} tasks", serialExecutorConcurrency); IExecutor[] executors = new IExecutor[serialExecutorConcurrency]; @@ -60,8 +58,7 @@ namespace CosmosBenchmark isWarmup: false, traceFailures: benchmarkConfig.TraceFailures, completionCallback: () => Interlocked.Decrement(ref this.pendingExecutorCount), - benchmarkConfig, - metricsCollectorProvider); + benchmarkConfig); } return await this.LogOutputStats( diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/SerialOperationExecutor.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/SerialOperationExecutor.cs index 3bb84e26f..200a647e7 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/SerialOperationExecutor.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/SerialOperationExecutor.cs @@ -13,7 +13,6 @@ namespace CosmosBenchmark internal class SerialOperationExecutor : IExecutor { private readonly IBenchmarkOperation operation; - private readonly string executorId; public SerialOperationExecutor( @@ -28,7 +27,6 @@ namespace CosmosBenchmark } public int SuccessOperationCount { get; private set; } - public int FailedOperationCount { get; private set; } public double TotalRuCharges { get; private set; } @@ -38,19 +36,15 @@ namespace CosmosBenchmark bool isWarmup, bool traceFailures, Action completionCallback, - BenchmarkConfig benchmarkConfig, - MetricsCollectorProvider metricsCollectorProvider) + BenchmarkConfig benchmarkConfig) { Trace.TraceInformation($"Executor {this.executorId} started"); - Trace.TraceInformation("Initializing counters and metrics."); - try { int currentIterationCount = 0; do { - IMetricsCollector metricsCollector = metricsCollectorProvider.GetMetricsCollector(this.operation); OperationResult? operationResult = null; await this.operation.PrepareAsync(); @@ -58,15 +52,12 @@ namespace CosmosBenchmark using (ITelemetrySpan telemetrySpan = TelemetrySpan.StartNew( benchmarkConfig, () => operationResult.Value, - disableTelemetry: isWarmup, - metricsCollector.RecordSuccessOpLatencyAndRps, - metricsCollector.RecordFailedOpLatencyAndRps)) + disableTelemetry: isWarmup)) { try { operationResult = await this.operation.ExecuteOnceAsync(); - - metricsCollector.CollectMetricsOnSuccess(); + telemetrySpan.MarkSuccess(); // Success case this.SuccessOperationCount++; @@ -79,13 +70,11 @@ namespace CosmosBenchmark } catch (Exception ex) { + telemetrySpan.MarkFailed(); if (traceFailures) { Trace.TraceInformation(ex.ToString()); } - telemetrySpan.MarkFailed(); - - metricsCollector.CollectMetricsOnFailure(); // failure case this.FailedOperationCount++; @@ -123,4 +112,4 @@ namespace CosmosBenchmark } } } -} \ No newline at end of file +} diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/TelemetrySpan.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/TelemetrySpan.cs index 4f8785860..ae03edcfa 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/TelemetrySpan.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/TelemetrySpan.cs @@ -19,8 +19,6 @@ namespace CosmosBenchmark private Stopwatch stopwatch; private Func lazyOperationResult; - private Action recordFailedOpLatencyAction; - private Action recordSuccessOpLatencyAction; private bool disableTelemetry; private bool isFailed = false; private BenchmarkConfig benchmarkConfig; @@ -28,9 +26,7 @@ namespace CosmosBenchmark public static ITelemetrySpan StartNew( BenchmarkConfig benchmarkConfig, Func lazyOperationResult, - bool disableTelemetry, - Action recordSuccessOpLatencyAction, - Action recordFailedOpLatencyAction) + bool disableTelemetry) { if (disableTelemetry || !TelemetrySpan.IncludePercentile) { @@ -42,17 +38,25 @@ namespace CosmosBenchmark benchmarkConfig = benchmarkConfig, stopwatch = Stopwatch.StartNew(), lazyOperationResult = lazyOperationResult, - recordSuccessOpLatencyAction = recordSuccessOpLatencyAction, - recordFailedOpLatencyAction = recordFailedOpLatencyAction, disableTelemetry = disableTelemetry }; } - public void MarkFailed() { this.isFailed = true; } + public void MarkFailed() + { + this.isFailed = true; + this.stopwatch.Stop(); + } + + public void MarkSuccess() + { + this.isFailed = false; + this.stopwatch.Stop(); + } public void Dispose() { - this.stopwatch.Stop(); + this.stopwatch.Stop(); // No-op in-case of MarkFailed or MarkSuccess prior call if (!this.disableTelemetry) { OperationResult operationResult = this.lazyOperationResult(); @@ -61,14 +65,13 @@ namespace CosmosBenchmark { RecordLatency(this.stopwatch.Elapsed.TotalMilliseconds); - if(this.isFailed) + if (this.isFailed) { - this.recordSuccessOpLatencyAction?.Invoke(this.stopwatch.Elapsed); + BenchmarkLatencyEventSource.Instance.OnOperationFailure((int)operationResult.OperationType, this.stopwatch.Elapsed.TotalMilliseconds); } else { - this.recordSuccessOpLatencyAction?.Invoke(this.stopwatch.Elapsed); - + BenchmarkLatencyEventSource.Instance.OnOperationSuccess((int)operationResult.OperationType, this.stopwatch.Elapsed.TotalMilliseconds); } } @@ -76,9 +79,8 @@ namespace CosmosBenchmark operationResult.DatabseName, operationResult.ContainerName, (int)this.stopwatch.ElapsedMilliseconds, - operationResult.LazyDiagnostics, + operationResult.LazyDiagnostics, this.benchmarkConfig.DiagnosticLatencyThresholdInMs); - } } @@ -104,16 +106,6 @@ namespace CosmosBenchmark return MathNet.Numerics.Statistics.Statistics.Percentile(latencyHistogram.Take(latencyIndex + 1), percentile); } - internal static double? GetLatencyQuantile(double quantile) - { - if (TelemetrySpan.latencyHistogram == null) - { - return null; - } - - return MathNet.Numerics.Statistics.Statistics.Quantile(latencyHistogram.Take(latencyIndex + 1), quantile); - } - private class NoOpDisposable : ITelemetrySpan { public static readonly NoOpDisposable Instance = new NoOpDisposable(); @@ -122,13 +114,19 @@ namespace CosmosBenchmark { } + public void MarkSuccess() + { + } + public void MarkFailed() { } } - public interface ITelemetrySpan : IDisposable { + public interface ITelemetrySpan : IDisposable + { + void MarkSuccess(); void MarkFailed(); } } -} +} \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/MetricCollectionWindow.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/MetricCollectionWindow.cs index 948ade075..fabdcfc75 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/MetricCollectionWindow.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/MetricCollectionWindow.cs @@ -11,38 +11,35 @@ namespace CosmosBenchmark /// internal class MetricCollectionWindow { - /// - /// The timestamp when window span is started. - /// - public DateTime Started { get; private set; } + private DateTime ValidTill { get; set; } - /// - /// The timestamp until which the current window span is not elapsed. - /// - public DateTime ValidTill { get; private set; } + private int MetricsReportingIntervalInSec { get; set; } /// /// Creates the instance of . /// /// Cosmos Benchmark configuration. - public MetricCollectionWindow(BenchmarkConfig config) + public MetricCollectionWindow(int metricsReportingIntervalInSec) { - this.Reset(config); + this.MetricsReportingIntervalInSec = metricsReportingIntervalInSec; + this.Reset(); } /// /// Indicates whether the current window is valid. /// - public bool IsValid => DateTime.UtcNow > this.ValidTill; + public bool IsInvalid() + { + return DateTime.UtcNow > this.ValidTill; + } /// /// Resets the started timestamp and valid till timespan. /// /// - public void Reset(BenchmarkConfig config) + public void Reset() { - this.Started = DateTime.UtcNow; - this.ValidTill = this.Started.AddSeconds(config.MetricsReportingIntervalInSec); + this.ValidTill = DateTime.UtcNow.AddSeconds(this.MetricsReportingIntervalInSec); } } } diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Program.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Program.cs index 744d53f2a..d1f8677cc 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Program.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Program.cs @@ -41,8 +41,7 @@ namespace CosmosBenchmark await AddAzureInfoToRunSummary(); MeterProvider meterProvider = BuildMeterProvider(config); - - MetricsCollectorProvider metricsCollectorProvider = new MetricsCollectorProvider(config, meterProvider); + CosmosBenchmarkEventListener listener = new CosmosBenchmarkEventListener(meterProvider, config); ThreadPool.SetMinThreads(config.MinThreadPoolSize, config.MinThreadPoolSize); @@ -62,7 +61,7 @@ namespace CosmosBenchmark Program program = new Program(); - RunSummary runSummary = await program.ExecuteAsync(config, metricsCollectorProvider); + RunSummary runSummary = await program.ExecuteAsync(config); if (!string.IsNullOrEmpty(config.DiagnosticsStorageConnectionString)) { @@ -90,25 +89,28 @@ namespace CosmosBenchmark /// private static MeterProvider BuildMeterProvider(BenchmarkConfig config) { + MeterProviderBuilder meterProviderBuilder = Sdk.CreateMeterProviderBuilder(); if (string.IsNullOrWhiteSpace(config.AppInsightsConnectionString)) { - return Sdk.CreateMeterProviderBuilder() - .AddMeter("CosmosBenchmarkInsertOperationMeter") - .AddMeter("CosmosBenchmarkQueryOperationMeter") - .AddMeter("CosmosBenchmarkReadOperationMeter") - .Build(); + foreach(string benchmarkName in MetricsCollector.GetBenchmarkMeterNames()) + { + meterProviderBuilder = meterProviderBuilder.AddMeter(benchmarkName); + }; + + return meterProviderBuilder.Build(); } OpenTelemetry.Trace.TracerProviderBuilder tracerProviderBuilder = Sdk.CreateTracerProviderBuilder() .AddAzureMonitorTraceExporter(); - return Sdk.CreateMeterProviderBuilder() - .AddAzureMonitorMetricExporter(configure: new Action( - (options) => options.ConnectionString = config.AppInsightsConnectionString)) - .AddMeter("CosmosBenchmarkInsertOperationMeter") - .AddMeter("CosmosBenchmarkQueryOperationMeter") - .AddMeter("CosmosBenchmarkReadOperationMeter") - .Build(); + meterProviderBuilder = meterProviderBuilder.AddAzureMonitorMetricExporter(configure: new Action( + (options) => options.ConnectionString = config.AppInsightsConnectionString)); + foreach (string benchmarkName in MetricsCollector.GetBenchmarkMeterNames()) + { + meterProviderBuilder = meterProviderBuilder.AddMeter(benchmarkName); + }; + + return meterProviderBuilder.Build(); } /// @@ -138,13 +140,11 @@ namespace CosmosBenchmark } } - /// /// Executing benchmarks for V2/V3 cosmosdb SDK. /// /// a Task object. - private async Task ExecuteAsync(BenchmarkConfig config, - MetricsCollectorProvider metricsCollectorProvider) + private async Task ExecuteAsync(BenchmarkConfig config) { // V3 SDK client initialization using (CosmosClient cosmosClient = config.CreateCosmosClient(config.Key)) @@ -200,7 +200,7 @@ namespace CosmosBenchmark } IExecutionStrategy execution = IExecutionStrategy.StartNew(benchmarkOperationFactory); - runSummary = await execution.ExecuteAsync(config, taskCount, opsPerTask, 0.01, metricsCollectorProvider); + runSummary = await execution.ExecuteAsync(config, taskCount, opsPerTask, 0.01); } if (config.CleanupOnFinish) @@ -224,8 +224,8 @@ namespace CosmosBenchmark Utility.TeeTraceInformation("Publishing results"); runSummary.Diagnostics = CosmosDiagnosticsLogger.GetDiagnostics(); await this.PublishResults( - config, - runSummary, + config, + runSummary, cosmosClient); } @@ -234,8 +234,8 @@ namespace CosmosBenchmark } private async Task PublishResults( - BenchmarkConfig config, - RunSummary runSummary, + BenchmarkConfig config, + RunSummary runSummary, CosmosClient benchmarkClient) { if (string.IsNullOrEmpty(config.ResultsEndpoint)) @@ -390,4 +390,4 @@ namespace CosmosBenchmark traceSource.Listeners.Clear(); } } -} \ No newline at end of file +} diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v2/InsertV2BenchmarkOperation.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v2/InsertV2BenchmarkOperation.cs index 124c84108..aafe6d4d4 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v2/InsertV2BenchmarkOperation.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v2/InsertV2BenchmarkOperation.cs @@ -55,6 +55,7 @@ namespace CosmosBenchmark { DatabseName = databsaeName, ContainerName = containerName, + OperationType = this.OperationType, RuCharges = ruCharges, LazyDiagnostics = () => itemResponse.RequestDiagnosticsString, }; diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v2/QueryStreamSinglePkV2BenchmarkOperation.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v2/QueryStreamSinglePkV2BenchmarkOperation.cs index b0f071f6d..222fa5fa3 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v2/QueryStreamSinglePkV2BenchmarkOperation.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v2/QueryStreamSinglePkV2BenchmarkOperation.cs @@ -82,6 +82,7 @@ namespace CosmosBenchmark { DatabseName = databsaeName, ContainerName = containerName, + OperationType = this.OperationType, RuCharges = totalCharge, LazyDiagnostics = lastDiagnostics, }; diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v2/QueryTSinglePkV2BenchmarkOperation.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v2/QueryTSinglePkV2BenchmarkOperation.cs index cd50a789b..7abb881ba 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v2/QueryTSinglePkV2BenchmarkOperation.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v2/QueryTSinglePkV2BenchmarkOperation.cs @@ -90,6 +90,7 @@ namespace CosmosBenchmark { DatabseName = databsaeName, ContainerName = containerName, + OperationType = this.OperationType, RuCharges = totalCharge, LazyDiagnostics = lastDiagnostics, }; diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v2/ReadFeedStreamV2BenchmarkOperation.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v2/ReadFeedStreamV2BenchmarkOperation.cs index 0a9b9e3d7..49e92195c 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v2/ReadFeedStreamV2BenchmarkOperation.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v2/ReadFeedStreamV2BenchmarkOperation.cs @@ -54,6 +54,7 @@ namespace CosmosBenchmark { DatabseName = databsaeName, ContainerName = containerName, + OperationType = this.OperationType, RuCharges = ruCharges, LazyDiagnostics = () => feedResponse.QueryMetrics.ToString(), }; diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v2/ReadNotExistsV2BenchmarkOperation.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v2/ReadNotExistsV2BenchmarkOperation.cs index 1d8b99449..825d10ab6 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v2/ReadNotExistsV2BenchmarkOperation.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v2/ReadNotExistsV2BenchmarkOperation.cs @@ -61,6 +61,7 @@ namespace CosmosBenchmark { DatabseName = databsaeName, ContainerName = containerName, + OperationType = this.OperationType, RuCharges = dce.RequestCharge, LazyDiagnostics = () => dce.ToString(), }; diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v2/ReadStreamExistsV2BenchmarkOperation.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v2/ReadStreamExistsV2BenchmarkOperation.cs index c7419db81..041d9e4ff 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v2/ReadStreamExistsV2BenchmarkOperation.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v2/ReadStreamExistsV2BenchmarkOperation.cs @@ -58,6 +58,7 @@ namespace CosmosBenchmark { DatabseName = databsaeName, ContainerName = containerName, + OperationType = this.OperationType, RuCharges = ruCharges, LazyDiagnostics = () => itemResponse.RequestDiagnosticsString, }; diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v2/ReadTExistsV2BenchmarkOperation.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v2/ReadTExistsV2BenchmarkOperation.cs index 7e161f8ff..7058f07f6 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v2/ReadTExistsV2BenchmarkOperation.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v2/ReadTExistsV2BenchmarkOperation.cs @@ -59,6 +59,7 @@ namespace CosmosBenchmark { DatabseName = databsaeName, ContainerName = containerName, + OperationType = this.OperationType, RuCharges = ruCharges, LazyDiagnostics = () => itemResponse.RequestDiagnosticsString, }; diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v3/InsertV3BenchmarkOperation.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v3/InsertV3BenchmarkOperation.cs index 504148bfd..cb3acc5d7 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v3/InsertV3BenchmarkOperation.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v3/InsertV3BenchmarkOperation.cs @@ -55,6 +55,7 @@ namespace CosmosBenchmark { DatabseName = databaseName, ContainerName = containerName, + OperationType = this.OperationType, RuCharges = ruCharges, CosmosDiagnostics = itemResponse.Diagnostics, LazyDiagnostics = () => itemResponse.Diagnostics.ToString(), diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v3/QueryTV3BenchmarkOperation.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v3/QueryTV3BenchmarkOperation.cs index f78153034..a40a8d083 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v3/QueryTV3BenchmarkOperation.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v3/QueryTV3BenchmarkOperation.cs @@ -108,6 +108,7 @@ namespace CosmosBenchmark { DatabseName = databaseName, ContainerName = containerName, + OperationType = this.OperationType, RuCharges = totalCharge, CosmosDiagnostics = lastDiagnostics, LazyDiagnostics = () => lastDiagnostics?.ToString(), diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v3/ReadFeedStreamV3BenchmarkOperation.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v3/ReadFeedStreamV3BenchmarkOperation.cs index 3b9c5cd80..cc6930545 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v3/ReadFeedStreamV3BenchmarkOperation.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v3/ReadFeedStreamV3BenchmarkOperation.cs @@ -59,6 +59,7 @@ namespace CosmosBenchmark { DatabseName = databsaeName, ContainerName = containerName, + OperationType = this.OperationType, RuCharges = feedResponse.Headers.RequestCharge, CosmosDiagnostics = feedResponse.Diagnostics, LazyDiagnostics = () => feedResponse.Diagnostics.ToString(), diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v3/ReadNotExistsV3BenchmarkOperation.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v3/ReadNotExistsV3BenchmarkOperation.cs index 1804ea985..c3a24f8dc 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v3/ReadNotExistsV3BenchmarkOperation.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v3/ReadNotExistsV3BenchmarkOperation.cs @@ -49,6 +49,7 @@ namespace CosmosBenchmark { DatabseName = databsaeName, ContainerName = containerName, + OperationType = this.OperationType, RuCharges = itemResponse.Headers.RequestCharge, CosmosDiagnostics = itemResponse.Diagnostics, LazyDiagnostics = () => itemResponse.Diagnostics.ToString(), diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v3/ReadStreamExistsV3BenchmarkOperation.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v3/ReadStreamExistsV3BenchmarkOperation.cs index fabe8f20b..752a773bd 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v3/ReadStreamExistsV3BenchmarkOperation.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v3/ReadStreamExistsV3BenchmarkOperation.cs @@ -56,6 +56,7 @@ namespace CosmosBenchmark { DatabseName = databsaeName, ContainerName = containerName, + OperationType = this.OperationType, RuCharges = itemResponse.Headers.RequestCharge, CosmosDiagnostics = itemResponse.Diagnostics, LazyDiagnostics = () => itemResponse.Diagnostics.ToString(), diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v3/ReadStreamExistsWithDiagnosticsV3BenchmarkOperation.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v3/ReadStreamExistsWithDiagnosticsV3BenchmarkOperation.cs index 74f2ac67f..5963f04c3 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v3/ReadStreamExistsWithDiagnosticsV3BenchmarkOperation.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v3/ReadStreamExistsWithDiagnosticsV3BenchmarkOperation.cs @@ -62,6 +62,7 @@ namespace CosmosBenchmark { DatabseName = databsaeName, ContainerName = containerName, + OperationType = this.OperationType, RuCharges = itemResponse.Headers.RequestCharge, CosmosDiagnostics = itemResponse.Diagnostics, LazyDiagnostics = () => itemResponse.Diagnostics.ToString(), diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v3/ReadTExistsV3BenchmarkOperation.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v3/ReadTExistsV3BenchmarkOperation.cs index bb6295356..1a1feccb8 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v3/ReadTExistsV3BenchmarkOperation.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v3/ReadTExistsV3BenchmarkOperation.cs @@ -55,6 +55,7 @@ namespace CosmosBenchmark { DatabseName = databsaeName, ContainerName = containerName, + OperationType = this.OperationType, RuCharges = itemResponse.Headers.RequestCharge, CosmosDiagnostics = itemResponse.Diagnostics, LazyDiagnostics = () => itemResponse.Diagnostics.ToString(),