[Internal] Benchmark tool: Adds Cosmos Benchmark Metrics (#3950)
* Adding metrics for Benchmark tool. * Adding OpenTelemetry. * Revert "Adding OpenTelemetry." This reverts commitc7da088469. * Telemetry for windowed percentiles. * OpenTelemetry, AppInsights and Dashboard. * Removing DiagnosticDataListener. * Code styling, comments and clean-up. * Fixing issues with dashboard. * Fixing positions of charts on the dashboard. * Fixing the dashboard. * Updating titles and subtitles. * Removing ILogger and other not required references. * Fixing code review points. * Fixing issues after rebase. * Removing unnecessary changes. * Fixing code review points. * Adding metrics for Benchmark tool. * Adding OpenTelemetry. * Revert "Adding OpenTelemetry." This reverts commitc7da088469. * Telemetry for windowed percentiles. * OpenTelemetry, AppInsights and Dashboard. * Removing DiagnosticDataListener. * Code styling, comments and clean-up. * Fixing issues with dashboard. * Fixing positions of charts on the dashboard. * Fixing the dashboard. * Updating titles and subtitles. * Removing ILogger and other not required references. * Fixing code review points. * Fixing issues after rebase. * Removing unnecessary changes. * Fixing code review points. * Fixing code review points. * make MetrcisCollectorProvider non static and remove locks * fix * fixes * use static class name TelemetrySpan.IncludePercentile * use app insights connection string * modified: Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Program.cs * modified: Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Program.cs * rename AppInsightsConnectionString * fix * fix comments * fix if AppInsights c string is not set * summary * fix * remove unnecessary collector types * remove unnecesary metere provicer * add event source * remove folder * fix * split success and failed latencies * fix * fix --------- Co-authored-by: David Chaava <chaava01@gmail.com> Co-authored-by: David Chaava <v-dchaava@microsoft.com>
This commit is contained in:
Mikhail Lipin
GitHub
Родитель
e708ec9025
Коммит
946dd4a95f
|
|
@ -6,12 +6,16 @@ namespace CosmosBenchmark
|
|||
{
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics;
|
||||
using System.Linq;
|
||||
using System.Runtime;
|
||||
using CommandLine;
|
||||
using Microsoft.Azure.Documents.Client;
|
||||
using Newtonsoft.Json;
|
||||
|
||||
/// <summary>
|
||||
/// Represents Benchmark Configuration
|
||||
/// </summary>
|
||||
public class BenchmarkConfig
|
||||
{
|
||||
private static readonly string UserAgentSuffix = "cosmosdbdotnetbenchmark";
|
||||
|
|
@ -123,6 +127,12 @@ namespace CosmosBenchmark
|
|||
[Option(Required = false, HelpText = "Container to publish results to")]
|
||||
public string ResultsContainer { get; set; } = "runsummary";
|
||||
|
||||
[Option(Required = false, HelpText = "Metrics reporting interval in seconds")]
|
||||
public int MetricsReportingIntervalInSec { get; set; } = 5;
|
||||
|
||||
[Option(Required = false, HelpText = "Application Insights connection string")]
|
||||
public string AppInsightsConnectionString { get; set; }
|
||||
|
||||
internal int GetTaskCount(int containerThroughput)
|
||||
{
|
||||
int taskCount = this.DegreeOfParallelism;
|
||||
|
|
@ -263,11 +273,11 @@ namespace CosmosBenchmark
|
|||
{
|
||||
foreach (Error e in errors)
|
||||
{
|
||||
Console.WriteLine(e.ToString());
|
||||
Trace.TraceInformation(e.ToString());
|
||||
}
|
||||
}
|
||||
|
||||
Environment.Exit(errors.Count());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -17,6 +17,7 @@
|
|||
</None>
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Azure.Monitor.OpenTelemetry.Exporter" Version="1.0.0-beta.13" />
|
||||
<PackageReference Include="CommandLineParser" Version="2.8.0" />
|
||||
<PackageReference Include="MathNet.Numerics" Version="4.15.0" />
|
||||
<PackageReference Include="Microsoft.Azure.DocumentDB.Core" Version="*" />
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ namespace CosmosBenchmark
|
|||
{
|
||||
using System;
|
||||
using System.Threading.Tasks;
|
||||
using OpenTelemetry.Metrics;
|
||||
|
||||
internal interface IExecutionStrategy
|
||||
{
|
||||
|
|
@ -19,7 +20,7 @@ namespace CosmosBenchmark
|
|||
BenchmarkConfig benchmarkConfig,
|
||||
int serialExecutorConcurrency,
|
||||
int serialExecutorIterationCount,
|
||||
double warmupFraction);
|
||||
|
||||
double warmupFraction,
|
||||
MetricsCollectorProvider metricsCollectorProvider);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -17,6 +17,8 @@ namespace CosmosBenchmark
|
|||
int iterationCount,
|
||||
bool isWarmup,
|
||||
bool traceFailures,
|
||||
Action completionCallback);
|
||||
Action completionCallback,
|
||||
BenchmarkConfig benchmarkConfig,
|
||||
MetricsCollectorProvider metricsCollectorProvider);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,36 @@
|
|||
//------------------------------------------------------------
|
||||
// Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
//------------------------------------------------------------
|
||||
|
||||
namespace CosmosBenchmark
|
||||
{
|
||||
using System;
|
||||
|
||||
/// <summary>
|
||||
/// Represents the metrics collector.
|
||||
/// </summary>
|
||||
public interface IMetricsCollector
|
||||
{
|
||||
/// <summary>
|
||||
/// Collects the number of successful operations.
|
||||
/// </summary>
|
||||
void CollectMetricsOnSuccess();
|
||||
|
||||
/// <summary>
|
||||
/// Collects the number of failed operations.
|
||||
/// </summary>
|
||||
void CollectMetricsOnFailure();
|
||||
|
||||
/// <summary>
|
||||
/// Records latency for success operations in milliseconda.
|
||||
/// </summary>
|
||||
/// <param name="milliseconds">The number of milliseconds to record.</param>
|
||||
void RecordSuccessOpLatencyAndRps(TimeSpan timeSpan);
|
||||
|
||||
/// <summary>
|
||||
/// Records latency for failed operations in milliseconda.
|
||||
/// </summary>
|
||||
/// <param name="milliseconds">The number of milliseconds to record.</param>
|
||||
void RecordFailedOpLatencyAndRps(TimeSpan timeSpan);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,166 @@
|
|||
//------------------------------------------------------------
|
||||
// Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
//------------------------------------------------------------
|
||||
|
||||
namespace CosmosBenchmark
|
||||
{
|
||||
using System;
|
||||
using System.Diagnostics.Metrics;
|
||||
|
||||
/// <summary>
|
||||
/// Represents the metrics collector.
|
||||
/// </summary>
|
||||
internal class MetricsCollector : IMetricsCollector
|
||||
{
|
||||
/// <summary>
|
||||
/// Represents the meter to collect metrics.
|
||||
/// </summary>
|
||||
private readonly Meter meter;
|
||||
|
||||
/// <summary>
|
||||
/// Represents the histogram for operation latency.
|
||||
/// </summary>
|
||||
private readonly Histogram<double> operationLatencyHistogram;
|
||||
|
||||
/// <summary>
|
||||
/// Represents the histogram for records per second metric.
|
||||
/// </summary>
|
||||
private readonly Histogram<double> rpsMetricNameHistogram;
|
||||
|
||||
/// <summary>
|
||||
/// Represents the histogram for failed operation latency.
|
||||
/// </summary>
|
||||
private readonly Histogram<double> operationFailedLatencyHistogram;
|
||||
|
||||
/// <summary>
|
||||
/// Represents the histogram failed operations for records per second metric.
|
||||
/// </summary>
|
||||
private readonly Histogram<double> rpsFailedMetricNameHistogram;
|
||||
|
||||
/// <summary>
|
||||
/// Represents the success operation counter.
|
||||
/// </summary>
|
||||
private readonly Counter<long> successOperationCounter;
|
||||
|
||||
/// <summary>
|
||||
/// Represents the failure operation counter.
|
||||
/// </summary>
|
||||
private readonly Counter<long> failureOperationCounter;
|
||||
|
||||
/// <summary>
|
||||
/// Represents latency in milliseconds metric gauge.
|
||||
/// </summary>
|
||||
/// <remarks>Please do not remove this as it used when collecting metrics..</remarks>
|
||||
private readonly ObservableGauge<double> latencyInMsMetricNameGauge;
|
||||
|
||||
/// <summary>
|
||||
/// Represents records per second metric gauge.
|
||||
/// </summary>
|
||||
/// <remarks>Please do not remove this as it used when collecting metrics..</remarks>
|
||||
private readonly ObservableGauge<double> rpsNameGauge;
|
||||
|
||||
|
||||
/// <summary>
|
||||
/// Represents latency in milliseconds metric gauge for failed operations.
|
||||
/// </summary>
|
||||
/// <remarks>Please do not remove this as it used when collecting metrics..</remarks>
|
||||
private readonly ObservableGauge<double> latencyInMsFailedMetricNameGauge;
|
||||
|
||||
/// <summary>
|
||||
/// Represents records per second metric gauge for failed operations.
|
||||
/// </summary>
|
||||
/// <remarks>Please do not remove this as it used when collecting metrics..</remarks>
|
||||
private readonly ObservableGauge<double> rpsFailedNameGauge;
|
||||
|
||||
/// <summary>
|
||||
/// Latency in milliseconds.
|
||||
/// </summary>
|
||||
private double latencyInMs;
|
||||
|
||||
/// <summary>
|
||||
/// Records per second.
|
||||
/// </summary>
|
||||
private double rps;
|
||||
|
||||
/// <summary>
|
||||
/// Latency in milliseconds.
|
||||
/// </summary>
|
||||
private double latencyFailedInMs;
|
||||
|
||||
/// <summary>
|
||||
/// Records per second.
|
||||
/// </summary>
|
||||
private double rpsFailed;
|
||||
|
||||
/// <summary>
|
||||
/// Initialize new instance of <see cref="MetricsCollector"/>.
|
||||
/// </summary>
|
||||
/// <param name="meter">OpenTelemetry meter.</param>
|
||||
public MetricsCollector(Meter meter, string prefix)
|
||||
{
|
||||
this.meter = meter;
|
||||
this.rpsMetricNameHistogram = meter.CreateHistogram<double>($"{prefix}OperationRpsHistogram");
|
||||
this.operationLatencyHistogram = meter.CreateHistogram<double>($"{prefix}OperationLatencyInMsHistogram");
|
||||
|
||||
this.rpsFailedMetricNameHistogram = meter.CreateHistogram<double>($"{prefix}FailedOperationRpsHistogram");
|
||||
this.operationFailedLatencyHistogram = meter.CreateHistogram<double>($"{prefix}FailedOperationLatencyInMsHistogram");
|
||||
|
||||
this.successOperationCounter = meter.CreateCounter<long>($"{prefix}OperationSuccess");
|
||||
this.failureOperationCounter = meter.CreateCounter<long>($"{prefix}OperationFailure");
|
||||
|
||||
this.latencyInMsMetricNameGauge = this.meter.CreateObservableGauge($"{prefix}OperationLatencyInMs",
|
||||
() => new Measurement<double>(this.latencyInMs));
|
||||
|
||||
this.rpsNameGauge = this.meter.CreateObservableGauge($"{prefix}OperationRps",
|
||||
() => new Measurement<double>(this.rps));
|
||||
|
||||
this.latencyInMsFailedMetricNameGauge = this.meter.CreateObservableGauge($"{prefix}FailedOperationLatencyInMs",
|
||||
() => new Measurement<double>(this.latencyInMs));
|
||||
|
||||
this.rpsFailedNameGauge = this.meter.CreateObservableGauge($"{prefix}FailedOperationRps",
|
||||
() => new Measurement<double>(this.rps));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Collects the number of successful operations.
|
||||
/// </summary>
|
||||
public void CollectMetricsOnSuccess()
|
||||
{
|
||||
this.successOperationCounter.Add(1);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Collects the number of failed operations.
|
||||
/// </summary>
|
||||
public void CollectMetricsOnFailure()
|
||||
{
|
||||
this.failureOperationCounter.Add(1);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Records success operation latency in milliseconds.
|
||||
/// </summary>
|
||||
/// <param name="milliseconds">The number of milliseconds to record.</param>
|
||||
public void RecordSuccessOpLatencyAndRps(
|
||||
TimeSpan timeSpan)
|
||||
{
|
||||
this.rps = 1000 / timeSpan.Milliseconds;
|
||||
this.latencyInMs = timeSpan.Milliseconds;
|
||||
this.rpsMetricNameHistogram.Record(this.rps);
|
||||
this.operationLatencyHistogram.Record(this.latencyInMs);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Records failed operation latency in milliseconds.
|
||||
/// </summary>
|
||||
/// <param name="milliseconds">The number of milliseconds to record.</param>
|
||||
public void RecordFailedOpLatencyAndRps(
|
||||
TimeSpan timeSpan)
|
||||
{
|
||||
this.rpsFailed = 1000 / timeSpan.Milliseconds;
|
||||
this.latencyFailedInMs = timeSpan.Milliseconds;
|
||||
this.rpsFailedMetricNameHistogram.Record(this.rpsFailed);
|
||||
this.operationFailedLatencyHistogram.Record(this.latencyFailedInMs);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,68 @@
|
|||
//------------------------------------------------------------
|
||||
// Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
//------------------------------------------------------------
|
||||
|
||||
namespace CosmosBenchmark
|
||||
{
|
||||
using System;
|
||||
using System.Diagnostics.Metrics;
|
||||
using OpenTelemetry.Metrics;
|
||||
|
||||
/// <summary>
|
||||
/// Represents the metrics collector provider.
|
||||
/// </summary>
|
||||
internal class MetricsCollectorProvider
|
||||
{
|
||||
private readonly MetricCollectionWindow metricCollectionWindow;
|
||||
|
||||
private readonly MetricsCollector insertOperationMetricsCollector;
|
||||
|
||||
private readonly MetricsCollector queryOperationMetricsCollector;
|
||||
|
||||
private readonly MetricsCollector readOperationMetricsCollector;
|
||||
|
||||
private readonly Meter insertOperationMeter = new("CosmosBenchmarkInsertOperationMeter");
|
||||
|
||||
private readonly Meter queryOperationMeter = new("CosmosBenchmarkQueryOperationMeter");
|
||||
|
||||
private readonly Meter readOperationMeter = new("CosmosBenchmarkReadOperationMeter");
|
||||
|
||||
private readonly MeterProvider meterProvider;
|
||||
|
||||
public MetricsCollectorProvider(BenchmarkConfig config, MeterProvider meterProvider)
|
||||
{
|
||||
this.meterProvider = meterProvider;
|
||||
this.insertOperationMetricsCollector ??= new MetricsCollector(this.insertOperationMeter, "Insert");
|
||||
this.queryOperationMetricsCollector ??= new MetricsCollector(this.queryOperationMeter, "Query");
|
||||
this.readOperationMetricsCollector ??= new MetricsCollector(this.readOperationMeter, "Read");
|
||||
this.metricCollectionWindow ??= new MetricCollectionWindow(config);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets the metric collector.
|
||||
/// </summary>
|
||||
/// <param name="benchmarkOperation">Benchmark operation.</param>
|
||||
/// <param name="config">Benchmark configuration.</param>
|
||||
/// <returns>Metrics collector.</returns>
|
||||
/// <exception cref="NotSupportedException">Thrown if provided benchmark operation is not covered supported to collect metrics.</exception>
|
||||
public IMetricsCollector GetMetricsCollector(IBenchmarkOperation benchmarkOperation, BenchmarkConfig config)
|
||||
{
|
||||
MetricCollectionWindow metricCollectionWindow = this.metricCollectionWindow;
|
||||
|
||||
// Reset metricCollectionWindow and flush.
|
||||
if (!metricCollectionWindow.IsValid)
|
||||
{
|
||||
this.meterProvider.ForceFlush();
|
||||
metricCollectionWindow.Reset(config);
|
||||
}
|
||||
|
||||
return benchmarkOperation.OperationType switch
|
||||
{
|
||||
BenchmarkOperationType.Insert => this.insertOperationMetricsCollector,
|
||||
BenchmarkOperationType.Query => this.queryOperationMetricsCollector,
|
||||
BenchmarkOperationType.Read => this.readOperationMetricsCollector,
|
||||
_ => throw new NotSupportedException($"The type of {nameof(benchmarkOperation)} is not supported for collecting metrics."),
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -11,6 +11,7 @@ namespace CosmosBenchmark
|
|||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Newtonsoft.Json;
|
||||
using OpenTelemetry.Metrics;
|
||||
|
||||
internal class ParallelExecutionStrategy : IExecutionStrategy
|
||||
{
|
||||
|
|
@ -28,7 +29,8 @@ namespace CosmosBenchmark
|
|||
BenchmarkConfig benchmarkConfig,
|
||||
int serialExecutorConcurrency,
|
||||
int serialExecutorIterationCount,
|
||||
double warmupFraction)
|
||||
double warmupFraction,
|
||||
MetricsCollectorProvider metricsCollectorProvider)
|
||||
{
|
||||
IExecutor warmupExecutor = new SerialOperationExecutor(
|
||||
executorId: "Warmup",
|
||||
|
|
@ -37,7 +39,9 @@ namespace CosmosBenchmark
|
|||
(int)(serialExecutorIterationCount * warmupFraction),
|
||||
isWarmup: true,
|
||||
traceFailures: benchmarkConfig.TraceFailures,
|
||||
completionCallback: () => { });
|
||||
completionCallback: () => { },
|
||||
benchmarkConfig,
|
||||
metricsCollectorProvider);
|
||||
|
||||
IExecutor[] executors = new IExecutor[serialExecutorConcurrency];
|
||||
for (int i = 0; i < serialExecutorConcurrency; i++)
|
||||
|
|
@ -54,11 +58,13 @@ namespace CosmosBenchmark
|
|||
iterationCount: serialExecutorIterationCount,
|
||||
isWarmup: false,
|
||||
traceFailures: benchmarkConfig.TraceFailures,
|
||||
completionCallback: () => Interlocked.Decrement(ref this.pendingExecutorCount));
|
||||
completionCallback: () => Interlocked.Decrement(ref this.pendingExecutorCount),
|
||||
benchmarkConfig,
|
||||
metricsCollectorProvider);
|
||||
}
|
||||
|
||||
return await this.LogOutputStats(
|
||||
benchmarkConfig,
|
||||
benchmarkConfig,
|
||||
executors);
|
||||
}
|
||||
|
||||
|
|
@ -160,4 +166,4 @@ namespace CosmosBenchmark
|
|||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -8,10 +8,12 @@ namespace CosmosBenchmark
|
|||
using System.Diagnostics;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Azure.Cosmos;
|
||||
using static CosmosBenchmark.TelemetrySpan;
|
||||
|
||||
internal class SerialOperationExecutor : IExecutor
|
||||
{
|
||||
private readonly IBenchmarkOperation operation;
|
||||
|
||||
private readonly string executorId;
|
||||
|
||||
public SerialOperationExecutor(
|
||||
|
|
@ -26,6 +28,7 @@ namespace CosmosBenchmark
|
|||
}
|
||||
|
||||
public int SuccessOperationCount { get; private set; }
|
||||
|
||||
public int FailedOperationCount { get; private set; }
|
||||
|
||||
public double TotalRuCharges { get; private set; }
|
||||
|
|
@ -34,27 +37,38 @@ namespace CosmosBenchmark
|
|||
int iterationCount,
|
||||
bool isWarmup,
|
||||
bool traceFailures,
|
||||
Action completionCallback)
|
||||
Action completionCallback,
|
||||
BenchmarkConfig benchmarkConfig,
|
||||
MetricsCollectorProvider metricsCollectorProvider)
|
||||
{
|
||||
Trace.TraceInformation($"Executor {this.executorId} started");
|
||||
|
||||
Trace.TraceInformation("Initializing counters and metrics.");
|
||||
|
||||
try
|
||||
{
|
||||
int currentIterationCount = 0;
|
||||
do
|
||||
{
|
||||
IMetricsCollector metricsCollector = metricsCollectorProvider.GetMetricsCollector(this.operation, benchmarkConfig);
|
||||
|
||||
OperationResult? operationResult = null;
|
||||
|
||||
await this.operation.PrepareAsync();
|
||||
|
||||
using (IDisposable telemetrySpan = TelemetrySpan.StartNew(
|
||||
using (ITelemetrySpan telemetrySpan = TelemetrySpan.StartNew(
|
||||
benchmarkConfig,
|
||||
() => operationResult.Value,
|
||||
disableTelemetry: isWarmup))
|
||||
disableTelemetry: isWarmup,
|
||||
metricsCollector.RecordSuccessOpLatencyAndRps,
|
||||
metricsCollector.RecordFailedOpLatencyAndRps))
|
||||
{
|
||||
try
|
||||
{
|
||||
operationResult = await this.operation.ExecuteOnceAsync();
|
||||
|
||||
metricsCollector.CollectMetricsOnSuccess();
|
||||
|
||||
// Success case
|
||||
this.SuccessOperationCount++;
|
||||
this.TotalRuCharges += operationResult.Value.RuCharges;
|
||||
|
|
@ -68,8 +82,11 @@ namespace CosmosBenchmark
|
|||
{
|
||||
if (traceFailures)
|
||||
{
|
||||
Console.WriteLine(ex.ToString());
|
||||
Trace.TraceInformation(ex.ToString());
|
||||
}
|
||||
telemetrySpan.MarkFailed();
|
||||
|
||||
metricsCollector.CollectMetricsOnFailure();
|
||||
|
||||
// failure case
|
||||
this.FailedOperationCount++;
|
||||
|
|
@ -96,10 +113,14 @@ namespace CosmosBenchmark
|
|||
|
||||
Trace.TraceInformation($"Executor {this.executorId} completed");
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
Trace.TraceInformation($"Error: {e.Message}");
|
||||
}
|
||||
finally
|
||||
{
|
||||
completionCallback();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -8,8 +8,9 @@ namespace CosmosBenchmark
|
|||
using System.Diagnostics;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using static CosmosBenchmark.TelemetrySpan;
|
||||
|
||||
internal struct TelemetrySpan : IDisposable
|
||||
internal class TelemetrySpan : ITelemetrySpan
|
||||
{
|
||||
private static double[] latencyHistogram;
|
||||
private static int latencyIndex = -1;
|
||||
|
|
@ -18,11 +19,18 @@ namespace CosmosBenchmark
|
|||
|
||||
private Stopwatch stopwatch;
|
||||
private Func<OperationResult> lazyOperationResult;
|
||||
private Action<TimeSpan> recordFailedOpLatencyAction;
|
||||
private Action<TimeSpan> recordSuccessOpLatencyAction;
|
||||
private bool disableTelemetry;
|
||||
private bool isFailed = false;
|
||||
private BenchmarkConfig benchmarkConfig;
|
||||
|
||||
public static IDisposable StartNew(
|
||||
public static ITelemetrySpan StartNew(
|
||||
BenchmarkConfig benchmarkConfig,
|
||||
Func<OperationResult> lazyOperationResult,
|
||||
bool disableTelemetry)
|
||||
bool disableTelemetry,
|
||||
Action<TimeSpan> recordSuccessOpLatencyAction,
|
||||
Action<TimeSpan> recordFailedOpLatencyAction)
|
||||
{
|
||||
if (disableTelemetry || !TelemetrySpan.IncludePercentile)
|
||||
{
|
||||
|
|
@ -31,12 +39,17 @@ namespace CosmosBenchmark
|
|||
|
||||
return new TelemetrySpan
|
||||
{
|
||||
benchmarkConfig = benchmarkConfig,
|
||||
stopwatch = Stopwatch.StartNew(),
|
||||
lazyOperationResult = lazyOperationResult,
|
||||
recordSuccessOpLatencyAction = recordSuccessOpLatencyAction,
|
||||
recordFailedOpLatencyAction = recordFailedOpLatencyAction,
|
||||
disableTelemetry = disableTelemetry
|
||||
};
|
||||
}
|
||||
|
||||
public void MarkFailed() { this.isFailed = true; }
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
this.stopwatch.Stop();
|
||||
|
|
@ -47,6 +60,16 @@ namespace CosmosBenchmark
|
|||
if (TelemetrySpan.IncludePercentile)
|
||||
{
|
||||
RecordLatency(this.stopwatch.Elapsed.TotalMilliseconds);
|
||||
|
||||
if(this.isFailed)
|
||||
{
|
||||
this.recordSuccessOpLatencyAction?.Invoke(TimeSpan.FromMilliseconds(this.stopwatch.Elapsed.TotalMilliseconds));
|
||||
}
|
||||
else
|
||||
{
|
||||
this.recordSuccessOpLatencyAction?.Invoke(TimeSpan.FromMilliseconds(this.stopwatch.Elapsed.TotalMilliseconds));
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
BenchmarkLatencyEventSource.Instance.LatencyDiagnostics(
|
||||
|
|
@ -65,13 +88,13 @@ namespace CosmosBenchmark
|
|||
|
||||
internal static void ResetLatencyHistogram(int totalNumberOfIterations)
|
||||
{
|
||||
latencyHistogram = new double[totalNumberOfIterations];
|
||||
TelemetrySpan.latencyHistogram = new double[totalNumberOfIterations];
|
||||
latencyIndex = -1;
|
||||
}
|
||||
|
||||
internal static double? GetLatencyPercentile(int percentile)
|
||||
{
|
||||
if (latencyHistogram == null)
|
||||
if (TelemetrySpan.latencyHistogram == null)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
|
@ -79,13 +102,31 @@ namespace CosmosBenchmark
|
|||
return MathNet.Numerics.Statistics.Statistics.Percentile(latencyHistogram.Take(latencyIndex + 1), percentile);
|
||||
}
|
||||
|
||||
private class NoOpDisposable : IDisposable
|
||||
internal static double? GetLatencyQuantile(double quantile)
|
||||
{
|
||||
if (TelemetrySpan.latencyHistogram == null)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
return MathNet.Numerics.Statistics.Statistics.Quantile(latencyHistogram.Take(latencyIndex + 1), quantile);
|
||||
}
|
||||
|
||||
private class NoOpDisposable : ITelemetrySpan
|
||||
{
|
||||
public static readonly NoOpDisposable Instance = new NoOpDisposable();
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
}
|
||||
|
||||
public void MarkFailed()
|
||||
{
|
||||
}
|
||||
}
|
||||
|
||||
public interface ITelemetrySpan : IDisposable {
|
||||
void MarkFailed();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,48 @@
|
|||
//------------------------------------------------------------
|
||||
// Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
//------------------------------------------------------------
|
||||
|
||||
namespace CosmosBenchmark
|
||||
{
|
||||
using System;
|
||||
|
||||
/// <summary>
|
||||
/// Represents the metric collection window (time span while accumulating and granulating the data)
|
||||
/// </summary>
|
||||
internal class MetricCollectionWindow
|
||||
{
|
||||
/// <summary>
|
||||
/// The timestamp when window span is started.
|
||||
/// </summary>
|
||||
public DateTime Started { get; private set; }
|
||||
|
||||
/// <summary>
|
||||
/// The timestamp until which the current window span is not elapsed.
|
||||
/// </summary>
|
||||
public DateTime ValidTill { get; private set; }
|
||||
|
||||
/// <summary>
|
||||
/// Creates the instance of <see cref="MetricCollectionWindow"/>.
|
||||
/// </summary>
|
||||
/// <param name="config">Cosmos Benchmark configuration.</param>
|
||||
public MetricCollectionWindow(BenchmarkConfig config)
|
||||
{
|
||||
this.Reset(config);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Indicates whether the current window is valid.
|
||||
/// </summary>
|
||||
public bool IsValid => DateTime.UtcNow > this.ValidTill;
|
||||
|
||||
/// <summary>
|
||||
/// Resets the started timestamp and valid till timespan.
|
||||
/// </summary>
|
||||
/// <param name="config"></param>
|
||||
public void Reset(BenchmarkConfig config)
|
||||
{
|
||||
this.Started = DateTime.UtcNow;
|
||||
this.ValidTill = this.Started.AddSeconds(config.MetricsReportingIntervalInSec);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -14,8 +14,13 @@ namespace CosmosBenchmark
|
|||
using System.Reflection;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Azure.Monitor.OpenTelemetry.Exporter;
|
||||
using Microsoft.Azure.Cosmos;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Newtonsoft.Json.Linq;
|
||||
using OpenTelemetry;
|
||||
using OpenTelemetry.Metrics;
|
||||
using Container = Microsoft.Azure.Cosmos.Container;
|
||||
|
||||
/// <summary>
|
||||
/// This sample demonstrates how to achieve high performance writes using Azure Comsos DB.
|
||||
|
|
@ -31,8 +36,12 @@ namespace CosmosBenchmark
|
|||
try
|
||||
{
|
||||
BenchmarkConfig config = BenchmarkConfig.From(args);
|
||||
await Program.AddAzureInfoToRunSummary();
|
||||
|
||||
await AddAzureInfoToRunSummary();
|
||||
|
||||
MeterProvider meterProvider = BuildMeterProvider(config);
|
||||
|
||||
MetricsCollectorProvider metricsCollectorProvider = new MetricsCollectorProvider(config, meterProvider);
|
||||
|
||||
ThreadPool.SetMinThreads(config.MinThreadPoolSize, config.MinThreadPoolSize);
|
||||
|
||||
if (config.EnableLatencyPercentiles)
|
||||
|
|
@ -45,7 +54,7 @@ namespace CosmosBenchmark
|
|||
|
||||
Program program = new Program();
|
||||
|
||||
RunSummary runSummary = await program.ExecuteAsync(config);
|
||||
RunSummary runSummary = await program.ExecuteAsync(config, metricsCollectorProvider);
|
||||
}
|
||||
finally
|
||||
{
|
||||
|
|
@ -58,6 +67,37 @@ namespace CosmosBenchmark
|
|||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Create a MeterProvider. If the App Insights connection string is not set, do not create an AppInsights Exporter.
|
||||
/// </summary>
|
||||
/// <returns></returns>
|
||||
private static MeterProvider BuildMeterProvider(BenchmarkConfig config)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(config.AppInsightsConnectionString))
|
||||
{
|
||||
return Sdk.CreateMeterProviderBuilder()
|
||||
.AddMeter("CosmosBenchmarkInsertOperationMeter")
|
||||
.AddMeter("CosmosBenchmarkQueryOperationMeter")
|
||||
.AddMeter("CosmosBenchmarkReadOperationMeter")
|
||||
.Build();
|
||||
}
|
||||
|
||||
OpenTelemetry.Trace.TracerProviderBuilder tracerProviderBuilder = Sdk.CreateTracerProviderBuilder()
|
||||
.AddAzureMonitorTraceExporter();
|
||||
|
||||
return Sdk.CreateMeterProviderBuilder()
|
||||
.AddAzureMonitorMetricExporter(configure: new Action<AzureMonitorExporterOptions>(
|
||||
(options) => options.ConnectionString = config.AppInsightsConnectionString))
|
||||
.AddMeter("CosmosBenchmarkInsertOperationMeter")
|
||||
.AddMeter("CosmosBenchmarkQueryOperationMeter")
|
||||
.AddMeter("CosmosBenchmarkReadOperationMeter")
|
||||
.Build();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Adds Azure VM information to run summary.
|
||||
/// </summary>
|
||||
/// <returns></returns>
|
||||
private static async Task AddAzureInfoToRunSummary()
|
||||
{
|
||||
using HttpClient httpClient = new HttpClient();
|
||||
|
|
@ -81,11 +121,13 @@ namespace CosmosBenchmark
|
|||
}
|
||||
}
|
||||
|
||||
|
||||
/// <summary>
|
||||
/// Executing benchmarks for V2/V3 cosmosdb SDK.
|
||||
/// </summary>
|
||||
/// <returns>a Task object.</returns>
|
||||
private async Task<RunSummary> ExecuteAsync(BenchmarkConfig config)
|
||||
private async Task<RunSummary> ExecuteAsync(BenchmarkConfig config,
|
||||
MetricsCollectorProvider metricsCollectorProvider)
|
||||
{
|
||||
// V3 SDK client initialization
|
||||
using (CosmosClient cosmosClient = config.CreateCosmosClient(config.Key))
|
||||
|
|
@ -137,7 +179,7 @@ namespace CosmosBenchmark
|
|||
}
|
||||
|
||||
IExecutionStrategy execution = IExecutionStrategy.StartNew(benchmarkOperationFactory);
|
||||
runSummary = await execution.ExecuteAsync(config, taskCount, opsPerTask, 0.01);
|
||||
runSummary = await execution.ExecuteAsync(config, taskCount, opsPerTask, 0.01, metricsCollectorProvider);
|
||||
}
|
||||
|
||||
if (config.CleanupOnFinish)
|
||||
|
|
@ -154,13 +196,12 @@ namespace CosmosBenchmark
|
|||
}
|
||||
runSummary.ConsistencyLevel = consistencyLevel;
|
||||
|
||||
|
||||
if (config.PublishResults)
|
||||
{
|
||||
runSummary.Diagnostics = CosmosDiagnosticsLogger.GetDiagnostics();
|
||||
await this.PublishResults(
|
||||
config,
|
||||
runSummary,
|
||||
config,
|
||||
runSummary,
|
||||
cosmosClient);
|
||||
}
|
||||
|
||||
|
|
@ -169,8 +210,8 @@ namespace CosmosBenchmark
|
|||
}
|
||||
|
||||
private async Task PublishResults(
|
||||
BenchmarkConfig config,
|
||||
RunSummary runSummary,
|
||||
BenchmarkConfig config,
|
||||
RunSummary runSummary,
|
||||
CosmosClient benchmarkClient)
|
||||
{
|
||||
if (string.IsNullOrEmpty(config.ResultsEndpoint))
|
||||
|
|
@ -266,8 +307,8 @@ namespace CosmosBenchmark
|
|||
{
|
||||
return await container.ReadContainerAsync();
|
||||
}
|
||||
catch(CosmosException ex) when (ex.StatusCode == HttpStatusCode.NotFound)
|
||||
{
|
||||
catch (CosmosException ex) when (ex.StatusCode == HttpStatusCode.NotFound)
|
||||
{
|
||||
// Show user cost of running this test
|
||||
double estimatedCostPerMonth = 0.06 * options.Throughput;
|
||||
double estimatedCostPerHour = estimatedCostPerMonth / (24 * 30);
|
||||
|
|
@ -288,4 +329,4 @@ namespace CosmosBenchmark
|
|||
traceSource.Listeners.Clear();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Загрузка…
Ссылка в новой задаче