Azure
/
azure-cosmos-dotnet-v3
зеркало из https://github.com/Azure/azure-cosmos-dotnet-v3.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

[Internal] Client Telemetry: Adds client config api call to get latest flag status (#4050) 

* first draft

* tets fix

* fix dependent projects

* reduce refresh time in tests

* fix tests and added comments

* fix diagnostic handler fix

* fix test

* adding test

* ret pushmove console

* fix test

* provide options to enable/disable this featire in benchmark and ctl proj

* updated trace message

Co-authored-by: Matias Quaranta <ealsur@users.noreply.github.com>

* remove import

* updated traces

Co-authored-by: Matias Quaranta <ealsur@users.noreply.github.com>

* test fix

* remove null assignment

* fix test

---------

Co-authored-by: Matias Quaranta <ealsur@users.noreply.github.com>
This commit is contained in:
Sourabh Jain 2023-08-31 23:25:30 +05:30 коммит произвёл GitHub
Родитель 3aced24bea
Коммит c72ebc8574
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: 4AEE18F83AFDEB23
23 изменённых файлов: 499 добавлений и 272 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

40
Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/BenchmarkConfig.cs
Просмотреть файл

@ -10,6 +10,7 @@ namespace CosmosBenchmark
using System.Linq;
using System.Runtime;
using CommandLine;
using Microsoft.Azure.Cosmos.Telemetry;
using Microsoft.Azure.Documents.Client;
using Newtonsoft.Json;
@ -102,18 +103,12 @@ namespace CosmosBenchmark
[Option(Required = false, HelpText = "Disable core SDK logging")]
public bool DisableCoreSdkLogging { get; set; }
[Option(Required = false, HelpText = "Enable Client Telemetry")]
public bool EnableTelemetry { get; set; }
[Option(Required = false, HelpText = "Enable Distributed Tracing")]
public bool EnableDistributedTracing { get; set; }
[Option(Required = false, HelpText = "Client Telemetry Schedule in Seconds")]
public int TelemetryScheduleInSec { get; set; }
[Option(Required = false, HelpText = "Client Telemetry Endpoint")]
public string TelemetryEndpoint { get; set; }
[Option(Required = false, HelpText = "Endpoint to publish results to")]
public string ResultsEndpoint { get; set; }
@ -143,6 +138,9 @@ namespace CosmosBenchmark
[Option(Required = false, HelpText = "Application Insights connection string")]
public string AppInsightsConnectionString { get; set; }
[Option(Required = false, HelpText = "Enable Client Telemetry Feature in SDK. Make sure you enable it from the portal also.")]
public bool EnableClientTelemetry { get; set; } = true;
internal int GetTaskCount(int containerThroughput)
{
int taskCount = this.DegreeOfParallelism;
@ -210,35 +208,21 @@ namespace CosmosBenchmark
internal Microsoft.Azure.Cosmos.CosmosClient CreateCosmosClient(string accountKey)
{
// Overwrite the default timespan if configured
if(this.TelemetryScheduleInSec > 0)
{
ClientTelemetryOptions.DefaultIntervalForTelemetryJob = TimeSpan.FromSeconds(this.TelemetryScheduleInSec);
}
Microsoft.Azure.Cosmos.CosmosClientOptions clientOptions = new Microsoft.Azure.Cosmos.CosmosClientOptions()
{
ApplicationName = this.GetUserAgentPrefix(),
MaxRetryAttemptsOnRateLimitedRequests = 0,
MaxRequestsPerTcpConnection = this.MaxRequestsPerTcpConnection,
MaxTcpConnectionsPerEndpoint = this.MaxTcpConnectionsPerEndpoint
MaxTcpConnectionsPerEndpoint = this.MaxTcpConnectionsPerEndpoint,
EnableClientTelemetry = this.EnableClientTelemetry
};
if (this.EnableTelemetry)
{
Environment.SetEnvironmentVariable(
Microsoft.Azure.Cosmos.Telemetry.ClientTelemetryOptions.EnvPropsClientTelemetryEnabled,
"true");
if (this.TelemetryScheduleInSec > 0)
{
Environment.SetEnvironmentVariable(
Microsoft.Azure.Cosmos.Telemetry.ClientTelemetryOptions.EnvPropsClientTelemetrySchedulingInSeconds,
Convert.ToString(this.TelemetryScheduleInSec));
}
if (!string.IsNullOrEmpty(this.TelemetryEndpoint))
{
Environment.SetEnvironmentVariable(
Microsoft.Azure.Cosmos.Telemetry.ClientTelemetryOptions.EnvPropsClientTelemetryEndpoint,
this.TelemetryEndpoint);
}
}
if (!string.IsNullOrWhiteSpace(this.ConsistencyLevel))
{
clientOptions.ConsistencyLevel = (Microsoft.Azure.Cosmos.ConsistencyLevel)Enum.Parse(typeof(Microsoft.Azure.Cosmos.ConsistencyLevel), this.ConsistencyLevel, ignoreCase: true);

11
Microsoft.Azure.Cosmos.Samples/Tools/CTL/CTLConfig.cs
Просмотреть файл

@ -97,18 +97,15 @@ namespace CosmosCTL
[Option("ctl_logging_context", Required = false, HelpText = "Defines a custom context to use on metrics")]
public string LogginContext { get; set; } = string.Empty;
[Option("ctl_telemetry_endpoint", Required = false, HelpText = "telemetry juno end point")]
public string TelemetryEndpoint { get; set; }
[Option("ctl_telemetry_schedule_in_sec", Required = false, HelpText = "telemetry task schedule time in sec")]
public string TelemetryScheduleInSeconds { get; set; }
[Option("ctl_reservoir_type", Required = false, HelpText = "Defines the reservoir type. Valid values are: Uniform, SlidingWindow and ExponentialDecay. The default value is SlidingWindow.")]
public ReservoirTypes ReservoirType { get; set; } = ReservoirTypes.SlidingWindow;
[Option("ctl_reservoir_sample_size", Required = false, HelpText = "The reservoir sample size.")]
public int ReservoirSampleSize { get; set; } = 1028;
[Option("ctl_enable_client_telemetry", Required = false, HelpText = "Enable Client Telemetry Feature in SDK. Make sure you enable it from the portal also.")]
public bool EnableClientTelemetry { get; set; } = true;
internal TimeSpan RunningTimeDurationAsTimespan { get; private set; } = TimeSpan.FromHours(10);
internal TimeSpan DiagnosticsThresholdDurationAsTimespan { get; private set; } = TimeSpan.FromSeconds(60);
@ -133,7 +130,7 @@ namespace CosmosCTL
CosmosClientOptions clientOptions = new CosmosClientOptions()
{
ApplicationName = CTLConfig.UserAgentSuffix,
EnableClientTelemetry = true
EnableClientTelemetry = this.EnableClientTelemetry
};
if (this.UseGatewayMode)

14
Microsoft.Azure.Cosmos.Samples/Tools/CTL/Program.cs
Просмотреть файл

@ -29,8 +29,6 @@ namespace CosmosCTL
{
CTLConfig config = CTLConfig.From(args);
SetEnvironmentVariables(config);
if (config.OutputEventTraces)
{
EnableTraceSourcesToConsole();
@ -54,12 +52,6 @@ namespace CosmosCTL
logger.LogInformation("Initialization completed.");
if(client.ClientOptions.EnableClientTelemetry.GetValueOrDefault()) {
logger.LogInformation("Telemetry is enabled for CTL.");
} else {
logger.LogInformation("Telemetry is disabled for CTL.");
}
List<Task> tasks = new List<Task>
{
scenario.RunAsync(
@ -148,12 +140,6 @@ namespace CosmosCTL
}
}
private static void SetEnvironmentVariables(CTLConfig config)
{
Environment.SetEnvironmentVariable(ClientTelemetryOptions.EnvPropsClientTelemetryEndpoint, config.TelemetryEndpoint);
Environment.SetEnvironmentVariable(ClientTelemetryOptions.EnvPropsClientTelemetrySchedulingInSeconds, config.TelemetryScheduleInSeconds);
}
private static IMetricsRoot ConfigureReporting(
CTLConfig config,
ILogger logger)

5
Microsoft.Azure.Cosmos/src/ConnectionPolicy.cs
Просмотреть файл

@ -48,7 +48,7 @@ namespace Microsoft.Azure.Cosmos
this.MaxConnectionLimit = defaultMaxConcurrentConnectionLimit;
this.RetryOptions = new RetryOptions();
this.EnableReadRequestsFallback = null;
this.EnableClientTelemetry = ClientTelemetryOptions.IsClientTelemetryEnabled();
this.EnableClientTelemetry = false; // by default feature flag is off
this.ServerCertificateCustomValidationCallback = null;
}
@ -211,6 +211,9 @@ namespace Microsoft.Azure.Cosmos
set;
}
/// <summary>
/// Gets or sets the flag to enable client telemetry feature.
/// </summary>
internal bool EnableClientTelemetry
{
get;

4
Microsoft.Azure.Cosmos/src/Handler/DiagnosticsHandler.cs
Просмотреть файл

@ -4,6 +4,7 @@
namespace Microsoft.Azure.Cosmos.Handlers
{
using System;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Azure.Cosmos.Handler;
@ -25,7 +26,8 @@ namespace Microsoft.Azure.Cosmos.Handlers
ResponseMessage responseMessage = await base.SendAsync(request, cancellationToken);
// Record the diagnostics on the response to get the CPU of when the request was executing
SystemUsageHistory systemUsageHistory = DiagnosticsHandlerHelper.Instance.GetDiagnosticsSystemHistory();
SystemUsageHistory systemUsageHistory = DiagnosticsHandlerHelper.GetInstance().GetDiagnosticsSystemHistory();
if (systemUsageHistory != null)
{
request.Trace.AddDatum(

59
Microsoft.Azure.Cosmos/src/Handler/DiagnosticsHandlerHelper.cs
Просмотреть файл

@ -8,7 +8,6 @@ namespace Microsoft.Azure.Cosmos.Handler
using System.Collections.Generic;
using Documents.Rntbd;
using Microsoft.Azure.Cosmos.Core.Trace;
using Microsoft.Azure.Cosmos.Telemetry;
/// <summary>
/// This is a helper class that creates a single static instance to avoid each
@ -22,34 +21,69 @@ namespace Microsoft.Azure.Cosmos.Handler
private const string Telemetrykey = "telemetry";
public static readonly TimeSpan DiagnosticsRefreshInterval = TimeSpan.FromSeconds(10);
private readonly SystemUsageRecorder diagnosticSystemUsageRecorder = new SystemUsageRecorder(
private static readonly SystemUsageRecorder DiagnosticSystemUsageRecorder = new SystemUsageRecorder(
identifier: Diagnostickey,
historyLength: 6,
refreshInterval: DiagnosticsHandlerHelper.DiagnosticsRefreshInterval);
private static readonly TimeSpan ClientTelemetryRefreshInterval = TimeSpan.FromSeconds(5);
private readonly SystemUsageRecorder telemetrySystemUsageRecorder = new SystemUsageRecorder(
private static readonly SystemUsageRecorder TelemetrySystemUsageRecorder = new SystemUsageRecorder(
identifier: Telemetrykey,
historyLength: 120,
refreshInterval: DiagnosticsHandlerHelper.ClientTelemetryRefreshInterval);
private static bool isDiagnosticsMonitoringEnabled = false;
private static bool isTelemetryMonitoringEnabled = false;
/// <summary>
/// Singleton to make sure only one instance of DiagnosticHandlerHelper is there.
/// The system usage collection is disabled for internal builds so it is set to null to avoid
/// compute for accidentally creating an instance or trying to use it.
/// </summary>
public static readonly DiagnosticsHandlerHelper Instance =
private static DiagnosticsHandlerHelper Instance =
#if INTERNAL
null;
#else
new DiagnosticsHandlerHelper();
#endif
private static bool isDiagnosticsMonitoringEnabled;
private static bool isTelemetryMonitoringEnabled;
private readonly SystemUsageMonitor systemUsageMonitor = null;
public static DiagnosticsHandlerHelper GetInstance()
{
return DiagnosticsHandlerHelper.Instance;
}
/// <summary>
/// Restart the monitor with client telemetry recorder if telemetry is enabled
/// </summary>
/// <param name="isClientTelemetryEnabled"></param>
public static void Refresh(bool isClientTelemetryEnabled)
{
if (isClientTelemetryEnabled != DiagnosticsHandlerHelper.isTelemetryMonitoringEnabled)
{
DiagnosticsHandlerHelper.Instance.StopSystemMonitor();
// Update telemetry flag
DiagnosticsHandlerHelper.isTelemetryMonitoringEnabled = isClientTelemetryEnabled;
// Create new instance, it will start a new system monitor job
DiagnosticsHandlerHelper.Instance = new DiagnosticsHandlerHelper();
}
}
private void StopSystemMonitor()
{
try
{
this.systemUsageMonitor?.Dispose();
}
catch (ObjectDisposedException ex)
{
DefaultTrace.TraceError($"Error while stopping system usage monitor. {0} ", ex);
}
}
/// <summary>
/// Start System Usage Monitor with Diagnostic and Telemetry Recorder if Telemetry is enabled
/// Otherwise Start System Usage Monitor with only Diagnostic Recorder
@ -61,16 +95,14 @@ namespace Microsoft.Azure.Cosmos.Handler
// If the CPU monitor fails for some reason don't block the application
try
{
DiagnosticsHandlerHelper.isTelemetryMonitoringEnabled = ClientTelemetryOptions.IsClientTelemetryEnabled();
List<SystemUsageRecorder> recorders = new List<SystemUsageRecorder>()
{
this.diagnosticSystemUsageRecorder,
DiagnosticsHandlerHelper.DiagnosticSystemUsageRecorder,
};
if (DiagnosticsHandlerHelper.isTelemetryMonitoringEnabled)
{
recorders.Add(this.telemetrySystemUsageRecorder);
recorders.Add(DiagnosticsHandlerHelper.TelemetrySystemUsageRecorder);
}
this.systemUsageMonitor = SystemUsageMonitor.CreateAndStart(recorders);
@ -82,7 +114,6 @@ namespace Microsoft.Azure.Cosmos.Handler
DefaultTrace.TraceError(ex.Message);
DiagnosticsHandlerHelper.isDiagnosticsMonitoringEnabled = false;
DiagnosticsHandlerHelper.isTelemetryMonitoringEnabled = false;
}
}
@ -99,7 +130,7 @@ namespace Microsoft.Azure.Cosmos.Handler
try
{
return this.diagnosticSystemUsageRecorder.Data;
return DiagnosticsHandlerHelper.DiagnosticSystemUsageRecorder.Data;
}
catch (Exception ex)
{
@ -123,7 +154,7 @@ namespace Microsoft.Azure.Cosmos.Handler
try
{
return this.telemetrySystemUsageRecorder.Data;
return DiagnosticsHandlerHelper.TelemetrySystemUsageRecorder.Data;
}
catch (Exception ex)
{

29
Microsoft.Azure.Cosmos/src/Resource/Settings/AccountClientConfiguration.cs Normal file
Просмотреть файл

@ -0,0 +1,29 @@
//------------------------------------------------------------
// Copyright (c) Microsoft Corporation. All rights reserved.
//------------------------------------------------------------
namespace Microsoft.Azure.Cosmos
{
using System.Collections.Generic;
using Microsoft.Azure.Documents;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
internal class AccountClientConfiguration
{
[JsonProperty(PropertyName = Constants.Properties.ClientTelemetryConfiguration)]
public ClientTelemetryConfiguration ClientTelemetryConfiguration { get; set; }
/// <summary>
/// This contains additional values for scenarios where the SDK is not aware of new fields.
/// This ensures that if resource is read and updated none of the fields will be lost in the process.
/// </summary>
[JsonExtensionData]
internal IDictionary<string, JToken> AdditionalProperties { get; private set; }
internal bool IsClientTelemetryEnabled()
{
return this.ClientTelemetryConfiguration.IsEnabled && this.ClientTelemetryConfiguration.Endpoint != null;
}
}
}

27
Microsoft.Azure.Cosmos/src/Resource/Settings/ClientTelemetryConfiguration.cs Normal file
Просмотреть файл

@ -0,0 +1,27 @@
//------------------------------------------------------------
// Copyright (c) Microsoft Corporation. All rights reserved.
//------------------------------------------------------------
namespace Microsoft.Azure.Cosmos
{
using System.Collections.Generic;
using Microsoft.Azure.Documents;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
internal class ClientTelemetryConfiguration
{
[JsonProperty(PropertyName = Constants.Properties.ClientTelemetryEnabled)]
public bool IsEnabled { get; set; }
[JsonProperty(PropertyName = Constants.Properties.ClientTelemetryEndpoint)]
public string Endpoint { get; set; }
/// <summary>
/// This contains additional values for scenarios where the SDK is not aware of new fields.
/// This ensures that if resource is read and updated none of the fields will be lost in the process.
/// </summary>
[JsonExtensionData]
internal IDictionary<string, JToken> AdditionalProperties { get; private set; }
}
}

16
Microsoft.Azure.Cosmos/src/Telemetry/ClientTelemetry.cs
Просмотреть файл

@ -26,11 +26,12 @@ namespace Microsoft.Azure.Cosmos.Telemetry
/// </summary>
internal class ClientTelemetry : IDisposable
{
private static readonly TimeSpan observingWindow = ClientTelemetryOptions.GetScheduledTimeSpan();
private static readonly TimeSpan observingWindow = ClientTelemetryOptions.DefaultIntervalForTelemetryJob;
private readonly ClientTelemetryProperties clientTelemetryInfo;
private readonly ClientTelemetryProcessor processor;
private readonly DiagnosticsHandlerHelper diagnosticsHelper;
private readonly string endpointUrl;
private readonly NetworkDataRecorder networkDataRecorder;
private readonly CancellationTokenSource cancellationTokenSource;
@ -63,6 +64,7 @@ namespace Microsoft.Azure.Cosmos.Telemetry
/// <param name="diagnosticsHelper"></param>
/// <param name="preferredRegions"></param>
/// <param name="globalEndpointManager"></param>
/// <param name="endpointUrl"></param>
/// <returns>ClientTelemetry</returns>
public static ClientTelemetry CreateAndStartBackgroundTelemetry(
string clientId,
@ -72,7 +74,8 @@ namespace Microsoft.Azure.Cosmos.Telemetry
AuthorizationTokenProvider authorizationTokenProvider,
DiagnosticsHandlerHelper diagnosticsHelper,
IReadOnlyList<string> preferredRegions,
GlobalEndpointManager globalEndpointManager)
GlobalEndpointManager globalEndpointManager,
string endpointUrl )
{
DefaultTrace.TraceInformation("Initiating telemetry with background task.");
@ -84,7 +87,8 @@ namespace Microsoft.Azure.Cosmos.Telemetry
authorizationTokenProvider,
diagnosticsHelper,
preferredRegions,
globalEndpointManager);
globalEndpointManager,
endpointUrl);
clientTelemetry.StartObserverTask();
@ -99,9 +103,12 @@ namespace Microsoft.Azure.Cosmos.Telemetry
AuthorizationTokenProvider authorizationTokenProvider,
DiagnosticsHandlerHelper diagnosticsHelper,
IReadOnlyList<string> preferredRegions,
GlobalEndpointManager globalEndpointManager)
GlobalEndpointManager globalEndpointManager,
string endpointUrl)
{
this.diagnosticsHelper = diagnosticsHelper ?? throw new ArgumentNullException(nameof(diagnosticsHelper));
this.endpointUrl = endpointUrl ?? throw new ArgumentNullException(nameof(endpointUrl));
this.globalEndpointManager = globalEndpointManager;
this.processor = new ClientTelemetryProcessor(httpClient, authorizationTokenProvider);
@ -177,6 +184,7 @@ namespace Microsoft.Azure.Cosmos.Telemetry
operationInfoSnapshot: operationInfoSnapshot,
cacheRefreshInfoSnapshot: cacheRefreshInfoSnapshot,
requestInfoSnapshot: requestInfoSnapshot,
endpointUrl: this.endpointUrl,
cancellationToken: cancellationToken.Token), cancellationToken.Token);
// Initiating Telemetry Data Processor task which will serialize and send telemetry information to Client Telemetry Service

67
Microsoft.Azure.Cosmos/src/Telemetry/ClientTelemetryOptions.cs
Просмотреть файл

@ -72,17 +72,13 @@ namespace Microsoft.Azure.Cosmos.Telemetry
internal const string IsThreadStarvingName = "SystemPool_IsThreadStarving_True";
internal const string IsThreadStarvingUnit = "Count";
internal const double DefaultTimeStampInSeconds = 600;
internal const double Percentile50 = 50.0;
internal const double Percentile90 = 90.0;
internal const double Percentile95 = 95.0;
internal const double Percentile99 = 99.0;
internal const double Percentile999 = 99.9;
internal const string DateFormat = "yyyy-MM-ddTHH:mm:ssZ";
internal const string EnvPropsClientTelemetrySchedulingInSeconds = "COSMOS.CLIENT_TELEMETRY_SCHEDULING_IN_SECONDS";
internal const string EnvPropsClientTelemetryEnabled = "COSMOS.CLIENT_TELEMETRY_ENABLED";
internal const string EnvPropsClientTelemetryVmMetadataUrl = "COSMOS.VM_METADATA_URL";
internal const string EnvPropsClientTelemetryEndpoint = "COSMOS.CLIENT_TELEMETRY_ENDPOINT";
internal const string EnvPropsClientTelemetryEnvironmentName = "COSMOS.ENVIRONMENT_NAME";
internal static readonly ResourceType AllowedResourceTypes = ResourceType.Document;
@ -99,53 +95,11 @@ namespace Microsoft.Azure.Cosmos.Telemetry
internal static readonly List<int> ExcludedStatusCodes = new List<int> { 404, 409, 412 };
internal static readonly int NetworkTelemetrySampleSize = 200;
internal static TimeSpan DefaultIntervalForTelemetryJob = TimeSpan.FromMinutes(10);
internal static int PayloadSizeThreshold = 1024 * 1024 * 2; // 2MB
internal static TimeSpan ClientTelemetryProcessorTimeOut = TimeSpan.FromMinutes(5);
private static Uri clientTelemetryEndpoint;
private static string environmentName;
private static TimeSpan scheduledTimeSpan = TimeSpan.Zero;
internal static bool IsClientTelemetryEnabled()
{
bool isTelemetryEnabled = ConfigurationManager
.GetEnvironmentVariable<bool>(ClientTelemetryOptions
.EnvPropsClientTelemetryEnabled, false);
DefaultTrace.TraceInformation($"Telemetry Flag is set to {isTelemetryEnabled}");
return isTelemetryEnabled;
}
internal static TimeSpan GetScheduledTimeSpan()
{
if (scheduledTimeSpan.Equals(TimeSpan.Zero))
{
double scheduledTimeInSeconds = ClientTelemetryOptions.DefaultTimeStampInSeconds;
try
{
scheduledTimeInSeconds = ConfigurationManager
.GetEnvironmentVariable<double>(
ClientTelemetryOptions.EnvPropsClientTelemetrySchedulingInSeconds,
ClientTelemetryOptions.DefaultTimeStampInSeconds);
if (scheduledTimeInSeconds <= 0)
{
throw new ArgumentException("Telemetry Scheduled time can not be less than or equal to 0.");
}
}
catch (Exception ex)
{
DefaultTrace.TraceError($"Error while getting telemetry scheduling configuration : {ex.Message}. Falling back to default configuration i.e. {scheduledTimeInSeconds}" );
}
scheduledTimeSpan = TimeSpan.FromSeconds(scheduledTimeInSeconds);
DefaultTrace.TraceInformation($"Telemetry Scheduled in Seconds {scheduledTimeSpan.TotalSeconds}");
}
return scheduledTimeSpan;
}
internal static string GetHostInformation(Compute vmInformation)
{
@ -155,23 +109,6 @@ namespace Microsoft.Azure.Cosmos.Telemetry
vmInformation?.AzEnvironment);
}
internal static Uri GetClientTelemetryEndpoint()
{
if (clientTelemetryEndpoint == null)
{
string uriProp = ConfigurationManager
.GetEnvironmentVariable<string>(
ClientTelemetryOptions.EnvPropsClientTelemetryEndpoint, null);
if (!String.IsNullOrEmpty(uriProp))
{
clientTelemetryEndpoint = new Uri(uriProp);
}
DefaultTrace.TraceInformation($"Telemetry Endpoint URL is {uriProp}");
}
return clientTelemetryEndpoint;
}
internal static string GetEnvironmentName()
{
if (String.IsNullOrEmpty(environmentName))

12
Microsoft.Azure.Cosmos/src/Telemetry/ClientTelemetryProcessor.cs
Просмотреть файл

@ -19,8 +19,6 @@ namespace Microsoft.Azure.Cosmos.Telemetry
internal class ClientTelemetryProcessor
{
private static readonly Uri endpointUrl = ClientTelemetryOptions.GetClientTelemetryEndpoint();
private readonly AuthorizationTokenProvider tokenProvider;
private readonly CosmosHttpClient httpClient;
@ -39,6 +37,7 @@ namespace Microsoft.Azure.Cosmos.Telemetry
ConcurrentDictionary<OperationInfo, (LongConcurrentHistogram latency, LongConcurrentHistogram requestcharget)> operationInfoSnapshot,
ConcurrentDictionary<CacheRefreshInfo, LongConcurrentHistogram> cacheRefreshInfoSnapshot,
IReadOnlyList<RequestInfo> requestInfoSnapshot,
string endpointUrl,
CancellationToken cancellationToken)
{
try
@ -48,7 +47,7 @@ namespace Microsoft.Azure.Cosmos.Telemetry
operationInfoSnapshot: operationInfoSnapshot,
cacheRefreshInfoSnapshot: cacheRefreshInfoSnapshot,
sampledRequestInfo: requestInfoSnapshot,
callback: async (payload) => await this.SendAsync(clientTelemetryInfo.GlobalDatabaseAccountName, payload, cancellationToken));
callback: async (payload) => await this.SendAsync(clientTelemetryInfo.GlobalDatabaseAccountName, payload, endpointUrl, cancellationToken));
}
catch (Exception ex)
{
@ -67,6 +66,7 @@ namespace Microsoft.Azure.Cosmos.Telemetry
private async Task SendAsync(
string globalDatabaseAccountName,
string jsonPayload,
string endpointUrl,
CancellationToken cancellationToken)
{
if (endpointUrl == null)
@ -77,12 +77,12 @@ namespace Microsoft.Azure.Cosmos.Telemetry
try
{
DefaultTrace.TraceInformation("Sending Telemetry Data to {0}", endpointUrl.AbsoluteUri);
DefaultTrace.TraceInformation("Sending Telemetry Data to {0}", endpointUrl);
using HttpRequestMessage request = new HttpRequestMessage
{
Method = HttpMethod.Post,
RequestUri = endpointUrl,
RequestUri = new Uri(endpointUrl),
Content = new StringContent(jsonPayload, Encoding.UTF8, "application/json")
};
@ -91,7 +91,7 @@ namespace Microsoft.Azure.Cosmos.Telemetry
INameValueCollection headersCollection = new StoreResponseNameValueCollection();
await this.tokenProvider.AddAuthorizationHeaderAsync(
headersCollection,
endpointUrl,
new Uri(endpointUrl),
"POST",
AuthorizationTokenType.PrimaryMasterKey);

108
Microsoft.Azure.Cosmos/src/Telemetry/TelemetryToServiceHelper.cs
Просмотреть файл

@ -5,12 +5,17 @@
namespace Microsoft.Azure.Cosmos.Telemetry
{
using System;
using System.Net.Http;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Azure.Cosmos.Core.Trace;
using Microsoft.Azure.Cosmos.Handler;
using Microsoft.Azure.Cosmos.Query.Core.Monads;
using Microsoft.Azure.Cosmos.Routing;
using Microsoft.Azure.Cosmos.Telemetry.Collector;
using Microsoft.Azure.Cosmos.Tracing;
using Microsoft.Azure.Documents;
using Microsoft.Azure.Documents.Collections;
internal class TelemetryToServiceHelper : IDisposable
{
@ -70,12 +75,69 @@ namespace Microsoft.Azure.Cosmos.Telemetry
TelemetryToServiceHelper helper = new TelemetryToServiceHelper(
clientId, connectionPolicy, cosmosAuthorization, httpClient, serviceEndpoint, globalEndpointManager, cancellationTokenSource);
helper.InitializeClientTelemetry();
_ = helper.RetrieveConfigAndInitiateTelemetryAsync(); // Let it run in backgroud
return helper;
#endif
}
private async Task RetrieveConfigAndInitiateTelemetryAsync()
{
try
{
Uri serviceEndpointWithPath = new Uri(this.serviceEnpoint + Paths.ClientConfigPathSegment);
TryCatch<AccountClientConfiguration> databaseAccountClientConfigs = await this.GetDatabaseAccountClientConfigAsync(this.cosmosAuthorization, this.httpClient, serviceEndpointWithPath);
if (databaseAccountClientConfigs.Succeeded)
{
this.InitializeClientTelemetry(databaseAccountClientConfigs.Result);
}
else if (!this.cancellationTokenSource.IsCancellationRequested)
{
DefaultTrace.TraceWarning($"Exception while calling client config " + databaseAccountClientConfigs.Exception.ToString());
}
}
catch (Exception ex)
{
DefaultTrace.TraceWarning($"Exception while running client config job " + ex.ToString());
}
}
private async Task<TryCatch<AccountClientConfiguration>> GetDatabaseAccountClientConfigAsync(AuthorizationTokenProvider cosmosAuthorization,
CosmosHttpClient httpClient,
Uri clientConfigEndpoint)
{
INameValueCollection headers = new RequestNameValueCollection();
await cosmosAuthorization.AddAuthorizationHeaderAsync(
headersCollection: headers,
clientConfigEndpoint,
HttpConstants.HttpMethods.Get,
AuthorizationTokenType.PrimaryMasterKey);
using (ITrace trace = Trace.GetRootTrace("Account Client Config Read", TraceComponent.Transport, TraceLevel.Info))
{
try
{
using (HttpResponseMessage responseMessage = await httpClient.GetAsync(
uri: clientConfigEndpoint,
additionalHeaders: headers,
resourceType: ResourceType.DatabaseAccount,
timeoutPolicy: HttpTimeoutPolicyControlPlaneRead.Instance,
clientSideRequestStatistics: null,
cancellationToken: default))
using (DocumentServiceResponse documentServiceResponse = await ClientExtensions.ParseResponseAsync(responseMessage))
{
return TryCatch<AccountClientConfiguration>.FromResult(CosmosResource.FromStream<AccountClientConfiguration>(documentServiceResponse));
}
}
catch (Exception ex)
{
DefaultTrace.TraceWarning($"Exception while calling client config " + ex.StackTrace);
return TryCatch<AccountClientConfiguration>.FromException(ex);
}
}
}
public ITelemetryCollector GetCollector()
{
return this.collector;
@ -89,28 +151,34 @@ namespace Microsoft.Azure.Cosmos.Telemetry
/// <summary>
/// Trigger Client Telemetry job when it is enabled and not already running.
/// </summary>
private void InitializeClientTelemetry()
private void InitializeClientTelemetry(AccountClientConfiguration clientConfig)
{
try
{
this.clientTelemetry = ClientTelemetry.CreateAndStartBackgroundTelemetry(
clientId: this.clientId,
httpClient: this.httpClient,
userAgent: this.connectionPolicy.UserAgentContainer.BaseUserAgent,
connectionMode: this.connectionPolicy.ConnectionMode,
authorizationTokenProvider: this.cosmosAuthorization,
diagnosticsHelper: DiagnosticsHandlerHelper.Instance,
preferredRegions: this.connectionPolicy.PreferredLocations,
globalEndpointManager: this.globalEndpointManager);
DiagnosticsHandlerHelper.Refresh(clientConfig.IsClientTelemetryEnabled());
this.collector = new TelemetryCollector(this.clientTelemetry, this.connectionPolicy);
DefaultTrace.TraceVerbose("Client Telemetry Enabled.");
}
catch (Exception ex)
if (clientConfig.IsClientTelemetryEnabled())
{
DefaultTrace.TraceWarning($"Error While starting Telemetry Job : {0}. Hence disabling Client Telemetry", ex.Message);
this.connectionPolicy.EnableClientTelemetry = false;
try
{
this.clientTelemetry = ClientTelemetry.CreateAndStartBackgroundTelemetry(
clientId: this.clientId,
httpClient: this.httpClient,
userAgent: this.connectionPolicy.UserAgentContainer.BaseUserAgent,
connectionMode: this.connectionPolicy.ConnectionMode,
authorizationTokenProvider: this.cosmosAuthorization,
diagnosticsHelper: DiagnosticsHandlerHelper.GetInstance(),
preferredRegions: this.connectionPolicy.PreferredLocations,
globalEndpointManager: this.globalEndpointManager,
endpointUrl: clientConfig.ClientTelemetryConfiguration.Endpoint);
this.collector = new TelemetryCollector(this.clientTelemetry, this.connectionPolicy);
DefaultTrace.TraceVerbose("Client Telemetry Enabled.");
}
catch (Exception ex)
{
DefaultTrace.TraceWarning($"Error While starting Telemetry Job : {0}. Hence disabling Client Telemetry", ex);
this.connectionPolicy.EnableClientTelemetry = false;
}
}
}

2
Microsoft.Azure.Cosmos/src/Tracing/TraceData/ClientSideRequestStatisticsTraceDatum.cs
Просмотреть файл

@ -419,7 +419,7 @@ namespace Microsoft.Azure.Cosmos.Tracing.TraceData
this.systemUsageHistory.Values.Count == 0 ||
this.systemUsageHistory.LastTimestamp + DiagnosticsHandlerHelper.DiagnosticsRefreshInterval < DateTime.UtcNow)
{
this.systemUsageHistory = DiagnosticsHandlerHelper.Instance.GetDiagnosticsSystemHistory();
this.systemUsageHistory = DiagnosticsHandlerHelper.GetInstance().GetDiagnosticsSystemHistory();
}
#endif
}

11
Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/ClientTelemetryReleaseTests.cs
Просмотреть файл

@ -36,20 +36,15 @@ namespace Microsoft.Azure.Cosmos.SDK.EmulatorTests
}
[ClassInitialize]
public static new void ClassInitialize(TestContext context)
public static void ClassInit(TestContext context)
{
ClientTelemetryTestsBase.ClassInitialize(context);
// It will go away in next PR
Environment.SetEnvironmentVariable(ClientTelemetryOptions.EnvPropsClientTelemetryEnabled, "true");
Environment.SetEnvironmentVariable(ClientTelemetryOptions.EnvPropsClientTelemetrySchedulingInSeconds, "1");
Environment.SetEnvironmentVariable(ClientTelemetryOptions.EnvPropsClientTelemetryEndpoint, "https://tools.cosmos.azure.com/api/clienttelemetry/trace");
}
[ClassCleanup]
public static new void FinalCleanup()
public static void ClassCleanUp()
{
ClientTelemetryTestsBase.FinalCleanup();
ClientTelemetryTestsBase.ClassCleanup();
}
[TestInitialize]

27
Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/ClientTelemetryTests.cs
Просмотреть файл

@ -5,11 +5,13 @@
namespace Microsoft.Azure.Cosmos.SDK.EmulatorTests
{
using Microsoft.Azure.Cosmos.Fluent;
using Microsoft.Azure.Cosmos.Telemetry;
using System.Net.Http;
using System.Net;
using System.Threading.Tasks;
using Microsoft.VisualStudio.TestTools.UnitTesting;
using Newtonsoft.Json;
using System.Text;
using Microsoft.Azure.Documents;
/// <summary>
/// In Emulator Mode, Run test against emulator and mock client telemetry service calls.
@ -21,10 +23,25 @@ namespace Microsoft.Azure.Cosmos.SDK.EmulatorTests
{
public override Task<HttpResponseMessage> HttpHandlerRequestCallbackChecks(HttpRequestMessage request)
{
if (request.RequestUri.AbsoluteUri.Equals(ClientTelemetryOptions.GetClientTelemetryEndpoint().AbsoluteUri))
if (request.RequestUri.AbsoluteUri.Equals(telemetryServiceEndpoint.AbsoluteUri))
{
return Task.FromResult(new HttpResponseMessage(HttpStatusCode.NoContent)); // In Emulator test, send hardcoded response status code as there is no real communication happens with client telemetry service
}
else if (request.RequestUri.AbsoluteUri.Contains(Paths.ClientConfigPathSegment))
{
HttpResponseMessage result = new HttpResponseMessage(HttpStatusCode.OK);
AccountClientConfiguration clientConfigProperties = new AccountClientConfiguration
{
ClientTelemetryConfiguration = new ClientTelemetryConfiguration
{
IsEnabled = true,
Endpoint = telemetryServiceEndpoint.AbsoluteUri
}
};
string payload = JsonConvert.SerializeObject(clientConfigProperties);
result.Content = new StringContent(payload, Encoding.UTF8, "application/json");
return Task.FromResult(result);
}
return null;
}
@ -35,15 +52,15 @@ namespace Microsoft.Azure.Cosmos.SDK.EmulatorTests
}
[ClassInitialize]
public static new void ClassInitialize(TestContext context)
public static void ClassInit(TestContext context)
{
ClientTelemetryTestsBase.ClassInitialize(context);
}
[ClassCleanup]
public static new void FinalCleanup()
public static void ClassCleanUp()
{
ClientTelemetryTestsBase.FinalCleanup();
ClientTelemetryTestsBase.ClassCleanup();
}
[TestInitialize]

74
Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/ClientTelemetryTestsBase.cs
Просмотреть файл

@ -27,7 +27,8 @@ namespace Microsoft.Azure.Cosmos.SDK.EmulatorTests
public abstract class ClientTelemetryTestsBase : BaseCosmosClientHelper
{
private static SystemUsageMonitor systemUsageMonitor;
protected static readonly Uri telemetryServiceEndpoint = new Uri("http://dummy.telemetry.service/api/url");
private static readonly List<string> preferredRegionList = new List<string>
{
Regions.EastUS,
@ -53,13 +54,13 @@ namespace Microsoft.Azure.Cosmos.SDK.EmulatorTests
public static void ClassInitialize(TestContext _)
{
Util.EnableClientTelemetryEnvironmentVariables();
ClientTelemetryOptions.DefaultIntervalForTelemetryJob = TimeSpan.FromSeconds(1);
}
SystemUsageMonitor oldSystemUsageMonitor = (SystemUsageMonitor)typeof(DiagnosticsHandlerHelper)
.GetField("systemUsageMonitor", BindingFlags.Instance | BindingFlags.NonPublic).GetValue(DiagnosticsHandlerHelper.Instance);
oldSystemUsageMonitor.Stop();
ClientTelemetryTestsBase.ResetSystemUsageMonitor(true);
public static void ClassCleanup()
{
//undone the changes done in ClassInitialize
ClientTelemetryOptions.DefaultIntervalForTelemetryJob = TimeSpan.FromMinutes(10);
}
public virtual void TestInitialize()
@ -70,7 +71,7 @@ namespace Microsoft.Azure.Cosmos.SDK.EmulatorTests
{
RequestCallBack = (request, cancellation) =>
{
if (request.RequestUri.AbsoluteUri.Equals(ClientTelemetryOptions.GetClientTelemetryEndpoint().AbsoluteUri))
if (request.RequestUri.AbsoluteUri.Equals(telemetryServiceEndpoint.AbsoluteUri))
{
string jsonObject = request.Content.ReadAsStringAsync().GetAwaiter().GetResult();
@ -83,7 +84,7 @@ namespace Microsoft.Azure.Cosmos.SDK.EmulatorTests
},
ResponseIntercepter = (response) =>
{
if (response.RequestMessage.RequestUri.AbsoluteUri.Equals(ClientTelemetryOptions.GetClientTelemetryEndpoint().AbsoluteUri))
if (response.RequestMessage != null && response.RequestMessage.RequestUri.AbsoluteUri.Equals(telemetryServiceEndpoint.AbsoluteUri))
{
Assert.AreEqual(HttpStatusCode.NoContent, response.StatusCode);
}
@ -92,7 +93,7 @@ namespace Microsoft.Azure.Cosmos.SDK.EmulatorTests
},
ExceptionIntercepter = (request, exception) =>
{
if (request.RequestUri.AbsoluteUri.Equals(ClientTelemetryOptions.GetClientTelemetryEndpoint().AbsoluteUri))
if (request.RequestUri.AbsoluteUri.Equals(telemetryServiceEndpoint.AbsoluteUri))
{
this.isClientTelemetryAPICallFailed = true;
}
@ -109,7 +110,7 @@ namespace Microsoft.Azure.Cosmos.SDK.EmulatorTests
return Task.FromResult(result);
}
if (request.RequestUri.AbsoluteUri.Equals(ClientTelemetryOptions.GetClientTelemetryEndpoint().AbsoluteUri))
if (request.RequestUri.AbsoluteUri.Equals(telemetryServiceEndpoint.AbsoluteUri))
{
string jsonObject = request.Content.ReadAsStringAsync().GetAwaiter().GetResult();
@ -123,7 +124,7 @@ namespace Microsoft.Azure.Cosmos.SDK.EmulatorTests
},
ResponseIntercepter = (response) =>
{
if (response.RequestMessage.RequestUri.AbsoluteUri.Equals(ClientTelemetryOptions.GetClientTelemetryEndpoint().AbsoluteUri))
if (response.RequestMessage != null && response.RequestMessage.RequestUri.AbsoluteUri.Equals(telemetryServiceEndpoint.AbsoluteUri))
{
Assert.AreEqual(HttpStatusCode.NoContent, response.StatusCode);
}
@ -131,7 +132,7 @@ namespace Microsoft.Azure.Cosmos.SDK.EmulatorTests
},
ExceptionIntercepter = (request, exception) =>
{
if (request.RequestUri.AbsoluteUri.Equals(ClientTelemetryOptions.GetClientTelemetryEndpoint().AbsoluteUri))
if (request.RequestUri.AbsoluteUri.Equals(telemetryServiceEndpoint.AbsoluteUri))
{
this.isClientTelemetryAPICallFailed = true;
}
@ -139,6 +140,7 @@ namespace Microsoft.Azure.Cosmos.SDK.EmulatorTests
};
this.cosmosClientBuilder = this.GetBuilder()
.WithTelemetryEnabled()
.WithApplicationPreferredRegions(ClientTelemetryTestsBase.preferredRegionList);
}
@ -146,34 +148,6 @@ namespace Microsoft.Azure.Cosmos.SDK.EmulatorTests
public abstract CosmosClientBuilder GetBuilder();
private static void ResetSystemUsageMonitor(bool isTelemetryEnabled)
{
ClientTelemetryTestsBase.systemUsageMonitor?.Stop();
FieldInfo diagnosticsHandlerHelperInstance = typeof(DiagnosticsHandlerHelper)
.GetField("isTelemetryMonitoringEnabled", BindingFlags.Instance | BindingFlags.Static | BindingFlags.NonPublic);
diagnosticsHandlerHelperInstance.SetValue(null, isTelemetryEnabled);
List<SystemUsageRecorder> recorders = new List<SystemUsageRecorder>()
{
(SystemUsageRecorder)typeof(DiagnosticsHandlerHelper)
.GetField("diagnosticSystemUsageRecorder",
BindingFlags.Instance | BindingFlags.NonPublic)
.GetValue(DiagnosticsHandlerHelper.Instance)
};
if (isTelemetryEnabled)
{
recorders.Add(
(SystemUsageRecorder)typeof(DiagnosticsHandlerHelper)
.GetField("telemetrySystemUsageRecorder",
BindingFlags.Instance | BindingFlags.NonPublic)
.GetValue(DiagnosticsHandlerHelper.Instance));
}
ClientTelemetryTestsBase.systemUsageMonitor = SystemUsageMonitor.CreateAndStart(recorders);
}
public virtual async Task Cleanup()
{
FieldInfo isInitializedField = typeof(VmMetadataApiHandler).GetField("isInitialized",
@ -185,16 +159,12 @@ namespace Microsoft.Azure.Cosmos.SDK.EmulatorTests
BindingFlags.Static |
BindingFlags.NonPublic);
azMetadataField.SetValue(null, null);
await base.TestCleanup();
Assert.IsFalse(this.isClientTelemetryAPICallFailed, $"Call to client telemetry service endpoint (i.e. {ClientTelemetryOptions.GetClientTelemetryEndpoint().AbsoluteUri}) failed");
Assert.IsFalse(this.isClientTelemetryAPICallFailed, $"Call to client telemetry service endpoint (i.e. {telemetryServiceEndpoint}) failed");
}
public static void FinalCleanup()
{
ClientTelemetryTestsBase.ResetSystemUsageMonitor(false);
}
public virtual async Task PointSuccessOperationsTest(ConnectionMode mode, bool isAzureInstance)
{
Container container = await this.CreateClientAndContainer(
@ -414,8 +384,6 @@ namespace Microsoft.Azure.Cosmos.SDK.EmulatorTests
public virtual async Task QueryOperationSinglePartitionTest(ConnectionMode mode)
{
Environment.SetEnvironmentVariable(ClientTelemetryOptions.EnvPropsClientTelemetrySchedulingInSeconds, "20");
Container container = await this.CreateClientAndContainer(mode);
ToDoActivity testItem = ToDoActivity.CreateRandomToDoActivity("MyTestPkValue", "MyTestItemId");
@ -471,7 +439,6 @@ namespace Microsoft.Azure.Cosmos.SDK.EmulatorTests
public virtual async Task QueryMultiPageSinglePartitionOperationTest(ConnectionMode mode)
{
Environment.SetEnvironmentVariable(ClientTelemetryOptions.EnvPropsClientTelemetrySchedulingInSeconds, "20");
Container container = await this.CreateClientAndContainer(mode: mode);
ItemRequestOptions requestOptions = new ItemRequestOptions()
@ -532,8 +499,6 @@ namespace Microsoft.Azure.Cosmos.SDK.EmulatorTests
public virtual async Task QueryOperationCrossPartitionTest(ConnectionMode mode)
{
Environment.SetEnvironmentVariable(ClientTelemetryOptions.EnvPropsClientTelemetrySchedulingInSeconds, "20");
ContainerInternal itemsCore = (ContainerInternal)await this.CreateClientAndContainer(
mode: mode,
isLargeContainer: true);
@ -674,7 +639,7 @@ namespace Microsoft.Azure.Cosmos.SDK.EmulatorTests
httpHandler.RequestCallBack = (request, cancellation) =>
{
if (request.RequestUri.AbsoluteUri.Equals(ClientTelemetryOptions.GetClientTelemetryEndpoint().AbsoluteUri))
if (request.RequestUri.AbsoluteUri.Equals(telemetryServiceEndpoint.AbsoluteUri))
{
string jsonObject = request.Content.ReadAsStringAsync().GetAwaiter().GetResult();
@ -696,11 +661,12 @@ namespace Microsoft.Azure.Cosmos.SDK.EmulatorTests
return Task.FromResult(result);
}
return null;
return this.HttpHandlerRequestCallbackChecks(request);
};
// Replacing originally initialized cosmos Builder with this one with new handler
this.cosmosClientBuilder = this.cosmosClientBuilder
.WithTelemetryEnabled()
.WithHttpClientFactory(() => new HttpClient(httpHandler));
Container container = await this.CreateClientAndContainer(

43
Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/SynchronizationContextTests.cs
Просмотреть файл

@ -6,9 +6,15 @@ namespace Microsoft.Azure.Cosmos.SDK.EmulatorTests
{
using System;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Azure.Cosmos.Linq;
using Microsoft.VisualStudio.TestTools.UnitTesting;
using Newtonsoft.Json;
using Microsoft.Azure.Documents;
[TestClass]
public class SynchronizationContextTests
@ -19,10 +25,29 @@ namespace Microsoft.Azure.Cosmos.SDK.EmulatorTests
[Timeout(30000)]
public void VerifySynchronizationContextDoesNotLock(bool withClientTelemetry)
{
if (withClientTelemetry)
HttpClientHandlerHelper httpHandler = new HttpClientHandlerHelper
{
Util.EnableClientTelemetryEnvironmentVariables();
}
RequestCallBack = (request, cancellation) =>
{
if (request.RequestUri.AbsoluteUri.Contains(Paths.ClientConfigPathSegment))
{
HttpResponseMessage result = new HttpResponseMessage(HttpStatusCode.OK);
AccountClientConfiguration clientConfigProperties = new AccountClientConfiguration
{
ClientTelemetryConfiguration = new ClientTelemetryConfiguration
{
IsEnabled = withClientTelemetry,
Endpoint = withClientTelemetry? "http://dummy.telemetry.endpoint/" : null
}
};
string payload = JsonConvert.SerializeObject(clientConfigProperties);
result.Content = new StringContent(payload, Encoding.UTF8, "application/json");
return Task.FromResult(result);
}
return null;
}
};
string databaseId = Guid.NewGuid().ToString();
SynchronizationContext prevContext = SynchronizationContext.Current;
@ -32,7 +57,10 @@ namespace Microsoft.Azure.Cosmos.SDK.EmulatorTests
SynchronizationContext.SetSynchronizationContext(syncContext);
syncContext.Post(_ =>
{
using (CosmosClient client = TestCommon.CreateCosmosClient())
using (CosmosClient client = TestCommon.CreateCosmosClient(
customizeClientBuilder: (builder) => builder
.WithTelemetryEnabled()
.WithHttpClientFactory(() => new HttpClient(httpHandler))))
{
Cosmos.Database database = client.CreateDatabaseAsync(databaseId).GetAwaiter().GetResult();
database = client.CreateDatabaseIfNotExistsAsync(databaseId).GetAwaiter().GetResult();
@ -124,7 +152,7 @@ namespace Microsoft.Azure.Cosmos.SDK.EmulatorTests
double cost = container.GetItemLinqQueryable<ToDoActivity>(
allowSynchronousQueryExecution: true).Select(x => x.cost).Sum();
ItemResponse<ToDoActivity> deleteResponse = container.DeleteItemAsync<ToDoActivity>(partitionKey: new Cosmos.PartitionKey(testItem.pk), id: testItem.id).ConfigureAwait(false).GetAwaiter().GetResult();
Assert.IsNotNull(deleteResponse);
}
@ -138,11 +166,6 @@ namespace Microsoft.Azure.Cosmos.SDK.EmulatorTests
{
client.GetDatabase(databaseId).DeleteAsync().GetAwaiter().GetResult();
}
if (withClientTelemetry)
{
Util.DisableClientTelemetryEnvironmentVariables();
}
}
}

2
Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Utils/HttpHandlerHelper.cs
Просмотреть файл

@ -52,6 +52,8 @@ namespace Microsoft.Azure.Cosmos.SDK.EmulatorTests
throw;
}
this.ExceptionIntercepter.Invoke(request, ex);
throw; // Anyway throw this exception
}
if (this.ResponseIntercepter != null)

14
Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Utils/Util.cs
Просмотреть файл

@ -522,20 +522,6 @@ namespace Microsoft.Azure.Cosmos.SDK.EmulatorTests
options.OfferThroughput);
}
internal static void EnableClientTelemetryEnvironmentVariables()
{
Environment.SetEnvironmentVariable(ClientTelemetryOptions.EnvPropsClientTelemetryEnabled, "true");
Environment.SetEnvironmentVariable(ClientTelemetryOptions.EnvPropsClientTelemetrySchedulingInSeconds, "1");
Environment.SetEnvironmentVariable(ClientTelemetryOptions.EnvPropsClientTelemetryEndpoint, "http://dummy.telemetry.endpoint/");
}
internal static void DisableClientTelemetryEnvironmentVariables()
{
Environment.SetEnvironmentVariable(ClientTelemetryOptions.EnvPropsClientTelemetryEnabled, null);
Environment.SetEnvironmentVariable(ClientTelemetryOptions.EnvPropsClientTelemetrySchedulingInSeconds, null);
Environment.SetEnvironmentVariable(ClientTelemetryOptions.EnvPropsClientTelemetryEndpoint, null);
}
private static TracerProvider OTelTracerProvider;
private static CustomListener TestListener;

67
Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Performance.Tests/Mocks/HttpHandlerHelper.cs Normal file
Просмотреть файл

@ -0,0 +1,67 @@
//------------------------------------------------------------
// Copyright (c) Microsoft Corporation. All rights reserved.
//------------------------------------------------------------
namespace Microsoft.Azure.Cosmos.Performance.Tests
{
using System;
using System.Net.Http;
using System.Threading;
using System.Threading.Tasks;
public class HttpClientHandlerHelper : DelegatingHandler
{
public HttpClientHandlerHelper() : base(new HttpClientHandler())
{
}
public Func<HttpRequestMessage, CancellationToken, Task<HttpResponseMessage>> RequestCallBack { get; set; }
public Func<HttpResponseMessage, Task<HttpResponseMessage>> ResponseIntercepter { get; set; }
public Action<HttpRequestMessage, Exception> ExceptionIntercepter { get; set; }
protected override async Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
{
HttpResponseMessage httpResponse = null;
if (this.RequestCallBack != null)
{
Task<HttpResponseMessage> response = this.RequestCallBack(request, cancellationToken);
if(response != null)
{
httpResponse = await response;
if (httpResponse != null)
{
if (this.ResponseIntercepter != null)
{
httpResponse = await this.ResponseIntercepter(httpResponse);
}
return httpResponse;
}
}
}
try
{
httpResponse = await base.SendAsync(request, cancellationToken);
}
catch (Exception ex) {
if (this.ExceptionIntercepter == null)
{
throw;
}
this.ExceptionIntercepter.Invoke(request, ex);
throw; // Anyway throw this exception
}
if (this.ResponseIntercepter != null)
{
httpResponse = await this.ResponseIntercepter(httpResponse);
}
return httpResponse;
}
}
}

42
Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Performance.Tests/Mocks/MockDocumentClient.cs
Просмотреть файл

@ -22,6 +22,9 @@ namespace Microsoft.Azure.Cosmos.Performance.Tests
using Microsoft.Azure.Cosmos.Query.Core.QueryPlan;
using Newtonsoft.Json;
using Microsoft.Azure.Cosmos.Telemetry;
using System.Net.Http;
using System.Net;
using System.Text;
internal class MockDocumentClient : DocumentClient, ICosmosAuthorizationTokenProvider
{
@ -51,7 +54,7 @@ namespace Microsoft.Azure.Cosmos.Performance.Tests
{
policy = new ConnectionPolicy
{
EnableClientTelemetry = isClientTelemetryEnabled.Value
EnableClientTelemetry = true // feature flag is always true
};
}
@ -59,6 +62,43 @@ namespace Microsoft.Azure.Cosmos.Performance.Tests
MockDocumentClient documentClient = new MockDocumentClient(policy);
CosmosClientBuilder cosmosClientBuilder = new CosmosClientBuilder("http://localhost", Convert.ToBase64String(Guid.NewGuid().ToByteArray()));
cosmosClientBuilder.WithConnectionModeDirect();
Uri telemetryServiceEndpoint = new Uri("https://dummy.endpoint.com/");
if (isClientTelemetryEnabled.HasValue)
{
// mock external calls
HttpClientHandlerHelper httpHandler = new HttpClientHandlerHelper
{
RequestCallBack = (request, cancellation) =>
{
if (request.RequestUri.AbsoluteUri.Equals(telemetryServiceEndpoint.AbsoluteUri))
{
return Task.FromResult(new HttpResponseMessage(HttpStatusCode.NoContent)); // In Emulator test, send hardcoded response status code as there is no real communication happens with client telemetry service
}
else if (request.RequestUri.AbsoluteUri.Contains(Paths.ClientConfigPathSegment))
{
HttpResponseMessage result = new HttpResponseMessage(HttpStatusCode.OK);
AccountClientConfiguration clientConfigProperties = new AccountClientConfiguration
{
ClientTelemetryConfiguration = new ClientTelemetryConfiguration
{
IsEnabled = isClientTelemetryEnabled.Value,
Endpoint = isClientTelemetryEnabled.Value?telemetryServiceEndpoint.AbsoluteUri: null
}
};
string payload = JsonConvert.SerializeObject(clientConfigProperties);
result.Content = new StringContent(payload, Encoding.UTF8, "application/json");
return Task.FromResult(result);
}
return null;
}
};
cosmosClientBuilder.WithHttpClientFactory(() => new HttpClient(httpHandler));
}
customizeClientBuilder?.Invoke(cosmosClientBuilder);
if (useCustomSerializer)

75
Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Diagnostics/DiagnosticHandlerHelperTests.cs Normal file
Просмотреть файл

@ -0,0 +1,75 @@
//------------------------------------------------------------
// Copyright (c) Microsoft Corporation. All rights reserved.
//------------------------------------------------------------
namespace Microsoft.Azure.Cosmos.Diagnostics
{
using System;
using System.Reflection;
using System.Threading.Tasks;
using Microsoft.Azure.Cosmos.Handler;
using Microsoft.VisualStudio.TestTools.UnitTesting;
[TestClass]
public class DiagnosticHandlerHelperTests
{
[ClassInitialize]
public static void Initialize(TestContext _)
{
DiagnosticHandlerHelperTests.ResetDiagnosticsHandlerHelper();
}
private static void ResetDiagnosticsHandlerHelper()
{
//Stop the job
DiagnosticsHandlerHelper helper = DiagnosticsHandlerHelper.GetInstance();
MethodInfo iMethod = helper.GetType().GetMethod("StopSystemMonitor", BindingFlags.NonPublic | BindingFlags.Instance);
iMethod.Invoke(helper, new object[] { });
//Reset the instance woth original value
FieldInfo field = typeof(DiagnosticsHandlerHelper).GetField("Instance",
BindingFlags.Static |
BindingFlags.NonPublic);
field.SetValue(null, Activator.CreateInstance(typeof(DiagnosticsHandlerHelper), true));
}
[TestMethod]
public void SingletonTest()
{
DiagnosticsHandlerHelper diagnosticHandlerHelper1 = DiagnosticsHandlerHelper.GetInstance();
DiagnosticsHandlerHelper diagnosticHandlerHelper2 = DiagnosticsHandlerHelper.GetInstance();
Assert.IsNotNull(diagnosticHandlerHelper1);
Assert.IsNotNull(diagnosticHandlerHelper2);
Assert.AreEqual(diagnosticHandlerHelper1, diagnosticHandlerHelper2, "Not Singleton");
}
[TestMethod]
public async Task RefreshTestAsync()
{
// Get default instance of DiagnosticsHandlerHelper with client telemetry disabled (default)
DiagnosticsHandlerHelper diagnosticHandlerHelper1 = DiagnosticsHandlerHelper.GetInstance();
await Task.Delay(10000); // warm up
Assert.IsNotNull(diagnosticHandlerHelper1.GetDiagnosticsSystemHistory());
int countBeforeRefresh = diagnosticHandlerHelper1.GetDiagnosticsSystemHistory().Values.Count;
// Refresh instance of DiagnosticsHandlerHelper with client telemetry enabled
DiagnosticsHandlerHelper.Refresh(true);
DiagnosticsHandlerHelper diagnosticHandlerHelper2 = DiagnosticsHandlerHelper.GetInstance();
int countAfterRefresh = diagnosticHandlerHelper1.GetDiagnosticsSystemHistory().Values.Count;
Console.WriteLine(countBeforeRefresh + " " + countAfterRefresh);
Assert.IsTrue(countBeforeRefresh <= countAfterRefresh, "After Refresh count should be greater than or equal to before refresh count");
Assert.AreNotEqual(diagnosticHandlerHelper1, diagnosticHandlerHelper2);
Assert.IsNotNull(diagnosticHandlerHelper2.GetDiagnosticsSystemHistory());
Assert.IsNotNull(diagnosticHandlerHelper2.GetClientTelemetrySystemHistory());
// Refresh instance of DiagnosticsHandlerHelper with client telemetry disabled
DiagnosticsHandlerHelper.Refresh(false);
DiagnosticsHandlerHelper diagnosticHandlerHelper3 = DiagnosticsHandlerHelper.GetInstance();
Assert.IsNotNull(diagnosticHandlerHelper3.GetDiagnosticsSystemHistory());
Assert.IsNull(diagnosticHandlerHelper3.GetClientTelemetrySystemHistory());
}
}
}

22
Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Telemetry/ClientTelemetryTests.cs
Просмотреть файл

@ -26,13 +26,6 @@ namespace Microsoft.Azure.Cosmos.Tests.Telemetry
[TestClass]
public class ClientTelemetryTests
{
[TestCleanup]
public void Cleanup()
{
Environment.SetEnvironmentVariable(ClientTelemetryOptions.EnvPropsClientTelemetryEnabled, null);
Environment.SetEnvironmentVariable(ClientTelemetryOptions.EnvPropsClientTelemetryEndpoint, null);
}
[TestMethod]
public void CheckMetricsAggregationLogic()
{
@ -126,7 +119,6 @@ namespace Microsoft.Azure.Cosmos.Tests.Telemetry
[DataRow(150, 0, 0)] // When only operation info is there in payload
public async Task CheckIfPayloadIsDividedCorrectlyAsync(int expectedOperationInfoSize, int expectedCacheRefreshInfoSize, int expectedRequestInfoSize)
{
Environment.SetEnvironmentVariable(ClientTelemetryOptions.EnvPropsClientTelemetryEndpoint, "http://dummy.telemetry.endpoint/");
ClientTelemetryOptions.PayloadSizeThreshold = 1024 * 15; //15 Kb
string data = File.ReadAllText("Telemetry/ClientTelemetryPayloadWithoutMetrics.json", Encoding.UTF8);
@ -245,7 +237,8 @@ namespace Microsoft.Azure.Cosmos.Tests.Telemetry
clientTelemetryProperties,
operationInfoSnapshot,
cacheRefreshInfoSnapshot,
requestInfoList,
requestInfoList,
"http://dummy.telemetry.endpoint/",
new CancellationTokenSource(ClientTelemetryOptions.ClientTelemetryProcessorTimeOut).Token);
Assert.AreEqual(expectedOperationInfoSize, actualOperationInfoSize, "Operation Info is not correct");
@ -256,8 +249,6 @@ namespace Microsoft.Azure.Cosmos.Tests.Telemetry
[TestMethod]
public async Task ClientTelmetryProcessor_should_timeout()
{
Environment.SetEnvironmentVariable(ClientTelemetryOptions.EnvPropsClientTelemetryEndpoint, "http://dummy.telemetry.endpoint/");
string data = File.ReadAllText("Telemetry/ClientTelemetryPayloadWithoutMetrics.json", Encoding.UTF8);
ClientTelemetryProperties clientTelemetryProperties = JsonConvert.DeserializeObject<ClientTelemetryProperties>(data);
@ -344,6 +335,7 @@ namespace Microsoft.Azure.Cosmos.Tests.Telemetry
operationInfoSnapshot,
cacheRefreshInfoSnapshot,
default,
"http://dummy.telemetry.endpoint/",
cts.Token);
});
@ -352,13 +344,5 @@ namespace Microsoft.Azure.Cosmos.Tests.Telemetry
processingTask: processorTask,
timeout: TimeSpan.FromTicks(1)));
}
[TestMethod]
[ExpectedException(typeof(FormatException))]
public void CheckMisconfiguredTelemetry_should_fail()
{
Environment.SetEnvironmentVariable(ClientTelemetryOptions.EnvPropsClientTelemetryEnabled, "non-boolean");
using CosmosClient client = MockCosmosUtil.CreateMockCosmosClient();
}
}
}