Preview - Updated docs/rules, new predicate
This commit is contained in:
Charles Torre
Родитель
51879b3d0a
Коммит
a1c3b905aa
|
|
@ -73,4 +73,18 @@ Mitigate(AppName="fabric:/MyApp42", MetricName="CpuPercent", MetricValue=?Metric
|
|||
RestartCodePackage().
|
||||
```
|
||||
|
||||
***Problem***: I want to check the value for the supplied resource metric (CpuPercent) and ensure that the usage is non-transient - that FabricObserver has generated at least 3 health reports for this issue in a 15 minute time span - before running the RestartCodePackage repair on any service belonging to the specified app.
|
||||
|
||||
***Solution***:
|
||||
```
|
||||
## Try to mitigate an SF Application in Error or Warning named fabric:/MyApp42 where one of its services is consuming too much CPU (as a percentage of total CPU)
|
||||
## and where at least 3 health events identifying this problem were produced in the last 15 minutes. This is useful to ensure you don't mitigate a transient (short-lived)
|
||||
## problem as they will self-correct.
|
||||
Mitigate(AppName="fabric:/MyApp42", MetricName="CpuPercent", MetricValue=?MetricValue) :- ?MetricValue >= 80,
|
||||
GetHealthEventHistory(?HealthEventCount, TimeRange=00:15:00),
|
||||
?HealthEventCount >= 3,
|
||||
TimeScopedRestartCodePackage(4, 01:00:00).
|
||||
```
|
||||
|
||||
|
||||
Please look through the [existing rules files](/FabricHealer/PackageRoot/Config/Rules) for real examples that have been tested. Simply modify the rules to meet your needs (like supplying your target app names, for example, and adjusting the simple logical constraints, if need be).
|
||||
|
|
|
|||
|
|
@ -51,7 +51,7 @@ namespace FHTest
|
|||
|
||||
// Set this to the full path to your Rules directory in the FabricHealer project's PackageRoot\Config directory.
|
||||
// e.g., if developing on Windows, then something like @"C:\Users\[me]\source\repos\service-fabric-healer\FabricHealer\PackageRoot\Config\Rules\";
|
||||
private const string FHRulesDirectory = @"C:\Users\[me]\source\repos\service-fabric-healer\FabricHealer\PackageRoot\Config\Rules\";
|
||||
private const string FHRulesDirectory = @"C:\Users\ctorre\source\repos\service-fabric-healer\FabricHealer\PackageRoot\Config\Rules\";
|
||||
|
||||
/* GuanLogic Tests */
|
||||
// TODO: Add more tests.
|
||||
|
|
@ -181,6 +181,7 @@ namespace FHTest
|
|||
|
||||
// Add external helper predicates.
|
||||
functorTable.Add(CheckFolderSizePredicateType.Singleton(RepairConstants.CheckFolderSize, repairTaskHelper, foHealthData));
|
||||
functorTable.Add(GetHealthEventHistoryPredicateType.Singleton(RepairConstants.GetHealthEventHistory, repairTaskHelper, foHealthData));
|
||||
functorTable.Add(GetRepairHistoryPredicateType.Singleton(RepairConstants.GetRepairHistory, repairTaskHelper, foHealthData));
|
||||
functorTable.Add(CheckInsideRunIntervalPredicateType.Singleton(RepairConstants.CheckInsideRunInterval, repairTaskHelper, foHealthData));
|
||||
functorTable.Add(EmitMessagePredicateType.Singleton(RepairConstants.EmitMessage, repairTaskHelper));
|
||||
|
|
|
|||
|
|
@ -677,6 +677,11 @@ namespace FabricHealer
|
|||
var supportedAppHealthStates = appHealthStates.Where(a => a.AggregatedHealthState == HealthState.Warning || a.AggregatedHealthState == HealthState.Error);
|
||||
var nodeList = await fabricClient.QueryManager.GetNodeListAsync().ConfigureAwait(false);
|
||||
|
||||
// Random wait to limit potential duplicate (concurrent) repair job creation from other FH instances.
|
||||
var random = new Random();
|
||||
int waitTimeMS = random.Next(random.Next(0, nodeCount * 100), 1000 * nodeCount);
|
||||
await Task.Delay(waitTimeMS, Token).ConfigureAwait(true);
|
||||
|
||||
foreach (var app in supportedAppHealthStates)
|
||||
{
|
||||
Token.ThrowIfCancellationRequested();
|
||||
|
|
@ -721,16 +726,12 @@ namespace FabricHealer
|
|||
{
|
||||
Token.ThrowIfCancellationRequested();
|
||||
|
||||
// Random wait to limit potential duplicate (concurrent) repair job creation from other FH instances.
|
||||
var random = new Random();
|
||||
int waitTimeMS = random.Next(random.Next(0, nodeCount * 100), 1000 * nodeCount);
|
||||
await Task.Delay(waitTimeMS, Token).ConfigureAwait(true);
|
||||
|
||||
if (string.IsNullOrWhiteSpace(evt.HealthInformation.Description))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
// If source of health data is not FabricObserver, move on.
|
||||
if (!JsonSerializationUtility.TryDeserialize(evt.HealthInformation.Description, out TelemetryData foHealthData))
|
||||
{
|
||||
continue;
|
||||
|
|
@ -760,7 +761,7 @@ namespace FabricHealer
|
|||
{
|
||||
foreach (var repair in fhRepairTasks)
|
||||
{
|
||||
var executorData = JsonSerializationUtility.TryDeserialize(repair.ExecutorData, out RepairExecutorData exData) ? exData : null;
|
||||
var executorData = JsonSerializationUtility.TryDeserialize(repair.ExecutorData, out RepairExecutorData exData) ? exData : null;
|
||||
|
||||
if (executorData?.RepairPolicy?.RepairAction != RepairActionType.RestartFabricNode &&
|
||||
executorData?.RepairPolicy?.RepairAction != RepairActionType.RestartProcess)
|
||||
|
|
@ -781,7 +782,7 @@ namespace FabricHealer
|
|||
}
|
||||
|
||||
repairRules = GetRepairRulesFromFOCode(foHealthData.Code, "fabric:/System");
|
||||
|
||||
|
||||
if (repairRules == null || repairRules?.Count == 0)
|
||||
{
|
||||
continue;
|
||||
|
|
@ -790,7 +791,7 @@ namespace FabricHealer
|
|||
repairId = $"{foHealthData.NodeName}_{foHealthData.SystemServiceProcessName}_{foHealthData.Code}";
|
||||
system = "System ";
|
||||
|
||||
var currentRepairs =
|
||||
var currentRepairs =
|
||||
await repairTaskEngine.GetFHRepairTasksCurrentlyProcessingAsync(RepairTaskEngine.FabricHealerExecutorName, Token).ConfigureAwait(true);
|
||||
|
||||
// Is a repair for the target app service instance already happening in the cluster?
|
||||
|
|
@ -814,7 +815,7 @@ namespace FabricHealer
|
|||
else
|
||||
{
|
||||
repairRules = GetRepairRulesFromFOCode(foHealthData.Code);
|
||||
|
||||
|
||||
// Nothing to do here.
|
||||
if (repairRules == null || repairRules?.Count == 0)
|
||||
{
|
||||
|
|
@ -828,7 +829,7 @@ namespace FabricHealer
|
|||
}
|
||||
|
||||
string serviceProcessName = $"{foHealthData.ServiceName?.Replace("fabric:/", "").Replace("/", "")}";
|
||||
var currentRepairs =
|
||||
var currentRepairs =
|
||||
await repairTaskEngine.GetFHRepairTasksCurrentlyProcessingAsync(RepairTaskEngine.FabricHealerExecutorName, Token).ConfigureAwait(true);
|
||||
|
||||
// This is the way each FH repair is ID'd. This data is stored in the related Repair Task's ExecutorData property.
|
||||
|
|
@ -848,6 +849,7 @@ namespace FabricHealer
|
|||
}
|
||||
|
||||
foHealthData.RepairId = repairId;
|
||||
foHealthData.HealthEventProperty = evt.HealthInformation.Property;
|
||||
string errOrWarn = "Error";
|
||||
|
||||
if (evt.HealthInformation.HealthState == HealthState.Warning)
|
||||
|
|
@ -865,6 +867,9 @@ namespace FabricHealer
|
|||
$"{repairRules.Count} Logic rules found for {system}Application-level repair.",
|
||||
Token).ConfigureAwait(true);
|
||||
|
||||
// Update the in-memory HealthEvent List.
|
||||
this.repairTaskManager.DetectedHealthEvents.Add(evt);
|
||||
|
||||
// Start the repair workflow.
|
||||
await repairTaskManager.StartRepairWorkflowAsync(foHealthData, repairRules, Token).ConfigureAwait(true);
|
||||
}
|
||||
|
|
@ -876,6 +881,11 @@ namespace FabricHealer
|
|||
// VM is using too much (based on user-supplied threshold value) of some monitored machine resource.
|
||||
private async Task ProcessNodeHealthAsync(IEnumerable<NodeHealthState> nodeHealthStates)
|
||||
{
|
||||
// Random wait to limit potential duplicate (concurrent) repair job creation from other FH instances.
|
||||
var random = new Random();
|
||||
int waitTimeMS = random.Next(random.Next(0, nodeCount * 100), 1000 * nodeCount);
|
||||
await Task.Delay(waitTimeMS, Token).ConfigureAwait(true);
|
||||
|
||||
var supportedNodeHealthStates = nodeHealthStates.Where(a => a.AggregatedHealthState == HealthState.Warning || a.AggregatedHealthState == HealthState.Error);
|
||||
|
||||
foreach (var node in supportedNodeHealthStates)
|
||||
|
|
@ -921,11 +931,6 @@ namespace FabricHealer
|
|||
{
|
||||
Token.ThrowIfCancellationRequested();
|
||||
|
||||
// Random wait to limit potential duplicate (concurrent) repair job creation from other FH instances.
|
||||
var random = new Random();
|
||||
int waitTimeMS = random.Next(random.Next(0, nodeCount * 100), 1000 * nodeCount);
|
||||
await Task.Delay(waitTimeMS, Token).ConfigureAwait(true);
|
||||
|
||||
if (string.IsNullOrWhiteSpace(evt.HealthInformation.Description))
|
||||
{
|
||||
continue;
|
||||
|
|
@ -980,6 +985,7 @@ namespace FabricHealer
|
|||
|
||||
string Id = $"VM_Repair_{foHealthData.Code}{foHealthData.NodeName}";
|
||||
foHealthData.RepairId = Id;
|
||||
foHealthData.HealthEventProperty = evt.HealthInformation.Property;
|
||||
string errOrWarn = "Error";
|
||||
|
||||
if (evt.HealthInformation.HealthState == HealthState.Warning)
|
||||
|
|
@ -996,6 +1002,9 @@ namespace FabricHealer
|
|||
$"{Environment.NewLine}" +
|
||||
$"VM repair policy is enabled. {repairRules.Count} Logic rules found for VM-level repair.",
|
||||
Token).ConfigureAwait(true);
|
||||
|
||||
// Update the in-memory HealthEvent List.
|
||||
this.repairTaskManager.DetectedHealthEvents.Add(evt);
|
||||
|
||||
// Start the repair workflow.
|
||||
await repairTaskManager.StartRepairWorkflowAsync(foHealthData, repairRules, Token).ConfigureAwait(true);
|
||||
|
|
@ -1013,6 +1022,11 @@ namespace FabricHealer
|
|||
return;
|
||||
}
|
||||
|
||||
// Random wait to limit potential duplicate (concurrent) repair job creation from other FH instances.
|
||||
var random = new Random();
|
||||
int waitTimeMS = random.Next(random.Next(0, nodeCount * 100), 1000 * nodeCount);
|
||||
await Task.Delay(waitTimeMS, Token).ConfigureAwait(true);
|
||||
|
||||
var repUnhealthyEvaluations = ((ReplicaHealthEvaluation)evaluation).UnhealthyEvaluations;
|
||||
|
||||
foreach (var healthEvaluation in repUnhealthyEvaluations)
|
||||
|
|
@ -1022,11 +1036,6 @@ namespace FabricHealer
|
|||
|
||||
Token.ThrowIfCancellationRequested();
|
||||
|
||||
// Random wait to limit potential duplicate (concurrent) repair job creation from other FH instances.
|
||||
var random = new Random();
|
||||
int waitTimeMS = random.Next(random.Next(0, nodeCount * 100), 1000 * nodeCount);
|
||||
await Task.Delay(waitTimeMS, Token).ConfigureAwait(true);
|
||||
|
||||
var service = await fabricClient.QueryManager.GetServiceNameAsync(
|
||||
eval.PartitionId,
|
||||
ConfigSettings.AsyncTimeout,
|
||||
|
|
@ -1156,7 +1165,6 @@ namespace FabricHealer
|
|||
case FOErrorWarningCodes.AppWarningTooManyActiveTcpPorts:
|
||||
case FOErrorWarningCodes.AppWarningTooManyOpenFileHandles:
|
||||
|
||||
|
||||
repairPolicySectionName = app == "fabric:/System" ? RepairConstants.SystemAppRepairPolicySectionName : RepairConstants.AppRepairPolicySectionName;
|
||||
break;
|
||||
|
||||
|
|
|
|||
|
|
@ -36,10 +36,27 @@
|
|||
|
||||
Mitigate() :- CheckInsideRunInterval(RunInterval=00:10:00), !.
|
||||
|
||||
|
||||
## Mitigation rules for multiple metrics and targets.
|
||||
|
||||
## CPU - Percent In Use - Specific application, any of its service processes.
|
||||
Mitigate(AppName="fabric:/CpuStress", MetricName="CpuPercent") :- TimeScopedRestartCodePackage(5, 01:00:00).
|
||||
## CPU - Percent In Use - Constrained on AppName and number of times FabricObserver generates an Error/Warning Health report for CpuPercent metric within a specified timespan.
|
||||
## This reads: Try to mitigate an SF Application in Error or Warning named fabric:/CpuStress where one of its services is consuming too much CPU (as a percentage of total CPU)
|
||||
## and where at least 3 health events identifying this problem were produced in the last 15 minutes. This is useful to ensure you don't mitigate a transient (short-lived)
|
||||
## problem as they will self-correct.
|
||||
Mitigate(AppName="fabric:/CpuStress", MetricName="CpuPercent") :- GetHealthEventHistory(?HealthEventCount, TimeRange=00:15:00), ?HealthEventCount >= 3,
|
||||
TimeScopedRestartCodePackage(4, 01:00:00).
|
||||
|
||||
## CPU - Percent In Use - Constrained on AppName = "fabric:/MyApp42", observed Metric value and health event count within specified time range.
|
||||
Mitigate(AppName="fabric:/MyApp42", MetricName="CpuPercent", MetricValue=?MetricValue) :- ?MetricValue >= 80,
|
||||
GetHealthEventHistory(?HealthEventCount, TimeRange=00:15:00),
|
||||
?HealthEventCount >= 3,
|
||||
TimeScopedRestartCodePackage(4, 01:00:00).
|
||||
|
||||
## CPU - Percent In Use - Specific application, any of its service processes. Your FO error/warning threshold alone prompts repair. This doesn't take into account transient misbehavior.
|
||||
Mitigate(AppName="fabric:/MyApp", MetricName="CpuPercent") :- TimeScopedRestartCodePackage(5, 01:00:00).
|
||||
|
||||
## CPU - Percent In Use - Any application's service that exceeds 90% cpu usage.
|
||||
Mitigate(MetricName="CpuPercent", MetricValue=?MetricValue) :- ?MetricValue >= 90, TimeScopedRestartCodePackage(5, 01:00:00).
|
||||
|
||||
## File Handles - Total allocated for Any SF Service Process belonging to the specified SF Application.
|
||||
## This is example of how to use the Guan system predicate, Contains. It takes two string args, the first is the substring to look for in the second arg.
|
||||
|
|
|
|||
|
|
@ -0,0 +1,84 @@
|
|||
// ------------------------------------------------------------
|
||||
// Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
// Licensed under the MIT License (MIT). See License.txt in the repo root for license information.
|
||||
// ------------------------------------------------------------
|
||||
|
||||
using System;
|
||||
using System.Threading.Tasks;
|
||||
using Guan.Common;
|
||||
using Guan.Logic;
|
||||
using FabricHealer.Utilities.Telemetry;
|
||||
using FabricHealer.Utilities;
|
||||
|
||||
namespace FabricHealer.Repair.Guan
|
||||
{
|
||||
public class GetHealthEventHistoryPredicateType : PredicateType
|
||||
{
|
||||
private static RepairTaskManager RepairTaskManager;
|
||||
private static TelemetryData FOHealthData;
|
||||
private static GetHealthEventHistoryPredicateType Instance;
|
||||
|
||||
private class Resolver : GroundPredicateResolver
|
||||
{
|
||||
public Resolver(CompoundTerm input, Constraint constraint, QueryContext context)
|
||||
: base(input, constraint, context, 1)
|
||||
{
|
||||
|
||||
}
|
||||
|
||||
protected override Task<Term> GetNextTermAsync()
|
||||
{
|
||||
long eventCount = 0;
|
||||
var timeRange = (TimeSpan)Input.Arguments[1].Value.GetEffectiveTerm().GetValue();
|
||||
|
||||
if (timeRange > TimeSpan.MinValue)
|
||||
{
|
||||
eventCount = RepairTaskManager.GetEntityHealthEventCountWithinTimeRange(FOHealthData.HealthEventProperty, timeRange);
|
||||
}
|
||||
else
|
||||
{
|
||||
string message = "You must supply a valid TimeSpan argument for GetHealthEventHistoryPredicateType. Default result has been supplied (0).";
|
||||
|
||||
RepairTaskManager.TelemetryUtilities.EmitTelemetryEtwHealthEventAsync(
|
||||
LogLevel.Info,
|
||||
$"GetHealthEventHistoryPredicateType::{FOHealthData.HealthEventProperty}",
|
||||
message,
|
||||
RepairTaskManager.Token).GetAwaiter().GetResult();
|
||||
}
|
||||
|
||||
var result = new CompoundTerm(Instance, null);
|
||||
|
||||
// By using "0" for name here means the rule can pass any name for this named variable arg as long as it is consistently used as such in the corresponding rule.
|
||||
result.AddArgument(new Constant(eventCount), "0");
|
||||
return Task.FromResult<Term>(result);
|
||||
}
|
||||
}
|
||||
|
||||
public static GetHealthEventHistoryPredicateType Singleton(string name, RepairTaskManager repairTaskManager, TelemetryData foHealthData)
|
||||
{
|
||||
RepairTaskManager = repairTaskManager;
|
||||
FOHealthData = foHealthData;
|
||||
|
||||
return Instance ??= new GetHealthEventHistoryPredicateType(name);
|
||||
}
|
||||
|
||||
private GetHealthEventHistoryPredicateType(string name)
|
||||
: base(name, true, 2, 2)
|
||||
{
|
||||
|
||||
}
|
||||
|
||||
public override PredicateResolver CreateResolver(CompoundTerm input, Constraint constraint, QueryContext context)
|
||||
{
|
||||
return new Resolver(input, constraint, context);
|
||||
}
|
||||
|
||||
public override void AdjustTerm(CompoundTerm term, Rule rule)
|
||||
{
|
||||
if (!(term.Arguments[0].Value is IndexedVariable))
|
||||
{
|
||||
throw new GuanException("The first argument of GetHealthEventHistoryPredicateType must be a variable: {0}", term);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -69,6 +69,7 @@ namespace FabricHealer.Repair
|
|||
// Helper Predicates.
|
||||
public const string CheckInsideRunInterval = "CheckInsideRunInterval";
|
||||
public const string CheckFolderSize = "CheckFolderSize";
|
||||
public const string GetHealthEventHistory = "GetHealthEventHistory";
|
||||
public const string GetRepairHistory = "GetRepairHistory";
|
||||
public const string EmitMessage = "EmitMessage";
|
||||
|
||||
|
|
|
|||
|
|
@ -24,19 +24,18 @@ namespace FabricHealer.Repair
|
|||
{
|
||||
public class RepairTaskManager : IRepairTasks
|
||||
{
|
||||
private readonly RepairTaskEngine repairTaskEngine;
|
||||
private readonly RepairExecutor RepairExec;
|
||||
private static readonly TimeSpan MaxWaitTimeForInfraRepairTaskCompleted = TimeSpan.FromHours(2);
|
||||
internal readonly List<HealthEvent> DetectedHealthEvents = new List<HealthEvent>();
|
||||
internal readonly StatelessServiceContext Context;
|
||||
internal readonly CancellationToken Token;
|
||||
internal readonly TelemetryUtilities TelemetryUtilities;
|
||||
public readonly FabricClient FabricClientInstance;
|
||||
|
||||
private TimeSpan AsyncTimeout
|
||||
{
|
||||
get;
|
||||
} = TimeSpan.FromSeconds(60);
|
||||
|
||||
private static readonly TimeSpan MaxWaitTimeForInfraRepairTaskCompleted = TimeSpan.FromHours(2);
|
||||
internal readonly FabricClient FabricClientInstance;
|
||||
private readonly RepairTaskEngine repairTaskEngine;
|
||||
private readonly RepairExecutor RepairExec;
|
||||
private readonly TimeSpan AsyncTimeout = TimeSpan.FromSeconds(60);
|
||||
private readonly DateTime HealthEventsListCreationTime = DateTime.UtcNow;
|
||||
private readonly TimeSpan MaxLifeTimeHealthEventsData = TimeSpan.FromDays(2);
|
||||
private DateTime LastHealthEventsListClearDateTime;
|
||||
|
||||
public RepairTaskManager(FabricClient fabricClient, StatelessServiceContext context, CancellationToken token)
|
||||
{
|
||||
|
|
@ -46,6 +45,7 @@ namespace FabricHealer.Repair
|
|||
RepairExec = new RepairExecutor(fabricClient, context, token);
|
||||
repairTaskEngine = new RepairTaskEngine(fabricClient);
|
||||
TelemetryUtilities = new TelemetryUtilities(fabricClient, context);
|
||||
LastHealthEventsListClearDateTime = HealthEventsListCreationTime;
|
||||
}
|
||||
|
||||
public async Task RemoveServiceFabricNodeStateAsync(string nodeName, CancellationToken cancellationToken)
|
||||
|
|
@ -165,6 +165,7 @@ namespace FabricHealer.Repair
|
|||
// Add external helper predicates.
|
||||
functorTable.Add(CheckFolderSizePredicateType.Singleton(RepairConstants.CheckFolderSize, this, foHealthData));
|
||||
functorTable.Add(GetRepairHistoryPredicateType.Singleton(RepairConstants.GetRepairHistory, this, foHealthData));
|
||||
functorTable.Add(GetHealthEventHistoryPredicateType.Singleton(RepairConstants.GetHealthEventHistory, this, foHealthData));
|
||||
functorTable.Add(CheckInsideRunIntervalPredicateType.Singleton(RepairConstants.CheckInsideRunInterval, this, foHealthData));
|
||||
functorTable.Add(EmitMessagePredicateType.Singleton(RepairConstants.EmitMessage, this));
|
||||
|
||||
|
|
@ -874,6 +875,36 @@ namespace FabricHealer.Repair
|
|||
return false;
|
||||
}
|
||||
|
||||
// Support for GetHealthEventHistoryPredicateType, which enables time-scoping logic rules based on health events related to specific SF entities/targets.
|
||||
internal int GetEntityHealthEventCountWithinTimeRange(string property, TimeSpan timeWindow)
|
||||
{
|
||||
int count = 0;
|
||||
var healthEvents = DetectedHealthEvents.Where(evt => evt.HealthInformation.Property == property);
|
||||
|
||||
if (healthEvents == null || !healthEvents.Any())
|
||||
{
|
||||
return count;
|
||||
}
|
||||
|
||||
foreach (HealthEvent healthEvent in healthEvents)
|
||||
{
|
||||
if (DateTime.UtcNow.Subtract(healthEvent.SourceUtcTimestamp) > timeWindow)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
count++;
|
||||
}
|
||||
|
||||
// Lifetime management of Health Events list data. Data is kept in-memory only for 2 days. If FH process restarts, data is not preserved.
|
||||
if (DateTime.UtcNow.Subtract(LastHealthEventsListClearDateTime) >= MaxLifeTimeHealthEventsData)
|
||||
{
|
||||
DetectedHealthEvents.Clear();
|
||||
LastHealthEventsListClearDateTime = DateTime.UtcNow;
|
||||
}
|
||||
|
||||
return count;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// This function checks to see if the target of a repair is healthy after the repair task completed.
|
||||
/// This will signal the result via telemetry and as a health event.
|
||||
|
|
|
|||
|
|
@ -105,6 +105,11 @@ namespace FabricHealer.Utilities.Telemetry
|
|||
get; set;
|
||||
}
|
||||
|
||||
public string HealthEventProperty
|
||||
{
|
||||
get; set;
|
||||
}
|
||||
|
||||
[JsonConstructor]
|
||||
public TelemetryData()
|
||||
{
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче