microsoft
/
service-fabric-healer
зеркало из https://github.com/microsoft/service-fabric-healer.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

Updated FHProxy, tests.

This commit is contained in:
Charles Torre 2022-06-20 16:07:32 -07:00
Родитель b19de45c76
Коммит 8d5362dd33
4 изменённых файлов: 103 добавлений и 39 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

94
FHTest/FHUnitTests.cs
Просмотреть файл

@ -21,6 +21,7 @@ using System.Fabric.Repair;
using System.Diagnostics;
using System.Fabric.Health;
using SupportedErrorCodes = FabricHealer.Utilities.SupportedErrorCodes;
using EntityType = FabricHealer.EntityType;
namespace FHTest
{
@ -110,6 +111,9 @@ namespace FHTest
public static async Task TestClassCleanupAsync()
{
await CleanupTestRepairJobsAsync();
// Ensure FHProxy cleans up its health reports.
FabricHealerProxy.Instance.Close();
}
/* GuanLogic Tests */
@ -364,7 +368,7 @@ namespace FHTest
return repairRules;
}
private async Task<(bool, TelemetryData data)>
private static async Task<(bool, TelemetryData data)>
IsEntityInWarningStateAsync(string appName = null, string serviceName = null, string nodeName = null)
{
EntityHealth healthData = null;
@ -440,7 +444,7 @@ namespace FHTest
static readonly RepairFacts RepairFactsMachineTarget = new RepairFacts
{
NodeName = NodeName,
EntityType = FabricHealer.EntityType.Machine,
EntityType = EntityType.Machine,
// Specifying Source is Required for unit tests.
// For unit tests, there is no FabricRuntime static, so FHProxy, which utilizes this type, will fail unless Source is provided here.
Source = "fabric:/Test"
@ -464,7 +468,7 @@ namespace FHTest
static readonly RepairFacts DiskRepairFacts = new RepairFacts
{
NodeName = NodeName,
EntityType = FabricHealer.EntityType.Disk,
EntityType = EntityType.Disk,
Metric = SupportedMetricNames.DiskSpaceUsageMb,
Code = SupportedErrorCodes.NodeWarningDiskSpaceMB,
// Specifying Source is Required for unit tests.
@ -479,11 +483,11 @@ namespace FHTest
RepairFactsMachineTarget,
RepairFactsNodeTarget,
RepairFactsServiceTarget,
SystemServiceRepairFacts,
SystemServiceRepairFacts
};
[TestMethod]
public async Task FabricHealerProxy_Restart_Service_Generates_Entity_Health_Warning()
public async Task FHProxy_Service_Facts_Generate_Entity_Health_Warning()
{
if (!IsLocalSFRuntimePresent())
{
@ -491,17 +495,18 @@ namespace FHTest
}
// This will put the entity into Warning with a specially-crafted Health Event description (serialized instance of ITelemetryData type).
await Proxy.Instance.RepairEntityAsync(RepairFactsServiceTarget, token);
var (generatedWarning, data) = await IsEntityInWarningStateAsync(null, RepairFactsServiceTarget.ServiceName);
await FabricHealerProxy.Instance.RepairEntityAsync(RepairFactsServiceTarget, token);
// FHProxy creates or renames Source with trailing id ("FabricHealerProxy");
Assert.IsTrue(RepairFactsServiceTarget.Source.EndsWith(FHProxyId));
var (generatedWarning, data) = await IsEntityInWarningStateAsync(null, RepairFactsServiceTarget.ServiceName);
Assert.IsTrue(generatedWarning);
Assert.IsTrue(data is TelemetryData);
Assert.IsTrue(data is not null);
}
[TestMethod]
public async Task FabricHealerProxy_Restart_Node_Generates_Entity_Health_Warning()
public async Task FHProxy_Node_Facts_Generates_Entity_Health_Warning()
{
if (!IsLocalSFRuntimePresent())
{
@ -509,17 +514,18 @@ namespace FHTest
}
// This will put the entity into Warning with a specially-crafted Health Event description (serialized instance of ITelemetryData type).
await Proxy.Instance.RepairEntityAsync(RepairFactsNodeTarget, token);
var (generatedWarning, data) = await IsEntityInWarningStateAsync(null, null, NodeName);
await FabricHealerProxy.Instance.RepairEntityAsync(RepairFactsNodeTarget, token);
// FHProxy creates or renames Source with trailing id ("FabricHealerProxy");
Assert.IsTrue(RepairFactsNodeTarget.Source.EndsWith(FHProxyId));
var (generatedWarning, data) = await IsEntityInWarningStateAsync(null, null, NodeName);
Assert.IsTrue(generatedWarning);
Assert.IsTrue(data is TelemetryData);
Assert.IsTrue(data is not null);
}
[TestMethod]
public async Task FHProxy_MissingFact_Generates_MissingRepairFactsException()
public async Task FHProxy_Missing_Fact_Generates_MissingRepairFactsException()
{
if (!IsLocalSFRuntimePresent())
{
@ -534,11 +540,22 @@ namespace FHTest
Source = "fabric:/Test"
};
await Assert.ThrowsExceptionAsync<MissingRepairFactsException>(async () => { await Proxy.Instance.RepairEntityAsync(repairFacts, token); });
await Assert.ThrowsExceptionAsync<MissingRepairFactsException>(async () =>
{
try
{
await FabricHealerProxy.Instance.RepairEntityAsync(repairFacts, token);
}
finally
{
// Ensure FHProxy cleans up its health reports.
FabricHealerProxy.Instance.Close();
}
});
}
[TestMethod]
public async Task FHProxy_MissingFact_Generates_ServiceNotFoundException()
public async Task FHProxy_Missing_Fact_Generates_ServiceNotFoundException()
{
if (!IsLocalSFRuntimePresent())
{
@ -554,11 +571,16 @@ namespace FHTest
Source = "fabric:/Test"
};
await Assert.ThrowsExceptionAsync<ServiceNotFoundException>(async () => { await Proxy.Instance.RepairEntityAsync(repairFacts, token); });
await Assert.ThrowsExceptionAsync<ServiceNotFoundException>(async () =>
{
await FabricHealerProxy.Instance.RepairEntityAsync(repairFacts, token);
});
}
[TestMethod]
public async Task FHProxy_MissingFact_Generates_NodeNotFoundException()
public async Task FHProxy_Missing_Fact_Generates_NodeNotFoundException()
{
if (!IsLocalSFRuntimePresent())
{
@ -571,7 +593,43 @@ namespace FHTest
// No need for Source here as an invalid node will be detected before the Source value matters.
};
await Assert.ThrowsExceptionAsync<NodeNotFoundException>(async () => { await Proxy.Instance.RepairEntityAsync(repairFacts, token); });
await Assert.ThrowsExceptionAsync<NodeNotFoundException>(async () =>
{
await FabricHealerProxy.Instance.RepairEntityAsync(repairFacts, token);
});
}
[TestMethod]
public async Task FHProxy_Multiple_Entity_Repair_Facts_Generate_Warnings()
{
if (!IsLocalSFRuntimePresent())
{
throw new InternalTestFailureException("You must run this test with an active local (dev) SF cluster.");
}
// This will put the entity into Warning with a specially-crafted Health Event description (serialized instance of ITelemetryData type).
await FabricHealerProxy.Instance.RepairEntityAsync(RepairFactsList, token);
foreach (var repair in RepairFactsList)
{
if (repair.ServiceName != null)
{
var (generatedWarningService, sdata) = await IsEntityInWarningStateAsync(null, repair.ServiceName);
Assert.IsTrue(generatedWarningService);
Assert.IsTrue(sdata is not null);
}
else if (repair.EntityType == EntityType.Disk || repair.EntityType == EntityType.Machine || repair.EntityType == EntityType.Node)
{
var (generatedWarningNode, ndata) = await IsEntityInWarningStateAsync(null, null, NodeName);
Assert.IsTrue(generatedWarningNode);
Assert.IsTrue(ndata is not null);
}
// FHProxy creates or renames Source with trailing id ("FabricHealerProxy");
Assert.IsTrue(repair.Source.EndsWith(FHProxyId));
}
}
}
}

8
FabricHealer.sln
Просмотреть файл

@ -47,16 +47,16 @@ Global
{9A19103F-16F7-4668-BE54-9A1E7A4F7556}.Debug|x64.Build.0 = Debug|x64
{9A19103F-16F7-4668-BE54-9A1E7A4F7556}.Release|Any CPU.ActiveCfg = Release|Any CPU
{9A19103F-16F7-4668-BE54-9A1E7A4F7556}.Release|Any CPU.Build.0 = Release|Any CPU
{9A19103F-16F7-4668-BE54-9A1E7A4F7556}.Release|x64.ActiveCfg = Release|Any CPU
{9A19103F-16F7-4668-BE54-9A1E7A4F7556}.Release|x64.Build.0 = Release|Any CPU
{9A19103F-16F7-4668-BE54-9A1E7A4F7556}.Release|x64.ActiveCfg = Release|x64
{9A19103F-16F7-4668-BE54-9A1E7A4F7556}.Release|x64.Build.0 = Release|x64
{8D9712BF-C026-4A36-B6D1-6345137D3B6F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{8D9712BF-C026-4A36-B6D1-6345137D3B6F}.Debug|Any CPU.Build.0 = Debug|Any CPU
{8D9712BF-C026-4A36-B6D1-6345137D3B6F}.Debug|x64.ActiveCfg = Debug|x64
{8D9712BF-C026-4A36-B6D1-6345137D3B6F}.Debug|x64.Build.0 = Debug|x64
{8D9712BF-C026-4A36-B6D1-6345137D3B6F}.Release|Any CPU.ActiveCfg = Release|Any CPU
{8D9712BF-C026-4A36-B6D1-6345137D3B6F}.Release|Any CPU.Build.0 = Release|Any CPU
{8D9712BF-C026-4A36-B6D1-6345137D3B6F}.Release|x64.ActiveCfg = Release|Any CPU
{8D9712BF-C026-4A36-B6D1-6345137D3B6F}.Release|x64.Build.0 = Release|Any CPU
{8D9712BF-C026-4A36-B6D1-6345137D3B6F}.Release|x64.ActiveCfg = Release|x64
{8D9712BF-C026-4A36-B6D1-6345137D3B6F}.Release|x64.Build.0 = Release|x64
{A977C8E0-2183-4845-95EA-7F3C3E795310}.Debug|Any CPU.ActiveCfg = Debug|x64
{A977C8E0-2183-4845-95EA-7F3C3E795310}.Debug|Any CPU.Build.0 = Debug|x64
{A977C8E0-2183-4845-95EA-7F3C3E795310}.Debug|Any CPU.Deploy.0 = Debug|x64

27
FabricHealerProxy/FabricHealer.cs → FabricHealerProxy/FabricHealerProxy.cs
Просмотреть файл

@ -25,10 +25,10 @@ namespace FabricHealer
/// that dictate which rules will be loaded by FabricHealer and passed to Guan for logic rule query execution that may or may not lead
/// to some repair (depends on the facts and the results of the logic programs that employ them).
/// </summary>
public sealed class Proxy
public sealed class FabricHealerProxy
{
private const string FHProxyId = "FabricHealerProxy";
private static Proxy instance;
private static FabricHealerProxy instance;
private static readonly FabricClientSettings settings = new FabricClientSettings
{
@ -49,7 +49,7 @@ namespace FabricHealer
CancellationTokenRegistration tokenRegistration;
CancellationTokenSource cts = null;
private Proxy()
private FabricHealerProxy()
{
if (repairDataHistory == null)
{
@ -58,9 +58,9 @@ namespace FabricHealer
}
/// <summary>
/// FabricHealerProxy.Proxy singleton. This is thread-safe.
/// FabricHealerProxy static singleton. This is thread-safe.
/// </summary>
public static Proxy Instance
public static FabricHealerProxy Instance
{
get
{
@ -70,7 +70,7 @@ namespace FabricHealer
{
if (instance == null)
{
instance = new Proxy();
instance = new FabricHealerProxy();
}
}
}
@ -574,6 +574,7 @@ namespace FabricHealer
{
Policy.Handle<FabricException>()
.Or<TimeoutException>()
.Or<HealthReportNotFoundException>()
.WaitAndRetry(
new[]
{
@ -599,7 +600,7 @@ namespace FabricHealer
var repairFacts = repairDataHistory.ElementAt(i).Value.RepairData;
var healthInformation = new HealthInformation(repairFacts.Source, repairFacts.Property, HealthState.Ok)
{
Description = $"Clearing existing {repairFacts.EntityType} health reports created by FabricHealerProxy",
Description = $"Clearing existing {repairFacts.EntityType} health report created by FabricHealerProxy",
TimeToLive = TimeSpan.FromMinutes(5),
RemoveWhenExpired = true
};
@ -650,6 +651,11 @@ namespace FabricHealer
break;
}
if (!VerifyHealthReportExistsAsync(repairFacts, fabricClient, cts.Token).Result)
{
throw new HealthReportNotFoundException();
}
_ = repairDataHistory.TryRemove(repairDataHistory.ElementAt(i).Key, out _);
--i;
}
@ -661,10 +667,11 @@ namespace FabricHealer
}
/// <summary>
/// Clears instance data created by FabricHealer.Proxy, including cleaning up any active Service Fabric health reports.
/// Note: calling Close does not cancel repairs that are in flight or in the FabricHealer internal repair queue.
/// Closes the FabricHealerProxy.Instance, disposes all IDisposable objects currently in memory, and clears all health data and health reports.
/// Note: calling Close does not cancel repairs that are in flight or in the FabricHealer internal repair queue. This function is called automatically when
/// a consuming SF service closes. You do not need to call this directly unless that is your intention.
/// </summary>
private void Close()
public void Close()
{
if (repairDataHistory != null)
{

13
FabricHealerProxy/README.md
Просмотреть файл

@ -21,7 +21,7 @@ using System.Fabric;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.ServiceFabric.Services.Runtime;
using FabricHealerProxy;
using FabricHealer;
namespace Stateless1
{
@ -137,18 +137,17 @@ namespace Stateless1
RepairFactsServiceTarget3,
RepairFactsServiceTarget4,
RepairFactsServiceTarget5,
RepairFactsServiceTarget6,
RepairFactsServiceTarget7
RepairFactsServiceTarget6
};
// This demonstrates which exceptions will be thrown by the API. The first three are FabricHealerProxy custom exceptions and represent user error (most likely).
// The last two are internal SF issues which will be thrown only after a series of retries. How to handle these is up to you.
try
{
await FabricHealer.Proxy.RepairEntityAsync(DiskRepairFacts, cancellationToken);
//await FabricHealer.Proxy.RepairEntityAsync(SystemServiceRepairFacts, cancellationToken);
//await FabricHealer.Proxy.RepairEntityAsync(RepairFactsMachineTarget, cancellationToken);
//await FabricHealer.Proxy.RepairEntityAsync(RepairFactsList, cancellationToken);
await FabricHealerProxy.Instance.RepairEntityAsync(RepairFactsServiceTarget7, cancellationToken);
await FabricHealerProxy.Instance.RepairEntityAsync(SystemServiceRepairFacts, cancellationToken);
await FabricHealerProxy.Instance.RepairEntityAsync(RepairFactsMachineTarget, cancellationToken);
await FabricHealerProxy.Instance.RepairEntityAsync(RepairFactsList, cancellationToken);
}
catch (MissingRepairFactsException)
{