зеркало из https://github.com/Azure/iotedge.git
[k8s] Map possible Pod/Container status to ModuleStatus (#2265)
* Mapping the pod status to the module status for Pending/Failed/Unknown * Created Kubernetes Environment & its Provider to handle the module status reports to the IOT hub * Updated the unit tests for handling the change in pod status to module status * Removed blank line * Pod status checks for container status before converting it to module status * Unit test to reflect the new model of determining module status * Update edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Kubernetes/KubernetesEnvironment.cs Co-Authored-By: Denis Molokanov <dmolokanov@users.noreply.github.com> * Updated unit tests * fixing the braces * fixed braces 2 * Fix build errors * Debug mode changes to test code flow * Updated status in debug mode * changed moduleruntime status instead of dockerruntimestatus which gets reported to IOT * Removed debug logs * Resolve conflicts * fix typo * Fix typo in unit test * More typo fix * Simplified unit test * Updated DockerModule priority parameter changes for k8s environment * Fix typo in unit test names * Updated the test cases with single responsbility approach * start time format changes * Refractoring Co-authored-by: Denis Molokanov <dmolokanov@users.noreply.github.com>
This commit is contained in:
gaya
GitHub
Родитель
b5f814cd88
Коммит
d824fb3991
|
|
@ -567,32 +567,32 @@ Global
|
|||
{08986FED-9283-47F1-B938-86D9C6E753E1}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{08986FED-9283-47F1-B938-86D9C6E753E1}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{08986FED-9283-47F1-B938-86D9C6E753E1}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
{47AA5965-5826-489D-B74B-314A61629121}.CheckInBuild|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{47AA5965-5826-489D-B74B-314A61629121}.CheckInBuild|Any CPU.Build.0 = Debug|Any CPU
|
||||
{47AA5965-5826-489D-B74B-314A61629121}.CheckInBuild|Any CPU.ActiveCfg = CheckInBuild|Any CPU
|
||||
{47AA5965-5826-489D-B74B-314A61629121}.CheckInBuild|Any CPU.Build.0 = CheckInBuild|Any CPU
|
||||
{47AA5965-5826-489D-B74B-314A61629121}.CodeCoverage|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{47AA5965-5826-489D-B74B-314A61629121}.CodeCoverage|Any CPU.Build.0 = Debug|Any CPU
|
||||
{47AA5965-5826-489D-B74B-314A61629121}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{47AA5965-5826-489D-B74B-314A61629121}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{47AA5965-5826-489D-B74B-314A61629121}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{47AA5965-5826-489D-B74B-314A61629121}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
{B736B03C-1185-4AE5-87B7-665C6790F50F}.CheckInBuild|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{B736B03C-1185-4AE5-87B7-665C6790F50F}.CheckInBuild|Any CPU.Build.0 = Debug|Any CPU
|
||||
{B736B03C-1185-4AE5-87B7-665C6790F50F}.CheckInBuild|Any CPU.ActiveCfg = CheckInBuild|Any CPU
|
||||
{B736B03C-1185-4AE5-87B7-665C6790F50F}.CheckInBuild|Any CPU.Build.0 = CheckInBuild|Any CPU
|
||||
{B736B03C-1185-4AE5-87B7-665C6790F50F}.CodeCoverage|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{B736B03C-1185-4AE5-87B7-665C6790F50F}.CodeCoverage|Any CPU.Build.0 = Debug|Any CPU
|
||||
{B736B03C-1185-4AE5-87B7-665C6790F50F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{B736B03C-1185-4AE5-87B7-665C6790F50F}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{B736B03C-1185-4AE5-87B7-665C6790F50F}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{B736B03C-1185-4AE5-87B7-665C6790F50F}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
{79540E35-45EA-4568-9AB7-C1CC0BFF73EE}.CheckInBuild|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{79540E35-45EA-4568-9AB7-C1CC0BFF73EE}.CheckInBuild|Any CPU.Build.0 = Debug|Any CPU
|
||||
{79540E35-45EA-4568-9AB7-C1CC0BFF73EE}.CheckInBuild|Any CPU.ActiveCfg = CheckInBuild|Any CPU
|
||||
{79540E35-45EA-4568-9AB7-C1CC0BFF73EE}.CheckInBuild|Any CPU.Build.0 = CheckInBuild|Any CPU
|
||||
{79540E35-45EA-4568-9AB7-C1CC0BFF73EE}.CodeCoverage|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{79540E35-45EA-4568-9AB7-C1CC0BFF73EE}.CodeCoverage|Any CPU.Build.0 = Debug|Any CPU
|
||||
{79540E35-45EA-4568-9AB7-C1CC0BFF73EE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{79540E35-45EA-4568-9AB7-C1CC0BFF73EE}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{79540E35-45EA-4568-9AB7-C1CC0BFF73EE}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{79540E35-45EA-4568-9AB7-C1CC0BFF73EE}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
{8181EB49-62CE-495B-8078-08DCF8C30541}.CheckInBuild|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{8181EB49-62CE-495B-8078-08DCF8C30541}.CheckInBuild|Any CPU.Build.0 = Debug|Any CPU
|
||||
{8181EB49-62CE-495B-8078-08DCF8C30541}.CheckInBuild|Any CPU.ActiveCfg = CheckInBuild|Any CPU
|
||||
{8181EB49-62CE-495B-8078-08DCF8C30541}.CheckInBuild|Any CPU.Build.0 = CheckInBuild|Any CPU
|
||||
{8181EB49-62CE-495B-8078-08DCF8C30541}.CodeCoverage|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{8181EB49-62CE-495B-8078-08DCF8C30541}.CodeCoverage|Any CPU.Build.0 = Debug|Any CPU
|
||||
{8181EB49-62CE-495B-8078-08DCF8C30541}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
|
|
|
|||
|
|
@ -82,5 +82,7 @@ namespace Microsoft.Azure.Devices.Edge.Agent.Kubernetes
|
|||
public const string EdgeK8sObjectOwnerUidKey = "EdgeK8sObjectOwnerUid";
|
||||
|
||||
public const string RunAsNonRootKey = "RunAsNonRoot";
|
||||
|
||||
public const string UnknownImage = "unknown";
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,165 @@
|
|||
// Copyright (c) Microsoft. All rights reserved.
|
||||
namespace Microsoft.Azure.Devices.Edge.Agent.Kubernetes
|
||||
{
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Azure.Devices.Edge.Agent.Core;
|
||||
using Microsoft.Azure.Devices.Edge.Agent.Docker;
|
||||
using Microsoft.Azure.Devices.Edge.Agent.Docker.Models;
|
||||
using Microsoft.Azure.Devices.Edge.Storage;
|
||||
using Microsoft.Azure.Devices.Edge.Util;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
/// <summary>
|
||||
/// This implementation gets the module runtime information from KubernetesRuntimeInfoProvider and
|
||||
/// the configuration information from the deploymentConfig.
|
||||
/// TODO: This could be made generic (not docker specific) and moved to Core.
|
||||
/// </summary>
|
||||
public class KubernetesEnvironment : IEnvironment
|
||||
{
|
||||
readonly IRuntimeInfoProvider moduleStatusProvider;
|
||||
readonly IEntityStore<string, ModuleState> moduleStateStore;
|
||||
readonly string operatingSystemType;
|
||||
readonly string architecture;
|
||||
readonly string version;
|
||||
readonly DeploymentConfig deploymentConfig;
|
||||
|
||||
public KubernetesEnvironment(
|
||||
IRuntimeInfoProvider moduleStatusProvider,
|
||||
DeploymentConfig deploymentConfig,
|
||||
IEntityStore<string, ModuleState> moduleStateStore,
|
||||
string operatingSystemType,
|
||||
string architecture,
|
||||
string version)
|
||||
{
|
||||
this.moduleStatusProvider = moduleStatusProvider;
|
||||
this.deploymentConfig = deploymentConfig;
|
||||
this.moduleStateStore = moduleStateStore;
|
||||
this.operatingSystemType = operatingSystemType;
|
||||
this.architecture = architecture;
|
||||
this.version = version;
|
||||
}
|
||||
|
||||
public async Task<ModuleSet> GetModulesAsync(CancellationToken token)
|
||||
{
|
||||
IEnumerable<ModuleRuntimeInfo> moduleStatuses = await this.moduleStatusProvider.GetModules(token);
|
||||
var modules = new List<IModule>();
|
||||
ModuleSet moduleSet = this.deploymentConfig.GetModuleSet();
|
||||
|
||||
foreach (ModuleRuntimeInfo moduleRuntimeInfo in moduleStatuses)
|
||||
{
|
||||
if (moduleRuntimeInfo.Type != "docker" || !(moduleRuntimeInfo is ModuleRuntimeInfo<DockerReportedConfig> dockerRuntimeInfo))
|
||||
{
|
||||
Events.InvalidModuleType(moduleRuntimeInfo);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!moduleSet.Modules.TryGetValue(dockerRuntimeInfo.Name, out IModule configModule) || !(configModule is DockerModule dockerModule))
|
||||
{
|
||||
dockerModule = new DockerModule(dockerRuntimeInfo.Name, string.Empty, ModuleStatus.Unknown, Core.RestartPolicy.Unknown, new DockerConfig(Constants.UnknownImage, new CreateContainerParameters()), ImagePullPolicy.OnCreate, Core.Constants.HighestPriority, new ConfigurationInfo(), null);
|
||||
}
|
||||
|
||||
Option<ModuleState> moduleStateOption = await this.moduleStateStore.Get(moduleRuntimeInfo.Name);
|
||||
ModuleState moduleState = moduleStateOption.GetOrElse(new ModuleState(0, moduleRuntimeInfo.ExitTime.GetOrElse(DateTime.MinValue)));
|
||||
|
||||
string image = !string.IsNullOrWhiteSpace(dockerRuntimeInfo.Config.Image) ? dockerRuntimeInfo.Config.Image : dockerModule.Config.Image;
|
||||
var dockerReportedConfig = new DockerReportedConfig(image, dockerModule.Config.CreateOptions, dockerRuntimeInfo.Config.ImageHash);
|
||||
IModule module;
|
||||
switch (moduleRuntimeInfo.Name)
|
||||
{
|
||||
case Core.Constants.EdgeHubModuleName:
|
||||
module = new EdgeHubDockerRuntimeModule(
|
||||
dockerModule.DesiredStatus,
|
||||
dockerModule.RestartPolicy,
|
||||
dockerReportedConfig,
|
||||
(int)dockerRuntimeInfo.ExitCode,
|
||||
moduleRuntimeInfo.Description,
|
||||
moduleRuntimeInfo.StartTime.GetOrElse(DateTime.MinValue),
|
||||
moduleRuntimeInfo.ExitTime.GetOrElse(DateTime.MinValue),
|
||||
moduleState.RestartCount,
|
||||
moduleState.LastRestartTimeUtc,
|
||||
moduleRuntimeInfo.ModuleStatus,
|
||||
dockerModule.ImagePullPolicy,
|
||||
dockerModule.Priority,
|
||||
dockerModule.ConfigurationInfo,
|
||||
dockerModule.Env);
|
||||
break;
|
||||
|
||||
case Core.Constants.EdgeAgentModuleName:
|
||||
module = new EdgeAgentDockerRuntimeModule(
|
||||
dockerReportedConfig,
|
||||
moduleRuntimeInfo.ModuleStatus,
|
||||
(int)dockerRuntimeInfo.ExitCode,
|
||||
moduleRuntimeInfo.Description,
|
||||
moduleRuntimeInfo.StartTime.GetOrElse(DateTime.MinValue),
|
||||
moduleRuntimeInfo.ExitTime.GetOrElse(DateTime.MinValue),
|
||||
dockerModule.ImagePullPolicy,
|
||||
dockerModule.ConfigurationInfo,
|
||||
dockerModule.Env);
|
||||
break;
|
||||
|
||||
default:
|
||||
module = new DockerRuntimeModule(
|
||||
moduleRuntimeInfo.Name,
|
||||
dockerModule.Version,
|
||||
dockerModule.DesiredStatus,
|
||||
dockerModule.RestartPolicy,
|
||||
dockerReportedConfig,
|
||||
(int)dockerRuntimeInfo.ExitCode,
|
||||
moduleRuntimeInfo.Description,
|
||||
moduleRuntimeInfo.StartTime.GetOrElse(DateTime.MinValue),
|
||||
moduleRuntimeInfo.ExitTime.GetOrElse(DateTime.MinValue),
|
||||
moduleState.RestartCount,
|
||||
moduleState.LastRestartTimeUtc,
|
||||
moduleRuntimeInfo.ModuleStatus,
|
||||
dockerModule.ImagePullPolicy,
|
||||
dockerModule.Priority,
|
||||
dockerModule.ConfigurationInfo,
|
||||
dockerModule.Env);
|
||||
break;
|
||||
}
|
||||
|
||||
modules.Add(module);
|
||||
}
|
||||
|
||||
return ModuleSet.Create(modules.ToArray());
|
||||
}
|
||||
|
||||
public Task<IRuntimeInfo> GetRuntimeInfoAsync()
|
||||
{
|
||||
IRuntimeInfo runtimeInfo = this.deploymentConfig.Runtime;
|
||||
if (runtimeInfo?.Type == "docker")
|
||||
{
|
||||
var platform = new DockerPlatformInfo(this.operatingSystemType, this.architecture, this.version);
|
||||
DockerRuntimeConfig config = (runtimeInfo as DockerRuntimeInfo)?.Config;
|
||||
runtimeInfo = new DockerReportedRuntimeInfo(runtimeInfo.Type, config, platform);
|
||||
}
|
||||
else if (runtimeInfo == null || runtimeInfo is UnknownRuntimeInfo)
|
||||
{
|
||||
var platform = new DockerPlatformInfo(this.operatingSystemType, this.architecture, this.version);
|
||||
runtimeInfo = new DockerReportedUnknownRuntimeInfo(platform);
|
||||
}
|
||||
|
||||
return Task.FromResult(runtimeInfo);
|
||||
}
|
||||
|
||||
static class Events
|
||||
{
|
||||
const int IdStart = AgentEventIds.DockerEnvironment;
|
||||
static readonly ILogger Log = Logger.Factory.CreateLogger<KubernetesEnvironment>();
|
||||
|
||||
enum EventIds
|
||||
{
|
||||
InvalidModuleType = IdStart
|
||||
}
|
||||
|
||||
public static void InvalidModuleType(ModuleRuntimeInfo moduleRuntimeInfo)
|
||||
{
|
||||
Log.LogWarning((int)EventIds.InvalidModuleType, $"Module {moduleRuntimeInfo.Name} has an invalid module type '{moduleRuntimeInfo.Type}'. Expected type 'docker'");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,55 @@
|
|||
// Copyright (c) Microsoft. All rights reserved.
|
||||
namespace Microsoft.Azure.Devices.Edge.Agent.Kubernetes
|
||||
{
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Azure.Devices.Edge.Agent.Core;
|
||||
using Microsoft.Azure.Devices.Edge.Storage;
|
||||
using Microsoft.Azure.Devices.Edge.Util;
|
||||
|
||||
public class KubernetesEnvironmentProvider : IEnvironmentProvider
|
||||
{
|
||||
readonly IRuntimeInfoProvider moduleStatusProvider;
|
||||
readonly IEntityStore<string, ModuleState> store;
|
||||
readonly string operatingSystemType;
|
||||
readonly string architecture;
|
||||
readonly string version;
|
||||
|
||||
KubernetesEnvironmentProvider(
|
||||
IRuntimeInfoProvider runtimeInfoProvider,
|
||||
IEntityStore<string, ModuleState> store,
|
||||
string operatingSystemType,
|
||||
string architecture,
|
||||
string version)
|
||||
{
|
||||
this.moduleStatusProvider = runtimeInfoProvider;
|
||||
this.store = Preconditions.CheckNotNull(store, nameof(store));
|
||||
this.operatingSystemType = operatingSystemType;
|
||||
this.architecture = architecture;
|
||||
this.version = version;
|
||||
}
|
||||
|
||||
public static async Task<KubernetesEnvironmentProvider> CreateAsync(
|
||||
IRuntimeInfoProvider runtimeInfoProvider,
|
||||
IEntityStore<string, ModuleState> store,
|
||||
CancellationToken token)
|
||||
{
|
||||
SystemInfo systemInfo = await Preconditions.CheckNotNull(runtimeInfoProvider, nameof(runtimeInfoProvider)).GetSystemInfo(token);
|
||||
return new KubernetesEnvironmentProvider(
|
||||
runtimeInfoProvider,
|
||||
store,
|
||||
systemInfo.OperatingSystemType,
|
||||
systemInfo.Architecture,
|
||||
systemInfo.Version);
|
||||
}
|
||||
|
||||
public IEnvironment Create(DeploymentConfig deploymentConfig) =>
|
||||
new KubernetesEnvironment(
|
||||
this.moduleStatusProvider,
|
||||
deploymentConfig,
|
||||
this.store,
|
||||
this.operatingSystemType,
|
||||
this.architecture,
|
||||
this.version);
|
||||
}
|
||||
}
|
||||
|
|
@ -16,7 +16,6 @@ namespace Microsoft.Azure.Devices.Edge.Agent.Kubernetes
|
|||
using Microsoft.Azure.Devices.Edge.Agent.Kubernetes.EdgeDeployment;
|
||||
using Microsoft.Azure.Devices.Edge.Util;
|
||||
using AgentDocker = Microsoft.Azure.Devices.Edge.Agent.Docker;
|
||||
|
||||
public class KubernetesRuntimeInfoProvider : IRuntimeInfoProvider, IRuntimeInfoSource
|
||||
{
|
||||
readonly string deviceNamespace;
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ namespace Microsoft.Azure.Devices.Edge.Agent.Kubernetes.EdgeDeployment
|
|||
public static ModuleRuntimeInfo ConvertToRuntime(this V1Pod pod, string name)
|
||||
{
|
||||
Option<V1ContainerStatus> containerStatus = GetContainerByName(name, pod);
|
||||
ReportedModuleStatus moduleStatus = ConvertPodStatusToModuleStatus(Option.Maybe(pod.Status));
|
||||
ReportedModuleStatus moduleStatus = ConvertPodStatusToModuleStatus(Option.Maybe(pod.Status), containerStatus);
|
||||
RuntimeData runtimeData = GetRuntimeData(containerStatus.OrDefault());
|
||||
|
||||
string moduleName = string.Empty;
|
||||
|
|
@ -43,7 +43,7 @@ namespace Microsoft.Azure.Devices.Edge.Agent.Kubernetes.EdgeDeployment
|
|||
return Option.Maybe(status);
|
||||
}
|
||||
|
||||
static ReportedModuleStatus ConvertPodStatusToModuleStatus(Option<V1PodStatus> podStatus)
|
||||
static ReportedModuleStatus ConvertPodStatusToModuleStatus(Option<V1PodStatus> podStatus, Option<V1ContainerStatus> containerStatus)
|
||||
{
|
||||
return podStatus.Map(
|
||||
status =>
|
||||
|
|
@ -51,17 +51,59 @@ namespace Microsoft.Azure.Devices.Edge.Agent.Kubernetes.EdgeDeployment
|
|||
switch (status.Phase)
|
||||
{
|
||||
case "Running":
|
||||
return new ReportedModuleStatus(ModuleStatus.Running, $"Started at {status.StartTime}");
|
||||
case "Failed":
|
||||
{
|
||||
return containerStatus.Map(c =>
|
||||
{
|
||||
if (c.State.Waiting != null)
|
||||
{
|
||||
return new ReportedModuleStatus(ModuleStatus.Backoff, $"Module in Back-off reason: {c.State.Waiting.Reason}");
|
||||
}
|
||||
else if (c.State.Terminated != null)
|
||||
{
|
||||
if (c.State.Terminated.ExitCode != 0)
|
||||
return new ReportedModuleStatus(ModuleStatus.Failed, $"Module Failed reason: {c.State.Terminated.Reason}");
|
||||
else
|
||||
return new ReportedModuleStatus(ModuleStatus.Stopped, $"Module Stopped reason: {c.State.Terminated.Reason}");
|
||||
}
|
||||
else
|
||||
{
|
||||
return new ReportedModuleStatus(ModuleStatus.Running, $"Started at {c.State.Running.StartedAt}");
|
||||
}
|
||||
}).GetOrElse(() => new ReportedModuleStatus(ModuleStatus.Failed, $"Module Failed with container status Unknown More Info: K8s reason: {status.Reason} with message: {status.Message}"));
|
||||
}
|
||||
|
||||
case "Pending":
|
||||
{
|
||||
return containerStatus.Map(c =>
|
||||
{
|
||||
if (c.State.Waiting != null)
|
||||
{
|
||||
return new ReportedModuleStatus(ModuleStatus.Backoff, $"Module in Back-off reason: {c.State.Waiting.Reason}");
|
||||
}
|
||||
else if (c.State.Terminated != null)
|
||||
{
|
||||
if (c.State.Terminated.ExitCode != 0)
|
||||
return new ReportedModuleStatus(ModuleStatus.Failed, $"Module Failed reason: {c.State.Terminated.Reason}");
|
||||
else
|
||||
return new ReportedModuleStatus(ModuleStatus.Stopped, $"Module Stopped reason: {c.State.Terminated.Reason}");
|
||||
}
|
||||
else
|
||||
{
|
||||
return new ReportedModuleStatus(ModuleStatus.Backoff, $"Started at {c.State.Running.StartedAt}");
|
||||
}
|
||||
}).GetOrElse(() => new ReportedModuleStatus(ModuleStatus.Failed, $"Module Failed with container status Unknown More Info: K8s reason: {status.Reason} with message: {status.Message}"));
|
||||
}
|
||||
|
||||
case "Unknown":
|
||||
return new ReportedModuleStatus(ModuleStatus.Failed, status.Reason);
|
||||
return new ReportedModuleStatus(ModuleStatus.Unknown, $"Module status Unknown reason: {status.Reason}");
|
||||
case "Succeeded":
|
||||
return new ReportedModuleStatus(ModuleStatus.Stopped, status.Reason);
|
||||
return new ReportedModuleStatus(ModuleStatus.Stopped, $"Module Stopped reason: {status.Reason} with message: {status.Message}");
|
||||
case "Failed":
|
||||
return new ReportedModuleStatus(ModuleStatus.Failed, $"Module Failed reason: {status.Reason} with message: {status.Message}");
|
||||
default:
|
||||
throw new InvalidOperationException($"Invalid pod status {status.Phase}");
|
||||
}
|
||||
}).GetOrElse(() => new ReportedModuleStatus(ModuleStatus.Failed, "Unable to get pod status"));
|
||||
}).GetOrElse(() => new ReportedModuleStatus(ModuleStatus.Unknown, "Unable to get pod status"));
|
||||
}
|
||||
|
||||
static RuntimeData GetRuntimeData(V1ContainerStatus status)
|
||||
|
|
|
|||
|
|
@ -227,17 +227,6 @@ namespace Microsoft.Azure.Devices.Edge.Agent.Service.Modules
|
|||
.As<IRuntimeInfoSource>()
|
||||
.SingleInstance();
|
||||
|
||||
// Task<IBackupSource>
|
||||
builder.Register(
|
||||
c =>
|
||||
{
|
||||
var serde = c.Resolve<ISerde<DeploymentConfigInfo>>();
|
||||
IDeploymentBackupSource backupSource = new DeploymentSecretBackup(Constants.EdgeAgentBackupName, this.deviceNamespace, this.moduleOwner, serde, c.Resolve<IKubernetes>());
|
||||
return Task.FromResult(backupSource);
|
||||
})
|
||||
.As<Task<IDeploymentBackupSource>>()
|
||||
.SingleInstance();
|
||||
|
||||
// KubernetesDeploymentProvider
|
||||
builder.Register(
|
||||
c => new KubernetesDeploymentMapper(
|
||||
|
|
@ -330,10 +319,9 @@ namespace Microsoft.Azure.Devices.Edge.Agent.Service.Modules
|
|||
{
|
||||
CancellationTokenSource tokenSource = new CancellationTokenSource(SystemInfoTimeout);
|
||||
var moduleStateStore = await c.Resolve<Task<IEntityStore<string, ModuleState>>>();
|
||||
var restartPolicyManager = c.Resolve<IRestartPolicyManager>();
|
||||
IRuntimeInfoProvider runtimeInfoProvider = c.Resolve<IRuntimeInfoProvider>();
|
||||
IEnvironmentProvider dockerEnvironmentProvider = await DockerEnvironmentProvider.CreateAsync(runtimeInfoProvider, moduleStateStore, restartPolicyManager, tokenSource.Token);
|
||||
return dockerEnvironmentProvider;
|
||||
IEnvironmentProvider kubernetesEnvironmentProvider = await KubernetesEnvironmentProvider.CreateAsync(runtimeInfoProvider, moduleStateStore, tokenSource.Token);
|
||||
return kubernetesEnvironmentProvider;
|
||||
})
|
||||
.As<Task<IEnvironmentProvider>>()
|
||||
.SingleInstance();
|
||||
|
|
|
|||
|
|
@ -160,108 +160,189 @@ namespace Microsoft.Azure.Devices.Edge.Agent.Kubernetes.Test
|
|||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ReturnsLastKnowModuleState()
|
||||
public static V1Pod CreatePodInPhaseWithContainerStatus(string podPhase, V1ContainerState containerState)
|
||||
=> new V1Pod
|
||||
{
|
||||
Metadata = new V1ObjectMeta
|
||||
{
|
||||
Name = "module-a-abc123",
|
||||
Labels = new Dictionary<string, string>
|
||||
{
|
||||
[KubernetesConstants.K8sEdgeModuleLabel] = "module-a"
|
||||
},
|
||||
Annotations = new Dictionary<string, string>
|
||||
{
|
||||
[KubernetesConstants.K8sEdgeOriginalModuleId] = "Module-A"
|
||||
}
|
||||
},
|
||||
Status = new V1PodStatus
|
||||
{
|
||||
Phase = podPhase,
|
||||
ContainerStatuses = new List<V1ContainerStatus>()
|
||||
{
|
||||
new V1ContainerStatus
|
||||
{
|
||||
Name = "module-a",
|
||||
State = containerState,
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
public static V1Pod CreatePodWithPodParametersOnly(string podPhase, string podReason, string podMessage)
|
||||
=> new V1Pod
|
||||
{
|
||||
Metadata = new V1ObjectMeta
|
||||
{
|
||||
Name = "module-a-abc123",
|
||||
Labels = new Dictionary<string, string>
|
||||
{
|
||||
[KubernetesConstants.K8sEdgeModuleLabel] = "module-a"
|
||||
},
|
||||
Annotations = new Dictionary<string, string>
|
||||
{
|
||||
[KubernetesConstants.K8sEdgeOriginalModuleId] = "Module-A"
|
||||
}
|
||||
},
|
||||
Status = new V1PodStatus { Phase = podPhase, Reason = podReason, Message = podMessage },
|
||||
};
|
||||
|
||||
public static IEnumerable<object[]> GetListOfPodsInRunningPhase()
|
||||
{
|
||||
return new[]
|
||||
{
|
||||
new object[]
|
||||
{
|
||||
CreatePodInPhaseWithContainerStatus("Running", new V1ContainerState(waiting: new V1ContainerStateWaiting("Waiting", "CrashBackLoopOff"))),
|
||||
"Module in Back-off reason: CrashBackLoopOff",
|
||||
ModuleStatus.Backoff
|
||||
},
|
||||
new object[]
|
||||
{
|
||||
CreatePodInPhaseWithContainerStatus("Running", new V1ContainerState(terminated: new V1ContainerStateTerminated(0, reason: "Completed"))),
|
||||
"Module Stopped reason: Completed",
|
||||
ModuleStatus.Stopped
|
||||
},
|
||||
new object[]
|
||||
{
|
||||
CreatePodInPhaseWithContainerStatus("Running", new V1ContainerState(terminated: new V1ContainerStateTerminated(139, reason: "Segmentation Fault"))),
|
||||
"Module Failed reason: Segmentation Fault",
|
||||
ModuleStatus.Failed
|
||||
},
|
||||
new object[]
|
||||
{
|
||||
CreatePodInPhaseWithContainerStatus("Running", new V1ContainerState(running: new V1ContainerStateRunning(startedAt: DateTime.Parse("2019-06-12T16:11:22Z")))),
|
||||
"Started at " + DateTime.Parse("2019-06-12T16:11:22Z"),
|
||||
ModuleStatus.Running
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[MemberData(nameof(GetListOfPodsInRunningPhase))]
|
||||
public async Task ReturnModuleStatusWhenPodIsRunning(V1Pod pod, string description, ModuleStatus status)
|
||||
{
|
||||
var client = new Mock<IKubernetes>(MockBehavior.Strict);
|
||||
var moduleManager = new Mock<IModuleManager>(MockBehavior.Strict);
|
||||
var runtimeInfo = new KubernetesRuntimeInfoProvider(Namespace, client.Object, moduleManager.Object);
|
||||
foreach (V1Pod pod in BuildPodList().Values)
|
||||
runtimeInfo.CreateOrUpdateAddPodInfo(pod);
|
||||
|
||||
ModuleRuntimeInfo info = (await runtimeInfo.GetModules(CancellationToken.None)).Single();
|
||||
|
||||
Assert.Equal(status, info.ModuleStatus);
|
||||
Assert.Equal(description, info.Description);
|
||||
}
|
||||
|
||||
public static IEnumerable<object[]> GetListOfPodsInPendingPhase()
|
||||
{
|
||||
return new[]
|
||||
{
|
||||
runtimeInfo.CreateOrUpdateAddPodInfo(pod);
|
||||
}
|
||||
new object[]
|
||||
{
|
||||
CreatePodInPhaseWithContainerStatus("Pending", new V1ContainerState(waiting: new V1ContainerStateWaiting("Waiting", "CrashBackLoopOff"))),
|
||||
"Module in Back-off reason: CrashBackLoopOff",
|
||||
ModuleStatus.Backoff
|
||||
},
|
||||
new object[]
|
||||
{
|
||||
CreatePodInPhaseWithContainerStatus("Pending", new V1ContainerState(terminated: new V1ContainerStateTerminated(0, reason: "Completed"))),
|
||||
"Module Stopped reason: Completed",
|
||||
ModuleStatus.Stopped
|
||||
},
|
||||
new object[]
|
||||
{
|
||||
CreatePodInPhaseWithContainerStatus("Pending", new V1ContainerState(terminated: new V1ContainerStateTerminated(139, reason: "Segmentation Fault"))),
|
||||
"Module Failed reason: Segmentation Fault",
|
||||
ModuleStatus.Failed
|
||||
},
|
||||
new object[]
|
||||
{
|
||||
CreatePodInPhaseWithContainerStatus("Pending", new V1ContainerState(running: new V1ContainerStateRunning(startedAt: DateTime.Parse("2019-06-12T16:11:22Z")))),
|
||||
"Started at " + DateTime.Parse("2019-06-12T16:11:22Z"),
|
||||
ModuleStatus.Backoff
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
Dictionary<string, V1Pod> modified = BuildPodList();
|
||||
[Theory]
|
||||
[MemberData(nameof(GetListOfPodsInPendingPhase))]
|
||||
public async Task ReturnModuleStatusWhenPodIsPending(V1Pod pod, string description, ModuleStatus status)
|
||||
{
|
||||
var client = new Mock<IKubernetes>(MockBehavior.Strict);
|
||||
var moduleManager = new Mock<IModuleManager>(MockBehavior.Strict);
|
||||
var runtimeInfo = new KubernetesRuntimeInfoProvider(Namespace, client.Object, moduleManager.Object);
|
||||
runtimeInfo.CreateOrUpdateAddPodInfo(pod);
|
||||
|
||||
DateTime agentStartTime = new DateTime(2019, 6, 11);
|
||||
modified["edgeagent"].Status.Phase = "Running";
|
||||
modified["edgeagent"].Status.StartTime = agentStartTime;
|
||||
ModuleRuntimeInfo info = (await runtimeInfo.GetModules(CancellationToken.None)).Single();
|
||||
|
||||
string pendingDescription = "0/1 node available";
|
||||
modified["edgehub"].Status.Phase = "Pending";
|
||||
modified["edgehub"].Status.Reason = pendingDescription;
|
||||
Assert.Equal(status, info.ModuleStatus);
|
||||
Assert.Equal(description, info.Description);
|
||||
}
|
||||
|
||||
string finishedDescription = "Pod finished";
|
||||
modified["simulatedtemperaturesensor"].Status.Phase = "Succeeded";
|
||||
modified["simulatedtemperaturesensor"].Status.Reason = finishedDescription;
|
||||
modified["simulatedtemperaturesensor"].Status.ContainerStatuses[1].State.Running = null;
|
||||
modified["simulatedtemperaturesensor"].Status.ContainerStatuses[1].State.Terminated = new V1ContainerStateTerminated(139, finishedAt: DateTime.Parse("2019-06-12T16:13:07Z"), startedAt: DateTime.Parse("2019-06-12T16:11:22Z"));
|
||||
|
||||
foreach (V1Pod pod in modified.Values)
|
||||
public static IEnumerable<object[]> GetListOfPodsInAbnormalPhase()
|
||||
{
|
||||
return new[]
|
||||
{
|
||||
runtimeInfo.CreateOrUpdateAddPodInfo(pod);
|
||||
}
|
||||
new object[]
|
||||
{
|
||||
CreatePodWithPodParametersOnly("Running", "Unschedulable", "persistentvolumeclaim module-a-pvc not found"),
|
||||
"Module Failed with container status Unknown More Info: K8s reason: Unschedulable with message: persistentvolumeclaim module-a-pvc not found",
|
||||
ModuleStatus.Failed
|
||||
},
|
||||
new object[]
|
||||
{
|
||||
CreatePodWithPodParametersOnly("Unknown", "Unknown", "Unknown"),
|
||||
"Module status Unknown reason: Unknown",
|
||||
ModuleStatus.Unknown
|
||||
},
|
||||
new object[]
|
||||
{
|
||||
CreatePodWithPodParametersOnly("Failed", "Terminated", "Non-zero exit code"),
|
||||
"Module Failed reason: Terminated with message: Non-zero exit code",
|
||||
ModuleStatus.Failed
|
||||
},
|
||||
new object[]
|
||||
{
|
||||
CreatePodWithPodParametersOnly("Succeeded", "Completed", "Zero exit code"),
|
||||
"Module Stopped reason: Completed with message: Zero exit code",
|
||||
ModuleStatus.Stopped
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
var modules = (await runtimeInfo.GetModules(CancellationToken.None)).ToList();
|
||||
Assert.Equal(3, modules.Count);
|
||||
[Theory]
|
||||
[MemberData(nameof(GetListOfPodsInAbnormalPhase))]
|
||||
public async Task ReturnModuleStatusWhenPodIsAbnormal(V1Pod pod, string description, ModuleStatus status)
|
||||
{
|
||||
var client = new Mock<IKubernetes>(MockBehavior.Strict);
|
||||
var moduleManager = new Mock<IModuleManager>(MockBehavior.Strict);
|
||||
var runtimeInfo = new KubernetesRuntimeInfoProvider(Namespace, client.Object, moduleManager.Object);
|
||||
runtimeInfo.CreateOrUpdateAddPodInfo(pod);
|
||||
|
||||
// Normal operation statuses
|
||||
foreach (var i in modules)
|
||||
{
|
||||
if (string.Equals("edgeAgent", i.Name))
|
||||
{
|
||||
Assert.Equal(ModuleStatus.Running, i.ModuleStatus);
|
||||
Assert.Equal($"Started at {agentStartTime.ToString()}", i.Description);
|
||||
}
|
||||
else if (string.Equals("edgeHub", i.Name))
|
||||
{
|
||||
Assert.Equal(ModuleStatus.Failed, i.ModuleStatus);
|
||||
Assert.Equal(pendingDescription, i.Description);
|
||||
Assert.Equal(Option.None<DateTime>(), i.ExitTime);
|
||||
}
|
||||
else if (string.Equals("SimulatedTemperatureSensor", i.Name))
|
||||
{
|
||||
Assert.Equal(ModuleStatus.Stopped, i.ModuleStatus);
|
||||
Assert.Equal(finishedDescription, i.Description);
|
||||
Assert.Equal(new DateTime(2019, 6, 12), i.StartTime.OrDefault().Date);
|
||||
Assert.Equal(new DateTime(2019, 6, 12), i.ExitTime.OrDefault().Date);
|
||||
}
|
||||
else
|
||||
{
|
||||
Assert.True(false, $"Missing module {i.Name} in validation");
|
||||
}
|
||||
ModuleRuntimeInfo info = (await runtimeInfo.GetModules(CancellationToken.None)).Single();
|
||||
|
||||
if (i is ModuleRuntimeInfo<DockerReportedConfig> d)
|
||||
{
|
||||
Assert.NotEqual("unknown:unknown", d.Config.Image);
|
||||
}
|
||||
}
|
||||
|
||||
string unknownDescription = "Could not reach pod";
|
||||
modified["edgeagent"].Status.Phase = "Unknown";
|
||||
modified["edgeagent"].Status.Reason = unknownDescription;
|
||||
|
||||
modified["edgehub"].Status = null;
|
||||
|
||||
foreach (V1Pod pod in modified.Values)
|
||||
{
|
||||
runtimeInfo.CreateOrUpdateAddPodInfo(pod);
|
||||
}
|
||||
|
||||
var abnormalModules = (await runtimeInfo.GetModules(CancellationToken.None)).ToList();
|
||||
Assert.Equal(3, modules.Count);
|
||||
|
||||
// Abnormal operation statuses
|
||||
foreach (var i in abnormalModules)
|
||||
{
|
||||
if (string.Equals("edgeAgent", i.Name))
|
||||
{
|
||||
Assert.Equal(ModuleStatus.Failed, i.ModuleStatus);
|
||||
Assert.Equal(unknownDescription, i.Description);
|
||||
}
|
||||
else if (string.Equals("edgeHub", i.Name))
|
||||
{
|
||||
Assert.Equal(ModuleStatus.Failed, i.ModuleStatus);
|
||||
Assert.Equal("Unable to get pod status", i.Description);
|
||||
}
|
||||
else if (string.Equals("SimulatedTemperatureSensor", i.Name))
|
||||
{
|
||||
}
|
||||
else
|
||||
{
|
||||
Assert.True(false, $"Missing module {i.Name} in validation");
|
||||
}
|
||||
}
|
||||
Assert.Equal(status, info.ModuleStatus);
|
||||
Assert.Equal(description, info.Description);
|
||||
}
|
||||
|
||||
static Dictionary<string, V1Pod> BuildPodList()
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче