Changes for autoscaling VMSS on simulation start and stop (#268)

This commit is contained in:
Avani Patel 2018-10-23 10:00:14 -07:00 коммит произвёл Harleen Thind
Родитель 9f51e51baf
Коммит 88678cc1ef
10 изменённых файлов: 282 добавлений и 20 удалений

2
.vscode/launch.json поставляемый
Просмотреть файл

@ -17,6 +17,8 @@
"PCS_STORAGEADAPTER_WEBSERVICE_URL": "http://localhost:9022/v1",
"PCS_STORAGEADAPTER_DOCUMENTDB_CONNSTRING": "your DocumentDb connection string",
"PCS_AZURE_STORAGE_ACCOUNT": "your Azure Storage Account connection string",
"PCS_RESOURCE_GROUP_LOCATION": "your Azure resource group location",
"PCS_VMSS_NAME": "your vm scale set name",
// Optional environment variables used in appsettings.ini
// For additonal optional settings, refer to comments in appsettings.ini

Просмотреть файл

@ -6,6 +6,7 @@ using System.Threading;
using System.Threading.Tasks;
using Microsoft.Azure.IoTSolutions.DeviceSimulation.PartitioningAgent;
using Microsoft.Azure.IoTSolutions.DeviceSimulation.Services;
using Microsoft.Azure.IoTSolutions.DeviceSimulation.Services.AzureManagementAdapter;
using Microsoft.Azure.IoTSolutions.DeviceSimulation.Services.Clustering;
using Microsoft.Azure.IoTSolutions.DeviceSimulation.Services.Concurrency;
using Microsoft.Azure.IoTSolutions.DeviceSimulation.Services.Diagnostics;
@ -14,6 +15,7 @@ using Microsoft.Azure.IoTSolutions.DeviceSimulation.Services.Runtime;
using Moq;
using PartitioningAgent.Test.helpers;
using Xunit;
using static Microsoft.Azure.IoTSolutions.DeviceSimulation.Services.Models.Simulation;
namespace PartitioningAgent.Test
{
@ -28,6 +30,7 @@ namespace PartitioningAgent.Test
private readonly Mock<IFactory> factory;
private readonly Mock<ILogger> log;
private readonly Mock<IDevices> devices;
private readonly Mock<IAzureManagementAdapterClient> azureManagementAdapterClient;
public AgentTest()
{
@ -38,6 +41,7 @@ namespace PartitioningAgent.Test
this.clusteringConfig = new Mock<IClusteringConfig>();
this.factory = new Mock<IFactory>();
this.log = new Mock<ILogger>();
this.azureManagementAdapterClient = new Mock<IAzureManagementAdapterClient>();
this.clusteringConfig.SetupGet(x => x.CheckIntervalMsecs).Returns(5);
this.thread.Setup(x => x.Sleep(It.IsAny<int>()))
@ -54,7 +58,8 @@ namespace PartitioningAgent.Test
this.thread.Object,
this.clusteringConfig.Object,
this.factory.Object,
this.log.Object);
this.log.Object,
this.azureManagementAdapterClient.Object);
this.devices = new Mock<IDevices>();
this.factory.Setup(x => x.Resolve<IDevices>()).Returns(this.devices.Object);
@ -610,6 +615,66 @@ namespace PartitioningAgent.Test
this.simulations.Verify(x => x.TryToSetDeviceDeletionCompleteAsync(simulation.Id), Times.Once);
}
[Fact, Trait(Constants.TYPE, Constants.UNIT_TEST)]
public void ItCalculatesRequiredNodes()
{
// Arrange
int expectedNodeCount = 9;
this.AfterStartRunOnlyOneLoop();
var deviceModels = new List<DeviceModelRef>
{
new DeviceModelRef { Id = "d1", Count = 50 },
new DeviceModelRef { Id = "d2", Count = 150 },
new DeviceModelRef { Id = "d3", Count = 200 },
};
var customDevices = new List<CustomDeviceRef>
{
new CustomDeviceRef { DeviceId = "1", DeviceModel = new DeviceModelRef { Id = "d1" } },
new CustomDeviceRef { DeviceId = "2", DeviceModel = new DeviceModelRef { Id = "d1" } },
new CustomDeviceRef { DeviceId = "3", DeviceModel = new DeviceModelRef { Id = "d2" } },
new CustomDeviceRef { DeviceId = "4", DeviceModel = new DeviceModelRef { Id = "d3" } },
new CustomDeviceRef { DeviceId = "5", DeviceModel = new DeviceModelRef { Id = "d3" } }
};
this.simulations.Setup(x => x.GetListAsync()).ReturnsAsync(new List<Simulation>
{
new Simulation
{
Id = Guid.NewGuid().ToString(),
Enabled = true,
StartTime = DateTimeOffset.UtcNow.AddHours(-2),
EndTime = DateTimeOffset.UtcNow.AddHours(1),
DevicesCreationComplete = true,
DeleteDevicesWhenSimulationEnds = true,
DeviceModels = deviceModels,
CustomDevices = customDevices
}
});
this.clusteringConfig.Setup(x => x.MaxDevicesPerNode).Returns(50);
this.TheCurrentNodeIsMaster();
// Act
this.target.StartAsync().CompleteOrTimeout();
// Assert
// Verify request to update autoscale settings is made when node count changes
this.azureManagementAdapterClient.Verify(x => x.CreateOrUpdateVmssAutoscaleSettingsAsync(It.Is<int>(a => a.Equals(expectedNodeCount))));
// Arrange
this.azureManagementAdapterClient.Invocations.Clear();
// Act
this.target.StartAsync().CompleteOrTimeout();
// Assert
// Verify request to update autoscale settings is not made when node count does not change
this.azureManagementAdapterClient.Verify(x => x.CreateOrUpdateVmssAutoscaleSettingsAsync(It.IsAny<int>()), Times.Never);
}
// Helper used to ensure that a task reaches an expected state
private static void WaitForTaskStatus(Task<Task> task, TaskStatus status, int time)
{

Просмотреть файл

@ -1,9 +1,11 @@
// Copyright (c) Microsoft. All rights reserved.
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using Microsoft.Azure.IoTSolutions.DeviceSimulation.Services;
using Microsoft.Azure.IoTSolutions.DeviceSimulation.Services.AzureManagementAdapter;
using Microsoft.Azure.IoTSolutions.DeviceSimulation.Services.Clustering;
using Microsoft.Azure.IoTSolutions.DeviceSimulation.Services.Concurrency;
using Microsoft.Azure.IoTSolutions.DeviceSimulation.Services.Diagnostics;
@ -20,13 +22,17 @@ namespace Microsoft.Azure.IoTSolutions.DeviceSimulation.PartitioningAgent
public class Agent : IPartitioningAgent
{
private const int DEFAULT_NODE_COUNT = 1;
private readonly IClusterNodes clusterNodes;
private readonly IDevicePartitions partitions;
private readonly ISimulations simulations;
private readonly IThreadWrapper thread;
private readonly IFactory factory;
private readonly ILogger log;
private readonly IAzureManagementAdapterClient azureManagementAdapter;
private readonly IClusteringConfig clusteringConfig;
private readonly int checkIntervalMsecs;
private int currentNodeCount;
private bool running;
@ -37,7 +43,8 @@ namespace Microsoft.Azure.IoTSolutions.DeviceSimulation.PartitioningAgent
IThreadWrapper thread,
IClusteringConfig clusteringConfig,
IFactory factory,
ILogger logger)
ILogger logger,
IAzureManagementAdapterClient azureManagementAdapter)
{
this.clusterNodes = clusterNodes;
this.partitions = partitions;
@ -45,8 +52,11 @@ namespace Microsoft.Azure.IoTSolutions.DeviceSimulation.PartitioningAgent
this.thread = thread;
this.factory = factory;
this.log = logger;
this.azureManagementAdapter = azureManagementAdapter;
this.clusteringConfig = clusteringConfig;
this.checkIntervalMsecs = clusteringConfig.CheckIntervalMsecs;
this.running = false;
this.currentNodeCount = DEFAULT_NODE_COUNT;
}
public async Task StartAsync()
@ -79,6 +89,9 @@ namespace Microsoft.Azure.IoTSolutions.DeviceSimulation.PartitioningAgent
await this.clusterNodes.RemoveStaleNodesAsync();
// Scale nodes in Vmss
await this.ScaleVmssNodes(activeSimulations);
// Create IoTHub devices for all the active simulations
await this.CreateDevicesAsync(activeSimulations);
@ -99,6 +112,43 @@ namespace Microsoft.Azure.IoTSolutions.DeviceSimulation.PartitioningAgent
{
this.running = false;
}
private async Task ScaleVmssNodes(IList<Simulation> activeSimulations)
{
// Default node count is 1
var nodeCount = DEFAULT_NODE_COUNT;
var maxDevicesPerNode = this.clusteringConfig.MaxDevicesPerNode;
if (activeSimulations.Count > 0)
{
var models = new List<Simulation.DeviceModelRef>();
var customDevices = 0;
foreach (var simulation in activeSimulations)
{
// Loop through all the device models used in the simulation
models = (from model in simulation.DeviceModels where model.Count > 0 select model).ToList();
// Count total custom devices
customDevices += simulation.CustomDevices.Count;
}
// Calculate the total number of devices
var totalDevices = models.Sum(model => model.Count) + customDevices;
// Calculate number of nodes required
nodeCount = maxDevicesPerNode > 0 ? (int)Math.Ceiling((double)totalDevices / maxDevicesPerNode) : DEFAULT_NODE_COUNT;
}
if (this.currentNodeCount != nodeCount)
{
// Send a request to update vmss auto scale settings to create vm instances
// TODO: when devices are added or removed, the number of VMs might need an update
await this.azureManagementAdapter.CreateOrUpdateVmssAutoscaleSettingsAsync(nodeCount);
this.currentNodeCount = nodeCount;
}
}
private async Task DeleteDevicesAsync(IList<Simulation> deletionRequiredSimulations)
{

Просмотреть файл

@ -0,0 +1,52 @@
// Copyright (c) Microsoft. All rights reserved.
using System.Collections.Generic;
using Newtonsoft.Json;
namespace Microsoft.Azure.IoTSolutions.DeviceSimulation.Services.AzureManagementAdapter
{
public class AutoScaleSettingsCreateOrUpdateRequestModel
{
[JsonProperty("location")]
public string Location { get; set; }
[JsonProperty(PropertyName = "properties")]
public Properties Properties { get; set; }
}
public class Properties
{
[JsonProperty("enabled")]
public bool Enabled { get; set; }
[JsonProperty("targetResourceUri")]
public string TargetResourceUri { get; set; }
[JsonProperty("profiles")]
public List<Profile> Profiles { get; set; }
}
public class Profile
{
[JsonProperty("name")]
public string Name { get; set; }
[JsonProperty("capacity")]
public Capacity Capacity { get; set; }
[JsonProperty("rules")]
public List<object> Rules { get; set; }
}
public class Capacity
{
[JsonProperty("minimum")]
public string Minimum { get; set; }
[JsonProperty("maximum")]
public string Maximum { get; set; }
[JsonProperty("default")]
public string Default { get; set; }
}
}

Просмотреть файл

@ -18,6 +18,7 @@ namespace Microsoft.Azure.IoTSolutions.DeviceSimulation.Services.AzureManagement
public interface IAzureManagementAdapterClient
{
Task<MetricsResponseListModel> PostAsync(MetricsRequestListModel requestList);
Task CreateOrUpdateVmssAutoscaleSettingsAsync(int vmCount);
}
public class AzureManagementAdapter : IAzureManagementAdapterClient
@ -56,16 +57,7 @@ namespace Microsoft.Azure.IoTSolutions.DeviceSimulation.Services.AzureManagement
/// <param name="requestList"></param>
public async Task<MetricsResponseListModel> PostAsync(MetricsRequestListModel requestList)
{
if (this.AccessTokenIsNullOrEmpty())
{
await this.GetAadTokenAsync();
}
// Renew access token 10 minutes before it's expire time
if (this.AccessTokenExpireSoon())
{
this.GetAadTokenAsync();
}
await this.CreateOrUpdateAccessTokenAsync();
if (requestList == null)
{
@ -97,6 +89,32 @@ namespace Microsoft.Azure.IoTSolutions.DeviceSimulation.Services.AzureManagement
return metricsResponseList;
}
public async Task CreateOrUpdateVmssAutoscaleSettingsAsync(int vmCount)
{
await this.CreateOrUpdateAccessTokenAsync();
var accessToken = $"Bearer {this.ReadSecureString(this.secureAccessToken)}";
var request = this.PrepareVmssAutoscaleSettingsRequest(accessToken, vmCount.ToString());
this.log.Debug("Azure Management request content", () => new { request.Content });
var response = await this.httpClient.PutAsync(request);
this.log.Debug("Azure management response", () => new { response });
// TODO: Exception handling for specific exceptions like not enough cores left in subscription.
this.ThrowIfError(response);
}
private async Task CreateOrUpdateAccessTokenAsync()
{
if (this.AccessTokenIsNullOrEmpty() || this.AccessTokenExpireSoon())
{
await this.GetAadTokenAsync();
}
}
private bool AccessTokenIsNullOrEmpty()
{
return this.secureAccessToken.Length == 0;
@ -116,11 +134,41 @@ namespace Microsoft.Azure.IoTSolutions.DeviceSimulation.Services.AzureManagement
request.SetUriFromString($"{this.config.AzureManagementAdapterApiUrl}/{path}");
request.Options.EnsureSuccess = false;
request.Options.Timeout = this.config.AzureManagementAdapterApiTimeout;
if (!this.config.AzureManagementAdapterApiUrl.ToLowerInvariant().StartsWith("https:"))
if (content != null)
{
throw new InvalidConfigurationException("Azure Management API url must start with https");
request.SetContent(content);
}
this.log.Debug("Azure Management request", () => new { request });
return request;
}
/// <summary>
/// https://docs.microsoft.com/en-us/rest/api/monitor/autoscalesettings/createorupdate
/// </summary>
private HttpRequest PrepareVmssAutoscaleSettingsRequest(string token, string vmCount)
{
var autoScaleSettingsName = "scalevmss";
var request = new HttpRequest();
request.AddHeader(HttpRequestHeader.Accept.ToString(), "application/json");
request.AddHeader(HttpRequestHeader.CacheControl.ToString(), "no-cache");
request.AddHeader(HttpRequestHeader.Authorization.ToString(), token);
request.SetUriFromString($"{this.config.AzureManagementAdapterApiUrl}/{this.GetVmssAutoScaleSettingsUrl(autoScaleSettingsName)}");
request.Options.EnsureSuccess = false;
request.Options.Timeout = this.config.AzureManagementAdapterApiTimeout;
var content = new AutoScaleSettingsCreateOrUpdateRequestModel();
content.Location = this.deploymentConfig.AzureResourceGroupLocation;
content.Properties = new Properties();
content.Properties.Enabled = true;
content.Properties.TargetResourceUri = this.GetVmssResourceUrl();
content.Properties.Profiles = new List<Profile>();
content.Properties.Profiles.Add(new Profile { Name = autoScaleSettingsName,
Capacity = new Capacity{ Minimum = vmCount, Maximum = vmCount, Default = vmCount },
Rules = new List<object>() });
if (content != null)
{
request.SetContent(content);
@ -135,10 +183,10 @@ namespace Microsoft.Azure.IoTSolutions.DeviceSimulation.Services.AzureManagement
{
if (!response.IsError) return;
this.log.Error("Metrics request error", () => new { response.Content });
this.diagnosticsLogger.LogServiceError("Metrics request error", new { response.Content });
this.log.Error("Management API request error", () => new { response.Content });
this.diagnosticsLogger.LogServiceError("Management API request error", new { response.Content });
throw new ExternalDependencyException(
new HttpRequestException($"Metrics request error: status code {response.StatusCode}"));
new HttpRequestException($"Management API request error: status code {response.StatusCode}"));
}
private string GetDefaultIoTHubMetricsUrl()
@ -150,6 +198,21 @@ namespace Microsoft.Azure.IoTSolutions.DeviceSimulation.Services.AzureManagement
$"$filter={this.GetDefaultMetricsQuery()}";
}
private string GetVmssResourceUrl()
{
return $"/subscriptions/{this.deploymentConfig.AzureSubscriptionId}" +
$"/resourceGroups/{this.deploymentConfig.AzureResourceGroup}" +
$"/providers/Microsoft.Compute/virtualMachineScaleSets/{this.deploymentConfig.AzureVmssName}";
}
private string GetVmssAutoScaleSettingsUrl(string name)
{
return $"/subscriptions/{this.deploymentConfig.AzureSubscriptionId}" +
$"/resourceGroups/{this.deploymentConfig.AzureResourceGroup}" +
$"/providers/microsoft.insights/autoscalesettings/{name}" +
$"?api-version=2015-04-01";
}
/// <summary>
/// TODO: Refactor this method when Azure Key Vault embeded into DS.
/// https://docs.microsoft.com/en-us/azure/key-vault/service-to-service-authentication

Просмотреть файл

@ -7,7 +7,9 @@ namespace Microsoft.Azure.IoTSolutions.DeviceSimulation.Services.Runtime
string AzureSubscriptionDomain { get; }
string AzureSubscriptionId { get; }
string AzureResourceGroup { get; }
string AzureResourceGroupLocation { get; }
string AzureIothubName { get; }
string AzureVmssName { get; }
string AadTenantId { get; }
string AadAppId { get; }
string AadAppSecret { get; }
@ -19,7 +21,9 @@ namespace Microsoft.Azure.IoTSolutions.DeviceSimulation.Services.Runtime
public string AzureSubscriptionDomain { get; set; }
public string AzureSubscriptionId { get; set; }
public string AzureResourceGroup { get; set; }
public string AzureResourceGroupLocation { get; set; }
public string AzureIothubName { get; set; }
public string AzureVmssName { get; set; }
public string AadTenantId { get; set; }
public string AadAppId { get; set; }
public string AadAppSecret { get; set; }

Просмотреть файл

@ -10,6 +10,8 @@
"PCS_STORAGEADAPTER_DOCUMENTDB_CONNSTRING": "your DocumentDb connection string",
"PCS_AZURE_STORAGE_ACCOUNT": "your Azure Storage Account connection string",
"PCS_STORAGEADAPTER_WEBSERVICE_URL": "http://localhost:9022/v1",
"PCS_RESOURCE_GROUP_LOCATION": "your Azure resource group location",
"PCS_VMSS_NAME": "your vm scale set name",
"PCS_LOG_LEVEL": "Debug",
"PCS_AUTH_REQUIRED": "false",
"PCS_AUTH_ISSUER": "",

Просмотреть файл

@ -128,8 +128,10 @@ namespace Microsoft.Azure.IoTSolutions.DeviceSimulation.WebService.Runtime
private const string AZURE_SUBSCRIPTION_DOMAIN = DEPLOYMENT_KEY + "azure_subscription_domain";
private const string AZURE_SUBSCRIPTION_ID = DEPLOYMENT_KEY + "azure_subscription_id";
private const string AZURE_RESOURCE_GROUP = DEPLOYMENT_KEY + "azure_resource_group";
private const string AZURE_RESOURCE_GROUP_LOCATION = DEPLOYMENT_KEY + "azure_resource_group_location";
private const string AZURE_IOTHUB_NAME = DEPLOYMENT_KEY + "azure_iothub_name";
private const string AZURE_VMSS_NAME = DEPLOYMENT_KEY + "azure_vmss_name";
private const string AZURE_ACTIVE_DIRECTORY_KEY = APPLICATION_KEY + "AzureActiveDirectory:";
private const string AAD_TENANT_ID = AZURE_ACTIVE_DIRECTORY_KEY + "tenant_id";
private const string AAD_APP_ID = AZURE_ACTIVE_DIRECTORY_KEY + "app_id";
@ -236,6 +238,16 @@ namespace Microsoft.Azure.IoTSolutions.DeviceSimulation.WebService.Runtime
"value in the 'appsettings.ini' configuration file.");
}
var azureManagementAdapterApiUrl = configData.GetString(AZURE_MANAGEMENT_ADAPTER_API_URL_KEY);
if (!azureManagementAdapterApiUrl.ToLowerInvariant().StartsWith("https:"))
{
throw new Exception("The service configuration is incomplete. " +
"Azure Management API url must start with https. " +
"For more information, see the environment variables " +
"used in project properties and the 'webservice_url' " +
"value in the 'appsettings.ini' configuration file.");
}
return new ServicesConfig
{
DeviceModelsFolder = MapRelativePath(configData.GetString(DEVICE_MODELS_FOLDER_KEY)),
@ -245,7 +257,7 @@ namespace Microsoft.Azure.IoTSolutions.DeviceSimulation.WebService.Runtime
IoTHubSdkDeviceClientTimeout = configData.GetOptionalUInt(IOTHUB_SDK_DEVICE_CLIENT_TIMEOUT_KEY),
StorageAdapterApiUrl = configData.GetString(STORAGE_ADAPTER_API_URL_KEY),
StorageAdapterApiTimeout = configData.GetInt(STORAGE_ADAPTER_API_TIMEOUT_KEY),
AzureManagementAdapterApiUrl = configData.GetString(AZURE_MANAGEMENT_ADAPTER_API_URL_KEY),
AzureManagementAdapterApiUrl = azureManagementAdapterApiUrl,
AzureManagementAdapterApiTimeout = configData.GetInt(AZURE_MANAGEMENT_ADAPTER_API_TIMEOUT_KEY),
AzureManagementAdapterApiVersion = configData.GetString(AZURE_MANAGEMENT_ADAPTER_API_VERSION),
TwinReadWriteEnabled = configData.GetBool(TWIN_READ_WRITE_ENABLED_KEY, true),
@ -320,7 +332,9 @@ namespace Microsoft.Azure.IoTSolutions.DeviceSimulation.WebService.Runtime
AzureSubscriptionDomain = configData.GetString(AZURE_SUBSCRIPTION_DOMAIN, "undefined.onmicrosoft.com"),
AzureSubscriptionId = configData.GetString(AZURE_SUBSCRIPTION_ID, Guid.Empty.ToString()),
AzureResourceGroup = configData.GetString(AZURE_RESOURCE_GROUP, "undefined"),
AzureResourceGroupLocation = configData.GetString(AZURE_RESOURCE_GROUP_LOCATION, "undefined"),
AzureIothubName = configData.GetString(AZURE_IOTHUB_NAME, "undefined"),
AzureVmssName = configData.GetString(AZURE_VMSS_NAME, "undefined"),
AadTenantId = configData.GetString(AAD_TENANT_ID, "undefined"),
AadAppId = configData.GetString(AAD_APP_ID, "undefined"),
AadAppSecret = configData.GetString(AAD_APP_SECRET, "undefined"),

Просмотреть файл

@ -12,6 +12,8 @@
<Variable name="ASPNETCORE_ENVIRONMENT" value="Development"/>
<Variable name="PCS_IOTHUB_CONNSTRING" value="your Azure IoT Hub connection string"/>
<Variable name="PCS_STORAGEADAPTER_WEBSERVICE_URL" value="http://localhost:9022/v1"/>
<Variable name="PCS_RESOURCE_GROUP_LOCATION" value="your Azure resource group location"/>
<Variable name="PCS_VMSS_NAME" value="your vm scale set name"/>
<Variable name="PCS_LOG_LEVEL" value="Debug"/>
<Variable name="PCS_AUTH_REQUIRED" value="false"/>
<Variable name="PCS_AUTH_ISSUER" value=""/>
@ -57,4 +59,4 @@
</Properties>
</MonoDevelop>
</ProjectExtensions>
</Project>
</Project>

Просмотреть файл

@ -172,6 +172,14 @@ azure_resource_group = "${?PCS_RESOURCE_GROUP}"
# The value is used to create a URL taking to the IoT Hub metrics in the Azure portal.
azure_iothub_name = "${?PCS_IOHUB_NAME}"
# Location where azure resource group is deployed, e.g "East US".
# The value is used to update auto scale settings on vmss on simulation start and stop.
azure_resource_group_location = "${?PCS_RESOURCE_GROUP_LOCATION}"
# Name of VMSS resource. E.g "test_vmss"
# The value is used to update auto scale settings on vmss on simulation start and stop.
azure_vmss_name = "${?PCS_VMSS_NAME}"
[DeviceSimulationService:Logging]
# Application log levels: Debug, Info, Warn, Error