From ff12c4de4cd97e46e0c79e1841e51cf4c65af255 Mon Sep 17 00:00:00 2001 From: SParekh Date: Thu, 7 May 2020 12:03:25 -0400 Subject: [PATCH] PrepareDedicatedHostGroup endpoint (#12) * Update to Azure runtime 3 and .net core 3.1 * Implement DedicatedHostEngine PrepareDedicatedHostGroup and corresponding tests * Abstract configuration * Exception handling and additional tests * Exception message and return on no DH created * Add Exception handling and diff status code for validation vs internal server error * Remove todo comment * Resolve PR comments * Add logs on determination of hosts to be added Co-authored-by: Sonali Parekh --- .../DedicatedHostsManagerFunctionClient.cs | 127 +++++++ DedicatedHostsManager.sln | 4 +- .../ComputeClient/DhmComputeClient.cs | 17 +- DedicatedHostsManager/Config.cs | 116 ++++++ DedicatedHostsManager/ConfigExtensions.cs | 39 ++ DedicatedHostsManager/Constants.cs | 2 + .../DedicatedHostEngine.cs | 335 +++++++++++++++--- .../DedicatedHostSelector.cs | 11 +- .../DedicatedHostEngine/Helper.cs | 25 ++ .../IDedicatedHostEngine.cs | 22 ++ .../DedicatedHostStateManager.cs | 19 +- .../DedicatedHostsFunction.cs | 133 ++++++- DedicatedHostsManager/Startup.cs | 10 +- DedicatedHostsManager/Sync/SyncProvider.cs | 15 +- .../DedicatedHostEngineTests.cs | 291 ++++++++++++++- .../DedicatedHostsManagerTests.csproj | 8 +- .../DedicatedHostsSelectorTests.cs | 18 +- .../PrepareDedicatedHostMappingConfig.json | 31 ++ README.md | 18 +- 19 files changed, 1109 insertions(+), 132 deletions(-) create mode 100644 DedicatedHostsManager/Config.cs create mode 100644 DedicatedHostsManager/ConfigExtensions.cs create mode 100644 DedicatedHostsManager/DedicatedHostEngine/Helper.cs create mode 100644 DedicatedHostsTests/TestData/PrepareDedicatedHostMappingConfig.json diff --git a/DedicatedHostManagerClients/DedicatedHostsManagerFunctionClient/DedicatedHostsManagerFunctionClient.cs b/DedicatedHostManagerClients/DedicatedHostsManagerFunctionClient/DedicatedHostsManagerFunctionClient.cs index 3e65066..b31804c 100644 --- a/DedicatedHostManagerClients/DedicatedHostsManagerFunctionClient/DedicatedHostsManagerFunctionClient.cs +++ b/DedicatedHostManagerClients/DedicatedHostsManagerFunctionClient/DedicatedHostsManagerFunctionClient.cs @@ -18,6 +18,9 @@ using System.Net.Http; using System.Text; using System.Threading.Tasks; using DedicatedHostClientHelpers; +using System.Net; +using System.Web.Http; +using System.Linq; namespace DedicatedHostsManagerFunctionClient { @@ -285,5 +288,129 @@ namespace DedicatedHostsManagerFunctionClient await _httpClient.GetAsync(deleteVmUri); return new OkObjectResult($"Deleted {vmName} VM."); } + + /// + /// Test Prepare Dedicated Host. + /// + /// HTTP request. + /// Logger. + /// + [FunctionName("TestPrepareDedicatedHostGroup")] + public async Task TestPrepareDedicatedHostGroup( + [HttpTrigger(AuthorizationLevel.Function, "get", "post", Route = null)] HttpRequest req, + ILogger log) + { + var parameters = req.GetQueryParameterDictionary(); + if (!parameters.ContainsKey(ResourceGroupName) || string.IsNullOrEmpty(parameters[ResourceGroupName])) + { + return new BadRequestObjectResult("Resource group name was missing in the query parameters."); + } + + if (!parameters.ContainsKey(HostGroupName) || string.IsNullOrEmpty(parameters[HostGroupName])) + { + return new BadRequestObjectResult("Host group name was missing in the query parameters."); + } + + if (!parameters.ContainsKey(VmCount) || !Int32.TryParse(parameters[VmCount], out int numVirtualMachines)) + { + return new BadRequestObjectResult("VmCount was missing in the query parameters."); + } + + if (!parameters.ContainsKey(VmSku) || string.IsNullOrEmpty(parameters[VmSku])) + { + return new BadRequestObjectResult("VM SKU was missing in the query parameters."); + } + + var authEndpoint = _configuration["AuthEndpoint"]; + var azureRmEndpoint = _configuration["AzureRmEndpoint"]; + var location = _configuration["Location"]; + var virtualMachineSize = parameters[VmSku]; + var tenantId = _configuration["TenantId"]; + var clientId = _configuration["ClientId"]; + var clientSecret = _configuration["FairfaxClientSecret"]; + var subscriptionId = _configuration["SubscriptionId"]; + var resourceGroupName = parameters[ResourceGroupName]; + var hostGroupName = parameters[HostGroupName]; + + log.LogInformation($"Generating auth token..."); + + var token = await TokenHelper.GetToken( + authEndpoint, + azureRmEndpoint, + tenantId, + clientId, + clientSecret); + var customTokenProvider = new AzureCredentials( + new TokenCredentials(token), + new TokenCredentials(token), + tenantId, + AzureEnvironment.FromName(_configuration["CloudName"])); + var client = RestClient + .Configure() + .WithEnvironment(AzureEnvironment.FromName(_configuration["CloudName"])) + .WithLogLevel(HttpLoggingDelegatingHandler.Level.Basic) + .WithCredentials(customTokenProvider) + .Build(); + + var azure = Azure.Authenticate(client, tenantId).WithSubscription(subscriptionId); + var computeManagementClient = new ComputeManagementClient(customTokenProvider) + { + SubscriptionId = subscriptionId, + BaseUri = new Uri(_configuration["ResourceManagerUri"]), + LongRunningOperationRetryTimeout = 5 + }; + + log.LogInformation($"Creating resource group ({resourceGroupName}), if needed"); + var resourceGroup = azure.ResourceGroups.Define(resourceGroupName) + .WithRegion(location) + .Create(); + log.LogInformation($"Creating host group ({hostGroupName}), if needed"); + var newDedicatedHostGroup = new DedicatedHostGroup() + { + Location = location, + PlatformFaultDomainCount = 1 + }; + await computeManagementClient.DedicatedHostGroups.CreateOrUpdateAsync( + resourceGroupName, + hostGroupName, + newDedicatedHostGroup); + + +#if DEBUG + var prepareDHGroup = + $"http://localhost:7071/api/PrepareDedicatedHostGroup" + + $"?token={token}" + + $"&cloudName={_configuration["CloudName"]}" + + $"&tenantId={tenantId}" + + $"&subscriptionId={subscriptionId}" + + $"&resourceGroup={resourceGroupName}" + + $"&vmSku={virtualMachineSize}" + + $"&dedicatedHostGroupName={hostGroupName}" + + $"&vmCount={numVirtualMachines}" + + $"&platformFaultDomain=0"; +#else + var prepareDHGroup = + _configuration["PrepareDHGroupUri"] + + $"&token={token}" + + $"&cloudName={_configuration["CloudName"]}" + + $"&tenantId={tenantId}" + + $"&subscriptionId={subscriptionId}" + + $"&resourceGroup={resourceGroupName}" + + $"&vmSku={virtualMachineSize}" + + $"&dedicatedHostGroupName={hostGroupName}" + + $"&vmCount={numVirtualMachines}" + + $"&platformFaultDomain=0"; +#endif + var response = await _httpClient.GetAsync(prepareDHGroup); + if (response.StatusCode != HttpStatusCode.OK) + { + return new ObjectResult(new { error = $"Exception thrown by {await response.Content.ReadAsStringAsync()}" }) + { + StatusCode = (int)response.StatusCode + }; + } + var dhCreated = await response.Content.ReadAsAsync>(); + return new OkObjectResult($"Prepared Dedicated Host Group completed successfully {string.Join(",", dhCreated.Select(c => c.Name))} VM."); + } } } diff --git a/DedicatedHostsManager.sln b/DedicatedHostsManager.sln index 1221a52..94cd314 100644 --- a/DedicatedHostsManager.sln +++ b/DedicatedHostsManager.sln @@ -1,7 +1,7 @@  Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio 15 -VisualStudioVersion = 15.0.28307.645 +# Visual Studio Version 16 +VisualStudioVersion = 16.0.29905.134 MinimumVisualStudioVersion = 10.0.40219.1 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DedicatedHostsManagerTests", "DedicatedHostsTests\DedicatedHostsManagerTests.csproj", "{FBCF3F2F-3178-439A-BE41-FE1A91B537F8}" EndProject diff --git a/DedicatedHostsManager/ComputeClient/DhmComputeClient.cs b/DedicatedHostsManager/ComputeClient/DhmComputeClient.cs index 7c5624d..42ea080 100644 --- a/DedicatedHostsManager/ComputeClient/DhmComputeClient.cs +++ b/DedicatedHostsManager/ComputeClient/DhmComputeClient.cs @@ -1,7 +1,6 @@ using Microsoft.Azure.Management.Compute; using Microsoft.Azure.Management.ResourceManager.Fluent; using Microsoft.Azure.Management.ResourceManager.Fluent.Authentication; -using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Logging; using Newtonsoft.Json; using Polly; @@ -21,21 +20,21 @@ namespace DedicatedHostsManager.ComputeClient { private static IComputeManagementClient _computeManagementClient; private readonly HttpClient _httpClient; - private readonly IConfiguration _configuration; + private readonly Config _config; private readonly ILogger _logger; /// /// Initializes a new instance of the DhmComputeClient class. /// - /// Configuration. + /// Configuration. /// Logger. /// To create an HTTP client. public DhmComputeClient( - IConfiguration configuration, + Config config, ILogger logger, IHttpClientFactory httpClientFactory) { - _configuration = configuration; + _config = config; _logger = logger; _httpClient = httpClientFactory.CreateClient(); } @@ -58,8 +57,8 @@ namespace DedicatedHostsManager.ComputeClient { SubscriptionId = subscriptionId, BaseUri = baseUri, - LongRunningOperationRetryTimeout = int.Parse(_configuration["ComputeClientLongRunningOperationRetryTimeoutSeconds"]), - HttpClient = { Timeout = TimeSpan.FromMinutes(int.Parse(_configuration["ComputeClientHttpTimeoutMin"])) } + LongRunningOperationRetryTimeout = _config.ComputeClientLongRunningOperationRetryTimeoutSeconds, + HttpClient = { Timeout = TimeSpan.FromMinutes(_config.ComputeClientHttpTimeoutMin) } }); } @@ -70,7 +69,7 @@ namespace DedicatedHostsManager.ComputeClient private async Task GetResourceManagerEndpoint(AzureEnvironment azureEnvironment) { - var armMetadataRetryCount = int.Parse(_configuration["GetArmMetadataRetryCount"]); + var armMetadataRetryCount = _config.GetArmMetadataRetryCount; HttpResponseMessage armResponseMessage = null; await Policy .Handle() @@ -82,7 +81,7 @@ namespace DedicatedHostsManager.ComputeClient $"Could not retrieve ARM metadata. Attempt #{r}/{armMetadataRetryCount}. Will try again in {ts.TotalSeconds} seconds. Exception={ex}")) .ExecuteAsync(async () => { - armResponseMessage = await _httpClient.GetAsync(_configuration["GetArmMetadataUrl"]); + armResponseMessage = await _httpClient.GetAsync(_config.GetArmMetadataUrl); }); if (armResponseMessage == null || armResponseMessage?.StatusCode != HttpStatusCode.OK) diff --git a/DedicatedHostsManager/Config.cs b/DedicatedHostsManager/Config.cs new file mode 100644 index 0000000..8773ae4 --- /dev/null +++ b/DedicatedHostsManager/Config.cs @@ -0,0 +1,116 @@ +using Newtonsoft.Json; +using System; +using System.Collections.Generic; +using System.Linq; + +namespace DedicatedHostsManager +{ + public class Config + { + public string LockContainerName { get; set; } + public int LockRetryCount { get; set; } + public int LockIntervalInSeconds { get; set; } + public int MinIntervalToCheckForVmInSeconds { get; set; } + public int MaxIntervalToCheckForVmInSeconds { get; set; } + public int RetryCountToCheckVmState { get; set; } + public int MaxRetriesToCreateVm { get; set; } + public int RedisConnectTimeoutMilliseconds { get; set; } + public int RedisSyncTimeoutMilliseconds { get; set; } + public int RedisConnectRetryCount { get; set; } + public int DhgCreateRetryCount { get; set; } + public int ComputeClientLongRunningOperationRetryTimeoutSeconds { get; set; } + public int ComputeClientHttpTimeoutMin { get; set; } + public int GetArmMetadataRetryCount { get; set; } + public int DedicatedHostCacheTtlMin { get; set; } + public string GetArmMetadataUrl { get; set; } + public string HostSelectorVmSize { get; set; } + public bool IsRunningInFairfax { get; set; } + public Connectionstrings ConnectionStrings { get; set; } + + public string VmToHostMapping + { + get { return "VirtualMachineToHostMapping to be referenced for details"; } + set + { + if (string.IsNullOrWhiteSpace(value)) + { + throw new Exception("VmToHostMapping must be specified in the configuration."); + } + + VirtualMachineToHostMapping = JsonConvert.DeserializeObject>(value); + } + } + + public Dictionary VirtualMachineToHostMapping { get; private set; } = new Dictionary(); + + public string DedicatedHostMapping + { + get { return "DedicatedHostConfigurationTable to be referenced for details"; } + set + { + if (string.IsNullOrWhiteSpace(value)) + { + throw new Exception("DedicatedHostMapping must be specified in the configuration."); + } + + var mapping = JsonConvert.DeserializeObject>(value); + + this.DedicatedHostConfigurationTable = mapping + .SelectMany(hm => hm.HostMapping + .Select(h => new { hm.Family, h.Region, h.Host.Type, h.Host.VmMapping })) + .SelectMany(vm => vm.VmMapping + .Select(fo => new DedicatedHostConfiguration() + { + DhFamily = vm.Family, + Location = vm.Region, + DhSku = vm.Type, + VmSku = fo.Size, + VmCapacity = fo.Capacity + } + )).ToList(); + } + } + + public IList DedicatedHostConfigurationTable { get; set; } = new List(); + + public class Connectionstrings + { + public string StorageConnectionString { get; set; } + public string RedisConnectionString { get; set; } + } + + public class DedicatedHostConfiguration + { + public string DhFamily { get; set; } + public string Location { get; set; } + public string DhSku { get; set; } + public string VmSku { get; set; } + public int VmCapacity { get; set; } + + public class JsonRepresentation + { + public string Family { get; set; } + public Hostmapping[] HostMapping { get; set; } + + public class Hostmapping + { + public string Region { get; set; } + public Host Host { get; set; } + } + + public class Host + { + public string Type { get; set; } + public VmMapping[] VmMapping { get; set; } + } + + public class VmMapping + { + public string Size { get; set; } + public int Capacity { get; set; } + } + } + } + } +} + diff --git a/DedicatedHostsManager/ConfigExtensions.cs b/DedicatedHostsManager/ConfigExtensions.cs new file mode 100644 index 0000000..ee5b7fd --- /dev/null +++ b/DedicatedHostsManager/ConfigExtensions.cs @@ -0,0 +1,39 @@ +using DedicatedHostsManager.ComputeClient; +using DedicatedHostsManager.DedicatedHostEngine; +using DedicatedHostsManager.DedicatedHostStateManager; +using DedicatedHostsManager.Sync; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; + +namespace DedicatedHostsManager +{ + public static class ConfigExtensions + { + public static void ConfigureCommonServices(this IServiceCollection services) + { + services.AddTransient(); + services.AddSingleton(p => p.GetService().CreatePocoConfig()); + services.AddHttpClient(); + services.AddSingleton(); + services.AddTransient(); + services.AddTransient(); + services.AddTransient(); + services.AddTransient(); + } + + private class ServiceFactory + { + private Config config = new Config(); + + public ServiceFactory(IConfiguration configuration) + { + config = configuration.Get(); + } + + public Config CreatePocoConfig() + { + return config; + } + } + } +} diff --git a/DedicatedHostsManager/Constants.cs b/DedicatedHostsManager/Constants.cs index 2e5bf0e..fcae329 100644 --- a/DedicatedHostsManager/Constants.cs +++ b/DedicatedHostsManager/Constants.cs @@ -18,5 +18,7 @@ public const string AvailabilityZone = "availabilityZone"; public const string Location = "location"; public const string PlatformFaultDomainCount = "platformFaultDomainCount"; + public const string PlatformFaultDomain = "platformFaultDomain"; + public const string VmCount = "vmCount"; } } \ No newline at end of file diff --git a/DedicatedHostsManager/DedicatedHostEngine/DedicatedHostEngine.cs b/DedicatedHostsManager/DedicatedHostEngine/DedicatedHostEngine.cs index 89a5731..8dfacc1 100644 --- a/DedicatedHostsManager/DedicatedHostEngine/DedicatedHostEngine.cs +++ b/DedicatedHostsManager/DedicatedHostEngine/DedicatedHostEngine.cs @@ -6,17 +6,16 @@ using Microsoft.Azure.Management.Compute.Models; using Microsoft.Azure.Management.ResourceManager.Fluent; using Microsoft.Azure.Management.ResourceManager.Fluent.Authentication; using Microsoft.Azure.Management.ResourceManager.Fluent.Core; -using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Logging; using Microsoft.Rest; using Microsoft.Rest.Azure; using Microsoft.WindowsAzure.Storage; -using Newtonsoft.Json; using Polly; using System; using System.Collections.Generic; using System.Diagnostics; using System.Linq; +using System.Net; using System.Threading; using System.Threading.Tasks; using SubResource = Microsoft.Azure.Management.Compute.Models.SubResource; @@ -29,11 +28,11 @@ namespace DedicatedHostsManager.DedicatedHostEngine public class DedicatedHostEngine : IDedicatedHostEngine { private readonly ILogger _logger; - private readonly IConfiguration _configuration; + private readonly Config _config; private readonly IDedicatedHostSelector _dedicatedHostSelector; private readonly ISyncProvider _syncProvider; private readonly IDedicatedHostStateManager _dedicatedHostStateManager; - private readonly IDhmComputeClient _dhmComputeClient; + private readonly IDhmComputeClient _dhmComputeClient; /// /// Initializes the Dedicated Host engine. @@ -45,15 +44,15 @@ namespace DedicatedHostsManager.DedicatedHostEngine /// Dedicated Host state manager. /// Dedicated Host compute client. public DedicatedHostEngine( - ILogger logger, - IConfiguration configuration, + ILogger logger, + Config config, IDedicatedHostSelector dedicatedHostSelector, ISyncProvider syncProvider, IDedicatedHostStateManager dedicatedHostStateManager, IDhmComputeClient dhmComputeClient) { _logger = logger; - _configuration = configuration; + _config = config; _dedicatedHostSelector = dedicatedHostSelector; _syncProvider = syncProvider; _dedicatedHostStateManager = dedicatedHostStateManager; @@ -76,8 +75,8 @@ namespace DedicatedHostsManager.DedicatedHostEngine string token, AzureEnvironment azureEnvironment, string tenantId, - string subscriptionId, - string resourceGroup, + string subscriptionId, + string resourceGroup, string dhgName, string azName, int platformFaultDomainCount, @@ -123,7 +122,7 @@ namespace DedicatedHostsManager.DedicatedHostEngine new TokenCredentials(token), tenantId, azureEnvironment); - + var newDedicatedHostGroup = new DedicatedHostGroup() { Location = location, @@ -132,12 +131,12 @@ namespace DedicatedHostsManager.DedicatedHostEngine if (!string.IsNullOrEmpty(azName)) { - newDedicatedHostGroup.Zones = new List{ azName }; + newDedicatedHostGroup.Zones = new List { azName }; } - var dhgCreateRetryCount = int.Parse(_configuration["DhgCreateRetryCount"]); + var dhgCreateRetryCount = _config.DhgCreateRetryCount; var computeManagementClient = await _dhmComputeClient.GetComputeManagementClient( - subscriptionId, + subscriptionId, azureCredentials, azureEnvironment); var response = new AzureOperationResponse(); @@ -178,9 +177,9 @@ namespace DedicatedHostsManager.DedicatedHostEngine string token, AzureEnvironment azureEnvironment, string tenantId, - string subscriptionId, - string resourceGroup, - string dhgName, + string subscriptionId, + string resourceGroup, + string dhgName, string dhName, string dhSku, string location) @@ -259,7 +258,7 @@ namespace DedicatedHostsManager.DedicatedHostEngine new DedicatedHost { Location = location, - Sku = new Sku() {Name = dhSku} + Sku = new Sku() { Name = dhSku } }, null); } @@ -351,17 +350,17 @@ namespace DedicatedHostsManager.DedicatedHostEngine azureEnvironment); VirtualMachine response = null; var vmProvisioningState = virtualMachine.ProvisioningState; - var minIntervalToCheckForVmInSeconds = int.Parse(_configuration["MinIntervalToCheckForVmInSeconds"]); - var maxIntervalToCheckForVmInSeconds = int.Parse(_configuration["MaxIntervalToCheckForVmInSeconds"]); - var retryCountToCheckVmState = int.Parse(_configuration["RetryCountToCheckVmState"]); - var maxRetriesToCreateVm = int.Parse(_configuration["MaxRetriesToCreateVm"]); - var dedicatedHostCacheTtlMin = int.Parse(_configuration["DedicatedHostCacheTtlMin"]); + var minIntervalToCheckForVmInSeconds = _config.MinIntervalToCheckForVmInSeconds; + var maxIntervalToCheckForVmInSeconds = _config.MaxIntervalToCheckForVmInSeconds; + var retryCountToCheckVmState = _config.RetryCountToCheckVmState; + var maxRetriesToCreateVm = _config.MaxRetriesToCreateVm; + var dedicatedHostCacheTtlMin = _config.DedicatedHostCacheTtlMin; var vmCreationRetryCount = 0; - + while ((string.IsNullOrEmpty(vmProvisioningState) || !string.Equals(vmProvisioningState, "Succeeded", StringComparison.InvariantCultureIgnoreCase)) && vmCreationRetryCount < maxRetriesToCreateVm) - { + { if (string.IsNullOrEmpty(vmProvisioningState)) { var dedicatedHostId = await GetDedicatedHostForVmPlacement( @@ -375,7 +374,7 @@ namespace DedicatedHostsManager.DedicatedHostEngine vmName, region.Name); - _dedicatedHostStateManager.MarkHostUsage(dedicatedHostId.ToLower(), DateTimeOffset.Now.ToString(), TimeSpan.FromMinutes(dedicatedHostCacheTtlMin)); + _dedicatedHostStateManager.MarkHostUsage(dedicatedHostId.ToLower(), DateTimeOffset.Now.ToString(), TimeSpan.FromMinutes(dedicatedHostCacheTtlMin)); virtualMachine.Host = new SubResource(dedicatedHostId); try { @@ -403,7 +402,7 @@ namespace DedicatedHostsManager.DedicatedHostEngine if (string.Equals(vmProvisioningState, "Failed", StringComparison.InvariantCultureIgnoreCase)) { _logger.LogMetric("VmProvisioningFailureCountMetric", 1); - _dedicatedHostStateManager.MarkHostAtCapacity(virtualMachine.Host.Id.ToLower(), DateTimeOffset.Now.ToString(), TimeSpan.FromMinutes(dedicatedHostCacheTtlMin)); + _dedicatedHostStateManager.MarkHostAtCapacity(virtualMachine.Host.Id.ToLower(), DateTimeOffset.Now.ToString(), TimeSpan.FromMinutes(dedicatedHostCacheTtlMin)); var dedicatedHostId = await GetDedicatedHostForVmPlacement( token, azureEnvironment, @@ -444,7 +443,7 @@ namespace DedicatedHostsManager.DedicatedHostEngine await Policy .Handle() .WaitAndRetryAsync( - retryCountToCheckVmState, + retryCountToCheckVmState, r => TimeSpan.FromSeconds(2 * r), onRetry: (ex, ts, r) => _logger.LogInformation( @@ -547,7 +546,7 @@ namespace DedicatedHostsManager.DedicatedHostEngine if (string.IsNullOrEmpty(matchingHostId)) { - var lockRetryCount = int.Parse(_configuration["LockRetryCount"]); + var lockRetryCount = _config.LockRetryCount; var hostGroupId = await GetDedicatedHostGroupId( token, azureEnvironment, @@ -559,7 +558,7 @@ namespace DedicatedHostsManager.DedicatedHostEngine await Policy .Handle() .WaitAndRetryAsync( - lockRetryCount, + lockRetryCount, r => TimeSpan.FromSeconds(2 * r), (ex, ts, r) => _logger.LogInformation($"Attempt #{r.Count}/{lockRetryCount}. Will try again in {ts.TotalSeconds} seconds. Exception={ex}")) .ExecuteAsync(async () => @@ -581,19 +580,7 @@ namespace DedicatedHostsManager.DedicatedHostEngine if (string.IsNullOrEmpty(matchingHostId)) { _logger.LogInformation($"Creating a new host."); - var vmToHostDictionary = JsonConvert.DeserializeObject>(_configuration["VmToHostMapping"]); - if (vmToHostDictionary == null || string.IsNullOrEmpty(vmToHostDictionary[requiredVmSize])) - { - throw new Exception($"Cannot find a dedicated host SKU for the {requiredVmSize}: vm to host mapping was null."); - } - - var hostSku = vmToHostDictionary[requiredVmSize]; - _logger.LogInformation($"Host SKU {hostSku} will be used to host VM SKU {requiredVmSize}."); - if (string.IsNullOrEmpty(hostSku)) - { - throw new Exception( - $"Cannot find a dedicated host SKU for the {requiredVmSize}: vm to host mapping was null."); - } + var hostSku = GetVmToHostMapping(requiredVmSize); var newDedicatedHostResponse = await CreateDedicatedHost( token, @@ -602,7 +589,7 @@ namespace DedicatedHostsManager.DedicatedHostEngine subscriptionId, resourceGroup, hostGroupName, - "host-" + (new Random().Next(100,999)), + "host-" + (new Random().Next(100, 999)), hostSku, location); @@ -634,7 +621,7 @@ namespace DedicatedHostsManager.DedicatedHostEngine _logger.LogMetric("GetDedicatedHostTimeSecondsMetric", innerLoopStopwatch.Elapsed.TotalSeconds); _logger.LogInformation($"GetDedicatedHost: Took {innerLoopStopwatch.Elapsed.TotalSeconds} seconds to find a matching host {matchingHostId} for {vmName} of {requiredVmSize} SKU."); return matchingHostId; - } + } /// /// List Dedicated Host groups. @@ -757,8 +744,8 @@ namespace DedicatedHostsManager.DedicatedHostEngine subscriptionId, azureCredentials, azureEnvironment); - var retryCountToCheckVm = int.Parse(_configuration["RetryCountToCheckVmState"]); - var dedicatedHostCacheTtlMin = int.Parse(_configuration["DedicatedHostCacheTtlMin"]); + var retryCountToCheckVm = _config.RetryCountToCheckVmState; + var dedicatedHostCacheTtlMin = _config.DedicatedHostCacheTtlMin; VirtualMachine virtualMachine = null; DedicatedHost dedicatedHost = null; string hostId = null; @@ -771,13 +758,13 @@ namespace DedicatedHostsManager.DedicatedHostEngine _logger.LogInformation( $"Could not get VM details for {vmName}. Attempt #{r}/{retryCountToCheckVm}. Will try again in {ts.TotalSeconds} seconds. Exception={ex}")) .ExecuteAsync(async () => - { - virtualMachine = await computeManagementClient.VirtualMachines.GetAsync(resourceGroup, vmName); - hostId = virtualMachine?.Host?.Id; - var hostName = hostId?.Split(new[] {'/'}).Last(); - await computeManagementClient.VirtualMachines.DeleteAsync(resourceGroup, vmName); - dedicatedHost = await computeManagementClient.DedicatedHosts.GetAsync(resourceGroup, dedicatedHostGroup, hostName, InstanceViewTypes.InstanceView); - }); + { + virtualMachine = await computeManagementClient.VirtualMachines.GetAsync(resourceGroup, vmName); + hostId = virtualMachine?.Host?.Id; + var hostName = hostId?.Split(new[] { '/' }).Last(); + await computeManagementClient.VirtualMachines.DeleteAsync(resourceGroup, vmName); + dedicatedHost = await computeManagementClient.DedicatedHosts.GetAsync(resourceGroup, dedicatedHostGroup, hostName, InstanceViewTypes.InstanceView); + }); if (string.IsNullOrEmpty(hostId)) { @@ -788,7 +775,7 @@ namespace DedicatedHostsManager.DedicatedHostEngine if (dedicatedHost?.VirtualMachines.Count == 0) { // Avoid locking for now; revisit if needed - _dedicatedHostStateManager.MarkHostForDeletion(hostId.ToLower(), DateTimeOffset.Now.ToString(), TimeSpan.FromMinutes(dedicatedHostCacheTtlMin)); + _dedicatedHostStateManager.MarkHostForDeletion(hostId.ToLower(), DateTimeOffset.Now.ToString(), TimeSpan.FromMinutes(dedicatedHostCacheTtlMin)); if (!_dedicatedHostStateManager.IsHostInUsage(hostId.ToLower())) { await computeManagementClient.DedicatedHosts.DeleteAsync(resourceGroup, dedicatedHostGroup, dedicatedHost.Name); @@ -797,6 +784,157 @@ namespace DedicatedHostsManager.DedicatedHostEngine } } + public async Task> PrepareDedicatedHostGroup( + string token, + AzureEnvironment azureEnvironment, + string tenantId, string subscriptionId, + string resourceGroup, + string dhGroupName, + string vmSku, + int vmInstances, + int? platformFaultDomain) + { + List dedicatedHosts = default; + + if (string.IsNullOrEmpty(token)) + { + throw new ArgumentNullException(nameof(token)); + } + + if (azureEnvironment == null) + { + throw new ArgumentNullException(nameof(azureEnvironment)); + } + + if (string.IsNullOrEmpty(tenantId)) + { + throw new ArgumentNullException(nameof(tenantId)); + } + + if (string.IsNullOrEmpty(subscriptionId)) + { + throw new ArgumentException(nameof(subscriptionId)); + } + + if (string.IsNullOrEmpty(resourceGroup)) + { + throw new ArgumentException(nameof(resourceGroup)); + } + + if (string.IsNullOrEmpty(dhGroupName)) + { + throw new ArgumentException(nameof(dhGroupName)); + } + + if (string.IsNullOrEmpty(vmSku)) + { + throw new ArgumentException(nameof(vmSku)); + } + + var azureCredentials = new AzureCredentials( + new TokenCredentials(token), + new TokenCredentials(token), + tenantId, + azureEnvironment); + + var computeManagementClient = await _dhmComputeClient.GetComputeManagementClient( + subscriptionId, + azureCredentials, + azureEnvironment); + + var dhgCreateRetryCount = _config.DhgCreateRetryCount; + var hostGroup = await GetDedicatedHostGroup(); + var location = hostGroup.Location; // Location of DH canot be different from Host Group. + var existingHostsOnDHGroup = await GetExistingHostsOnDHGroup(); + + var (dhSku, vmCapacityPerHost) = GetVmCapacityPerHost(location, vmSku); + + var numOfDedicatedHostsByFaultDomain = this.CalculatePlatformFaultDomainToHost( + hostGroup, + existingHostsOnDHGroup, + vmSku, + vmInstances, + vmCapacityPerHost, + platformFaultDomain); + + await CreateDedicatedHosts(); + + return dedicatedHosts ?? new List(); + + async Task GetDedicatedHostGroup() + { + var response = await Helper.ExecuteAsyncWithRetry>( + funcToexecute: () => computeManagementClient.DedicatedHostGroups.GetWithHttpMessagesAsync(resourceGroup, dhGroupName), + logHandler: (retryMsg) => _logger.LogInformation($"Get Dedicated Host Group '{dhGroupName} failed.' {retryMsg}"), + exceptionFilter: ce => !ce.Message.Contains("not found")); + return response.Body; + } + + async Task> GetExistingHostsOnDHGroup() + { + var hostsInHostGroup = await this._dedicatedHostSelector.ListDedicatedHosts( + token, + azureEnvironment, + tenantId, + subscriptionId, + resourceGroup, + dhGroupName); + + var taskList = hostsInHostGroup.Select( + dedicatedHost => Helper.ExecuteAsyncWithRetry>( + () => computeManagementClient.DedicatedHosts.GetWithHttpMessagesAsync( + resourceGroup, + dhGroupName, + dedicatedHost.Name, + InstanceViewTypes.InstanceView), + (retryMsg) => _logger.LogInformation($"Get details for Dedicated Host '{dedicatedHost.Name} failed.' {retryMsg}"))); + + var response = await Task.WhenAll(taskList); + return response.Select(r => r.Body).ToList(); + } + + async Task CreateDedicatedHosts() + { + if (numOfDedicatedHostsByFaultDomain.Any()) + { + var createDhHostTasks = numOfDedicatedHostsByFaultDomain + .SelectMany(c => Enumerable.Repeat(c.fd, c.numberOfHosts)) + .Select(pfd => Helper.ExecuteAsyncWithRetry>( + funcToexecute: () => computeManagementClient.DedicatedHosts.CreateOrUpdateWithHttpMessagesAsync( + resourceGroup, + dhGroupName, + "host-" + (new Random().Next(100, 999)), + new DedicatedHost + { + Location = location, + Sku = new Sku() { Name = dhSku }, + PlatformFaultDomain = pfd + }), + logHandler: (retryMsg) => _logger.LogInformation($"Create host on Dedicated Host Group Fault Domain {pfd} failed.' {retryMsg}"), + retryCount: _config.DhgCreateRetryCount)); + + var bulkTask = Task.WhenAll(createDhHostTasks); + try + { + var response = await bulkTask; + dedicatedHosts = response.Select(c => c.Body).ToList(); + _logger.LogInformation(@$"Following dedicated hosts created created successfully : {string.Join(",", dedicatedHosts.Select(d => d.Name))}"); + } + catch (Exception ex) + { + if (bulkTask?.Exception?.InnerExceptions != null && bulkTask.Exception.InnerExceptions.Any()) + { + throw new Exception($"Creation of Dedicated Host failed with exceptions : \n {string.Join(",\n", bulkTask.Exception.InnerExceptions.Select(c => c?.Message + "\n"))}"); + } + else + { + throw new Exception($"Unexpected exception thrown {ex?.Message}"); + } + } + } + } + } + /// /// Gets the ID for a Dedicated Host group. /// @@ -826,5 +964,94 @@ namespace DedicatedHostsManager.DedicatedHostEngine azureEnvironment); return (await computeManagementClient.DedicatedHostGroups.GetAsync(resourceGroupName, hostGroupName)).Id; } + + private IList<(int fd, int numberOfHosts)> CalculatePlatformFaultDomainToHost( + DedicatedHostGroup dhGroup, + IList existingHostsOnDHGroup, + string vmSku, + int vmInstancesRequested, + int vmCapacityPerHost, + int? platformFaultDomain) + { + var dedicatedHosts = new List<(int fd, int numberOfHosts)>(); + var platformFaultDomains = DetermineFaultDomainsForPlacement(dhGroup.PlatformFaultDomainCount, platformFaultDomain); + var vmsRequiredPerFaultDomain = (int)Math.Round((decimal)vmInstancesRequested / platformFaultDomains.Length, MidpointRounding.ToPositiveInfinity); + + foreach (var fd in platformFaultDomains) + { + var dhInFaultDomain = existingHostsOnDHGroup.Where(c => c.PlatformFaultDomain == fd); + var availableVMCapacityInFaultDomain = 0; + foreach (var host in dhInFaultDomain) + { + // Existing hosts can be different sku then DH Sku determined for VM size. + availableVMCapacityInFaultDomain += (int)(host.InstanceView?.AvailableCapacity?.AllocatableVMs? + .FirstOrDefault(v => v.VmSize.Equals(vmSku, StringComparison.InvariantCultureIgnoreCase))?.Count ?? 0); + } + + if (vmsRequiredPerFaultDomain > availableVMCapacityInFaultDomain) + { + var fdHostsToBeAdded = (int)(Math.Round(((decimal)vmsRequiredPerFaultDomain - availableVMCapacityInFaultDomain) / vmCapacityPerHost, MidpointRounding.ToPositiveInfinity)); + dedicatedHosts.Add((fd, fdHostsToBeAdded)); + _logger.LogInformation(@$"{fdHostsToBeAdded} Hosts to be added to PlatformFaultDomain - '{fd}'"); + } + } + + return dedicatedHosts; + + int[] DetermineFaultDomainsForPlacement(int dhGroupFaultDomainCount, int? platformFaultDomain) + { + if (platformFaultDomain != null && platformFaultDomain > (dhGroupFaultDomainCount - 1)) + { + throw new Exception($"Invalid requested Platform Fault domain -Dedicated Host Group Fault Domains = {dhGroupFaultDomainCount}, requested Platform Fault domain = {platformFaultDomain}"); + } + + return platformFaultDomain switch + { + 0 => new int[] { 0 }, + 1 => new int[] { 1 }, + 2 => new int[] { 2 }, + _ => Enumerable.Range(0, dhGroupFaultDomainCount).ToArray() // As # of small, perf of using Range not significant + }; + } + } + + private (string dhSku, int vmCapacity) GetVmCapacityPerHost(string location, string vmSku) + { + var matchingConfig = _config.DedicatedHostConfigurationTable + .Where(c => (c.Location == "default" || c.Location.Equals(location, StringComparison.OrdinalIgnoreCase)) + && c.VmSku == vmSku); + + if (!matchingConfig.Any()) + { + throw new Exception($"DhSku mapping not found for default OR Location {location} / VM Sku {vmSku}"); + } + + var regionSpecific = matchingConfig.SingleOrDefault(c => c.Location == location); + if (regionSpecific != null) + { + return (regionSpecific.DhSku, regionSpecific.VmCapacity); + } + var defaultSetting = matchingConfig.Single(c => c.Location == "default"); + return (defaultSetting.DhSku, defaultSetting.VmCapacity); + } + + private string GetVmToHostMapping(string vmSku) + { + var vmToHostDictionary = _config.VirtualMachineToHostMapping; + if (vmToHostDictionary == null || string.IsNullOrEmpty(vmToHostDictionary[vmSku])) + { + throw new Exception($"Cannot find a dedicated host SKU for the {vmSku}: vm to host mapping was null."); + } + + var hostSku = vmToHostDictionary[vmSku]; + _logger.LogInformation($"Host SKU {hostSku} will be used to host VM SKU {vmSku}."); + if (string.IsNullOrEmpty(hostSku)) + { + throw new Exception( + $"Cannot find a dedicated host SKU for the {vmSku}: vm to host mapping was null."); + } + + return hostSku; + } } } diff --git a/DedicatedHostsManager/DedicatedHostEngine/DedicatedHostSelector.cs b/DedicatedHostsManager/DedicatedHostEngine/DedicatedHostSelector.cs index 8e557f6..0daf2b8 100644 --- a/DedicatedHostsManager/DedicatedHostEngine/DedicatedHostSelector.cs +++ b/DedicatedHostsManager/DedicatedHostEngine/DedicatedHostSelector.cs @@ -12,7 +12,6 @@ using System.Collections.Generic; using System.Linq; using System.Threading; using System.Threading.Tasks; -using Microsoft.Extensions.Configuration; namespace DedicatedHostsManager.DedicatedHostEngine { @@ -22,7 +21,7 @@ namespace DedicatedHostsManager.DedicatedHostEngine public class DedicatedHostSelector : IDedicatedHostSelector { private readonly ILogger _logger; - private readonly IConfiguration _configuration; + private readonly Config _config; private readonly IDedicatedHostStateManager _dedicatedHostStateManager; private readonly IDhmComputeClient _dhmComputeClient; @@ -31,18 +30,18 @@ namespace DedicatedHostsManager.DedicatedHostEngine /// /// Logger. /// Dedicated Host state management. - /// Configuration. + /// Configuration. /// Dedicated Host compute client. public DedicatedHostSelector( ILogger logger, IDedicatedHostStateManager dedicatedHostStateManager, - IConfiguration configuration, + Config config, IDhmComputeClient dhmComputeClient) { _logger = logger; _dedicatedHostStateManager = dedicatedHostStateManager; _dhmComputeClient = dhmComputeClient; - _configuration = configuration; + _config = config; } /// @@ -156,7 +155,7 @@ namespace DedicatedHostsManager.DedicatedHostEngine foreach (var host in hostList) { var count = host.InstanceView?.AvailableCapacity?.AllocatableVMs? - .First(v => v.VmSize.Equals(_configuration["HostSelectorVmSize"], StringComparison.InvariantCultureIgnoreCase)).Count; + .First(v => v.VmSize.Equals(_config.HostSelectorVmSize, StringComparison.InvariantCultureIgnoreCase)).Count; if (count < minCount) { minCount = count; diff --git a/DedicatedHostsManager/DedicatedHostEngine/Helper.cs b/DedicatedHostsManager/DedicatedHostEngine/Helper.cs new file mode 100644 index 0000000..30243c3 --- /dev/null +++ b/DedicatedHostsManager/DedicatedHostEngine/Helper.cs @@ -0,0 +1,25 @@ +using Polly; +using System; +using System.Threading.Tasks; + +namespace DedicatedHostsManager.DedicatedHostEngine +{ + public class Helper + { + private const int DefaultNetworkFailureRetryCount = 2; + + public static Task ExecuteAsyncWithRetry( + Func> funcToexecute, + Action logHandler, + Func exceptionFilter = null, + int retryCount = DefaultNetworkFailureRetryCount) where TException : Exception + { + return Policy.Handle(ce => exceptionFilter != null ? exceptionFilter(ce) : true) + .WaitAndRetryAsync( + retryCount, + r => TimeSpan.FromSeconds(2 * r), + (ex, ts, r) => logHandler($"Attempt #{r}/{retryCount}. Will try again in {ts.TotalSeconds} seconds. Exception={ex}")) + .ExecuteAsync(() => funcToexecute()); + } + } +} diff --git a/DedicatedHostsManager/DedicatedHostEngine/IDedicatedHostEngine.cs b/DedicatedHostsManager/DedicatedHostEngine/IDedicatedHostEngine.cs index 4031d4d..ec00339 100644 --- a/DedicatedHostsManager/DedicatedHostEngine/IDedicatedHostEngine.cs +++ b/DedicatedHostsManager/DedicatedHostEngine/IDedicatedHostEngine.cs @@ -139,5 +139,27 @@ namespace DedicatedHostsManager.DedicatedHostEngine string resourceGroup, string dedicatedHostGroup, string vmName); + + /// + /// Creates a Dedicated Host. + /// + /// Auth token. + /// Azure cloud. + /// Tenant ID. + /// Subscription ID. + /// Resource group. + /// Dedicated Host group name. + /// Dedicated Host name. + /// Virtual Machine SKU to be hosted on Dedicated Hosts + Task> PrepareDedicatedHostGroup( + string token, + AzureEnvironment azureEnvironment, + string tenantId, + string subscriptionId, + string resourceGroup, + string dhgName, + string vmSku, + int vmInstances, + int? platformFaultDomain); } } \ No newline at end of file diff --git a/DedicatedHostsManager/DedicatedHostStateManager/DedicatedHostStateManager.cs b/DedicatedHostsManager/DedicatedHostStateManager/DedicatedHostStateManager.cs index 3463600..25612f6 100644 --- a/DedicatedHostsManager/DedicatedHostStateManager/DedicatedHostStateManager.cs +++ b/DedicatedHostsManager/DedicatedHostStateManager/DedicatedHostStateManager.cs @@ -1,5 +1,4 @@ -using Microsoft.Extensions.Configuration; -using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging; using StackExchange.Redis; using System; @@ -11,7 +10,7 @@ namespace DedicatedHostsManager.DedicatedHostStateManager public class DedicatedHostStateManager : IDedicatedHostStateManager { private static readonly object LockObject = new object(); - private readonly IConfiguration _configuration; + private readonly Config _config; private readonly ILogger _logger; private readonly int _hostCapacityDbIndex; private readonly int _hostDeletionDbIndex; @@ -22,13 +21,13 @@ namespace DedicatedHostsManager.DedicatedHostStateManager /// /// Initializes the state manager. /// - /// Configuration. + /// Configuration. /// Logging. - public DedicatedHostStateManager(IConfiguration configuration, ILogger logger) + public DedicatedHostStateManager(Config config, ILogger logger) { - _configuration = configuration; + _config = config; _logger = logger; - _redisCacheConnection = _configuration.GetConnectionString("RedisConnectionString"); + _redisCacheConnection = _config.ConnectionStrings.RedisConnectionString; _hostCapacityDbIndex = 0; _hostDeletionDbIndex = 1; _hostUsageDbIndex = 2; @@ -55,9 +54,9 @@ namespace DedicatedHostsManager.DedicatedHostStateManager _connectionMultiplexer?.Dispose(); var configurationOptions = ConfigurationOptions.Parse(_redisCacheConnection); - configurationOptions.ConnectTimeout = int.Parse(_configuration["RedisConnectTimeoutMilliseconds"]); - configurationOptions.SyncTimeout = int.Parse(_configuration["RedisSyncTimeoutMilliseconds"]); - configurationOptions.ConnectRetry = int.Parse(_configuration["RedisConnectRetryCount"]); + configurationOptions.ConnectTimeout = _config.RedisConnectTimeoutMilliseconds; + configurationOptions.SyncTimeout = _config.RedisSyncTimeoutMilliseconds; + configurationOptions.ConnectRetry = _config.RedisConnectRetryCount; configurationOptions.AbortOnConnectFail = false; configurationOptions.Ssl = true; _connectionMultiplexer = ConnectionMultiplexer.Connect(configurationOptions); diff --git a/DedicatedHostsManager/DedicatedHostsFunction.cs b/DedicatedHostsManager/DedicatedHostsFunction.cs index 7e30182..150c713 100644 --- a/DedicatedHostsManager/DedicatedHostsFunction.cs +++ b/DedicatedHostsManager/DedicatedHostsFunction.cs @@ -6,11 +6,11 @@ using Microsoft.Azure.Management.ResourceManager.Fluent; using Microsoft.Azure.Management.ResourceManager.Fluent.Core; using Microsoft.Azure.WebJobs; using Microsoft.Azure.WebJobs.Extensions.Http; -using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Logging; using Newtonsoft.Json; using System; using System.Diagnostics; +using System.Net; using System.Threading.Tasks; using ExecutionContext = Microsoft.Azure.WebJobs.ExecutionContext; @@ -22,17 +22,14 @@ namespace DedicatedHostsManager public class DedicatedHostsFunction { private readonly IDedicatedHostEngine _dedicatedHostEngine; - private readonly IConfiguration _configuration; - + /// /// Initialization. /// /// Dedicated Host Engine. - /// Configuration. - public DedicatedHostsFunction(IDedicatedHostEngine dedicatedHostEngine, IConfiguration configuration) + public DedicatedHostsFunction(IDedicatedHostEngine dedicatedHostEngine) { _dedicatedHostEngine = dedicatedHostEngine; - _configuration = configuration; } /// @@ -241,5 +238,129 @@ namespace DedicatedHostsManager return new BadRequestObjectResult(exception.ToString()); } } + + + /// + /// This call to the Dedicated Host Manager library will prepare the host group by creating sufficient number of dedicated hosts so that a future call to VM or VMSS creation will be successful. + /// + /// HTTP request. + /// Logger. + /// Function execution context. + [FunctionName("PrepareDedicatedHostGroup")] + public async Task PrepareDedicatedHostGroup( + [HttpTrigger(AuthorizationLevel.Function, "get", "post", Route = null)] + HttpRequest req, + ILogger log, + ExecutionContext context) + { + var parameters = req.GetQueryParameterDictionary(); + + if (!parameters.ContainsKey(Constants.CloudName) || string.IsNullOrEmpty(parameters[Constants.CloudName])) + { + return new BadRequestObjectResult("CloudName was missing in the query parameters."); + } + + if (!parameters.ContainsKey(Constants.TenantId) || string.IsNullOrEmpty(parameters[Constants.TenantId])) + { + return new BadRequestObjectResult("TenantId was missing in the query parameters."); + } + + if (!parameters.ContainsKey(Constants.Token) || string.IsNullOrEmpty(parameters[Constants.Token])) + { + return new BadRequestObjectResult("Token was missing in the query parameters."); + } + + if (!parameters.ContainsKey(Constants.SubscriptionId) || + string.IsNullOrEmpty(parameters[Constants.SubscriptionId])) + { + return new BadRequestObjectResult("Subscription ID was missing in the query parameters."); + } + + if (!parameters.ContainsKey(Constants.ResourceGroup) || + string.IsNullOrEmpty(parameters[Constants.ResourceGroup])) + { + return new BadRequestObjectResult("Resource group was missing in the query parameters."); + } + + if (!parameters.ContainsKey(Constants.DedicatedHostGroupName) || + string.IsNullOrEmpty(parameters[Constants.DedicatedHostGroupName])) + { + return new BadRequestObjectResult("Dedicated host group was missing in the query parameters."); + } + + if (!parameters.ContainsKey(Constants.VmSku) || string.IsNullOrEmpty(parameters[Constants.VmSku])) + { + return new BadRequestObjectResult("VmSku was missing in the query parameters."); + } + + if (!parameters.ContainsKey(Constants.VmCount) || !int.TryParse(parameters[Constants.VmCount], out int vmCount)) + { + return new BadRequestObjectResult("VmCount was missing in the query parameters or not numeric value"); + } + + int? platformFaultDomain; + if (!parameters.ContainsKey(Constants.PlatformFaultDomain)) + { + platformFaultDomain = null; + } + else if (int.TryParse(parameters[Constants.PlatformFaultDomain], out int parsedFD) && parsedFD >= 0 && parsedFD <= 2) + { + platformFaultDomain = parsedFD; + } + else + { + return new BadRequestObjectResult("PlatformFaultDomain if specificed must be a value between 0-2"); + } + + var sw = Stopwatch.StartNew(); + try + { + var requestBody = await req.ReadAsStringAsync(); + var virtualMachine = JsonConvert.DeserializeObject(requestBody); + var cloudName = parameters[Constants.CloudName]; + AzureEnvironment azureEnvironment = null; + if (cloudName.Equals("AzureGlobalCloud", StringComparison.InvariantCultureIgnoreCase) + || cloudName.Equals("AzureCloud", StringComparison.InvariantCultureIgnoreCase)) + { + azureEnvironment = AzureEnvironment.AzureGlobalCloud; + } + else + { + azureEnvironment = AzureEnvironment.FromName(cloudName); + } + + var prepareDedicatedHostGroupResponse = await _dedicatedHostEngine.PrepareDedicatedHostGroup( + parameters[Constants.Token], + azureEnvironment, + parameters[Constants.TenantId], + parameters[Constants.SubscriptionId], + parameters[Constants.ResourceGroup], + parameters[Constants.DedicatedHostGroupName], + parameters[Constants.VmSku], + vmCount, + platformFaultDomain); + + log.LogInformation( + $"PrepareDedicatedHostGroup: Took {sw.Elapsed.TotalSeconds}s"); + log.LogMetric("PrepareDedicatedHostGroupTimeSecondsMetric", sw.Elapsed.TotalSeconds); + log.LogMetric("PrepareDedicatedHostGroupSuccessCountMetric", 1); + + return new OkObjectResult(prepareDedicatedHostGroupResponse); + } + catch (ArgumentException exception) + { + log.LogError( + $"PrepareDedicatedHostGroup: Validation Error creating {parameters[Constants.DedicatedHostGroupName]}, time spent: {sw.Elapsed.TotalSeconds}s, Exception: {exception}"); + log.LogMetric("PrepareDedicatedHostGroupFailureCountMetric", 1); + return new BadRequestObjectResult(exception.ToString()); + } + catch (Exception exception) + { + log.LogError( + $"PrepareDedicatedHostGroup: Error creating {parameters[Constants.DedicatedHostGroupName]}, time spent: {sw.Elapsed.TotalSeconds}s, Exception: {exception}"); + log.LogMetric("PrepareDedicatedHostGroupFailureCountMetric", 1); + return new ObjectResult(exception.ToString()) { StatusCode = (int)HttpStatusCode.InternalServerError }; + } + } } } diff --git a/DedicatedHostsManager/Startup.cs b/DedicatedHostsManager/Startup.cs index 79e6908..571765a 100644 --- a/DedicatedHostsManager/Startup.cs +++ b/DedicatedHostsManager/Startup.cs @@ -2,10 +2,6 @@ using System.Collections.Generic; using System.Linq; using DedicatedHostsManager; -using DedicatedHostsManager.ComputeClient; -using DedicatedHostsManager.DedicatedHostEngine; -using DedicatedHostsManager.DedicatedHostStateManager; -using DedicatedHostsManager.Sync; using Microsoft.ApplicationInsights.AspNetCore; using Microsoft.ApplicationInsights.Channel; using Microsoft.ApplicationInsights.Extensibility; @@ -91,11 +87,7 @@ namespace DedicatedHostsManager }); builder.Services.AddHttpClient(); - builder.Services.AddSingleton(); - builder.Services.AddTransient(); - builder.Services.AddTransient(); - builder.Services.AddTransient(); - builder.Services.AddTransient(); + builder.Services.ConfigureCommonServices(); } } } diff --git a/DedicatedHostsManager/Sync/SyncProvider.cs b/DedicatedHostsManager/Sync/SyncProvider.cs index 9008537..33399b6 100644 --- a/DedicatedHostsManager/Sync/SyncProvider.cs +++ b/DedicatedHostsManager/Sync/SyncProvider.cs @@ -1,6 +1,5 @@ using System; using System.Threading.Tasks; -using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Logging; using Microsoft.WindowsAzure.Storage; using Microsoft.WindowsAzure.Storage.Blob; @@ -12,7 +11,7 @@ namespace DedicatedHostsManager.Sync /// public class SyncProvider : ISyncProvider { - private readonly IConfiguration _configuration; + private readonly Config _config; private readonly ILogger _logger; private readonly CloudBlobContainer _cloudBlobContainer; private string _lease; @@ -20,15 +19,15 @@ namespace DedicatedHostsManager.Sync /// /// Initialize sync provider. /// - /// Configuration. + /// Configuration. /// Logging. - public SyncProvider(IConfiguration configuration, ILogger logger) + public SyncProvider(Config config, ILogger logger) { - _configuration = configuration; + _config = config; _logger = logger; - var storageAccount = CloudStorageAccount.Parse(_configuration.GetConnectionString("StorageConnectionString")); + var storageAccount = CloudStorageAccount.Parse(_config.ConnectionStrings.StorageConnectionString); var blobClient = storageAccount.CreateCloudBlobClient(); - _cloudBlobContainer = blobClient.GetContainerReference(_configuration["LockContainerName"]); + _cloudBlobContainer = blobClient.GetContainerReference(_config.LockContainerName); } /// @@ -44,7 +43,7 @@ namespace DedicatedHostsManager.Sync await blockBlob.UploadTextAsync(blobName); } - var lockIntervalInSeconds = int.Parse(_configuration["LockIntervalInSeconds"]); + var lockIntervalInSeconds = _config.LockIntervalInSeconds; _lease = await blockBlob.AcquireLeaseAsync(TimeSpan.FromSeconds(lockIntervalInSeconds), null); _logger.LogInformation($"Acquired lock for {blockBlob}"); } diff --git a/DedicatedHostsTests/DedicatedHostEngineTests.cs b/DedicatedHostsTests/DedicatedHostEngineTests.cs index 3561c14..6cc8d09 100644 --- a/DedicatedHostsTests/DedicatedHostEngineTests.cs +++ b/DedicatedHostsTests/DedicatedHostEngineTests.cs @@ -1,11 +1,14 @@ +using DedicatedHostsManager; using DedicatedHostsManager.ComputeClient; using DedicatedHostsManager.DedicatedHostEngine; using DedicatedHostsManager.DedicatedHostStateManager; using DedicatedHostsManager.Sync; +using FluentAssertions; using Microsoft.Azure.Management.Compute; using Microsoft.Azure.Management.Compute.Models; using Microsoft.Azure.Management.ResourceManager.Fluent; using Microsoft.Azure.Management.ResourceManager.Fluent.Authentication; +using Microsoft.Azure.Management.ResourceManager.Fluent.Core; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Logging; using Microsoft.Rest.Azure; @@ -13,6 +16,8 @@ using Moq; using Newtonsoft.Json; using System.Collections.Generic; using System.IO; +using System.Linq; +using System.Net; using System.Threading; using System.Threading.Tasks; using Xunit; @@ -35,15 +40,15 @@ namespace DedicatedHostsManagerTests private const string Location = "test-Location"; private const string HostGroupName = "test-dhg"; private const string VmSize = "Standard-D2s-v3"; - private static AzureOperationResponse _dedicatedHostGroupResponseMock; + private static AzureOperationResponse _dedicatedHostGroupResponseMock; [Fact] public async Task CreateDedicatedHostGroupTest() { var mockDhg = new DedicatedHostGroup(Location, PlatformFaultDomainCount, null, HostGroupName); var loggerMock = new Mock>(); - var configurationMock = new Mock(); - configurationMock.Setup(s => s["DhgCreateRetryCount"]).Returns("1"); + var config = new Config(); + config.DhgCreateRetryCount = 1; var dedicatedHostSelectorMock = new Mock(); var syncProviderMock = new Mock(); var dedicatedHostStateManagerMock = new Mock(); @@ -72,7 +77,7 @@ namespace DedicatedHostsManagerTests var dedicatedHostEngineTest = new DedicatedHostEngineTest( loggerMock.Object, - configurationMock.Object, + config, dedicatedHostSelectorMock.Object, syncProviderMock.Object, dedicatedHostStateManagerMock.Object, @@ -97,7 +102,7 @@ namespace DedicatedHostsManagerTests public async Task GetDedicatedHostForVmPlacementTest() { var loggerMock = new Mock>(); - var configurationMock = new Mock(); + var configurationMock = new Mock(); var dedicatedHostSelectorMock = new Mock(); var syncProviderMock = new Mock(); var dedicatedHostStateManagerMock = new Mock(); @@ -138,7 +143,7 @@ namespace DedicatedHostsManagerTests .ReturnsAsync("/subscriptions/6e412d70-9128-48a7-97b4-04e5bd35cefc/resourceGroups/63296244-ce2c-46d8-bc36-3e558792fbee/providers/Microsoft.Compute/hostGroups/citrix-dhg/hosts/20887a6e-0866-4bae-82b7-880839d9e76b"); var dedicatedHostEngine = new DedicatedHostEngine( - loggerMock.Object, + loggerMock.Object, configurationMock.Object, dedicatedHostSelectorMock.Object, syncProviderMock.Object, @@ -150,24 +155,284 @@ namespace DedicatedHostsManagerTests Assert.Equal(host, "/subscriptions/6e412d70-9128-48a7-97b4-04e5bd35cefc/resourceGroups/63296244-ce2c-46d8-bc36-3e558792fbee/providers/Microsoft.Compute/hostGroups/citrix-dhg/hosts/20887a6e-0866-4bae-82b7-880839d9e76b"); } + [Theory] + [MemberData(nameof(PrepareDedicatedHostGroupTestData))] + public async Task PrepareDedicatedHostGroup_Scenarios_Test(string location, int platformFDCount, int? platformFD, int vmInstanceToCreate, List existingHosts, List expectedHostsToBeCreated) + { + // Arrange + var hostGroupName = "TestDH"; + var loggerMock = new Mock>(); + var config = new Config(); + var dedicatedHostSelectorMock = new Mock(); + var syncProviderMock = new Mock(); + var dedicatedHostStateManagerMock = new Mock(); + var dhmComputeClientMock = new Mock(); + var computeManagementClientMock = new Mock(); + + // *** Mock Configuration + config.DedicatedHostMapping = File.ReadAllText(@"TestData\PrepareDedicatedHostMappingConfig.json"); + // *** Mock Get Host Group call + computeManagementClientMock + .Setup( + s => s.DedicatedHostGroups.GetWithHttpMessagesAsync( + It.IsAny(), + It.IsAny(), + null, + It.IsAny())) + .ReturnsAsync(new AzureOperationResponse() { + Response = new System.Net.Http.HttpResponseMessage(HttpStatusCode.OK), + Body = new DedicatedHostGroup(location, platformFDCount, null, hostGroupName) }); + + // *** Mock Existing Host information call + existingHosts.ForEach(dh => + computeManagementClientMock + .Setup( + s => s.DedicatedHosts.GetWithHttpMessagesAsync( + It.IsAny(), + hostGroupName, + dh.Name, + InstanceViewTypes.InstanceView, + null, + It.IsAny())) + .ReturnsAsync(new AzureOperationResponse() { Body = existingHosts.Single(d => d.Name == dh.Name) })); + + // *** Mock Create Dedicated Host Call + computeManagementClientMock + .Setup(s => s.DedicatedHosts.CreateOrUpdateWithHttpMessagesAsync( + It.IsAny(), + hostGroupName, + It.IsAny(), + It.IsAny(), + null, + It.IsAny())) + .ReturnsAsync((string rg, string dhgName, string dhName, DedicatedHost dh, Dictionary> headers, CancellationToken ctk) => + { + return new AzureOperationResponse() + { + Body = new DedicatedHost(location: dh.Location, sku: dh.Sku, name: dhName, platformFaultDomain: dh.PlatformFaultDomain) + }; + }); + + dhmComputeClientMock.Setup(s => + s.GetComputeManagementClient(It.IsAny(), It.IsAny(), + It.IsAny())) + .ReturnsAsync(computeManagementClientMock.Object); + + // *** Mock List Hosts call + dedicatedHostSelectorMock + .Setup( + s => s.ListDedicatedHosts( + It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), hostGroupName)) + .ReturnsAsync(existingHosts); + + var dedicatedHostEngine = new DedicatedHostEngine( + loggerMock.Object, + config, + dedicatedHostSelectorMock.Object, + syncProviderMock.Object, + dedicatedHostStateManagerMock.Object, + dhmComputeClientMock.Object); + + // Act + var addedHosts = await dedicatedHostEngine.PrepareDedicatedHostGroup( + Token, + AzureEnvironment.AzureUSGovernment, + TenantId, + SubscriptionId, + ResourceGroup, + hostGroupName, + "Standard_D2s_v3", + vmInstanceToCreate, + platformFD); + + (addedHosts ?? (new List())) + .Select(p => new { p.Location, p.PlatformFaultDomain, p.Sku }) + .Should().BeEquivalentTo(expectedHostsToBeCreated + .Select(p => new { p.Location, p.PlatformFaultDomain, p.Sku })); + } + private class DedicatedHostEngineTest : DedicatedHostEngine { public DedicatedHostEngineTest( - ILogger logger, - IConfiguration configuration, - IDedicatedHostSelector dedicatedHostSelector, + ILogger logger, + Config config, + IDedicatedHostSelector dedicatedHostSelector, ISyncProvider syncProvider, IDedicatedHostStateManager dedicatedHostStateManager, - IDhmComputeClient dhmComputeClient) + IDhmComputeClient dhmComputeClient) : base( - logger, - configuration, - dedicatedHostSelector, + logger, + config, + dedicatedHostSelector, syncProvider, dedicatedHostStateManager, dhmComputeClient) { } } + + public static IEnumerable PrepareDedicatedHostGroupTestData => + new List + { + // Single PlatformFaultDomain in DH Group + new object[] { + Region.GovernmentUSVirginia.Name, + 1, + 0, + 50, // 50 - 10 = 40 Required with 32 instance capacity, 2 host required + new List(){ + new DedicatedHost( + location: Region.GovernmentUSVirginia.Name, + sku: new Sku(){Name="NotRelevant"}, + name: "", + platformFaultDomain: 0, + instanceView: new DedicatedHostInstanceView(availableCapacity: new DedicatedHostAvailableCapacity( + new List() + { + new DedicatedHostAllocatableVM("Standard_D2s_v3", 10) + }))) + }, + new List() + { + new DedicatedHost( + location: Region.GovernmentUSVirginia.Name, + sku: new Sku(){Name="DSv3-Type1"}, + platformFaultDomain: 0), + new DedicatedHost( + location: Region.GovernmentUSVirginia.Name, + sku: new Sku(){Name="DSv3-Type1"}, + platformFaultDomain: 0), + } + }, + // Existing Hosts has enough capacity + new object[] { + Region.GovernmentUSVirginia.Name, + 1, + 0, + 5, // 10 Available, 5 required, 0 host required + new List(){ + new DedicatedHost( + location: Region.GovernmentUSVirginia.Name, + sku: new Sku(){Name="NotRelevant"}, + name: "", + platformFaultDomain: 0, + instanceView: new DedicatedHostInstanceView(availableCapacity: new DedicatedHostAvailableCapacity( + new List() + { + new DedicatedHostAllocatableVM("Standard_D2s_v3", 10) + }))) + }, + new List() + { + } + }, + // PlatformFaultDomainCount for DHG > 1, null in PlatformFaultDomain specified, should round assign equal hosts + new object[] { + Region.GovernmentUSVirginia.Name, + 2, + null, + 40, // 40 / 2 Fault Domains = 20 each required to be added per FD + new List(){ + new DedicatedHost( + location: Region.GovernmentUSVirginia.Name, + sku: new Sku(){Name="NotRelevant"}, + name: "Host-1", + platformFaultDomain: 0, + instanceView: new DedicatedHostInstanceView(availableCapacity: new DedicatedHostAvailableCapacity( + new List() + { + new DedicatedHostAllocatableVM("Standard_D2s_v3", 10) + }))), + new DedicatedHost( + location: Region.GovernmentUSVirginia.Name, + sku: new Sku(){Name="NotRelevant"}, + name: "Host-2", + platformFaultDomain: 1, + instanceView: new DedicatedHostInstanceView(availableCapacity: new DedicatedHostAvailableCapacity( + new List() + { + new DedicatedHostAllocatableVM("Standard_D2s_v3", 30) + }))) + }, + new List() + { + new DedicatedHost( + location: Region.GovernmentUSVirginia.Name, + sku: new Sku(){Name="DSv3-Type1"}, + platformFaultDomain: 0) + } + }, + // PlatformFaultDomainCount for DHG > 1, 0 in PlatformFaultDomain specified, should add hosts for all to requested PlatformFaultDomain ONLY + new object[] { + Region.GovernmentUSVirginia.Name, + 2, + 0, + 60, // 60 / 1 Fault Domains = 60 required to be added per FD 0 + new List(){ + new DedicatedHost( + location: Region.GovernmentUSVirginia.Name, + sku: new Sku(){Name="NotRelevant"}, + name: "Host-1", + platformFaultDomain: 0, + instanceView: new DedicatedHostInstanceView(availableCapacity: new DedicatedHostAvailableCapacity( + new List() + { + new DedicatedHostAllocatableVM("Standard_D2s_v3", 10) + }))), + new DedicatedHost( + location: Region.GovernmentUSVirginia.Name, + sku: new Sku(){Name="NotRelevant"}, + name: "Host-2", + platformFaultDomain: 1, + instanceView: new DedicatedHostInstanceView(availableCapacity: new DedicatedHostAvailableCapacity( + new List() + { + new DedicatedHostAllocatableVM("Standard_D2s_v3", 10) + }))) + }, + new List() + { + new DedicatedHost( + location: Region.GovernmentUSVirginia.Name, + sku: new Sku(){Name="DSv3-Type1"}, + platformFaultDomain: 0), + new DedicatedHost( + location: Region.GovernmentUSVirginia.Name, + sku: new Sku(){Name="DSv3-Type1"}, + platformFaultDomain: 0) + } + }, + // PlatformFaultDomainCount for DHG > 1, No existing Hosts, should create as expected + new object[] { + Region.GovernmentUSVirginia.Name, + 1, + null, + 30, // 30 / 1 Fault Domains = 30 requires creating 1 host + new List(){ + }, + new List() + { + new DedicatedHost( + location: Region.GovernmentUSVirginia.Name, + sku: new Sku(){Name="DSv3-Type1"}, + platformFaultDomain: 0) } + }, + // Region specific configuration to be applied if location specific config exists + new object[] { + Region.GovernmentUSIowa.Name, + 1, + null, + 30, // 30 / 1 Fault Domains -> requires creating 1 Host but of type "DSv3-type2" + new List(){ + }, + new List() + { + new DedicatedHost( + location: Region.GovernmentUSIowa.Name, + sku: new Sku(){Name="DSv3-Type2"}, + platformFaultDomain: 0) + } + } + }; } } diff --git a/DedicatedHostsTests/DedicatedHostsManagerTests.csproj b/DedicatedHostsTests/DedicatedHostsManagerTests.csproj index 434894f..a173858 100644 --- a/DedicatedHostsTests/DedicatedHostsManagerTests.csproj +++ b/DedicatedHostsTests/DedicatedHostsManagerTests.csproj @@ -8,6 +8,7 @@ + @@ -22,10 +23,6 @@ - - - - Always @@ -33,6 +30,9 @@ Always + + Always + diff --git a/DedicatedHostsTests/DedicatedHostsSelectorTests.cs b/DedicatedHostsTests/DedicatedHostsSelectorTests.cs index 8b5b8e6..2cd010d 100644 --- a/DedicatedHostsTests/DedicatedHostsSelectorTests.cs +++ b/DedicatedHostsTests/DedicatedHostsSelectorTests.cs @@ -14,9 +14,9 @@ using System.IO; using System.Linq; using System.Threading; using System.Threading.Tasks; -using Microsoft.Extensions.Configuration; using Xunit; -using JsonConvert = Newtonsoft.Json.JsonConvert; +using Newtonsoft.Json; +using DedicatedHostsManager; namespace DedicatedHostsManagerTests { @@ -39,7 +39,7 @@ namespace DedicatedHostsManagerTests const string expectedHostId = "/subscriptions/6e412d70-9128-48a7-97b4-04e5bd35cefc/resourceGroups/63296244-ce2c-46d8-bc36-3e558792fbee/providers/Microsoft.Compute/hostGroups/citrix-dhg/hosts/20887a6e-0866-4bae-82b7-880839d9e76b"; var loggerMock = new Mock>(); var dedicatedHostStateManagerMock = new Mock(); - var configurationMock = new Mock(); + var config = new Config(); var dhmComputeClientMock = new Mock(); dedicatedHostStateManagerMock.Setup(s => s.IsHostAtCapacity(It.IsAny())).Returns(false); var dedicatedHostList = @@ -84,7 +84,7 @@ namespace DedicatedHostsManagerTests var dedicatedHostSelector = new DedicatedHostSelectorTest( loggerMock.Object, dedicatedHostStateManagerMock.Object, - configurationMock.Object, + config, dhmComputeClientMock.Object); var actualHostId = await dedicatedHostSelector.SelectDedicatedHost( Token, @@ -105,8 +105,8 @@ namespace DedicatedHostsManagerTests var loggerMock = new Mock>(); var dedicatedHostStateManagerMock = new Mock(); var dhmComputeClientMock = new Mock(); - var configurationMock = new Mock(); - configurationMock.Setup(s => s["HostSelectorVmSize"]).Returns("Standard_D2s_v3"); + var config = new Config(); + config.HostSelectorVmSize = "Standard_D2s_v3"; var dedicatedHostList = JsonConvert.DeserializeObject>( File.ReadAllText(@"TestData\dedicatedHostsInput2.json")); @@ -114,7 +114,7 @@ namespace DedicatedHostsManagerTests var dedicatedHostSelector = new DedicatedHostSelectorTest( loggerMock.Object, dedicatedHostStateManagerMock.Object, - configurationMock.Object, + config, dhmComputeClientMock.Object); var actualHostId = dedicatedHostSelector.SelectMostPackedHost(dedicatedHostList); @@ -126,9 +126,9 @@ namespace DedicatedHostsManagerTests public DedicatedHostSelectorTest( ILogger logger, IDedicatedHostStateManager dedicatedHostStateManager, - IConfiguration configuration, + Config config, IDhmComputeClient dhmComputeClient) - : base(logger, dedicatedHostStateManager, configuration, dhmComputeClient) + : base(logger, dedicatedHostStateManager, config, dhmComputeClient) { } diff --git a/DedicatedHostsTests/TestData/PrepareDedicatedHostMappingConfig.json b/DedicatedHostsTests/TestData/PrepareDedicatedHostMappingConfig.json new file mode 100644 index 0000000..191706c --- /dev/null +++ b/DedicatedHostsTests/TestData/PrepareDedicatedHostMappingConfig.json @@ -0,0 +1,31 @@ +[ + { + "family": "DSv3", + "hostMapping": [ + { + "region": "default", + "host": { + "type": "DSv3-Type1", + "vmMapping": [ + { + "size": "Standard_D2s_v3", + "capacity": 32 + } + ] + } + }, + { + "region": "usgoviowa", + "host": { + "type": "DSv3-Type2", + "vmMapping": [ + { + "size": "Standard_D2s_v3", + "capacity": 32 + } + ] + } + } + ] + } +] \ No newline at end of file diff --git a/README.md b/README.md index c650ba3..8476818 100644 --- a/README.md +++ b/README.md @@ -2,8 +2,14 @@ # Dedicated Hosts Manager Azure Dedicated Host (DH) provides physical servers that host one or more Azure virtual machines; host-level isolation means that capacity is dedicated to your organization and servers are not shared with other customers. To use DH, users currently need to manage DH themselves - e.g. when to spin up or spin down Hosts, determine VM placement on Hosts, bin pack VMs compactly on Hosts to minimize Host usage and optimize for cost, or use another Host selection strategy, manage VM creation traffic burst scenarios, etc. -The Dedicated Hosts Manager library abstracts Host Management logic from users, and makes it easy for users to use DH. Users only need to specify the number and SKU of VMs that need to be allocated, and this library takes care of the rest. This library is packaged as an Azure Function that can be deployed in your subscription, and is easy to integrate with . The library is extensible and allows for customizing Host selection logic. +The Dedicated Hosts Manager library abstracts Host Management logic from users, and makes it easy for users to use DH. +* **CreateVm**: Users only need to specify the number and SKU of VMs that need to be allocated, and this library takes care of the rest. +* **DeleteVM** Users need to specify VM to delete and library will also deallocate any Host that have no allocated VM.s +* **PrepareDedicatedHostGroup** This call to the Dedicated Host Manager library will prepare the host group by creating sufficient number of dedicated hosts so that a future call to VM or VMSS creation will be successful. + +This library is packaged as an Azure Function that can be deployed in your subscription, and is easy to integrate with . The library is extensible and allows for customizing Host selection logic. + # Support * Solution supports Azure Function runtime v3 * Developed and tested using VS 2019 @@ -92,7 +98,11 @@ The Dedicated Hosts Manager library abstracts Host Management logic from users, "name": "VmToHostMapping", "value": "{\"Standard_D2s_v3\":\"DSv3-Type1\",\"Standard_D4s_v3\":\"DSv3-Type1\",\"Standard_D8s_v3\":\"DSv3-Type1\",\"Standard_D16s_v3\":\"DSv3-Type1\",\"Standard_D32-8s_v3\":\"DSv3-Type1\",\"Standard_D32-16s_v3\":\"DSv3-Type1\",\"Standard_D32s_v3\":\"DSv3-Type1\",\"Standard_D48s_v3\":\"DSv3-Type1\",\"Standard_D64-16s_v3\":\"DSv3-Type1\",\"Standard_D64-32s_v3\":\"DSv3-Type1\",\"Standard_D64s_v3\":\"DSv3-Type1\",\"Standard_E2s_v3\":\"ESv3-Type1\",\"Standard_E4s_v3\":\"ESv3-Type1\",\"Standard_E8s_v3\":\"ESv3-Type1\",\"Standard_E16s_v3\":\"ESv3-Type1\",\"Standard_E32-8s_v3\":\"ESv3-Type1\",\"Standard_E32-16s_v3\":\"ESv3-Type1\",\"Standard_E32s_v3\":\"ESv3-Type1\",\"Standard_E48s_v3\":\"ESv3-Type1\",\"Standard_E64-16s_v3\":\"ESv3-Type1\",\"Standard_E64-32s_v3\":\"ESv3-Type1\",\"Standard_E64s_v3\":\"ESv3-Type1\",\"Standard_F2s_v3\":\"FSv2-Type2\",\"Standard_F4s_v3\":\"FSv2-Type2\",\"Standard_F8s_v3\":\"FSv2-Type2\",\"Standard_F16s_v3\":\"FSv2-Type2\",\"Standard_F32-8s_v3\":\"FSv2-Type2\",\"Standard_F32-16s_v3\":\"FSv2-Type2\",\"Standard_F32s_v3\":\"FSv2-Type2\",\"Standard_F48s_v3\":\"FSv2-Type2\",\"Standard_F64-16s_v3\":\"FSv2-Type2\",\"Standard_F64-32s_v3\":\"FSv2-Type2\",\"Standard_F64s_v3\":\"FSv2-Type2\"}", } - + , + { + "name": "DedicatedHostMapping", + "value": "[{\"family\":\"DSv3\",\"hostMapping\":[{\"region\":\"default\",\"host\":{\"type\":\"DSv3-Type1\",\"vmMapping\":[{\"size\":\"Standard_D2s_v3\",\"capacity\":32},{\"size\":\"Standard_D4s_v3\",\"capacity\":16},{\"size\":\"Standard_D8s_v3\",\"capacity\":8},{\"size\":\"Standard_D16s_v3\",\"capacity\":4},{\"size\":\"Standard_D32s_v3\",\"capacity\":2},{\"size\":\"Standard_D48s_v3\",\"capacity\":1},{\"size\":\"Standard_D64s_v3\",\"capacity\":1}]}}]}]", + } ``` _Connection strings:_ ```json @@ -135,6 +145,10 @@ The Dedicated Hosts Manager library abstracts Host Management logic from users, "name": "DhmDeleteVmnUri", "value": "",