Dynamic Concurrency support (#2720)

TestChildProcess/Program.cs

@@ -0,0 +1,13 @@
+using System;
+
+namespace TestChildProcess
+{
+    class Program
+    {
+        static void Main(string[] args)
+        {
+            Console.WriteLine("Child process started");
+            Console.ReadLine();
+        }
+    }
+}

TestChildProcess/TestChildProcess.csproj

@@ -0,0 +1,8 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>netcoreapp3.1</TargetFramework>
+  </PropertyGroup>
+
+</Project>

WebJobs.sln

@@ -51,7 +51,9 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "TestProjects", "TestProject
 EndProject
 Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "FSharpFunctions", "test\TestProjects\FSharpFunctions\FSharpFunctions.fsproj", "{11702A4B-8402-4082-BE38-4F0C2CBBF61D}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Benchmarks", "test\Benchmarks\Benchmarks.csproj", "{78F8086D-8313-477A-B24F-E475A690880A}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Benchmarks", "test\Benchmarks\Benchmarks.csproj", "{78F8086D-8313-477A-B24F-E475A690880A}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "TestChildProcess", "TestChildProcess\TestChildProcess.csproj", "{CEBC078A-9DD2-4558-829B-DBAE6BEA4264}"
 EndProject
 Global
 	GlobalSection(SharedMSBuildProjectFiles) = preSolution
@@ -134,6 +136,10 @@ Global
 		{78F8086D-8313-477A-B24F-E475A690880A}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{78F8086D-8313-477A-B24F-E475A690880A}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{78F8086D-8313-477A-B24F-E475A690880A}.Release|Any CPU.Build.0 = Release|Any CPU
+		{CEBC078A-9DD2-4558-829B-DBAE6BEA4264}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{CEBC078A-9DD2-4558-829B-DBAE6BEA4264}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{CEBC078A-9DD2-4558-829B-DBAE6BEA4264}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{CEBC078A-9DD2-4558-829B-DBAE6BEA4264}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -150,6 +156,7 @@ Global
 		{C5E1A8E8-711F-4377-A8BD-7DB58E6C580D} = {639967B0-0544-4C52-94AC-9A3D25E33256}
 		{11702A4B-8402-4082-BE38-4F0C2CBBF61D} = {C5E1A8E8-711F-4377-A8BD-7DB58E6C580D}
 		{78F8086D-8313-477A-B24F-E475A690880A} = {639967B0-0544-4C52-94AC-9A3D25E33256}
+		{CEBC078A-9DD2-4558-829B-DBAE6BEA4264} = {C5E1A8E8-711F-4377-A8BD-7DB58E6C580D}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {371BFA14-0980-4A43-A18A-CA1C1A9CB784}

appveyor.yml

@@ -18,7 +18,7 @@ install:
 - ps: |
     $env:CommitHash = "$env:APPVEYOR_REPO_COMMIT"
 
-    .\dotnet-install.ps1 -Version 2.1.300 -Architecture x86
+    .\dotnet-install.ps1 -Version 3.1.410 -Architecture x86
 build_script:
 - ps: |
     $buildNumber = 0

build/common.props

@@ -1,9 +1,9 @@
 <Project>
   <PropertyGroup>
     <!-- Packages can have independent versions and only increment when released -->
-    <Version>3.0.29$(VersionSuffix)</Version>
+    <Version>3.0.30$(VersionSuffix)</Version>
     <ExtensionsStorageVersion>4.0.5$(VersionSuffix)</ExtensionsStorageVersion>
-    <HostStorageVersion>4.0.3$(VersionSuffix)</HostStorageVersion>
+    <HostStorageVersion>4.0.4$(VersionSuffix)</HostStorageVersion>
     <LoggingVersion>4.0.2$(VersionSuffix)</LoggingVersion>
     
     <TargetFramework>netstandard2.0</TargetFramework>

src/Microsoft.Azure.WebJobs.Host.Storage/BlobStorageConcurrencyStatusRepository.cs

@@ -0,0 +1,115 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT License. See License.txt in the project root for license information.
+
+#nullable enable
+
+using System;
+using System.IO;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.Azure.Storage;
+using Microsoft.Azure.Storage.Blob;
+using Microsoft.Azure.WebJobs.Host.Executors;
+using Microsoft.Azure.WebJobs.Host.Scale;
+using Microsoft.Azure.WebJobs.Logging;
+using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.Logging;
+using Newtonsoft.Json;
+
+namespace Microsoft.Azure.WebJobs.Host
+{
+    internal class BlobStorageConcurrencyStatusRepository : IConcurrencyStatusRepository
+    {
+        private const string HostContainerName = "azure-webjobs-hosts";
+        private readonly IHostIdProvider _hostIdProvider;
+        private readonly IConfiguration _configuration;
+        private readonly ILogger _logger;
+        private CloudBlobContainer? _blobContainer;
+
+        public BlobStorageConcurrencyStatusRepository(IConfiguration configuration, IHostIdProvider hostIdProvider, ILoggerFactory loggerFactory)
+        {
+            _configuration = configuration;
+            _hostIdProvider = hostIdProvider;
+            _logger = loggerFactory.CreateLogger(LogCategories.Concurrency);
+        }
+
+        public async Task<HostConcurrencySnapshot?> ReadAsync(CancellationToken cancellationToken)
+        {
+            string blobPath = await GetBlobPathAsync(cancellationToken);
+
+            try
+            {
+                CloudBlobContainer? container = await GetContainerAsync(cancellationToken);
+                if (container != null)
+                {
+                    CloudBlockBlob blob = container.GetBlockBlobReference(blobPath);
+                    string content = await blob.DownloadTextAsync(cancellationToken);
+                    if (!string.IsNullOrEmpty(content))
+                    {
+                        var result = JsonConvert.DeserializeObject<HostConcurrencySnapshot>(content);
+                        return result;
+                    }
+                }
+            }
+            catch (StorageException stex) when (stex.RequestInformation?.HttpStatusCode == 404)
+            {
+                return null;
+            }
+            catch (Exception e)
+            {
+                _logger.LogError(e, $"Error reading snapshot blob {blobPath}");
+                throw e;
+            }
+
+            return null;
+        }
+
+        public async Task WriteAsync(HostConcurrencySnapshot snapshot, CancellationToken cancellationToken)
+        {
+            string blobPath = await GetBlobPathAsync(cancellationToken);
+
+            try
+            {
+                CloudBlobContainer? container = await GetContainerAsync(cancellationToken);
+                if (container != null)
+                {
+                    CloudBlockBlob blob = container.GetBlockBlobReference(blobPath);
+
+                    using (StreamWriter writer = new StreamWriter(await blob.OpenWriteAsync(cancellationToken)))
+                    {
+                        var content = JsonConvert.SerializeObject(snapshot);
+                        await writer.WriteAsync(content);
+                    }
+                }
+            }
+            catch (Exception e)
+            {
+                _logger.LogError(e, $"Error writing snapshot blob {blobPath}");
+                throw e;
+            }
+        }
+
+        internal async Task<CloudBlobContainer?> GetContainerAsync(CancellationToken cancellationToken)
+        {
+            if (_blobContainer == null)
+            {
+                string storageConnectionString = _configuration.GetWebJobsConnectionString(ConnectionStringNames.Storage);
+                if (!string.IsNullOrEmpty(storageConnectionString) && CloudStorageAccount.TryParse(storageConnectionString, out CloudStorageAccount account))
+                {
+                    var client = account.CreateCloudBlobClient();
+                    _blobContainer = client.GetContainerReference(HostContainerName);
+
+                    await _blobContainer.CreateIfNotExistsAsync(cancellationToken);
+                }
+            }
+
+            return _blobContainer;
+        }
+
+        internal async Task<string> GetBlobPathAsync(CancellationToken cancellationToken)
+        {
+            string hostId = await _hostIdProvider.GetHostIdAsync(cancellationToken);
+            return $"concurrency/{hostId}/concurrencyStatus.json";
+        }
+    }
+}

src/Microsoft.Azure.WebJobs.Host.Storage/StorageServiceCollectionExtensions.cs

@@ -9,6 +9,7 @@ using Microsoft.Azure.WebJobs;
 using Microsoft.Azure.WebJobs.Host;
 using Microsoft.Azure.WebJobs.Host.Config;
 using Microsoft.Azure.WebJobs.Host.Loggers;
+using Microsoft.Azure.WebJobs.Host.Scale;
 using Microsoft.Azure.WebJobs.Host.Storage;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.DependencyInjection.Extensions;
@@ -50,6 +51,8 @@ namespace Microsoft.Extensions.Hosting
             services.TryAddEnumerable(ServiceDescriptor.Transient<IConfigureOptions<JobHostInternalStorageOptions>, CoreWebJobsOptionsSetup<JobHostInternalStorageOptions>>());
 
             services.TryAddSingleton<IDelegatingHandlerProvider, DefaultDelegatingHandlerProvider>();
+
+            services.AddSingleton<IConcurrencyStatusRepository, BlobStorageConcurrencyStatusRepository>();
         }
 
         // This is only called if the host didn't already provide an implementation

src/Microsoft.Azure.WebJobs.Host.Storage/WebJobs.Host.Storage.csproj

@@ -9,6 +9,7 @@
     <Version>$(HostStorageVersion)</Version>
     <InformationalVersion>$(Version) Commit hash: $(CommitHash)</InformationalVersion>
     <TargetFramework>netstandard2.0</TargetFramework>
+    <LangVersion>latest</LangVersion>
     <AssemblyName>Microsoft.Azure.WebJobs.Host.Storage</AssemblyName>
   </PropertyGroup>  
   

src/Microsoft.Azure.WebJobs.Host/Config/ConcurrencyOptionsSetup.cs

@@ -0,0 +1,97 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT License. See License.txt in the project root for license information.
+
+using System;
+using Microsoft.Azure.WebJobs.Host.Scale;
+using Microsoft.Extensions.Options;
+
+namespace Microsoft.Azure.WebJobs.Host.Config
+{
+    internal class ConcurrencyOptionsSetup : IConfigureOptions<ConcurrencyOptions>
+    {
+        private const int BytesPerGB = 1024 * 1024 * 1024;
+
+        public void Configure(ConcurrencyOptions options)
+        {
+            // TODO: Once Memory monitoring is public add this back.
+            // For now, the memory throttle is internal only for testing.
+            // https://github.com/Azure/azure-webjobs-sdk/issues/2733
+            //ConfigureMemoryOptions(options);
+        }
+
+        internal static void ConfigureMemoryOptions(ConcurrencyOptions options)
+        {
+            string sku = Utility.GetWebsiteSku();
+            int numCores = Utility.GetEffectiveCoresCount();
+            ConfigureMemoryOptions(options, sku, numCores);
+        }
+
+        internal static void ConfigureMemoryOptions(ConcurrencyOptions options, string sku, int numCores)
+        {
+            long memoryLimitBytes = GetMemoryLimitBytes(sku, numCores);
+            if (memoryLimitBytes > 0)
+            {
+                // if we're able to determine the memory limit, apply it
+                options.TotalAvailableMemoryBytes = memoryLimitBytes;
+            }
+        }
+
+        internal static long GetMemoryLimitBytes(string sku, int numCores)
+        {
+            if (!string.IsNullOrEmpty(sku))
+            {
+                float memoryGBPerCore = GetMemoryGBPerCore(sku);
+
+                if (memoryGBPerCore > 0)
+                {
+                    double memoryLimitBytes = memoryGBPerCore * numCores * BytesPerGB;
+
+                    if (string.Equals(sku, "IsolatedV2", StringComparison.OrdinalIgnoreCase) && numCores == 8)
+                    {
+                        // special case for upper tier IsolatedV2 where GB per Core
+                        // isn't cleanly linear
+                        memoryLimitBytes = (float)23 * BytesPerGB;
+                    }
+
+                    return (long)memoryLimitBytes;
+                }
+            }
+
+            // unable to determine memory limit
+            return -1;
+        }
+
+        internal static float GetMemoryGBPerCore(string sku)
+        {
+            if (string.IsNullOrEmpty(sku))
+            {
+                return -1;
+            }
+
+            // These memory allowances are based on published limits:
+            // Dynamic SKU: https://docs.microsoft.com/en-us/azure/azure-functions/functions-scale#service-limits
+            // Premium SKU: https://docs.microsoft.com/en-us/azure/azure-functions/functions-premium-plan?tabs=portal#available-instance-skus
+            // Dedicated SKUs: https://azure.microsoft.com/en-us/pricing/details/app-service/windows/
+            switch (sku.ToLower())
+            {
+                case "free":
+                case "shared":
+                    return 1;
+                case "dynamic":
+                    return 1.5F;
+                case "basic":
+                case "standard":
+                    return 1.75F;
+                case "premiumv2":
+                case "isolated":
+                case "elasticpremium":
+                    return 3.5F;
+                case "premiumv3":
+                case "isolatedv2":
+                    return 4;
+                default:
+                    return -1;
+            }
+        }
+    }
+}

src/Microsoft.Azure.WebJobs.Host/Config/PrimaryHostCoordinatorOptionsSetup.cs

@@ -0,0 +1,30 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT License. See License.txt in the project root for license information.
+
+using System;
+using Microsoft.Azure.WebJobs.Host.Scale;
+using Microsoft.Azure.WebJobs.Hosting;
+using Microsoft.Extensions.Options;
+
+namespace Microsoft.Azure.WebJobs.Host.Config
+{
+    internal class PrimaryHostCoordinatorOptionsSetup : IConfigureOptions<PrimaryHostCoordinatorOptions>
+    {
+        private readonly IOptions<ConcurrencyOptions> _concurrencyOptions;
+
+        public PrimaryHostCoordinatorOptionsSetup(IOptions<ConcurrencyOptions> concurrencyOptions)
+        {
+            _concurrencyOptions = concurrencyOptions;
+        }
+
+        public void Configure(PrimaryHostCoordinatorOptions options)
+        {
+            // in most WebJobs SDK scenarios, primary host coordination is not needed
+            // however, some features require it
+            if (_concurrencyOptions.Value.DynamicConcurrencyEnabled)
+            {
+                options.Enabled = true;
+            }
+        }
+    }
+}

src/Microsoft.Azure.WebJobs.Host/Constants.cs

@@ -9,8 +9,11 @@ namespace Microsoft.Azure.WebJobs.Host
         public const string EnvironmentSettingName = "AzureWebJobsEnv";
         public const string DevelopmentEnvironmentValue = "Development";
         public const string DynamicSku = "Dynamic";
+        public const string ElasticPremiumSku = "ElasticPremium";
         public const string AzureWebsiteSku = "WEBSITE_SKU";
         public const string AzureWebJobsShutdownFile = "WEBJOBS_SHUTDOWN_FILE";
+        public const string AzureWebsiteInstanceId = "WEBSITE_INSTANCE_ID";
+        public const string AzureWebsiteContainerName = "CONTAINER_NAME";
         public const string DateTimeFormatString = "yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fffK";
     }
 }

src/Microsoft.Azure.WebJobs.Host/Executors/FunctionExecutor.cs

@@ -14,6 +14,7 @@ using Microsoft.Azure.WebJobs.Host.Bindings;
 using Microsoft.Azure.WebJobs.Host.Indexers;
 using Microsoft.Azure.WebJobs.Host.Loggers;
 using Microsoft.Azure.WebJobs.Host.Protocols;
+using Microsoft.Azure.WebJobs.Host.Scale;
 using Microsoft.Azure.WebJobs.Host.Timers;
 using Microsoft.Azure.WebJobs.Logging;
 using Microsoft.Extensions.Logging;
@@ -29,6 +30,7 @@ namespace Microsoft.Azure.WebJobs.Host.Executors
         private readonly ILogger _resultsLogger;
         private readonly IEnumerable<IFunctionFilter> _globalFunctionFilters;
         private readonly IDrainModeManager _drainModeManager;
+        private readonly ConcurrencyManager _concurrencyManager;
 
         private readonly Dictionary<string, object> _inputBindingScope = new Dictionary<string, object>
         {
@@ -50,6 +52,7 @@ namespace Microsoft.Azure.WebJobs.Host.Executors
                 IFunctionOutputLogger functionOutputLogger,
                 IWebJobsExceptionHandler exceptionHandler,
                 IAsyncCollector<FunctionInstanceLogEntry> functionEventCollector,
+                ConcurrencyManager concurrencyManager,
                 ILoggerFactory loggerFactory = null,
                 IEnumerable<IFunctionFilter> globalFunctionFilters = null,
                 IDrainModeManager drainModeManager = null)
@@ -62,6 +65,7 @@ namespace Microsoft.Azure.WebJobs.Host.Executors
             _resultsLogger = _loggerFactory.CreateLogger(LogCategories.Results);
             _globalFunctionFilters = globalFunctionFilters ?? Enumerable.Empty<IFunctionFilter>();
             _drainModeManager = drainModeManager;
+            _concurrencyManager = concurrencyManager ?? throw new ArgumentNullException(nameof(concurrencyManager));
         }
 
         public HostOutputMessage HostOutputMessage
@@ -79,9 +83,15 @@ namespace Microsoft.Azure.WebJobs.Host.Executors
             ExceptionDispatchInfo exceptionInfo = null;
             string functionStartedMessageId = null;
             FunctionInstanceLogEntry instanceLogEntry = null;
+            Stopwatch sw = Stopwatch.StartNew();
 
             try
             {
+                if (_concurrencyManager.Enabled)
+                {
+                    _concurrencyManager.FunctionStarted(functionInstanceEx.FunctionDescriptor.Id);
+                }
+
                 using (_resultsLogger?.BeginFunctionScope(functionInstanceEx, HostOutputMessage.HostInstanceId))
                 using (parameterHelper)
                 {
@@ -130,6 +140,12 @@ namespace Microsoft.Azure.WebJobs.Host.Executors
             }
             finally
             {
+                sw.Stop();
+                if (_concurrencyManager.Enabled)
+                {
+                    _concurrencyManager.FunctionCompleted(functionInstanceEx.FunctionDescriptor.Id, sw.Elapsed);
+                }
+
                 ((IDisposable)functionInstanceEx)?.Dispose();
             }
 

src/Microsoft.Azure.WebJobs.Host/Hosting/IPrimaryHostStateProvider.cs

@@ -0,0 +1,22 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT License. See License.txt in the project root for license information.
+
+// Type was moved from https://github.com/Azure/azure-functions-host/blob/dev/src/WebJobs.Script/Host/IPrimaryHostStateProvider.cs
+
+namespace Microsoft.Azure.WebJobs.Hosting
+{
+    /// <summary>
+    /// Provides access to the primary host state. When an application is running on multiple
+    /// scaled out instances, only one instance will be primary.
+    /// </summary>
+    /// <remarks>
+    /// See <see cref="PrimaryHostCoordinatorOptions"/> for more information.
+    /// </remarks>
+    public interface IPrimaryHostStateProvider
+    {
+        /// <summary>
+        /// Gets or sets a value indicating whether the currently running host is "Primary".
+        /// </summary>
+        bool IsPrimary { get; set; }
+    }
+}

src/Microsoft.Azure.WebJobs.Host/Hosting/PrimaryHostCoordinator.cs

@@ -0,0 +1,250 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT License. See License.txt in the project root for license information.
+
+// Type was moved from https://github.com/Azure/azure-functions-host/blob/dev/src/WebJobs.Script/Host/PrimaryHostCoordinator.cs
+
+using System;
+using System.Globalization;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.Azure.WebJobs.Host;
+using Microsoft.Azure.WebJobs.Host.Executors;
+using Microsoft.Azure.WebJobs.Logging;
+using Microsoft.Extensions.Hosting;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
+
+namespace Microsoft.Azure.WebJobs.Hosting
+{
+    /// <summary>
+    /// Responsible for determining which host instance is the "Primary".
+    /// </summary>
+    internal sealed class PrimaryHostCoordinator : IHostedService, IDisposable
+    {
+        internal const string LockBlobName = "host";
+
+        private readonly Timer _timer;
+        private readonly TimeSpan _leaseTimeout;
+        private readonly IHostIdProvider _hostIdProvider;
+        private readonly TimeSpan _renewalInterval;
+        private readonly TimeSpan _leaseRetryInterval;
+        private readonly ILogger _logger;
+        private readonly IPrimaryHostStateProvider _primaryHostStateProvider;
+        private readonly string _websiteInstanceId;
+        private readonly IOptions<PrimaryHostCoordinatorOptions> _options;
+        private readonly IDistributedLockManager _lockManager;
+
+        private IDistributedLock _lockHandle; // If non-null, then we own the lock.
+        private bool _disposed;
+        private bool _processingLease;
+        private DateTime _lastRenewal;
+        private TimeSpan _lastRenewalLatency;
+
+        private string _hostId;
+
+        public PrimaryHostCoordinator(IOptions<PrimaryHostCoordinatorOptions> coordinatorOptions, IHostIdProvider hostIdProvider, IDistributedLockManager lockManager,
+            IPrimaryHostStateProvider primaryHostStateProvider, ILoggerFactory loggerFactory)
+        {
+            _leaseTimeout = coordinatorOptions.Value.LeaseTimeout;
+            _hostIdProvider = hostIdProvider;
+            _websiteInstanceId = Utility.GetInstanceId();
+            _options = coordinatorOptions;
+
+            _lockManager = lockManager;
+            if (lockManager == null)
+            {
+                throw new ArgumentNullException(nameof(lockManager));
+            }
+
+            // Renew the lease three seconds before it expires
+            _renewalInterval = _options.Value.RenewalInterval ?? _leaseTimeout.Add(TimeSpan.FromSeconds(-3));
+
+            // Attempt to acquire a lease every 5 seconds
+            _leaseRetryInterval = TimeSpan.FromSeconds(5);
+
+            _timer = new Timer(ProcessLeaseTimerTick);
+
+            _logger = loggerFactory.CreateLogger(LogCategories.HostGeneral);
+            _primaryHostStateProvider = primaryHostStateProvider;
+        }
+
+        internal bool LeaseTimerRunning { get; private set; }
+
+        private bool HasLease => _lockHandle != null;
+
+        internal IDistributedLock LockHandle
+        {
+            get
+            {
+                return _lockHandle;
+            }
+
+            set
+            {
+                _lockHandle = value;
+                _primaryHostStateProvider.IsPrimary = HasLease;
+            }
+        }
+
+        private void ProcessLeaseTimerTick(object state)
+        {
+            if (_processingLease)
+            {
+                return;
+            }
+
+            _processingLease = true;
+
+            AcquireOrRenewLeaseAsync()
+                .ContinueWith(t =>
+                {
+                    if (t.IsFaulted)
+                    {
+                        t.Exception.Handle(e =>
+                        {
+                            ProcessLeaseError(Utility.FlattenException(e));
+                            return true;
+                        });
+                    }
+
+                    _processingLease = false;
+                }, TaskContinuationOptions.ExecuteSynchronously);
+        }
+
+        private async Task AcquireOrRenewLeaseAsync()
+        {
+            if (_hostId == null)
+            {
+                _hostId = await _hostIdProvider.GetHostIdAsync(CancellationToken.None);
+            }
+
+            string lockName = GetBlobName(_hostId);
+
+            DateTime requestStart = DateTime.UtcNow;
+            if (HasLease)
+            {
+                try
+                {
+                    await _lockManager.RenewAsync(LockHandle, CancellationToken.None);
+
+                    _lastRenewal = DateTime.UtcNow;
+                    _lastRenewalLatency = _lastRenewal - requestStart;
+                }
+                catch
+                {
+                    // The lease was 'stolen'. Log details for debugging.
+                    string lastRenewalFormatted = _lastRenewal.ToString("yyyy-MM-ddTHH:mm:ss.FFFZ", CultureInfo.InvariantCulture);
+                    int millisecondsSinceLastSuccess = (int)(DateTime.UtcNow - _lastRenewal).TotalMilliseconds;
+                    int lastRenewalMilliseconds = (int)_lastRenewalLatency.TotalMilliseconds;
+                    ProcessLeaseError($"Another host has acquired the lease. The last successful renewal completed at {lastRenewalFormatted} ({millisecondsSinceLastSuccess} milliseconds ago) with a duration of {lastRenewalMilliseconds} milliseconds.");
+                }
+            }
+            else
+            {
+                string proposedLeaseId = _websiteInstanceId;
+                LockHandle = await _lockManager.TryLockAsync(null, lockName, _websiteInstanceId, proposedLeaseId, _leaseTimeout, CancellationToken.None);
+                if (LockHandle == null)
+                {
+                    // We didn't have the lease and failed to acquire it. Common if somebody else already has it.
+                    // This is normal and does not warrant any logging.
+                    return;
+                }
+
+                _lastRenewal = DateTime.UtcNow;
+                _lastRenewalLatency = _lastRenewal - requestStart;
+
+                _logger.PrimaryHostCoordinatorLockLeaseAcquired(_websiteInstanceId);
+
+                // We've successfully acquired the lease, change the timer to use our renewal interval
+                SetTimerInterval(_renewalInterval);
+            }
+        }
+
+        // The StorageDistributedLockManager will put things under the /locks path automatically
+        internal static string GetBlobName(string hostId) => $"{hostId}/{LockBlobName}";
+
+        private void ProcessLeaseError(string reason)
+        {
+            if (HasLease)
+            {
+                ResetLease();
+
+                _logger.PrimaryHostCoordinatorFailedToRenewLockLease(reason);
+            }
+            else
+            {
+                _logger.PrimaryHostCoordinatorFailedToAcquireLockLease(_websiteInstanceId, reason);
+            }
+        }
+
+        private void ResetLease()
+        {
+            LockHandle = null;
+            SetTimerInterval(_leaseRetryInterval);
+        }
+
+        private void SetTimerInterval(TimeSpan interval, TimeSpan? dueTimeout = null)
+        {
+            if (!_disposed)
+            {
+                _timer.Change(dueTimeout ?? interval, interval);
+            }
+        }
+
+        private void TryReleaseLeaseIfOwned()
+        {
+            try
+            {
+                if (HasLease)
+                {
+                    Task.Run(() => _lockManager.ReleaseLockAsync(_lockHandle, CancellationToken.None)).GetAwaiter().GetResult();
+
+                    _logger.PrimaryHostCoordinatorReleasedLocklLease(_websiteInstanceId);
+                }
+            }
+            catch (Exception)
+            {
+                // Best effort, the lease will expire if we fail to release it.
+            }
+        }
+
+        private void Dispose(bool disposing)
+        {
+            if (!_disposed)
+            {
+                if (disposing)
+                {
+                    _timer.Dispose();
+
+                    TryReleaseLeaseIfOwned();
+                }
+
+                _disposed = true;
+            }
+        }
+
+        public void Dispose()
+        {
+            Dispose(true);
+        }
+
+        public Task StartAsync(CancellationToken cancellationToken)
+        {
+            if (_options.Value.Enabled)
+            {
+                _timer.Change(_leaseRetryInterval, _leaseRetryInterval);
+                LeaseTimerRunning = true;
+            }
+            
+            return Task.CompletedTask;
+        }
+
+        public Task StopAsync(CancellationToken cancellationToken)
+        {
+            _timer.Change(Timeout.Infinite, Timeout.Infinite);
+            LeaseTimerRunning = false;
+
+            return Task.CompletedTask;
+        }
+    }
+}

src/Microsoft.Azure.WebJobs.Host/Hosting/PrimaryHostCoordinatorOptions.cs

@@ -0,0 +1,41 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT License. See License.txt in the project root for license information.
+
+// Type was moved from https://github.com/Azure/azure-functions-host/blob/dev/src/WebJobs.Script/Host/PrimaryHostCoordinatorOptions.cs
+
+using System;
+
+namespace Microsoft.Azure.WebJobs.Hosting
+{
+    public class PrimaryHostCoordinatorOptions
+    {
+        private TimeSpan _leaseTimeout = TimeSpan.FromSeconds(15);
+
+        public PrimaryHostCoordinatorOptions()
+        {
+            Enabled = false;
+        }
+
+        public bool Enabled { get; set; }
+
+        public TimeSpan LeaseTimeout
+        {
+            get
+            {
+                return _leaseTimeout;
+            }
+
+            set
+            {
+                if (value < TimeSpan.FromSeconds(15) || value > TimeSpan.FromSeconds(60))
+                {
+                    throw new ArgumentOutOfRangeException(nameof(LeaseTimeout), $"The {nameof(LeaseTimeout)} should be between 15 and 60 seconds but was '{value}'");
+                }
+
+                _leaseTimeout = value;
+            }
+        }
+
+        public TimeSpan? RenewalInterval { get; set; } = null;
+    }
+}

src/Microsoft.Azure.WebJobs.Host/Hosting/PrimaryHostStateProvider.cs

@@ -0,0 +1,12 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT License. See License.txt in the project root for license information.
+
+// Type was moved from https://github.com/Azure/azure-functions-host/blob/dev/src/WebJobs.Script/Host/PrimaryHostStateProvider.cs
+
+namespace Microsoft.Azure.WebJobs.Hosting
+{
+    internal class PrimaryHostStateProvider : IPrimaryHostStateProvider
+    {
+        public bool IsPrimary { get; set; }
+    }
+}

src/Microsoft.Azure.WebJobs.Host/Hosting/WebJobsServiceCollectionExtensions.cs

@@ -31,6 +31,7 @@ namespace Microsoft.Azure.WebJobs
     public static class WebJobsServiceCollectionExtensions
     {
         private const string SingletonConfigSectionName = "Singleton";
+        private const string ConcurrencyConfigSectionName = "Concurrency";
 
         /// <summary>
         /// Adds the WebJobs services to the provided <see cref="IServiceCollection"/>.
@@ -86,6 +87,9 @@ namespace Microsoft.Azure.WebJobs
             services.TryAddSingleton<IDistributedLockManager, InMemoryDistributedLockManager>();
             services.TryAddSingleton<IScaleMonitorManager, ScaleMonitorManager>();
 
+            services.AddSingleton<IPrimaryHostStateProvider, PrimaryHostStateProvider>();
+            services.AddSingleton<IHostedService, PrimaryHostCoordinator>();
+
             // $$$ Can we remove these completely? 
             services.TryAddSingleton<DefaultTriggerBindingFactory>();
             services.TryAddSingleton<ITriggerBindingProvider>(p => p.GetRequiredService<DefaultTriggerBindingFactory>().Create());
@@ -115,6 +119,24 @@ namespace Microsoft.Azure.WebJobs
             services.AddSingleton<IHostedService, OptionsLoggingService>();
             services.AddSingleton<IOptionsFormatter<LoggerFilterOptions>, LoggerFilterOptionsFormatter>();
 
+            // Concurrency management
+            services.TryAddSingleton<IConcurrencyStatusRepository, NullConcurrencyStatusRepository>();
+            services.TryAddSingleton<IHostProcessMonitor, DefaultHostProcessMonitor>();
+            services.TryAddSingleton<IConcurrencyThrottleManager, DefaultConcurrencyThrottleManager>();
+            services.TryAddEnumerable(ServiceDescriptor.Singleton<IConcurrencyThrottleProvider, HostHealthThrottleProvider>());
+            services.TryAddEnumerable(ServiceDescriptor.Singleton<IConcurrencyThrottleProvider, ThreadPoolStarvationThrottleProvider>());
+            services.TryAddSingleton<ConcurrencyManager>();
+            services.TryAddEnumerable(ServiceDescriptor.Singleton<IHostedService, ConcurrencyManagerService>());
+
+            services.ConfigureOptions<ConcurrencyOptionsSetup>();
+            services.ConfigureOptions<PrimaryHostCoordinatorOptionsSetup>();
+            services.AddOptions<ConcurrencyOptions>()
+                .Configure<IConfiguration>((options, config) =>
+                {
+                    var section = config.GetWebJobsRootConfiguration().GetSection(ConcurrencyConfigSectionName);
+                    section.Bind(options);
+                });
+
             services.AddOptions<SingletonOptions>()
                 .Configure<IHostingEnvironment>((options, env) =>
                 {

src/Microsoft.Azure.WebJobs.Host/Loggers/Logger/Constants/LogCategories.cs

@@ -13,6 +13,8 @@ namespace Microsoft.Azure.WebJobs.Logging
         private static readonly Regex _userFunctionRegex = new Regex(@"^Function\.[^\s\.]+\.User$");
         private static readonly Regex _functionRegex = new Regex(@"^Function\.[^\s\.]+$");
 
+        public const string HostGeneral = "Host.General";
+
         /// <summary>
         /// The category for all logs written by the function host during startup and shutdown. This
         /// includes indexing and configuration logs.
@@ -44,6 +46,16 @@ namespace Microsoft.Azure.WebJobs.Logging
         /// </summary>
         public const string Bindings = "Host.Bindings";
 
+        /// <summary>
+        /// The category for scale related logs.
+        /// </summary>
+        public const string Scale = "Host.Scale";
+
+        /// <summary>
+        /// The category for logs related to concurrency management.
+        /// </summary>
+        public const string Concurrency = "Host.Concurrency";
+
         /// <summary>
         /// The category for function binding access stats.
         /// </summary>

src/Microsoft.Azure.WebJobs.Host/Loggers/Logger/LoggerExtensions.cs

@@ -21,6 +21,160 @@ namespace Microsoft.Extensions.Logging
            new EventId(325, nameof(LogFunctionRetryAttempt)),
            "Waiting for `{nextDelay}` before retrying function execution. Next attempt: '{attempt}'. Max retry count: '{retryStrategy.MaxRetryCount}'");
 
+        private static readonly Action<ILogger, int, string, double, double, Exception> _hostProcessCpuStats =
+           LoggerMessage.Define<int, string, double, double>(
+           LogLevel.Debug,
+           new EventId(326, nameof(HostProcessCpuStats)),
+           "[HostMonitor] Host process CPU stats (PID {pid}): History=({formattedCpuLoadHistory}), AvgCpuLoad={avgCpuLoad}, MaxCpuLoad={maxCpuLoad}");
+
+        private static readonly Action<ILogger, double, float, Exception> _hostCpuThresholdExceeded =
+           LoggerMessage.Define<double, float>(
+           LogLevel.Warning,
+           new EventId(327, nameof(HostCpuThresholdExceeded)),
+           "[HostMonitor] Host CPU threshold exceeded ({aggregateCpuLoad} >= {cpuThreshold})");
+
+        private static readonly Action<ILogger, double, Exception> _hostAggregateCpuLoad =
+           LoggerMessage.Define<double>(
+           LogLevel.Debug,
+           new EventId(328, nameof(HostAggregateCpuLoad)),
+           "[HostMonitor] Host aggregate CPU load {aggregateCpuLoad}");
+
+        private static readonly Action<ILogger, int, string, double, double, Exception> _hostProcessMemoryUsage =
+           LoggerMessage.Define<int, string, double, double>(
+           LogLevel.Debug,
+           new EventId(329, nameof(HostProcessMemoryUsage)),
+           "[HostMonitor] Host process memory usage (PID {pid}): History=({formattedMemoryUsageHistory}), AvgUsage={avgMemoryUsage}, MaxUsage={maxMemoryUsage}");
+
+        private static readonly Action<ILogger, double, double, Exception> _hostMemoryThresholdExceeded =
+           LoggerMessage.Define<double, double>(
+           LogLevel.Warning,
+           new EventId(330, nameof(HostMemoryThresholdExceeded)),
+           "[HostMonitor] Host memory threshold exceeded ({aggregateMemoryUsage} >= {memoryThreshold})");
+
+        private static readonly Action<ILogger, double, int, Exception> _hostAggregateMemoryUsage =
+           LoggerMessage.Define<double, int>(
+           LogLevel.Debug,
+           new EventId(331, nameof(HostAggregateMemoryUsage)),
+           "[HostMonitor] Host aggregate memory usage {aggregateMemoryUsage} ({percentageOfMax}% of threshold)");
+
+        private static readonly Action<ILogger, string, int, int, Exception> _hostConcurrencyStatus =
+           LoggerMessage.Define<string, int, int>(
+           LogLevel.Debug,
+           new EventId(332, nameof(HostConcurrencyStatus)),
+           "{functionId} Concurrency: {concurrency}, OutstandingInvocations: {outstandingInvocations}");
+
+        private static readonly Action<ILogger, Exception> _hostThreadStarvation =
+           LoggerMessage.Define(
+           LogLevel.Warning,
+           new EventId(333, nameof(HostThreadStarvation)),
+           "Possible thread pool starvation detected.");
+
+        private static readonly Action<ILogger, string, Exception> _primaryHostCoordinatorFailedToRenewLockLease =
+            LoggerMessage.Define<string>(
+            LogLevel.Information,
+            new EventId(334, nameof(PrimaryHostCoordinatorFailedToRenewLockLease)),
+            "Failed to renew host lock lease: {reason}");
+
+        private static readonly Action<ILogger, string, string, Exception> _primaryHostCoordinatorFailedToAcquireLockLease =
+            LoggerMessage.Define<string, string>(
+            LogLevel.Debug,
+            new EventId(335, nameof(PrimaryHostCoordinatorFailedToAcquireLockLease)),
+            "Host instance '{websiteInstanceId}' failed to acquire host lock lease: {reason}");
+
+        private static readonly Action<ILogger, string, Exception> _primaryHostCoordinatorReleasedLocklLease =
+            LoggerMessage.Define<string>(
+            LogLevel.Debug,
+            new EventId(336, nameof(PrimaryHostCoordinatorReleasedLocklLease)),
+            "Host instance '{websiteInstanceId}' released lock lease.");
+
+        private static readonly Action<ILogger, string, Exception> _primaryHostCoordinatorLockLeaseAcquired =
+            LoggerMessage.Define<string>(
+            LogLevel.Information,
+            new EventId(337, nameof(PrimaryHostCoordinatorLockLeaseAcquired)),
+            "Host lock lease acquired by instance ID '{websiteInstanceId}'.");
+
+        private static readonly Action<ILogger, string, string, Exception> _functionConcurrencyDecrease =
+           LoggerMessage.Define<string, string>(
+           LogLevel.Debug,
+           new EventId(338, nameof(FunctionConcurrencyDecrease)),
+           "{functionId} Decreasing concurrency (Enabled throttles: {enabledThrottles})");
+
+        private static readonly Action<ILogger, string, Exception> _functionConcurrencyIncrease =
+           LoggerMessage.Define<string>(
+           LogLevel.Debug,
+           new EventId(338, nameof(FunctionConcurrencyIncrease)),
+           "{functionId} Increasing concurrency");
+
+        public static void PrimaryHostCoordinatorLockLeaseAcquired(this ILogger logger, string websiteInstanceId)
+        {
+            _primaryHostCoordinatorLockLeaseAcquired(logger, websiteInstanceId, null);
+        }
+
+        public static void PrimaryHostCoordinatorFailedToRenewLockLease(this ILogger logger, string reason)
+        {
+            _primaryHostCoordinatorFailedToRenewLockLease(logger, reason, null);
+        }
+
+        public static void PrimaryHostCoordinatorFailedToAcquireLockLease(this ILogger logger, string websiteInstanceId, string reason)
+        {
+            _primaryHostCoordinatorFailedToAcquireLockLease(logger, websiteInstanceId, reason, null);
+        }
+
+        public static void PrimaryHostCoordinatorReleasedLocklLease(this ILogger logger, string websiteInstanceId)
+        {
+            _primaryHostCoordinatorReleasedLocklLease(logger, websiteInstanceId, null);
+        }
+
+        public static void HostThreadStarvation(this ILogger logger)
+        {
+            _hostThreadStarvation(logger, null);
+        }
+
+        public static void HostConcurrencyStatus(this ILogger logger, string functionId, int concurrency, int outstandingInvocations)
+        {
+            _hostConcurrencyStatus(logger, functionId, concurrency, outstandingInvocations, null);
+        }
+
+        public static void HostAggregateCpuLoad(this ILogger logger, double aggregateCpuLoad)
+        {
+            _hostAggregateCpuLoad(logger, aggregateCpuLoad, null);
+        }
+
+        public static void HostProcessCpuStats(this ILogger logger, int pid, string formattedCpuLoadHistory, double avgCpuLoad, double maxCpuLoad)
+        {
+            _hostProcessCpuStats(logger, pid, formattedCpuLoadHistory, avgCpuLoad, maxCpuLoad, null);
+        }
+
+        public static void HostCpuThresholdExceeded(this ILogger logger, double aggregateCpuLoad, float cpuThreshold)
+        {
+            _hostCpuThresholdExceeded(logger, aggregateCpuLoad, cpuThreshold, null);
+        }
+
+        public static void HostAggregateMemoryUsage(this ILogger logger, double aggregateMemoryUsage, int percentageOfMax)
+        {
+            _hostAggregateMemoryUsage(logger, aggregateMemoryUsage, percentageOfMax, null);
+        }
+
+        public static void HostProcessMemoryUsage(this ILogger logger, int pid, string formattedMemoryUsageHistory, double avgMemoryUsage, double maxMemoryUsage)
+        {
+            _hostProcessMemoryUsage(logger, pid, formattedMemoryUsageHistory, avgMemoryUsage, maxMemoryUsage, null);
+        }
+
+        public static void HostMemoryThresholdExceeded(this ILogger logger, double aggregateMemoryUsage, double memoryThreshold)
+        {
+            _hostMemoryThresholdExceeded(logger, aggregateMemoryUsage, memoryThreshold, null);
+        }
+
+        public static void FunctionConcurrencyDecrease(this ILogger logger, string functionId, string enabledThrottles)
+        {
+            _functionConcurrencyDecrease(logger, functionId, enabledThrottles, null);
+        }
+
+        public static void FunctionConcurrencyIncrease(this ILogger logger, string functionId)
+        {
+            _functionConcurrencyIncrease(logger, functionId, null);
+        }
+
         /// <summary>
         /// Logs a metric value.
         /// </summary>

src/Microsoft.Azure.WebJobs.Host/Scale/ConcurrencyManager.cs

@@ -0,0 +1,242 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT License. See License.txt in the project root for license information.
+
+#nullable enable
+
+using System;
+using System.Collections.Concurrent;
+using System.Linq;
+using Microsoft.Azure.WebJobs.Logging;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
+
+namespace Microsoft.Azure.WebJobs.Host.Scale
+{
+    /// <summary>
+    /// Used to implement collaborative dynamic concurrency management between the host and function triggers.
+    /// Function listeners can call <see cref="GetStatus"/> within their listener polling loops to determine the
+    /// amount of new work that can be fetched. The manager internally adjusts concurrency based on various
+    /// health heuristics.
+    /// </summary>
+    public class ConcurrencyManager
+    {
+        internal const int MinConsecutiveIncreaseLimit = 5;
+        internal const int MinConsecutiveDecreaseLimit = 3;
+
+        private readonly ILogger _logger;
+        private readonly IOptions<ConcurrencyOptions> _options;
+        private readonly IConcurrencyThrottleManager _concurrencyThrottleManager;
+        private readonly bool _enabled;
+
+        private ConcurrencyThrottleAggregateStatus? _throttleStatus;
+
+#nullable disable
+        // for mock testing only
+        internal ConcurrencyManager()
+        {
+        }
+#nullable enable
+
+        public ConcurrencyManager(IOptions<ConcurrencyOptions> options, ILoggerFactory loggerFactory, IConcurrencyThrottleManager concurrencyThrottleManager)
+        {
+            _options = options ?? throw new ArgumentNullException(nameof(options));
+            _concurrencyThrottleManager = concurrencyThrottleManager ?? throw new ArgumentNullException(nameof(concurrencyThrottleManager));
+
+            if (loggerFactory == null)
+            {
+                throw new ArgumentNullException(nameof(loggerFactory));
+            }
+            _logger = loggerFactory.CreateLogger(LogCategories.Concurrency);
+            _enabled = _options.Value.DynamicConcurrencyEnabled;
+
+            EffectiveCoresCount = Utility.GetEffectiveCoresCount();
+        }
+
+        /// <summary>
+        /// Gets a value indicating whether dynamic concurrency is enabled.
+        /// </summary>
+        public bool Enabled => _enabled;
+
+        /// <summary>
+        /// Gets a value indicating whether any throttles are currently enabled.
+        /// </summary>
+        internal virtual bool ThrottleEnabled => _throttleStatus?.State == ThrottleState.Enabled;
+
+        internal int EffectiveCoresCount { get; set; }
+
+        internal ConcurrentDictionary<string, ConcurrencyStatus> ConcurrencyStatuses = new ConcurrentDictionary<string, ConcurrencyStatus>(StringComparer.OrdinalIgnoreCase);
+
+        /// <summary>
+        /// Gets the concurrency status for the specified function. 
+        /// </summary>
+        /// <param name="functionId">This should be the full ID, as returned by <see cref="FunctionDescriptor.ID"/>.</param>
+        /// <returns>The updated concurrency status.</returns>
+        /// <remarks>
+        /// This method shouldn't be called concurrently for the same function ID.
+        /// </remarks>
+        public ConcurrencyStatus GetStatus(string functionId)
+        {
+            if (string.IsNullOrEmpty(functionId))
+            {
+                throw new ArgumentNullException(nameof(functionId));
+            }
+
+            // because this method won't be called concurrently for the same function ID, we can make
+            // updates to the function specific status below without locking.
+            var functionConcurrencyStatus = GetFunctionConcurrencyStatus(functionId);
+
+            if (!functionConcurrencyStatus.CanAdjustConcurrency())
+            {
+                // if we've made an adjustment recently for this function, just return the
+                // current status
+                return functionConcurrencyStatus;
+            }
+
+            // determine whether any throttles are currently enabled
+            _throttleStatus = _concurrencyThrottleManager.GetStatus();
+            if (_throttleStatus.State == ThrottleState.Unknown)
+            {
+                // if we're un an unknown state, we'll make no moves
+                // however, we will continue to take work at the current concurrency level
+                return functionConcurrencyStatus;
+            }
+
+            if (_throttleStatus.State == ThrottleState.Disabled)
+            {
+                if (CanIncreaseConcurrency(functionConcurrencyStatus))
+                {
+                    _logger.FunctionConcurrencyIncrease(functionId);
+                    functionConcurrencyStatus.IncreaseConcurrency();
+                }
+            }
+            else if (CanDecreaseConcurrency(functionConcurrencyStatus))
+            {
+                string enabledThrottles = _throttleStatus.EnabledThrottles != null ? string.Join(",", _throttleStatus.EnabledThrottles) : string.Empty;
+                _logger.FunctionConcurrencyDecrease(functionId, enabledThrottles);
+
+                functionConcurrencyStatus.DecreaseConcurrency();
+            }
+
+            _logger.HostConcurrencyStatus(functionId, functionConcurrencyStatus.CurrentConcurrency, functionConcurrencyStatus.OutstandingInvocations);
+
+            return functionConcurrencyStatus;
+        }
+
+        internal virtual HostConcurrencySnapshot GetSnapshot()
+        {
+            var functionSnapshots = ConcurrencyStatuses.Values.ToDictionary(p => p.FunctionId, q => new FunctionConcurrencySnapshot { Concurrency = q.CurrentConcurrency }, StringComparer.OrdinalIgnoreCase);
+            var hostSnapshot = new HostConcurrencySnapshot
+            {
+                Timestamp = DateTime.UtcNow,
+                NumberOfCores = EffectiveCoresCount,
+                FunctionSnapshots = functionSnapshots
+            };
+            return hostSnapshot;
+        }
+
+        internal virtual void ApplySnapshot(HostConcurrencySnapshot hostSnapshot)
+        {
+            if (hostSnapshot == null)
+            {
+                throw new ArgumentNullException(nameof(hostSnapshot));
+            }
+
+            if (hostSnapshot.FunctionSnapshots != null)
+            {
+                foreach (var functionSnapshot in hostSnapshot.FunctionSnapshots)
+                {
+                    int concurrency = GetCoreAdjustedConcurrency(functionSnapshot.Value.Concurrency, hostSnapshot.NumberOfCores, EffectiveCoresCount);
+                    functionSnapshot.Value.Concurrency = concurrency;
+
+                    // Since we may be initializing for functions that haven't run yet, if the snapshot contains
+                    // stale functions, they'll be added. When we write snapshots, we prune stale entries though.
+                    var concurrencyStatus = GetFunctionConcurrencyStatus(functionSnapshot.Key);
+                    _logger.LogInformation($"Applying status snapshot for function {functionSnapshot.Key} (Concurrency: {functionSnapshot.Value.Concurrency})");
+                    concurrencyStatus.ApplySnapshot(functionSnapshot.Value);
+                }
+            }
+        }
+
+        internal static int GetCoreAdjustedConcurrency(int concurrency, int otherCores, int cores)
+        {
+            if (cores != otherCores)
+            {
+                // To allow for variance across machines, we compute the effective concurrency
+                // based on number of cores. When running in an App Service plan, all instances will have
+                // the same VM specs. When running in the Consumption plan, VMs may differ. In the latter case,
+                // if the snapshot was taken on a VM with a different core count than ours, the adjusted
+                // concurency we compute may not be optimal, but it's a starting point that we'll dynamically
+                // adjust from as needed.
+                float concurrencyPerCore = (float)concurrency / otherCores;
+                int adjustedConcurrency = (int)(cores * concurrencyPerCore);
+
+                return Math.Max(1, adjustedConcurrency);
+            }
+
+            return concurrency;
+        }
+
+        private bool CanIncreaseConcurrency(ConcurrencyStatus concurrencyStatus)
+        {
+            // we're in a throttle disabled state
+            if (_throttleStatus?.ConsecutiveCount < MinConsecutiveIncreaseLimit)
+            {
+                // only increase if we've been healthy for a while
+                return false;
+            }
+
+            return concurrencyStatus.CanIncreaseConcurrency(_options.Value.MaximumFunctionConcurrency);
+        }
+
+        private bool CanDecreaseConcurrency(ConcurrencyStatus concurrencyStatus)
+        {
+            // we're in a throttle enabled state
+            if (_throttleStatus?.ConsecutiveCount < MinConsecutiveDecreaseLimit)
+            {
+                // only decrease if we've been unhealthy for a while
+                return false;
+            }
+
+            return concurrencyStatus.CanDecreaseConcurrency();
+        }
+
+        internal void FunctionStarted(string functionId)
+        {
+            if (string.IsNullOrEmpty(functionId))
+            {
+                throw new ArgumentNullException(nameof(functionId));
+            }
+            
+            // Here we TryGet rather than GetOrAdd so that we're only performing
+            // this overhead for functions that are actually using dynamic concurrency.
+            // For DC enabled functions, we won't see an invocation until after the first
+            // call to GetStatus, which will create the ConcurrencyStatus.
+            if (ConcurrencyStatuses.TryGetValue(functionId, out ConcurrencyStatus concurrencyStatus))
+            {
+                concurrencyStatus.FunctionStarted();
+            }
+        }
+
+        internal void FunctionCompleted(string functionId, TimeSpan latency)
+        {
+            if (string.IsNullOrEmpty(functionId))
+            {
+                throw new ArgumentNullException(nameof(functionId));
+            }
+
+            // Here we TryGet rather than GetOrAdd so that we're only performing
+            // this overhead for functions that are actually using dynamic concurrency.
+            // For DC enabled functions, we won't see an invocation until after the first
+            // call to GetStatus, which will create the ConcurrencyStatus.
+            if (ConcurrencyStatuses.TryGetValue(functionId, out ConcurrencyStatus concurrencyStatus))
+            {
+                concurrencyStatus.FunctionCompleted(latency);
+            }
+        }
+
+        private ConcurrencyStatus GetFunctionConcurrencyStatus(string functionId)
+        {
+            return ConcurrencyStatuses.GetOrAdd(functionId, new ConcurrencyStatus(functionId, this));
+        }
+    }
+}

src/Microsoft.Azure.WebJobs.Host/Scale/ConcurrencyManagerService.cs

@@ -0,0 +1,164 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT License. See License.txt in the project root for license information.
+
+#nullable enable
+
+using System;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using System.Timers;
+using Microsoft.Azure.WebJobs.Host.Indexers;
+using Microsoft.Azure.WebJobs.Hosting;
+using Microsoft.Azure.WebJobs.Logging;
+using Microsoft.Extensions.Hosting;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
+
+namespace Microsoft.Azure.WebJobs.Host.Scale
+{
+    /// <summary>
+    /// Service responsible for startup time application of Dynamic Concurrency snapshots,
+    /// as well as periodic background persistence of status snapshots.
+    /// </summary>
+    internal class ConcurrencyManagerService : IHostedService, IDisposable
+    {
+        private readonly ILogger _logger;
+        private readonly IOptions<ConcurrencyOptions> _options;
+        private readonly IConcurrencyStatusRepository _statusRepository;
+        private readonly System.Timers.Timer _statusPersistenceTimer;
+        private readonly ConcurrencyManager _concurrencyManager;
+        private readonly IFunctionIndexProvider _functionIndexProvider;
+        private readonly IPrimaryHostStateProvider _primaryHostStateProvider;
+
+        private HostConcurrencySnapshot? _lastSnapshot;
+        private bool _disposed;
+
+        public ConcurrencyManagerService(IOptions<ConcurrencyOptions> options, ILoggerFactory loggerFactory, ConcurrencyManager concurrencyManager, IConcurrencyStatusRepository statusRepository, IFunctionIndexProvider functionIndexProvider, IPrimaryHostStateProvider primaryHostStateProvider)
+        {
+            _options = options ?? throw new ArgumentNullException(nameof(options));
+            _concurrencyManager = concurrencyManager ?? throw new ArgumentNullException(nameof(concurrencyManager));
+            _statusRepository = statusRepository ?? throw new ArgumentNullException(nameof(statusRepository));
+            _functionIndexProvider = functionIndexProvider ?? throw new ArgumentNullException(nameof(functionIndexProvider));
+            _primaryHostStateProvider = primaryHostStateProvider ?? throw new ArgumentNullException(nameof(primaryHostStateProvider));
+
+            if (loggerFactory == null)
+            {
+                throw new ArgumentNullException(nameof(loggerFactory));
+            }
+            _logger = loggerFactory.CreateLogger(LogCategories.Concurrency);
+
+            _statusPersistenceTimer = new System.Timers.Timer
+            {
+                AutoReset = false,
+                Interval = 10000
+            };
+            _statusPersistenceTimer.Elapsed += OnPersistenceTimer;
+        }
+
+        internal System.Timers.Timer StatusPersistenceTimer => _statusPersistenceTimer;
+
+        public async Task StartAsync(CancellationToken cancellationToken)
+        {
+            if (_options.Value.DynamicConcurrencyEnabled && _options.Value.SnapshotPersistenceEnabled)
+            {
+                await ApplySnapshotAsync();
+
+                _statusPersistenceTimer.Start();
+            }
+        }
+
+        public Task StopAsync(CancellationToken cancellationToken)
+        {
+            _statusPersistenceTimer?.Stop();
+            return Task.CompletedTask;
+        }
+
+        private async Task ApplySnapshotAsync()
+        {
+            try
+            {
+                // one time application of status snapshot on startup
+                var snapshot = await _statusRepository.ReadAsync(CancellationToken.None);
+                if (snapshot != null)
+                {
+                    _lastSnapshot = snapshot;
+                    _concurrencyManager.ApplySnapshot(snapshot);
+                }
+            }
+            catch (Exception ex)
+            {
+                _logger.LogError(ex, "Error applying concurrency snapshot.");
+            }
+        }
+
+        internal async void OnPersistenceTimer(object sender, ElapsedEventArgs e)
+        {
+            await OnPersistenceTimer();
+        }
+
+        internal async Task OnPersistenceTimer()
+        {
+            if (_disposed)
+            {
+                return;
+            }
+
+            try
+            {
+                if (_primaryHostStateProvider.IsPrimary)
+                {
+                    await WriteSnapshotAsync();
+                }
+            }
+            catch (Exception ex)
+            {
+                // Don't allow background exceptions to escape
+                _logger.LogError(ex, "Error persisting concurrency snapshot.");
+            }
+
+            if (!_disposed)
+            {
+                _statusPersistenceTimer.Start();
+            }
+        }
+
+        internal async Task WriteSnapshotAsync()
+        {
+            var snapshot = _concurrencyManager.GetSnapshot();
+
+            // only persist snapshots for functions that are in our current index
+            // this ensures we prune any stale entries from a previously applied snapshot
+            var functionIndex = await _functionIndexProvider.GetAsync(CancellationToken.None);
+            var functions = functionIndex.ReadAll().ToLookup(p => p.Descriptor.Id, StringComparer.OrdinalIgnoreCase);
+            snapshot.FunctionSnapshots = snapshot.FunctionSnapshots.Where(p => functions.Contains(p.Key)).ToDictionary(p => p.Key, p => p.Value);
+
+            if (!snapshot.Equals(_lastSnapshot))
+            {
+                await _statusRepository.WriteAsync(snapshot, CancellationToken.None);
+
+                _lastSnapshot = snapshot;
+            }
+        }
+
+        protected virtual void Dispose(bool disposing)
+        {
+            if (!_disposed)
+            {
+                if (disposing)
+                {
+                    _statusPersistenceTimer.Dispose();
+                }
+
+                _disposed = true;
+            }
+        }
+
+        public void Dispose()
+        {
+            // Do not change this code. Put cleanup code in 'Dispose(bool disposing)' method
+            Dispose(disposing: true);
+            GC.SuppressFinalize(this);
+        }
+    }
+}

src/Microsoft.Azure.WebJobs.Host/Scale/ConcurrencyOptions.cs

@@ -0,0 +1,161 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT License. See License.txt in the project root for license information.
+
+using System;
+using Microsoft.Azure.WebJobs.Hosting;
+using Newtonsoft.Json;
+using Newtonsoft.Json.Linq;
+
+namespace Microsoft.Azure.WebJobs.Host.Scale
+{
+    /// <summary>
+    /// Options used to configure dynamic concurrency control.
+    /// </summary>
+    public class ConcurrencyOptions : IOptionsFormatter
+    {
+        private int _maximumFunctionConcurrency;
+        private long _totalAvaliableMemoryBytes;
+        private float _memoryThreshold;
+        private float _cpuThreshold;
+
+        public ConcurrencyOptions()
+        {
+            SnapshotPersistenceEnabled = true;
+            _maximumFunctionConcurrency = 500;
+            _totalAvaliableMemoryBytes = -1;
+            _cpuThreshold = 0.80F;
+            _memoryThreshold = 0.80F;
+        }
+
+        /// <summary>
+        /// Gets or sets a value indicating whether the dynamic concurrency control feature
+        /// is enabled.
+        /// </summary>
+        public bool DynamicConcurrencyEnabled { get; set; }
+
+        /// <summary>
+        /// Gets or sets the maximum concurrency that will be enforced per function.
+        /// Set to -1 to indicate no limit.
+        /// </summary>
+        public int MaximumFunctionConcurrency 
+        { 
+            get
+            {
+                return _maximumFunctionConcurrency;
+            }
+            set
+            {
+                if (value <= 0 && value != -1)
+                {
+                    throw new ArgumentOutOfRangeException(nameof(MaximumFunctionConcurrency));
+                }
+
+                _maximumFunctionConcurrency = value;
+            }
+        }
+
+        /// <summary>
+        /// Gets or sets the total amount of physical memory the host has access to.
+        /// This value is used in conjunction with <see cref="MemoryThreshold"/> to
+        /// determine when memory based throttling will kick in.
+        /// A value of -1 indicates that the available memory limit is unknown, and
+        /// memory based throtting will be disabled.
+        /// </summary>
+        /// <remarks>
+        /// When deployed to App Service, this value will be defaulted based on the SKU
+        /// and other plan info.
+        /// </remarks>
+        internal long TotalAvailableMemoryBytes
+        { 
+            get
+            {
+                return _totalAvaliableMemoryBytes;
+            }
+            set
+            {
+                if (value <= 0 && value != -1)
+                {
+                    throw new ArgumentOutOfRangeException(nameof(TotalAvailableMemoryBytes));
+                }
+
+                _totalAvaliableMemoryBytes = value;
+            }
+        }
+
+        /// <summary>
+        /// Gets or sets the memory threshold dictating when memory based throttling
+        /// will kick in. The value should be between 0 and 1 (exclusive) indicating a percentage
+        /// of <see cref="TotalAvailableMemoryBytes"/>.
+        /// Set to -1 to disable memory based throttling.
+        /// </summary>
+        internal float MemoryThreshold 
+        {
+            get
+            {
+                return _memoryThreshold;
+            }
+            set
+            {
+                if ((value <= 0 && value != -1) || (value >= 1))
+                {
+                    throw new ArgumentOutOfRangeException(nameof(MemoryThreshold));
+                }
+
+                _memoryThreshold = value;
+            }
+        }
+
+        /// <summary>
+        /// Gets or sets the CPU threshold dictating when cpu based throttling
+        /// will kick in. Value should be between 0 and 1 indicating a cpu percentage.
+        /// </summary>
+        public float CPUThreshold
+        {
+            get
+            {
+                return _cpuThreshold;
+            }
+            set
+            {
+                if (value <= 0 || value >= 1)
+                {
+                    throw new ArgumentOutOfRangeException(nameof(CPUThreshold));
+                }
+
+                _cpuThreshold = value;
+            }
+        }
+
+        /// <summary>
+        /// Gets or sets a value indicating whether concurrency status snapshots will be periodically
+        /// written to persistent storage, to enable hosts to remember and apply previously learned levels
+        /// on startup.
+        /// </summary>
+        public bool SnapshotPersistenceEnabled { get; set; }
+
+        internal bool MemoryThrottleEnabled
+        {
+            get
+            {
+                return TotalAvailableMemoryBytes > 0 && MemoryThreshold > 0;
+            }
+        }
+
+        public string Format()
+        {
+            JObject options = new JObject
+            {
+                { nameof(DynamicConcurrencyEnabled), DynamicConcurrencyEnabled },
+                { nameof(MaximumFunctionConcurrency), MaximumFunctionConcurrency },
+                { nameof(TotalAvailableMemoryBytes), TotalAvailableMemoryBytes },
+                // TODO: Once Memory monitoring is public add this back
+                // https://github.com/Azure/azure-webjobs-sdk/issues/2733
+                //{ nameof(MemoryThreshold), MemoryThreshold },
+                { nameof(CPUThreshold), CPUThreshold },
+                { nameof(SnapshotPersistenceEnabled), SnapshotPersistenceEnabled }
+            };
+
+            return options.ToString(Formatting.Indented);
+        }
+    }
+}

src/Microsoft.Azure.WebJobs.Host/Scale/ConcurrencyStatus.cs

@@ -0,0 +1,318 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT License. See License.txt in the project root for license information.
+
+using System;
+using System.Diagnostics;
+using System.Threading;
+
+namespace Microsoft.Azure.WebJobs.Host.Scale
+{
+    /// <summary>
+    /// Represents the concurrency status for a function.
+    /// </summary>
+    public class ConcurrencyStatus
+    {
+        internal const int NextStatusDelayDefaultSeconds = 1;
+        internal const int DefaultFailedAdjustmentQuietWindowSeconds = 30;
+        internal const int AdjustmentRunWindowSeconds = 10;
+        internal const int DefaultMinAdjustmentFrequencySeconds = 5;
+        internal const int MaxAdjustmentDelta = 5;
+
+        private readonly ConcurrencyManager _concurrencyManager;
+        private readonly object _syncLock = new object();
+
+        private int _adjustmentRunDirection;
+        private int _adjustmentRunCount;
+
+        /// <summary>
+        /// Constructs a new instance.
+        /// </summary>
+        /// <param name="functionId">The function ID this status is for.</param>
+        /// <param name="concurrencyManager">The <see cref="ConcurrencyManager"/>.</param>
+        public ConcurrencyStatus(string functionId, ConcurrencyManager concurrencyManager)
+        {
+            _concurrencyManager = concurrencyManager ?? throw new ArgumentNullException(nameof(concurrencyManager));
+
+            if (string.IsNullOrEmpty(functionId))
+            {
+                throw new ArgumentNullException(nameof(functionId));
+            }
+            FunctionId = functionId;
+            
+            CurrentConcurrency = 1;
+            OutstandingInvocations = 0;
+            MaxConcurrentExecutionsSinceLastAdjustment = 0;
+            _adjustmentRunDirection = 0;
+
+            // We don't start the decrease stopwatch initially because we only want
+            // to take it into consideration when a decrease has actually happened.
+            // We start the adjustment stopwatch immediately, because we want don't
+            // want the first adjustment to happen immediately.
+            LastConcurrencyAdjustmentStopwatch = Stopwatch.StartNew();
+            LastConcurrencyDecreaseStopwatch = new Stopwatch();
+        }
+
+        /// <summary>
+        /// Gets the function ID this status is for.
+        /// </summary>
+        public string FunctionId { get; }
+
+        /// <summary>
+        /// Gets the current number of new invocations of this function the host can process.
+        /// When throttling is enabled, this may return 0 meaning no new invocations should be
+        /// started.
+        /// </summary>
+        public int AvailableInvocationCount
+        {
+            get
+            {
+                if (_concurrencyManager.ThrottleEnabled || OutstandingInvocations > CurrentConcurrency)
+                {
+                    // we can't take any work right now
+                    return 0;
+                }
+                else
+                {
+                    // no throttles are enabled, so we can take work up to the current concurrency level
+                    return CurrentConcurrency - OutstandingInvocations;
+                }
+            }
+        }
+
+        /// <summary>
+        /// Gets the current concurrency level for this function. This adjusts
+        /// dynamically over time.
+        /// </summary>
+        public int CurrentConcurrency { get; internal set; }
+
+        /// <summary>
+        /// Gets the current number of in progress invocations of this function.
+        /// </summary>
+        public int OutstandingInvocations { get; internal set; }
+
+        /// <summary>
+        /// Gets the recommended delay for the next time <see cref="ConcurrencyManager.GetStatus(string)"/> should be called
+        /// for this function.
+        /// </summary>
+        public TimeSpan NextStatusDelay
+        {
+            get
+            {
+                if (AvailableInvocationCount == 0)
+                {
+                    // currently hardcoded, but can be made dynamic/configurable in the future
+                    // if the host is currently throttling or otherwise can't take any more work,
+                    // caller should delay the next status call, to give time for the situation
+                    // to change
+                    return TimeSpan.FromSeconds(NextStatusDelayDefaultSeconds);
+                }
+                else
+                {
+                    return TimeSpan.Zero;
+                }
+            }
+        }
+
+        /// <summary>
+        /// Gets a Stopwatch measuring the time since concurrency was last adjusted either up
+        /// or down for this function.
+        /// </summary>
+        internal Stopwatch LastConcurrencyAdjustmentStopwatch { get; }
+
+        /// <summary>
+        /// Gets a Stopwatch measuring the time since concurrency was last adjusted down
+        /// for this function.
+        /// </summary>
+        internal Stopwatch LastConcurrencyDecreaseStopwatch { get; }
+
+        /// <summary>
+        /// Gets or sets the highest actual invocation concurrency observed since the last
+        /// concurrency adjustment.
+        /// </summary>
+        internal int MaxConcurrentExecutionsSinceLastAdjustment { get; set; }
+
+        /// <summary>
+        /// Gets or sets the number of function invocations since the last time
+        /// concurrency was adjusted for this function.
+        /// </summary>
+        internal int InvocationsSinceLastAdjustment { get; set; }
+
+        /// <summary>
+        /// Gets or sets the total amount of time the function has run since the last time
+        /// concurrency was adjusted.
+        /// </summary>
+        internal double TotalInvocationTimeSinceLastAdjustmentMs { get; set; }
+
+        internal void ApplySnapshot(FunctionConcurrencySnapshot snapshot)
+        {
+            if (snapshot == null)
+            {
+                throw new ArgumentNullException(nameof(snapshot));
+            }
+
+            lock (_syncLock)
+            {
+                if (snapshot.Concurrency > CurrentConcurrency)
+                {
+                    CurrentConcurrency = snapshot.Concurrency;
+                }
+            }
+        }
+
+        internal bool CanAdjustConcurrency()
+        {
+            // Don't adjust too often, either up or down if we've made an adjustment recently.
+            TimeSpan timeSinceLastAdjustment = LastConcurrencyAdjustmentStopwatch.Elapsed;
+            TimeSpan minAdjustmentFrequency = GetLatencyAdjustedInterval(TimeSpan.FromSeconds(ProcessMonitor.DefaultSampleIntervalSeconds * 2), TimeSpan.FromSeconds(DefaultMinAdjustmentFrequencySeconds), 1);
+
+            return timeSinceLastAdjustment > minAdjustmentFrequency;
+        }
+
+        internal bool CanDecreaseConcurrency()
+        {
+            return CurrentConcurrency > 1;
+        }
+
+        internal bool CanIncreaseConcurrency(int maxDegreeOfParallelism)
+        {
+            var timeSinceLastDecrease = LastConcurrencyDecreaseStopwatch.IsRunning ? LastConcurrencyDecreaseStopwatch.Elapsed : TimeSpan.MaxValue;
+            TimeSpan minDecreaseQuietWindow = GetLatencyAdjustedInterval(TimeSpan.FromSeconds(ProcessMonitor.DefaultSampleIntervalSeconds * 10), TimeSpan.FromSeconds(DefaultFailedAdjustmentQuietWindowSeconds), 10);
+            if (timeSinceLastDecrease < minDecreaseQuietWindow)
+            {
+                // if we've had a recent failed adjustment, we'll avoid any increases for a while
+                return false;
+            }
+
+            if (MaxConcurrentExecutionsSinceLastAdjustment < CurrentConcurrency)
+            {
+                // We only want to increase if we're fully utilizing our current concurrency level.
+                // E.g. if we increased to a high concurrency level, then events slowed to a trickle,
+                // we wouldn't want to keep increasing.
+                return false;
+            }
+
+            // a max parallelism of -1 indicates unbounded
+            return maxDegreeOfParallelism == -1 || CurrentConcurrency < maxDegreeOfParallelism;
+        }
+
+        internal TimeSpan GetLatencyAdjustedInterval(TimeSpan minInterval, TimeSpan defaultInterval, int latencyMultiplier)
+        {
+            TimeSpan resultInterval = defaultInterval;
+
+            if (InvocationsSinceLastAdjustment > 0)
+            {
+                // Compute based on function latency, so faster functions can adjust more often, allowing their concurrency
+                // to "break away" from longer running, heavier functions.
+                // While a longer running function might not actually be a problem (e.g. might be I/O bound),
+                // this latency based prioritization is still beneficial. It allows fast functions to give
+                // us fast feedback. Worst case, for long running functions we just revert back to the default
+                // interval. So this is an optimization applied when possible.
+                int avgInvocationLatencyMs = GetAverageInvocationLatencyMS();
+                int computedIntervalMS = (int)minInterval.TotalMilliseconds + latencyMultiplier * avgInvocationLatencyMs;
+                resultInterval = TimeSpan.FromMilliseconds(Math.Min(computedIntervalMS, (int)defaultInterval.TotalMilliseconds));
+            }
+
+            return resultInterval;
+        }
+
+        internal void IncreaseConcurrency()
+        {
+            int delta = GetNextAdjustment(1);
+            AdjustConcurrency(delta);
+        }
+
+        internal void DecreaseConcurrency()
+        {
+            int delta = GetNextAdjustment(-1);
+            AdjustConcurrency(delta);
+        }
+
+        internal void FunctionStarted()
+        {
+            lock (_syncLock)
+            {
+                OutstandingInvocations++;
+
+                if (OutstandingInvocations > MaxConcurrentExecutionsSinceLastAdjustment)
+                {
+                    // record the high water mark for utilized concurrency this interval
+                    MaxConcurrentExecutionsSinceLastAdjustment = OutstandingInvocations;
+                }
+            }
+        }
+
+        internal void FunctionCompleted(TimeSpan latency)
+        {
+            lock (_syncLock)
+            {
+                TotalInvocationTimeSinceLastAdjustmentMs += latency.TotalMilliseconds;
+                InvocationsSinceLastAdjustment++;
+                if (OutstandingInvocations > 0)
+                {
+                    OutstandingInvocations--;
+                }
+            }
+        }
+
+        internal int GetNextAdjustment(int direction)
+        {
+            // keep track of consecutive adjustment runs in the same direction
+            // so we can increase velocity
+            TimeSpan timeSinceLastAdjustment = LastConcurrencyAdjustmentStopwatch.Elapsed;
+            int adjustmentRunCount = _adjustmentRunCount;
+            if ((_adjustmentRunDirection != 0 && _adjustmentRunDirection != direction) || timeSinceLastAdjustment.TotalSeconds > AdjustmentRunWindowSeconds)
+            {
+                // clear our adjustment run if we change direction or too
+                // much time has elapsed since last change
+                // when we change directions, our last move might have been large,
+                // but well move back in the other direction slowly
+                adjustmentRunCount = _adjustmentRunCount = 0;
+            }
+            else
+            {
+                // increment for next cycle
+                _adjustmentRunCount++;
+            }
+            _adjustmentRunDirection = direction;
+
+            // based on consecutive moves in the same direction, we'll adjust velocity
+            int speedFactor = Math.Min(MaxAdjustmentDelta, adjustmentRunCount);
+            return direction * (1 + speedFactor);
+        }
+
+        private void AdjustConcurrency(int delta)
+        {
+            if (delta < 0)
+            {
+                // if we're adjusting down, restart the stopwatch to delay any further
+                // increase attempts for a period to allow things to stabilize at the new
+                // concurrency level
+                LastConcurrencyDecreaseStopwatch.Restart();
+            }
+
+            LastConcurrencyAdjustmentStopwatch.Restart();
+
+            // ensure we don't adjust below 1
+            int newConcurrency = CurrentConcurrency + delta;
+            newConcurrency = Math.Max(1, newConcurrency);
+
+            lock (_syncLock)
+            {
+                CurrentConcurrency = newConcurrency;
+                MaxConcurrentExecutionsSinceLastAdjustment = 0;
+                InvocationsSinceLastAdjustment = 0;
+                TotalInvocationTimeSinceLastAdjustmentMs = 0;
+            }
+        }
+
+        private int GetAverageInvocationLatencyMS()
+        {
+            int avgInvocationLatencyMs;
+            lock (_syncLock)
+            {
+                avgInvocationLatencyMs = (int)TotalInvocationTimeSinceLastAdjustmentMs / InvocationsSinceLastAdjustment;
+            }
+            return avgInvocationLatencyMs;
+        }
+    }
+}

src/Microsoft.Azure.WebJobs.Host/Scale/ConcurrencyThrottleAggregateStatus.cs

@@ -0,0 +1,31 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT License. See License.txt in the project root for license information.
+
+#nullable enable
+
+using System.Collections.Generic;
+
+namespace Microsoft.Azure.WebJobs.Host.Scale
+{
+    /// <summary>
+    /// Represents an aggregated throttle result.
+    /// </summary>
+    public class ConcurrencyThrottleAggregateStatus
+    {
+        /// <summary>
+        /// Gets or sets current aggregate throttle state.
+        /// </summary>
+        public ThrottleState State { get; set; }
+
+        /// <summary>
+        /// Gets or sets the collection of currently enabled throttles.
+        /// </summary>
+        public ICollection<string>? EnabledThrottles { get; set; }
+
+        /// <summary>
+        /// Gets or sets the number of times the <see cref="State"/> has been in the current
+        /// state consecutively.
+        /// </summary>
+        public int ConsecutiveCount { get; set; }
+    }
+}

src/Microsoft.Azure.WebJobs.Host/Scale/ConcurrencyThrottleStatus.cs

@@ -0,0 +1,25 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT License. See License.txt in the project root for license information.
+
+#nullable enable
+
+using System.Collections.Generic;
+
+namespace Microsoft.Azure.WebJobs.Host.Scale
+{
+    /// <summary>
+    /// Represents an throttle result.
+    /// </summary>
+    public class ConcurrencyThrottleStatus
+    {
+        /// <summary>
+        /// Gets or sets current throttle state.
+        /// </summary>
+        public ThrottleState State { get; set; }
+
+        /// <summary>
+        /// Gets or sets the collection of currently enabled throttles.
+        /// </summary>
+        public ICollection<string>? EnabledThrottles { get; set; }
+    }
+}

src/Microsoft.Azure.WebJobs.Host/Scale/DefaultConcurrencyThrottleManager.cs

@@ -0,0 +1,117 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT License. See License.txt in the project root for license information.
+
+#nullable enable
+
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Linq;
+using Microsoft.Azure.WebJobs.Logging;
+using Microsoft.Extensions.Logging;
+
+namespace Microsoft.Azure.WebJobs.Host.Scale
+{
+    internal class DefaultConcurrencyThrottleManager : IConcurrencyThrottleManager
+    {
+        private readonly TimeSpan _minUpdateInterval = TimeSpan.FromMilliseconds(1000);
+        private readonly IEnumerable<IConcurrencyThrottleProvider> _throttleProviders;
+        private readonly ILogger _logger;
+
+        private object _syncLock = new object();
+        private ThrottleState _throttleState;
+        private List<string>? _enabledThrottles;
+        private int _consecutiveCount;
+
+        public DefaultConcurrencyThrottleManager(IEnumerable<IConcurrencyThrottleProvider> throttleProviders, ILoggerFactory loggerFactory)
+        {
+            if (throttleProviders == null)
+            {
+                throw new ArgumentNullException(nameof(throttleProviders));
+            }
+            _throttleProviders = throttleProviders.ToList();
+
+            if (loggerFactory == null)
+            {
+                throw new ArgumentNullException(nameof(loggerFactory));
+            }
+            _logger = loggerFactory.CreateLogger(LogCategories.Concurrency);
+
+            LastThrottleCheckStopwatch = Stopwatch.StartNew();
+        }
+
+        internal Stopwatch LastThrottleCheckStopwatch { get; }
+
+        public ConcurrencyThrottleAggregateStatus GetStatus()
+        {
+            // throttle querying of throttle providers so we're not calling them too often
+            if (LastThrottleCheckStopwatch.Elapsed > _minUpdateInterval)
+            {
+                UpdateThrottleState();
+            }
+
+            ConcurrencyThrottleAggregateStatus status;
+            lock (_syncLock)
+            {
+                status = new ConcurrencyThrottleAggregateStatus
+                {
+                    State = _throttleState,
+                    EnabledThrottles = _enabledThrottles?.AsReadOnly(),
+                    ConsecutiveCount = _consecutiveCount
+                };
+            }
+
+            return status;
+        }
+
+        private void UpdateThrottleState()
+        {
+            IEnumerable<ConcurrencyThrottleStatus> throttleResults = _throttleProviders.Select(p => p.GetStatus(_logger));
+
+            bool throttleEnabled = throttleResults.Any(p => p.State == ThrottleState.Enabled);
+            ThrottleState newThrottleState;
+            if (throttleEnabled)
+            {
+                // if any throttles are enabled, we're in an enabled state
+                newThrottleState = ThrottleState.Enabled;
+            }
+            else if (throttleResults.Any(p => p.State == ThrottleState.Unknown))
+            {
+                // if no throttles are enabled, but at least 1 is in an unknown state
+                // we're in an unknown state
+                newThrottleState = ThrottleState.Unknown;
+            }
+            else
+            {
+                // all throttles are disabled
+                newThrottleState = ThrottleState.Disabled;
+            }
+
+            List<string>? newEnabledThrottles = null;
+            if (newThrottleState == ThrottleState.Enabled)
+            {
+                newEnabledThrottles = throttleResults.Where(p => p.EnabledThrottles != null).SelectMany(p => p.EnabledThrottles).Distinct().ToList();
+            }
+
+            lock (_syncLock)
+            {
+                if (newThrottleState == _throttleState)
+                {
+                    // throttle state has remained the same since the last time we checked
+                    // so we're in a run
+                    _consecutiveCount++;
+                }
+                else
+                {
+                    // throttle state has changed, so any run has ended
+                    _consecutiveCount = 0;
+                }
+
+                _throttleState = newThrottleState;
+                _enabledThrottles = newEnabledThrottles;
+            }
+
+            LastThrottleCheckStopwatch.Restart();
+        }
+    }
+}

src/Microsoft.Azure.WebJobs.Host/Scale/DefaultHostProcessMonitor.cs

@@ -0,0 +1,292 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT License. See License.txt in the project root for license information.
+
+// Some of the logic in this file was moved from https://github.com/Azure/azure-functions-host/blob/852eed9ceef6ef56b431428a8eb31f1bd9c97f3b/src/WebJobs.Script/Scale/HostPerformanceManager.cs 
+
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Linq;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
+
+namespace Microsoft.Azure.WebJobs.Host.Scale
+{
+    internal class DefaultHostProcessMonitor : IHostProcessMonitor, IDisposable
+    {
+        internal const string CpuLimitName = "CPU";
+        internal const string MemoryLimitName = "Memory";
+        internal const int MinSampleCount = 5;
+
+        private readonly long _maxMemoryThresholdBytes;
+        private readonly ProcessMonitor _hostProcessMonitor;
+        private readonly List<ProcessMonitor> _childProcessMonitors = new List<ProcessMonitor>();
+        private readonly IOptions<ConcurrencyOptions> _options;
+        private readonly object _syncLock = new object();
+
+        private bool _disposed;
+
+        public DefaultHostProcessMonitor(IOptions<ConcurrencyOptions> options, ProcessMonitor processMonitor = null)
+        {
+            _options = options ?? throw new ArgumentNullException(nameof(options));
+            _hostProcessMonitor = processMonitor ?? new ProcessMonitor(Process.GetCurrentProcess());
+
+            if (_options.Value.MemoryThrottleEnabled)
+            {
+                _maxMemoryThresholdBytes = (long) (_options.Value.TotalAvailableMemoryBytes * (double)_options.Value.MemoryThreshold);
+            }
+        }
+
+        public void RegisterChildProcess(Process process)
+        {
+            if (process == null)
+            {
+                throw new ArgumentNullException(nameof(process));
+            }
+
+            var monitor = new ProcessMonitor(process);
+            RegisterChildProcessMonitor(monitor);
+        }
+
+        internal void RegisterChildProcessMonitor(ProcessMonitor monitor)
+        {
+            lock (_syncLock)
+            {
+                _childProcessMonitors.Add(monitor);
+            }
+        }
+
+        public void UnregisterChildProcess(Process process)
+        {
+            if (process == null)
+            {
+                throw new ArgumentNullException(nameof(process));
+            }
+
+            ProcessMonitor monitor = null;
+            lock (_syncLock)
+            {
+                monitor = _childProcessMonitors.SingleOrDefault(p => p.Process == process);
+                if (monitor != null)
+                {
+                    _childProcessMonitors.Remove(monitor);
+                }
+            }
+
+            monitor?.Dispose();
+        }
+
+        public HostProcessStatus GetStatus(ILogger logger = null)
+        {
+            var healthResult = new HostProcessStatus
+            {
+                State = HostHealthState.Unknown
+            };
+
+            // get the current stats for the host process
+            ProcessStats hostProcessStats = _hostProcessMonitor.GetStats();
+
+            // get the current stats for any child processes
+            RemoveExitedChildProcesses();
+            ProcessMonitor[] currChildProcessMonitors;
+            lock (_syncLock)
+            {
+                // snapshot the current set of child monitors
+                currChildProcessMonitors = _childProcessMonitors.ToArray();
+            }
+            IEnumerable<ProcessStats> childProcessStats = currChildProcessMonitors.Select(p => p.GetStats()).ToList();
+
+            var exceededLimits = new List<string>();
+            HostHealthState cpuStatus = GetCpuStatus(hostProcessStats, childProcessStats, logger);
+            var statuses = new List<HostHealthState>
+            {
+                cpuStatus
+            };
+            if (cpuStatus == HostHealthState.Overloaded)
+            {
+                exceededLimits.Add(CpuLimitName);
+            }
+
+            HostHealthState memoryStatus = GetMemoryStatus(hostProcessStats, childProcessStats, logger);
+            statuses.Add(memoryStatus);
+            if (memoryStatus == HostHealthState.Overloaded)
+            {
+                exceededLimits.Add(MemoryLimitName);
+            }
+
+            if (statuses.All(p => p == HostHealthState.Unknown))
+            {
+                healthResult.State = HostHealthState.Unknown;
+            }
+            else if (statuses.Any(p => p == HostHealthState.Overloaded))
+            {
+                healthResult.State = HostHealthState.Overloaded;
+            }
+            else
+            {
+                healthResult.State = HostHealthState.Ok;
+            }
+
+            healthResult.ExceededLimits = exceededLimits.AsReadOnly();
+
+            return healthResult;
+        }
+
+        private HostHealthState GetMemoryStatus(ProcessStats hostProcessStats, IEnumerable<ProcessStats> childProcessStats, ILogger logger = null)
+        {
+            HostHealthState status = HostHealthState.Unknown;
+
+            // if memory throttling is not enabled return immediately
+            if (!_options.Value.MemoryThrottleEnabled)
+            {
+                return status;
+            }
+
+            // First compute Memory usage for any registered child processes.
+            // Here and below we wait until we get enough samples before making
+            // any health decisions. This ensures that we've waited a short period
+            // after startup to allow usage to stabilize.
+            double currentChildMemoryUsageTotal = 0;
+            foreach (var currentChildStats in childProcessStats.Where(p => p.MemoryUsageHistory.Count() >= MinSampleCount))
+            {
+                // take the last N samples
+                int currChildProcessMemoryStatsCount = currentChildStats.MemoryUsageHistory.Count();
+                var currChildMemoryStats = currentChildStats.MemoryUsageHistory.TakeLastN(MinSampleCount);
+                var currChildMemoryStatsAverage = currChildMemoryStats.Average();
+
+                currentChildMemoryUsageTotal += currChildMemoryStats.Last();
+
+                string formattedLoadHistory = string.Join(",", currChildMemoryStats);
+                logger?.HostProcessMemoryUsage(currentChildStats.ProcessId, formattedLoadHistory, currChildMemoryStatsAverage, currChildMemoryStats.Max());
+            }
+
+            // calculate the aggregate usage across host + child processes
+            int hostProcessMemoryStatsCount = hostProcessStats.MemoryUsageHistory.Count();
+            if (hostProcessMemoryStatsCount >= MinSampleCount)
+            {
+                var lastSamples = hostProcessStats.MemoryUsageHistory.TakeLastN(MinSampleCount);
+
+                string formattedUsageHistory = string.Join(",", lastSamples);
+                var hostAverageMemoryUsageBytes = lastSamples.Average();
+                logger?.HostProcessMemoryUsage(hostProcessStats.ProcessId, formattedUsageHistory, Math.Round(hostAverageMemoryUsageBytes), lastSamples.Max());
+
+                // For memory limit, unlike cpu we don't want to compare the average against the limit,
+                // we want to use the last/current. Memory needs to be a harder limit that we want to avoid hitting
+                // otherwise the host could see OOM exceptions.
+                double currentMemoryUsage = currentChildMemoryUsageTotal + lastSamples.Last();
+
+                // compute the current total memory usage for host + children
+                var percentageOfMax = (int)(100 * (currentMemoryUsage / _maxMemoryThresholdBytes));
+                logger?.HostAggregateMemoryUsage(currentMemoryUsage, percentageOfMax);
+
+                // if the average is above our threshold, return true (we're overloaded)
+                if (currentMemoryUsage >= _maxMemoryThresholdBytes)
+                {
+                    // TODO: As part of enabling memory throttling review the use of GC.Collect here.
+                    // https://github.com/Azure/azure-webjobs-sdk/issues/2733
+                    GC.Collect();
+
+                    logger?.HostMemoryThresholdExceeded(currentMemoryUsage, _maxMemoryThresholdBytes);
+                    return HostHealthState.Overloaded;
+                }
+                else
+                {
+                    return HostHealthState.Ok;
+                }
+            }
+
+            return status;
+        }
+
+        private HostHealthState GetCpuStatus(ProcessStats hostProcessStats, IEnumerable<ProcessStats> childProcessStats, ILogger logger = null)
+        {
+            HostHealthState status = HostHealthState.Unknown;
+
+            // First compute CPU usage for any registered child processes.
+            // here and below, we wait until we get enough samples before making
+            // any health decisions, to ensure that we have enough data to make
+            // an informed decision.
+            double childAverageCpuTotal = 0;
+            var averageChildCpuStats = new List<double>();
+            foreach (var currentStatus in childProcessStats.Where(p => p.CpuLoadHistory.Count() >= MinSampleCount))
+            {
+                // take the last N samples
+                int currChildProcessCpuStatsCount = currentStatus.CpuLoadHistory.Count();
+                var currChildCpuStats = currentStatus.CpuLoadHistory.TakeLastN(MinSampleCount);
+                var currChildCpuStatsAverage = currChildCpuStats.Average();
+                averageChildCpuStats.Add(currChildCpuStatsAverage);
+
+                string formattedLoadHistory = string.Join(",", currChildCpuStats);
+                logger?.HostProcessCpuStats(currentStatus.ProcessId, formattedLoadHistory, currChildCpuStatsAverage, currChildCpuStats.Max());
+            }
+            childAverageCpuTotal = averageChildCpuStats.Sum();
+
+            // Calculate the aggregate load of host + child processes
+            int hostProcessCpuStatsCount = hostProcessStats.CpuLoadHistory.Count();
+            if (hostProcessCpuStatsCount >= MinSampleCount)
+            {
+                var lastSamples = hostProcessStats.CpuLoadHistory.TakeLastN(MinSampleCount);
+
+                string formattedLoadHistory = string.Join(",", lastSamples);
+                var hostAverageCpu = lastSamples.Average();
+                logger?.HostProcessCpuStats(hostProcessStats.ProcessId, formattedLoadHistory, Math.Round(hostAverageCpu), Math.Round(lastSamples.Max()));
+
+                // compute the aggregate average CPU usage for host + children for the last MinSampleCount samples
+                var aggregateAverage = Math.Round(hostAverageCpu + childAverageCpuTotal);
+                logger?.HostAggregateCpuLoad(aggregateAverage);
+
+                // if the average is above our threshold, return true (we're overloaded)
+                var adjustedThreshold = _options.Value.CPUThreshold * 100;
+                if (aggregateAverage >= adjustedThreshold)
+                {
+                    logger?.HostCpuThresholdExceeded(aggregateAverage, adjustedThreshold);
+                    return HostHealthState.Overloaded;
+                }
+                else
+                {
+                    return HostHealthState.Ok;
+                }
+            }
+
+            return status;
+        }
+
+        private void RemoveExitedChildProcesses()
+        {
+            var exitedChildMonitors = _childProcessMonitors.Where(p => p.Process.HasExited).ToArray();
+            if (exitedChildMonitors.Length > 0)
+            {
+                lock (_syncLock)
+                {
+                    foreach (var exitedChildMonitor in exitedChildMonitors)
+                    {
+                        _childProcessMonitors.Remove(exitedChildMonitor);
+                    }
+                }
+            }
+        }
+
+        protected virtual void Dispose(bool disposing)
+        {
+            if (!_disposed)
+            {
+                if (disposing)
+                {
+                    _hostProcessMonitor?.Dispose();
+
+                    foreach (var childMonitor in _childProcessMonitors)
+                    {
+                        childMonitor?.Dispose();
+                    }
+                }
+
+                _disposed = true;
+            }
+        }
+
+        public void Dispose()
+        {
+            Dispose(true);
+        }
+    }
+}

src/Microsoft.Azure.WebJobs.Host/Scale/DefaultProcessMetricsProvider.cs

@@ -0,0 +1,40 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT License. See License.txt in the project root for license information.
+
+// This type was moved from https://github.com/Azure/azure-functions-host/blob/dev/src/WebJobs.Script/Scale/DefaultProcessMetricsProvider.cs
+
+
+using System;
+using System.Diagnostics;
+
+namespace Microsoft.Azure.WebJobs.Host.Scale
+{
+    /// <summary>
+    /// Default implementation, that just delegates to the underlying Process.
+    /// </summary>
+    internal class DefaultProcessMetricsProvider : IProcessMetricsProvider
+    {
+        private readonly Process _process;
+
+        public DefaultProcessMetricsProvider(Process process)
+        {
+            _process = process ?? throw new ArgumentNullException(nameof(process));
+        }
+
+        public TimeSpan TotalProcessorTime
+        {
+            get
+            {
+                return _process.TotalProcessorTime;
+            }
+        }
+
+        public long PrivateMemoryBytes
+        {
+            get
+            {
+                return _process.PrivateMemorySize64;
+            }
+        }
+    }
+}

src/Microsoft.Azure.WebJobs.Host/Scale/FunctionConcurrencySnapshot.cs

@@ -0,0 +1,38 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT License. See License.txt in the project root for license information.
+
+namespace Microsoft.Azure.WebJobs.Host.Scale
+{
+    /// <summary>
+    /// Represents a snapshot of the current concurrency status of a function.
+    /// </summary>
+    public class FunctionConcurrencySnapshot
+    {
+        /// <summary>
+        /// Gets or sets the concurrency level of the function.
+        /// </summary>
+        public int Concurrency { get; set; }
+
+        public override bool Equals(object obj) => Equals(obj as FunctionConcurrencySnapshot);
+
+        private bool Equals(FunctionConcurrencySnapshot other)
+        {
+            if (other == null)
+            {
+                return false;
+            }
+
+            if (object.ReferenceEquals(this, other))
+            {
+                return true;
+            }
+
+            return other.Concurrency == Concurrency;
+        }
+
+        public override int GetHashCode()
+        {
+            return Concurrency.GetHashCode();
+        }
+    }
+}

src/Microsoft.Azure.WebJobs.Host/Scale/HostConcurrencySnapshot.cs

@@ -0,0 +1,98 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT License. See License.txt in the project root for license information.
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+
+namespace Microsoft.Azure.WebJobs.Host.Scale
+{
+    /// <summary>
+    /// Represents a snapshot of the current concurrency status of the host.
+    /// </summary>
+    public class HostConcurrencySnapshot
+    {
+        /// <summary>
+        /// Gets or sets the timestamp the snapshot was taken.
+        /// </summary>
+        public DateTime Timestamp { get; set; }
+
+        /// <summary>
+        /// Gets or sets the number of cores the host machine has.
+        /// </summary>
+        public int NumberOfCores { get; set; }
+
+        /// <summary>
+        /// Gets the collection of current function concurrency snapshots, indexed
+        /// by function ID.
+        /// </summary>
+        public Dictionary<string, FunctionConcurrencySnapshot> FunctionSnapshots { get; set; }
+
+        public override bool Equals(object obj) => Equals(obj as HostConcurrencySnapshot);
+
+        private bool Equals(HostConcurrencySnapshot other)
+        {
+            if (other == null)
+            {
+                return false;
+            }
+
+            if (object.ReferenceEquals(this, other))
+            {
+                return true;
+            }
+
+            if (NumberOfCores != other.NumberOfCores)
+            {
+                return false;
+            }
+
+            if ((FunctionSnapshots == null && other.FunctionSnapshots != null && other.FunctionSnapshots.Count > 0) ||
+                (FunctionSnapshots != null && FunctionSnapshots.Count > 0 && other.FunctionSnapshots == null))
+            {
+                return false;
+            }
+
+            if (FunctionSnapshots != null && other.FunctionSnapshots != null)
+            {
+                if (FunctionSnapshots.Count() != other.FunctionSnapshots.Count() ||
+                    FunctionSnapshots.Keys.Union(other.FunctionSnapshots.Keys, StringComparer.OrdinalIgnoreCase).Count() != FunctionSnapshots.Count())
+                {
+                    // function set aren't equal
+                    return false;
+                }
+
+                // we know we have the same set of functions in both 
+                // we now want to compare each function snapshot
+                foreach (var functionId in FunctionSnapshots.Keys)
+                {
+                    if (other.FunctionSnapshots.TryGetValue(functionId, out FunctionConcurrencySnapshot otherFunctionSnapshot) &&
+                        FunctionSnapshots.TryGetValue(functionId, out FunctionConcurrencySnapshot functionSnapshot) &&
+                        !functionSnapshot.Equals(otherFunctionSnapshot))
+                    {
+                        return false;
+                    }
+                }
+            }
+            
+            // if none of the above checks have returned false, the snapshots
+            // are equal
+            return true;
+        }
+
+        public override int GetHashCode()
+        {
+            int hashCode = NumberOfCores.GetHashCode();
+
+            if (FunctionSnapshots != null)
+            {
+                foreach (var functionSnapshot in FunctionSnapshots)
+                {
+                    hashCode |= functionSnapshot.Key.GetHashCode() ^ functionSnapshot.Value.GetHashCode();
+                }
+            }
+
+            return hashCode;
+        }
+    }
+}

src/Microsoft.Azure.WebJobs.Host/Scale/HostHealthState.cs

@@ -0,0 +1,26 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT License. See License.txt in the project root for license information.
+
+namespace Microsoft.Azure.WebJobs.Host.Scale
+{
+    /// <summary>
+    /// Enumeration of the possible host health status states.
+    /// </summary>
+    public enum HostHealthState
+    {
+        /// <summary>
+        /// Not enough information to determine the state.
+        /// </summary>
+        Unknown,
+
+        /// <summary>
+        /// The host is under pressure and is currently overloaded.
+        /// </summary>
+        Overloaded,
+
+        /// <summary>
+        /// The host is currently in a healthy state.
+        /// </summary>
+        Ok
+    }
+}

src/Microsoft.Azure.WebJobs.Host/Scale/HostHealthThrottleProvider.cs

@@ -0,0 +1,48 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT License. See License.txt in the project root for license information.
+
+#nullable enable
+
+using System;
+using Microsoft.Extensions.Logging;
+
+namespace Microsoft.Azure.WebJobs.Host.Scale
+{
+    /// <summary>
+    /// This throttle provider monitors host process health.
+    /// </summary>
+    internal class HostHealthThrottleProvider : IConcurrencyThrottleProvider
+    {
+        private readonly IHostProcessMonitor _hostProcessMonitor;
+
+        public HostHealthThrottleProvider(IHostProcessMonitor hostProcessMonitor)
+        {
+            _hostProcessMonitor = hostProcessMonitor ?? throw new ArgumentNullException(nameof(hostProcessMonitor));
+        }
+
+        public ConcurrencyThrottleStatus GetStatus(ILogger? logger = null)
+        {
+            var processStatus = _hostProcessMonitor.GetStatus(logger);
+
+            var status = new ConcurrencyThrottleStatus
+            {
+                EnabledThrottles = processStatus.ExceededLimits
+            };
+
+            switch (processStatus.State)
+            {
+                case HostHealthState.Overloaded:
+                    status.State = ThrottleState.Enabled;
+                    break;
+                case HostHealthState.Ok:
+                    status.State = ThrottleState.Disabled;
+                    break;
+                default:
+                    status.State = ThrottleState.Unknown;
+                    break;
+            }
+
+            return status;
+        }
+    }
+}

src/Microsoft.Azure.WebJobs.Host/Scale/HostProcessStatus.cs

@@ -0,0 +1,23 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT License. See License.txt in the project root for license information.
+
+using System.Collections.Generic;
+
+namespace Microsoft.Azure.WebJobs.Host.Scale
+{
+    /// <summary>
+    /// Represents the health status of the host process.
+    /// </summary>
+    public class HostProcessStatus
+    {
+        /// <summary>
+        /// Gets the current health status of the host.
+        /// </summary>
+        public HostHealthState State { get; set; }
+
+        /// <summary>
+        /// Gets the collection of currently exceeded limits.
+        /// </summary>
+        public ICollection<string> ExceededLimits { get; set; }
+    }
+}

src/Microsoft.Azure.WebJobs.Host/Scale/IConcurrencyStatusRepository.cs

@@ -0,0 +1,31 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT License. See License.txt in the project root for license information.
+
+#nullable enable
+
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace Microsoft.Azure.WebJobs.Host.Scale
+{
+    /// <summary>
+    /// Provides functionality for <see cref="HostConcurrencySnapshot"/> persistence.
+    /// </summary>
+    public interface IConcurrencyStatusRepository
+    {
+        /// <summary>
+        /// Writes the specified snapshot.
+        /// </summary>
+        /// <param name="snapshot">The snapshot to persist.</param>
+        /// <param name="cancellationToken">A cancellation token.</param>
+        /// <returns>A task that completes when the write is finished.</returns>
+        Task WriteAsync(HostConcurrencySnapshot snapshot, CancellationToken cancellationToken);
+
+        /// <summary>
+        /// Reads the last host concurrency snapshot if present.
+        /// </summary>
+        /// <param name="cancellationToken">A cancellation token.</param>
+        /// <returns>A task that returns the snapshot if present, or null.</returns>
+        Task<HostConcurrencySnapshot?> ReadAsync(CancellationToken cancellationToken);
+    }
+}

src/Microsoft.Azure.WebJobs.Host/Scale/IConcurrencyThrottleManager.cs

@@ -0,0 +1,18 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT License. See License.txt in the project root for license information.
+
+namespace Microsoft.Azure.WebJobs.Host.Scale
+{
+    /// <summary>
+    /// Provides aggregated access to all registered <see cref="IConcurrencyThrottleProvider"/> instances.
+    /// </summary>
+    public interface IConcurrencyThrottleManager
+    {
+        /// <summary>
+        /// Gets a an aggregate throttle status by querying all registered <see cref="IConcurrencyThrottleProvider"/>.
+        /// instances.
+        /// </summary>
+        /// <returns>The current <see cref="ConcurrencyThrottleAggregateStatus"/>.</returns>
+        ConcurrencyThrottleAggregateStatus GetStatus();
+    }
+}

src/Microsoft.Azure.WebJobs.Host/Scale/IConcurrencyThrottleProvider.cs

@@ -0,0 +1,23 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT License. See License.txt in the project root for license information.
+
+#nullable enable
+
+using Microsoft.Extensions.Logging;
+
+namespace Microsoft.Azure.WebJobs.Host.Scale
+{
+    /// <summary>
+    /// Defines an interface for providing throttle signals to <see cref="ConcurrencyManager"/> to allow
+    /// concurrency to be dynamically adjusted at runtime based on throttle state.
+    /// </summary>
+    public interface IConcurrencyThrottleProvider
+    {
+        /// <summary>
+        /// Returns the current throttle status for this provider.
+        /// </summary>
+        /// <param name="logger">Optional logger to write throttle status to.</param>
+        /// <returns>The current <see cref="ConcurrencyThrottleStatus"/>.</returns>
+        ConcurrencyThrottleStatus GetStatus(ILogger? logger = null);
+    }
+}

src/Microsoft.Azure.WebJobs.Host/Scale/IHostProcessMonitor.cs

@@ -0,0 +1,34 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT License. See License.txt in the project root for license information.
+
+using System.Diagnostics;
+using Microsoft.Extensions.Logging;
+
+namespace Microsoft.Azure.WebJobs.Host.Scale
+{
+    /// <summary>
+    /// Defines a service that can be used to monitor process health for the
+    /// primary host and any child processes.
+    /// </summary>
+    public interface IHostProcessMonitor
+    {
+        /// <summary>
+        /// Register the specified child process for monitoring.
+        /// </summary>
+        /// <param name="process">The process to register.</param>
+        void RegisterChildProcess(Process process);
+
+        /// <summary>
+        /// Unregister the specified child process from monitoring.
+        /// </summary>
+        /// <param name="process">The process to unregister.</param>
+        void UnregisterChildProcess(Process process);
+
+        /// <summary>
+        /// Get the current host health status.
+        /// </summary>
+        /// <param name="logger">If specified, results will be logged to this logger.</param>
+        /// <returns>The status.</returns>
+        HostProcessStatus GetStatus(ILogger logger = null);
+    }
+}

src/Microsoft.Azure.WebJobs.Host/Scale/IProcessMetricsProvider.cs

@@ -0,0 +1,25 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT License. See License.txt in the project root for license information.
+
+// Type was moved from https://github.com/Azure/azure-functions-host/blob/dev/src/WebJobs.Script/Scale/IProcessMetricsProvider.cs
+
+using System;
+
+namespace Microsoft.Azure.WebJobs.Host.Scale
+{
+    /// <summary>
+    /// Provider for process level metrics.
+    /// </summary>
+    internal interface IProcessMetricsProvider
+    {
+        /// <summary>
+        /// Gets the total processor time for the process.
+        /// </summary>
+        TimeSpan TotalProcessorTime { get; }
+
+        /// <summary>
+        /// Gets the amount of private memory currently used by the process.
+        /// </summary>
+        long PrivateMemoryBytes { get; }
+    }
+}

src/Microsoft.Azure.WebJobs.Host/Scale/NullConcurrencyStatusRepository.cs

@@ -0,0 +1,23 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT License. See License.txt in the project root for license information.
+
+#nullable enable
+
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace Microsoft.Azure.WebJobs.Host.Scale
+{
+    internal class NullConcurrencyStatusRepository : IConcurrencyStatusRepository
+    {
+        public Task<HostConcurrencySnapshot?> ReadAsync(CancellationToken cancellationToken)
+        {
+            return Task.FromResult<HostConcurrencySnapshot?>(null);
+        }
+
+        public Task WriteAsync(HostConcurrencySnapshot snapshot, CancellationToken cancellationToken)
+        {
+            return Task.CompletedTask;
+        }
+    }
+}

src/Microsoft.Azure.WebJobs.Host/Scale/ProcessMonitor.cs

@@ -0,0 +1,195 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT License. See License.txt in the project root for license information.
+
+// Type was moved from https://github.com/Azure/azure-functions-host/blob/dev/src/WebJobs.Script/Scale/ProcessMonitor.cs
+
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Threading;
+
+namespace Microsoft.Azure.WebJobs.Host.Scale
+{
+    internal class ProcessMonitor : IDisposable
+    {
+        internal const int SampleHistorySize = 10;
+        internal const int DefaultSampleIntervalSeconds = 1;
+
+        private readonly List<double> _cpuLoadHistory = new List<double>();
+        private readonly List<long> _memoryUsageHistory = new List<long>();
+        private readonly int _effectiveCores;
+        private readonly Process _process;
+        private readonly bool _autoStart;
+        private readonly IProcessMetricsProvider _processMetricsProvider;
+
+        private Timer _timer;
+        private TimeSpan? _lastProcessorTime;
+        private DateTime _lastSampleTime;
+        private bool _disposed = false;
+        private TimeSpan? _interval;
+        private object _syncLock = new object();
+
+        // for mock testing only
+        internal ProcessMonitor()
+        {
+        }
+
+        public ProcessMonitor(Process process, TimeSpan? interval = null)
+            : this(process, new DefaultProcessMetricsProvider(process), interval)
+        {
+        }
+
+        public ProcessMonitor(Process process, IProcessMetricsProvider processMetricsProvider, TimeSpan? interval = null, int? effectiveCores = null, bool autoStart = true)
+        {
+            _process = process ?? throw new ArgumentNullException(nameof(process));
+            _interval = interval ?? TimeSpan.FromSeconds(DefaultSampleIntervalSeconds);
+            _processMetricsProvider = processMetricsProvider;
+            _effectiveCores = effectiveCores ?? Utility.GetEffectiveCoresCount();
+            _autoStart = autoStart;
+        }
+
+        /// <summary>
+        /// The process being monitored.
+        /// </summary>
+        public virtual Process Process => _process;
+
+        internal void EnsureTimerStarted()
+        {
+            if (_timer == null)
+            {
+                lock (_syncLock)
+                {
+                    if (_timer == null)
+                    {
+                        _timer = new Timer(OnTimer, null, TimeSpan.Zero, _interval.Value);
+                    }
+                }
+            }
+        }
+
+        public virtual ProcessStats GetStats()
+        {
+            if (_autoStart)
+            {
+                // we only start the timer on demand, ensuring that if stats aren't being queried,
+                // we're not performing unnecessary background work
+                EnsureTimerStarted();
+            }
+
+            ProcessStats stats = null;
+            lock (_syncLock)
+            {
+                stats = new ProcessStats
+                {
+                    ProcessId = _process.Id,
+                    CpuLoadHistory = _cpuLoadHistory.ToArray(),
+                    MemoryUsageHistory = _memoryUsageHistory.ToArray()
+                };
+            }
+            return stats;
+        }
+
+        private void OnTimer(object state)
+        {
+            if (_disposed || _process.HasExited)
+            {
+                return;
+            }
+
+            try
+            {
+                _process.Refresh();
+
+                SampleProcessMetrics();
+            }
+            catch
+            {
+                // Don't allow background exceptions to escape
+                // E.g. when a child process we're monitoring exits,
+                // we may process exceptions until the timer stops.
+            }
+        }
+
+        internal void SampleProcessMetrics()
+        {
+            var currSampleTime = DateTime.UtcNow;
+            var currSampleDuration = currSampleTime - _lastSampleTime;
+
+            SampleProcessMetrics(currSampleDuration);
+
+            _lastSampleTime = currSampleTime;
+        }
+
+        internal void SampleProcessMetrics(TimeSpan currSampleDuration)
+        {
+            SampleCPULoad(currSampleDuration);
+            SampleMemoryUsage();
+        }
+
+        internal void SampleCPULoad(TimeSpan currSampleDuration)
+        {
+            var currProcessorTime = _processMetricsProvider.TotalProcessorTime;
+
+            if (_lastProcessorTime != null)
+            {
+                double cpuLoad = CalculateCpuLoad(currSampleDuration, currProcessorTime, _lastProcessorTime.Value, _effectiveCores);
+                AddSample(_cpuLoadHistory, cpuLoad);
+            }
+
+            _lastProcessorTime = currProcessorTime;
+        }
+
+        internal static double CalculateCpuLoad(TimeSpan sampleDuration, TimeSpan currProcessorTime, TimeSpan lastProcessorTime, int coreCount)
+        {
+            // total processor time used for this sample across all cores
+            var currSampleProcessorTime = (currProcessorTime - lastProcessorTime).TotalMilliseconds;
+
+            // max possible processor time for this sample across all cores
+            var totalSampleProcessorTime = coreCount * sampleDuration.TotalMilliseconds;
+
+            // percentage of the max is our actual usage
+            double cpuLoad = currSampleProcessorTime / totalSampleProcessorTime;
+            cpuLoad = Math.Round(cpuLoad * 100);
+
+            // in some high load scenarios or when the host is undergoing thread pool starvation
+            // we've seen the above calculation return > 100. So we apply a Min here so our load
+            // is never above 100%.
+            return Math.Min(100, cpuLoad);
+        }
+
+        internal void SampleMemoryUsage()
+        {
+            AddSample(_memoryUsageHistory, _processMetricsProvider.PrivateMemoryBytes);
+        }
+
+        private void AddSample<T>(List<T> samples, T sample)
+        {
+            lock (_syncLock)
+            {
+                if (samples.Count == SampleHistorySize)
+                {
+                    samples.RemoveAt(0);
+                }
+                samples.Add(sample);
+            }
+        }
+
+        protected virtual void Dispose(bool disposing)
+        {
+            if (!_disposed)
+            {
+                if (disposing)
+                {
+                    _timer?.Dispose();
+                }
+
+                _disposed = true;
+            }
+        }
+
+        public void Dispose()
+        {
+            Dispose(true);
+        }
+    }
+}

src/Microsoft.Azure.WebJobs.Host/Scale/ProcessStats.cs

@@ -0,0 +1,29 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT License. See License.txt in the project root for license information.
+
+// Type was moved from https://github.com/Azure/azure-functions-host/blob/dev/src/WebJobs.Script/Scale/ProcessStats.cs
+
+using System.Collections.Generic;
+
+namespace Microsoft.Azure.WebJobs.Host.Scale
+{
+    internal class ProcessStats
+    {
+        /// <summary>
+        /// Gets or sets the ID of the process these stats are for.
+        /// </summary>
+        public int ProcessId { get; set; }
+
+        /// <summary>
+        /// Gets or sets the cpu load history collection. Each sample
+        /// is a usage percentage.
+        /// </summary>
+        public IEnumerable<double> CpuLoadHistory { get; set; }
+
+        /// <summary>
+        /// Gets or sets the memory history collection. Each sample
+        /// is in bytes.
+        /// </summary>
+        public IEnumerable<long> MemoryUsageHistory { get; set; }
+    }
+}

src/Microsoft.Azure.WebJobs.Host/Scale/ThreadPoolStarvationThrottleProvider.cs

@@ -0,0 +1,129 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT License. See License.txt in the project root for license information.
+
+#nullable enable
+
+using System;
+using System.Collections.Generic;
+using System.Collections.ObjectModel;
+using System.Threading;
+using Microsoft.Extensions.Logging;
+
+namespace Microsoft.Azure.WebJobs.Host.Scale
+{
+    /// <summary>
+    /// This throttle provider monitors for thread starvation signals. For a healthy signal, it relies on its
+    /// internal timer being run consistently. Thus it acts as a "canary in the coal mine" (https://en.wikipedia.org/wiki/Sentinel_species)
+    /// for thread pool starvation situations.
+    /// </summary>
+    internal class ThreadPoolStarvationThrottleProvider : IConcurrencyThrottleProvider, IDisposable
+    {
+        internal const string ThreadPoolStarvationThrottleName = "ThreadPoolStarvation";
+
+        private const int IntervalMS = 100;
+        private const double FailureThreshold = 0.5;
+
+        private readonly ReadOnlyCollection<string> _exceededThrottles;
+        private readonly object _syncLock = new object();
+
+        private bool _disposedValue;
+        private Timer? _timer;
+        private int _invocations;
+        private DateTime _lastCheck;
+
+        public ThreadPoolStarvationThrottleProvider()
+        {
+            _exceededThrottles = new List<string> { ThreadPoolStarvationThrottleName }.AsReadOnly();
+        }
+
+        public void OnTimer(object state)
+        {
+            lock (_syncLock)
+            {
+                _invocations++;
+            }
+        }
+
+        private void EnsureTimerStarted()
+        {
+            if (_timer == null)
+            {
+                lock (_syncLock)
+                {
+                    if (_timer == null)
+                    {
+                        _timer = new Timer(OnTimer, null, 0, IntervalMS);
+                        _lastCheck = DateTime.UtcNow;
+                    }
+                }
+            }
+        }
+
+        public ConcurrencyThrottleStatus GetStatus(ILogger? logger = null)
+        {
+            // we only start the timer on demand, ensuring that if state isn't being queried,
+            // we're not performing unnecessary background work
+            EnsureTimerStarted();
+
+            int missedCount;
+            int expectedCount;
+
+            lock (_syncLock)
+            {
+                // determine how many occurrences we expect to have had since
+                // the last check
+                TimeSpan duration = DateTime.UtcNow - _lastCheck;
+                expectedCount = (int)Math.Floor(duration.TotalMilliseconds / IntervalMS);
+
+                // calculate how many we missed
+                missedCount = Math.Max(0, expectedCount - _invocations);
+
+                _invocations = 0;
+                _lastCheck = DateTime.UtcNow;
+            }
+
+            // if the number of missed occurrences is over threshold
+            // we know things are unhealthy
+            var status = new ConcurrencyThrottleStatus
+            {
+                State = ThrottleState.Disabled
+            };
+            int failureThreshold = (int)(expectedCount * FailureThreshold);
+            if (expectedCount > 0 && missedCount > failureThreshold)
+            {
+                logger?.HostThreadStarvation();
+                status.State = ThrottleState.Enabled;
+                status.EnabledThrottles = _exceededThrottles;
+            }
+
+            return status;
+        }
+
+        // for testing only
+        internal void ResetInvocations()
+        {
+            lock (_syncLock)
+            {
+                _invocations = 0;
+            }
+        }
+
+        protected virtual void Dispose(bool disposing)
+        {
+            if (!_disposedValue)
+            {
+                if (disposing)
+                {
+                    _timer?.Dispose();
+                }
+                _disposedValue = true;
+            }
+        }
+
+        public void Dispose()
+        {
+            Dispose(disposing: true);
+            GC.SuppressFinalize(this);
+        }
+    }
+}

src/Microsoft.Azure.WebJobs.Host/Scale/ThrottleState.cs

@@ -0,0 +1,26 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT License. See License.txt in the project root for license information.
+
+namespace Microsoft.Azure.WebJobs.Host.Scale
+{
+    /// <summary>
+    /// Enumeration of possible states a <see cref="IConcurrencyThrottleProvider"/> can be in.
+    /// </summary>
+    public enum ThrottleState
+    {
+        /// <summary>
+        /// The throttle provider doesn't have enough data yet to make a throttle determination.
+        /// </summary>
+        Unknown,
+
+        /// <summary>
+        /// The throttle is enabled.
+        /// </summary>
+        Enabled,
+
+        /// <summary>
+        /// The throttle is disabled.
+        /// </summary>
+        Disabled
+    }
+}

src/Microsoft.Azure.WebJobs.Host/Utility.cs

@@ -1,7 +1,11 @@
 // Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the MIT License. See License.txt in the project root for license information.
 
+using System;
+using System.Collections.Generic;
+using System.Linq;
 using System.Reflection;
+using System.Text;
 using Microsoft.Azure.WebJobs.Description;
 using Microsoft.Azure.WebJobs.Host;
 using Microsoft.Azure.WebJobs.Host.Config;
@@ -10,8 +14,111 @@ using Microsoft.Extensions.Configuration;
 
 namespace Microsoft.Azure.WebJobs
 {
-    internal class Utility
+    internal static class Utility
     {
+        public static string GetInstanceId()
+        {
+            string instanceId = Environment.GetEnvironmentVariable(Constants.AzureWebsiteInstanceId)
+                     ?? GetStableHash(Environment.MachineName).ToString("X").PadLeft(32, '0');
+
+            return instanceId.Substring(0, Math.Min(instanceId.Length, 32));
+        }
+
+        /// <summary>
+        /// Computes a stable non-cryptographic hash
+        /// </summary>
+        /// <param name="value">The string to use for computation</param>
+        /// <returns>A stable, non-cryptographic, hash</returns>
+        internal static int GetStableHash(string value)
+        {
+            if (value == null)
+            {
+                throw new ArgumentNullException(nameof(value));
+            }
+
+            unchecked
+            {
+                int hash = 23;
+                foreach (char c in value)
+                {
+                    hash = (hash * 31) + c;
+                }
+                return hash;
+            }
+        }
+
+        public static string FlattenException(Exception ex, Func<string, string> sourceFormatter = null, bool includeSource = true)
+        {
+            StringBuilder flattenedErrorsBuilder = new StringBuilder();
+            string lastError = null;
+            sourceFormatter = sourceFormatter ?? ((s) => s);
+
+            if (ex is AggregateException)
+            {
+                ex = ex.InnerException;
+            }
+
+            do
+            {
+                StringBuilder currentErrorBuilder = new StringBuilder();
+                if (includeSource && !string.IsNullOrEmpty(ex.Source))
+                {
+                    currentErrorBuilder.AppendFormat("{0}: ", sourceFormatter(ex.Source));
+                }
+
+                currentErrorBuilder.Append(ex.Message);
+
+                if (!ex.Message.EndsWith("."))
+                {
+                    currentErrorBuilder.Append(".");
+                }
+
+                // sometimes inner exceptions are exactly the same
+                // so first check before duplicating
+                string currentError = currentErrorBuilder.ToString();
+                if (lastError == null ||
+                    string.Compare(lastError.Trim(), currentError.Trim()) != 0)
+                {
+                    if (flattenedErrorsBuilder.Length > 0)
+                    {
+                        flattenedErrorsBuilder.Append(" ");
+                    }
+                    flattenedErrorsBuilder.Append(currentError);
+                }
+
+                lastError = currentError;
+            }
+            while ((ex = ex.InnerException) != null);
+
+            return flattenedErrorsBuilder.ToString();
+        }
+
+        public static int GetEffectiveCoresCount()
+        {
+            // When not running on VMSS, the dynamic plan has some limits that mean that a given instance is using effectively a single core,
+            // so we should not use Environment.Processor count in this case.
+            var effectiveCores = (IsConsumptionSku() && !IsVMSS()) ? 1 : Environment.ProcessorCount;
+            return effectiveCores;
+        }
+
+        public static string GetWebsiteSku()
+        {
+            return Environment.GetEnvironmentVariable(Constants.AzureWebsiteSku);
+        }
+
+        public static bool IsConsumptionSku()
+        {
+            string value = GetWebsiteSku();
+            return string.Equals(value, Constants.DynamicSku, StringComparison.OrdinalIgnoreCase);
+        }
+
+        public static bool IsVMSS()
+        {
+            string value = Environment.GetEnvironmentVariable("RoleInstanceId");
+            return value != null && value.IndexOf("HostRole", StringComparison.OrdinalIgnoreCase) >= 0;
+        }
+
+
         public static IConfigurationSection GetExtensionConfigurationSection<T>(IConfiguration configuration) where T : IExtensionConfigProvider
         {
             return configuration.GetWebJobsExtensionConfigurationSection(GetExtensionConfigurationSectionName<T>());
@@ -41,6 +148,18 @@ namespace Microsoft.Azure.WebJobs
             return (functionNameAttribute != null) ? functionNameAttribute.Name : methodInfo.GetShortName();
         }
 
+        public static IEnumerable<TElement> TakeLastN<TElement>(this IEnumerable<TElement> source, int take)
+        {
+            if (take < 0)
+            {
+                throw new ArgumentOutOfRangeException(nameof(take));
+            }
+
+            int skipCount = Math.Max(0, source.Count() - take);
+
+            return source.Skip(skipCount).Take(take);
+        }
+
         private static string GetExtensionConfigurationSectionName<TExtension>() where TExtension : IExtensionConfigProvider
         {
             var type = typeof(TExtension).GetTypeInfo();

src/Microsoft.Azure.WebJobs.Host/WebJobs.Host.csproj

@@ -3,6 +3,7 @@
   <Import Project="..\..\build\common.props" />
   <PropertyGroup>
     <TargetFramework>netstandard2.0</TargetFramework>
+    <LangVersion>latest</LangVersion>
     <PackageId>Microsoft.Azure.WebJobs</PackageId>
     <Description>This package contains the runtime assemblies for Microsoft.Azure.WebJobs.Host. It also adds rich diagnostics capabilities which makes it easier to monitor the WebJobs in the dashboard. For more information, please visit http://go.microsoft.com/fwlink/?LinkID=320971</Description>
     <AssemblyName>Microsoft.Azure.WebJobs.Host</AssemblyName>

test/Benchmarks/Benchmarks.csproj

@@ -1,9 +1,9 @@
-<Project Sdk="Microsoft.NET.Sdk">
+<Project Sdk="Microsoft.NET.Sdk">
   <Import Project="..\..\build\common.props" />
 
   <PropertyGroup>
     <OutputType>Exe</OutputType>
-    <TargetFramework>netcoreapp2.1</TargetFramework>
+    <TargetFramework>netcoreapp3.1</TargetFramework>
     <StartupObject>Benchmarks.Program</StartupObject>
   </PropertyGroup>
 

test/Microsoft.Azure.WebJobs.Host.EndToEndTests/ApplicationInsights/ApplicationInsightsEndToEndTests.cs

@@ -127,7 +127,7 @@ namespace Microsoft.Azure.WebJobs.Host.EndToEndTests
                 await host.GetJobHost().CallAsync(methodInfo, new { input = "function input" });
                 await host.StopAsync();
 
-                Assert.Equal(16, _channel.Telemetries.Count);
+                Assert.Equal(17, _channel.Telemetries.Count);
 
                 // Validate the request
                 RequestTelemetry request = _channel.Telemetries
@@ -150,19 +150,20 @@ namespace Microsoft.Azure.WebJobs.Host.EndToEndTests
                 string expectedOperationId = request.Context.Operation.Id;
 
                 ValidateTrace(telemetries[0], "ApplicationInsightsLoggerOptions", "Microsoft.Azure.WebJobs.Hosting.OptionsLoggingService");
-                ValidateTrace(telemetries[1], "Executed ", expectedFunctionCategory, testName, invocationId, expectedOperationId, request.Id);
-                ValidateTrace(telemetries[2], "Executing ", expectedFunctionCategory, testName, invocationId, expectedOperationId, request.Id);
-                ValidateTrace(telemetries[3], "Found the following functions:\r\n", LogCategories.Startup);
-                ValidateTrace(telemetries[4], "FunctionResultAggregatorOptions", "Microsoft.Azure.WebJobs.Hosting.OptionsLoggingService");
-                ValidateTrace(telemetries[5], "Job host started", LogCategories.Startup);
-                ValidateTrace(telemetries[6], "Job host stopped", LogCategories.Startup);
-                ValidateTrace(telemetries[7], "Logger", expectedFunctionUserCategory, testName, invocationId, expectedOperationId, request.Id, hasCustomScope: true);
-                ValidateTrace(telemetries[8], "LoggerFilterOptions", "Microsoft.Azure.WebJobs.Hosting.OptionsLoggingService");
+                ValidateTrace(telemetries[1], "ConcurrencyOptions", "Microsoft.Azure.WebJobs.Hosting.OptionsLoggingService");
+                ValidateTrace(telemetries[2], "Executed ", expectedFunctionCategory, testName, invocationId, expectedOperationId, request.Id);
+                ValidateTrace(telemetries[3], "Executing ", expectedFunctionCategory, testName, invocationId, expectedOperationId, request.Id);
+                ValidateTrace(telemetries[4], "Found the following functions:\r\n", LogCategories.Startup);
+                ValidateTrace(telemetries[5], "FunctionResultAggregatorOptions", "Microsoft.Azure.WebJobs.Hosting.OptionsLoggingService");
+                ValidateTrace(telemetries[6], "Job host started", LogCategories.Startup);
+                ValidateTrace(telemetries[7], "Job host stopped", LogCategories.Startup);
+                ValidateTrace(telemetries[8], "Logger", expectedFunctionUserCategory, testName, invocationId, expectedOperationId, request.Id, hasCustomScope: true);
                 ValidateTrace(telemetries[9], "LoggerFilterOptions", "Microsoft.Azure.WebJobs.Hosting.OptionsLoggingService");
-                ValidateTrace(telemetries[10], "SingletonOptions", "Microsoft.Azure.WebJobs.Hosting.OptionsLoggingService");
-                ValidateTrace(telemetries[11], "Starting JobHost", "Microsoft.Azure.WebJobs.Hosting.JobHostService");
-                ValidateTrace(telemetries[12], "Stopping JobHost", "Microsoft.Azure.WebJobs.Hosting.JobHostService");
-                ValidateTrace(telemetries[13], "Trace", expectedFunctionUserCategory, testName, invocationId, expectedOperationId, request.Id);
+                ValidateTrace(telemetries[10], "LoggerFilterOptions", "Microsoft.Azure.WebJobs.Hosting.OptionsLoggingService");
+                ValidateTrace(telemetries[11], "SingletonOptions", "Microsoft.Azure.WebJobs.Hosting.OptionsLoggingService");
+                ValidateTrace(telemetries[12], "Starting JobHost", "Microsoft.Azure.WebJobs.Hosting.JobHostService");
+                ValidateTrace(telemetries[13], "Stopping JobHost", "Microsoft.Azure.WebJobs.Hosting.JobHostService");
+                ValidateTrace(telemetries[14], "Trace", expectedFunctionUserCategory, testName, invocationId, expectedOperationId, request.Id);
 
                 // We should have 1 custom metric.
                 MetricTelemetry metric = _channel.Telemetries
@@ -204,7 +205,7 @@ namespace Microsoft.Azure.WebJobs.Host.EndToEndTests
                 await Assert.ThrowsAsync<FunctionInvocationException>(() => host.GetJobHost().CallAsync(methodInfo, new { input = "function input" }));
                 await host.StopAsync();
 
-                Assert.Equal(19, _channel.Telemetries.Count);
+                Assert.Equal(20, _channel.Telemetries.Count);
 
                 // Validate the request
                 RequestTelemetry request = _channel.Telemetries
@@ -227,20 +228,21 @@ namespace Microsoft.Azure.WebJobs.Host.EndToEndTests
                 string expectedOperationId = request.Context.Operation.Id;
 
                 ValidateTrace(telemetries[0], "ApplicationInsightsLoggerOptions", "Microsoft.Azure.WebJobs.Hosting.OptionsLoggingService");
-                ValidateTrace(telemetries[1], "Error", expectedFunctionUserCategory, testName, invocationId, expectedOperationId, request.Id, expectedLogLevel: LogLevel.Error);
-                ValidateTrace(telemetries[2], "Executed", expectedFunctionCategory, testName, invocationId, expectedOperationId, request.Id, expectedLogLevel: LogLevel.Error);
-                ValidateTrace(telemetries[3], "Executing", expectedFunctionCategory, testName, invocationId, expectedOperationId, request.Id);
-                ValidateTrace(telemetries[4], "Found the following functions:\r\n", LogCategories.Startup);
-                ValidateTrace(telemetries[5], "FunctionResultAggregatorOptions", "Microsoft.Azure.WebJobs.Hosting.OptionsLoggingService");
-                ValidateTrace(telemetries[6], "Job host started", LogCategories.Startup);
-                ValidateTrace(telemetries[7], "Job host stopped", LogCategories.Startup);
-                ValidateTrace(telemetries[8], "Logger", expectedFunctionUserCategory, testName, invocationId, expectedOperationId, request.Id, hasCustomScope: true);
-                ValidateTrace(telemetries[9], "LoggerFilterOptions", "Microsoft.Azure.WebJobs.Hosting.OptionsLoggingService");
+                ValidateTrace(telemetries[1], "ConcurrencyOptions", "Microsoft.Azure.WebJobs.Hosting.OptionsLoggingService");
+                ValidateTrace(telemetries[2], "Error", expectedFunctionUserCategory, testName, invocationId, expectedOperationId, request.Id, expectedLogLevel: LogLevel.Error);
+                ValidateTrace(telemetries[3], "Executed", expectedFunctionCategory, testName, invocationId, expectedOperationId, request.Id, expectedLogLevel: LogLevel.Error);
+                ValidateTrace(telemetries[4], "Executing", expectedFunctionCategory, testName, invocationId, expectedOperationId, request.Id);
+                ValidateTrace(telemetries[5], "Found the following functions:\r\n", LogCategories.Startup);
+                ValidateTrace(telemetries[6], "FunctionResultAggregatorOptions", "Microsoft.Azure.WebJobs.Hosting.OptionsLoggingService");
+                ValidateTrace(telemetries[7], "Job host started", LogCategories.Startup);
+                ValidateTrace(telemetries[8], "Job host stopped", LogCategories.Startup);
+                ValidateTrace(telemetries[9], "Logger", expectedFunctionUserCategory, testName, invocationId, expectedOperationId, request.Id, hasCustomScope: true);
                 ValidateTrace(telemetries[10], "LoggerFilterOptions", "Microsoft.Azure.WebJobs.Hosting.OptionsLoggingService");
-                ValidateTrace(telemetries[11], "SingletonOptions", "Microsoft.Azure.WebJobs.Hosting.OptionsLoggingService");
-                ValidateTrace(telemetries[12], "Starting JobHost", "Microsoft.Azure.WebJobs.Hosting.JobHostService");
-                ValidateTrace(telemetries[13], "Stopping JobHost", "Microsoft.Azure.WebJobs.Hosting.JobHostService");
-                ValidateTrace(telemetries[14], "Trace", expectedFunctionUserCategory, testName, invocationId, expectedOperationId, request.Id);
+                ValidateTrace(telemetries[11], "LoggerFilterOptions", "Microsoft.Azure.WebJobs.Hosting.OptionsLoggingService");
+                ValidateTrace(telemetries[12], "SingletonOptions", "Microsoft.Azure.WebJobs.Hosting.OptionsLoggingService");
+                ValidateTrace(telemetries[13], "Starting JobHost", "Microsoft.Azure.WebJobs.Hosting.JobHostService");
+                ValidateTrace(telemetries[14], "Stopping JobHost", "Microsoft.Azure.WebJobs.Hosting.JobHostService");
+                ValidateTrace(telemetries[15], "Trace", expectedFunctionUserCategory, testName, invocationId, expectedOperationId, request.Id);
 
                 // Validate the exception
                 ExceptionTelemetry[] exceptions = _channel.Telemetries
@@ -256,7 +258,7 @@ namespace Microsoft.Azure.WebJobs.Host.EndToEndTests
 
         [Theory]
         [InlineData(LogLevel.None, 0, 0)]
-        [InlineData(LogLevel.Information, 5, 10)] // 5 start, 2 stop, 4x traces per request, 1x requests
+        [InlineData(LogLevel.Information, 5, 11)] // 5 start, 2 stop, 4x traces per request, 1x requests
         [InlineData(LogLevel.Warning, 2, 0)] // 2x warning trace per request
         public async Task QuickPulse_Works_EvenIfFiltered(LogLevel defaultLevel, int tracesPerRequest, int additionalTraces)
         {

test/Microsoft.Azure.WebJobs.Host.EndToEndTests/AsyncChainEndToEndTests.cs

@@ -172,6 +172,14 @@ namespace Microsoft.Azure.WebJobs.Host.EndToEndTests
                     "  \"FlushTimeout\": \"00:00:30\",",
                     "  \"IsEnabled\": false",
                     "}",
+                    "ConcurrencyOptions",
+                    "{",
+                    "  \"DynamicConcurrencyEnabled\": false",
+                    "  \"MaximumFunctionConcurrency\": 500",
+                    "  \"TotalAvailableMemoryBytes\": -1",
+                    "  \"CPUThreshold\": 0.8",
+                    "  \"SnapshotPersistenceEnabled\": true",
+                    "}",
                     "BlobsOptions",
                     "{",
                     "  \"CentralizedPoisonQueue\": false",

test/Microsoft.Azure.WebJobs.Host.EndToEndTests/BlobStorageConcurrencyStatusRepositoryTests.cs

@@ -0,0 +1,145 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT License. See License.txt in the project root for license information.
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.Azure.Storage.Blob;
+using Microsoft.Azure.WebJobs.Host.Executors;
+using Microsoft.Azure.WebJobs.Host.Scale;
+using Microsoft.Azure.WebJobs.Host.TestCommon;
+using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.Logging;
+using Moq;
+using Newtonsoft.Json;
+using Xunit;
+
+namespace Microsoft.Azure.WebJobs.Host.EndToEndTests
+{
+    [Trait(TestTraits.CategoryTraitName, TestTraits.DynamicConcurrency)]
+    public class BlobStorageConcurrencyStatusRepositoryTests
+    {
+        private const string TestHostId = "test123";
+        private readonly BlobStorageConcurrencyStatusRepository _repository;
+        private readonly LoggerFactory _loggerFactory;
+        private readonly TestLoggerProvider _loggerProvider;
+        private readonly HostConcurrencySnapshot _testSnapshot;
+        private readonly Mock<IHostIdProvider> _mockHostIdProvider;
+
+        public BlobStorageConcurrencyStatusRepositoryTests()
+        {
+            _testSnapshot = new HostConcurrencySnapshot
+            {
+                NumberOfCores = 4,
+                Timestamp = DateTime.UtcNow
+            };
+            _testSnapshot.FunctionSnapshots = new Dictionary<string, FunctionConcurrencySnapshot>
+            {
+                { "function0", new FunctionConcurrencySnapshot { Concurrency = 5 } },
+                { "function1", new FunctionConcurrencySnapshot { Concurrency = 10 } },
+                { "function2", new FunctionConcurrencySnapshot { Concurrency = 15 } }
+            };
+
+            _loggerFactory = new LoggerFactory();
+            _loggerProvider = new TestLoggerProvider();
+            _loggerFactory.AddProvider(_loggerProvider);
+
+            IConfiguration configuration = new ConfigurationBuilder().AddEnvironmentVariables().Build();
+            _mockHostIdProvider = new Mock<IHostIdProvider>(MockBehavior.Strict);
+            _mockHostIdProvider.Setup(p => p.GetHostIdAsync(CancellationToken.None)).ReturnsAsync(TestHostId);
+
+            _repository = new BlobStorageConcurrencyStatusRepository(configuration, _mockHostIdProvider.Object, _loggerFactory);
+        }
+
+        [Fact]
+        public async Task GetContainerAsync_ReturnsExpectedContainer()
+        {
+            CloudBlobContainer container = await _repository.GetContainerAsync(CancellationToken.None);
+            Assert.Equal(HostContainerNames.Hosts, container.Name);
+        }
+
+        [Fact]
+        public async Task GetBlobPathAsync_ReturnsExpectedPath()
+        {
+            string path = await _repository.GetBlobPathAsync(CancellationToken.None);
+
+            Assert.Equal($"concurrency/{TestHostId}/concurrencyStatus.json", path);
+        }
+
+        [Fact]
+        public async Task WriteAsync_WritesExpectedBlob()
+        {
+            await DeleteTestBlobsAsync();
+
+            var path = await _repository.GetBlobPathAsync(CancellationToken.None);
+            CloudBlobContainer container = await _repository.GetContainerAsync(CancellationToken.None);
+            CloudBlockBlob blob = container.GetBlockBlobReference(path);
+            bool exists = await blob.ExistsAsync();
+            Assert.False(exists);
+
+            await _repository.WriteAsync(_testSnapshot, CancellationToken.None);
+
+            exists = await blob.ExistsAsync();
+            Assert.True(exists);
+
+            var content = await blob.DownloadTextAsync();
+            var result = JsonConvert.DeserializeObject<HostConcurrencySnapshot>(content);
+
+            Assert.True(_testSnapshot.Equals(result));
+        }
+
+        [Fact]
+        public async Task ReadAsyncAsync_ReadsExpectedBlob()
+        {
+            await DeleteTestBlobsAsync();
+
+            string path = await _repository.GetBlobPathAsync(CancellationToken.None);
+            CloudBlobContainer container = await _repository.GetContainerAsync(CancellationToken.None);
+            CloudBlockBlob blob = container.GetBlockBlobReference(path);
+
+            string content = JsonConvert.SerializeObject(_testSnapshot);
+            await blob.UploadTextAsync(content);
+
+            var result = await _repository.ReadAsync(CancellationToken.None);
+
+            Assert.True(_testSnapshot.Equals(result));
+        }
+
+        [Fact]
+        public async Task ReadAsyncAsync_NoSnapshot_ReturnsNull()
+        {
+            await DeleteTestBlobsAsync();
+
+            var result = await _repository.ReadAsync(CancellationToken.None);
+
+            Assert.Null(result);
+        }
+
+        [Fact]
+        public async Task NoStorageConnection_HandledGracefully()
+        {
+            IConfiguration configuration = new ConfigurationBuilder().Build();
+            var localRepository = new BlobStorageConcurrencyStatusRepository(configuration, _mockHostIdProvider.Object, _loggerFactory);
+
+            var container = await localRepository.GetContainerAsync(CancellationToken.None);
+            Assert.Null(container);
+
+            await localRepository.WriteAsync(new HostConcurrencySnapshot(), CancellationToken.None);
+
+            var snapshot = await localRepository.ReadAsync(CancellationToken.None);
+            Assert.Null(snapshot);
+        }
+
+        private async Task DeleteTestBlobsAsync()
+        {
+            CloudBlobContainer container = await _repository.GetContainerAsync(CancellationToken.None);
+            var blobs = container.ListBlobs($"concurrency/{TestHostId}", useFlatBlobListing: true);
+            foreach (var blob in blobs.Cast<CloudBlockBlob>())
+            {
+                await blob.DeleteAsync();
+            }
+        }
+    }
+}

test/Microsoft.Azure.WebJobs.Host.EndToEndTests/DynamicConcurrencyEndToEndTests.cs

@@ -0,0 +1,754 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT License. See License.txt in the project root for license information.
+
+using System;
+using System.Collections.Concurrent;
+using System.Collections.Generic;
+using System.Linq;
+using System.Reflection;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.Azure.WebJobs.Description;
+using Microsoft.Azure.WebJobs.Host.Bindings;
+using Microsoft.Azure.WebJobs.Host.Config;
+using Microsoft.Azure.WebJobs.Host.Executors;
+using Microsoft.Azure.WebJobs.Host.Listeners;
+using Microsoft.Azure.WebJobs.Host.Protocols;
+using Microsoft.Azure.WebJobs.Host.Scale;
+using Microsoft.Azure.WebJobs.Host.TestCommon;
+using Microsoft.Azure.WebJobs.Host.Triggers;
+using Microsoft.Azure.WebJobs.Logging;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Hosting;
+using Microsoft.Extensions.Logging;
+using Xunit;
+
+namespace Microsoft.Azure.WebJobs.Host.EndToEndTests
+{
+    [Trait(TestTraits.CategoryTraitName, TestTraits.DynamicConcurrency)]
+    public class DynamicConcurrencyEndToEndTests
+    {
+        private const string TestHostId = "e2etesthost";
+
+        public DynamicConcurrencyEndToEndTests()
+        {
+            TestEventSource.Reset();
+            TestJobs.InvokeCount = 0;
+        }
+
+        [Fact]
+        public async Task DynamicConcurrencyEnabled_HighCpu_Throttles()
+        {
+            string functionName = nameof(TestJobs.ConcurrencyTest_HighCpu);
+            string functionId = GetFunctionId(functionName);
+            int eventCount = 100;
+            AddTestEvents("concurrency-work-items-1", eventCount);
+
+            // force an initial concurrency to ensure throttles are hit relatively quickly
+            IHost host = CreateTestJobHost<TestJobs>();
+            var concurrencyManager = host.GetServiceOrNull<ConcurrencyManager>();
+            int initialConcurrency = 5;
+            ApplyTestSnapshot(concurrencyManager, highCpuConcurrency: initialConcurrency);
+
+            host.Start();
+
+            await TestHelpers.Await(() =>
+            {
+                // wait until we've processed some events and we've throttled down
+                var logs = GetConcurrencyLogs(host);
+                var concurrencyDecreaseLogs = logs.Where(p => p.FormattedMessage.Contains($"{functionId} Decreasing concurrency")).ToArray();
+                var throttleLogs = logs.Where(p => p.Level == LogLevel.Warning &&
+                    (p.FormattedMessage.Contains("Host CPU threshold exceeded") || p.FormattedMessage.Contains("thread pool starvation detected"))).ToArray();
+                bool complete = TestJobs.InvokeCount > 5 && throttleLogs.Length > 0 && concurrencyDecreaseLogs.Length > 0;
+
+                return Task.FromResult(complete);
+            });
+
+            await host.StopAsync();
+
+            var functionSnapshot = GetFunctionSnapshotOrNull(concurrencyManager, functionName);
+
+            // verify concurrency was limited
+            Assert.True(functionSnapshot.Concurrency <= initialConcurrency);
+
+            host.Dispose();
+        }
+
+        [Fact]
+        public async Task DynamicConcurrencyEnabled_HighMemory_MemoryThrottleEnabled_Throttles()
+        {
+            string functionName = nameof(TestJobs.ConcurrencyTest_HighMemory);
+            string functionId = GetFunctionId(functionName);
+            int eventCount = 100;
+            AddTestEvents("concurrency-work-items-3", eventCount);
+
+            // enable the memory throttle with a low value so it'll be enabled
+            int totalAvailableMemoryBytes = 500 * 1024 * 1024;
+
+            // force an initial concurrency to ensure throttles are hit relatively quickly
+            IHost host = CreateTestJobHost<TestJobs>(totalAvailableMemoryBytes: totalAvailableMemoryBytes);
+            var concurrencyManager = host.GetServiceOrNull<ConcurrencyManager>();
+            int initialConcurrency = 5;
+            ApplyTestSnapshot(concurrencyManager, highMemoryConcurrency: initialConcurrency);
+
+            host.Start();
+
+            await TestHelpers.Await(() =>
+            {
+                // wait until we've processed some events and we've throttled down
+                var logs = GetConcurrencyLogs(host);
+                var concurrencyDecreaseLogs = logs.Where(p => p.FormattedMessage.Contains($"{functionId} Decreasing concurrency")).ToArray();
+                var throttleLogs = logs.Where(p => p.Level == LogLevel.Warning && p.FormattedMessage.Contains("Host memory threshold exceeded")).ToArray();
+                bool complete = TestJobs.InvokeCount > 5 && throttleLogs.Length > 0 && concurrencyDecreaseLogs.Length > 0;
+
+                return Task.FromResult(complete);
+            }, timeout: 90 * 1000);
+
+            await host.StopAsync();
+
+            host.Dispose();
+        }
+
+        [Fact]
+        public async Task DynamicConcurrencyEnabled_HighMemory_MemoryThrottleDisabled_Throttles()
+        {
+            string functionName = nameof(TestJobs.ConcurrencyTest_HighMemory);
+            string functionId = GetFunctionId(functionName);
+            int eventCount = 100;
+            AddTestEvents("concurrency-work-items-3", eventCount);
+
+            // force an initial concurrency to ensure throttles are hit relatively quickly
+            IHost host = CreateTestJobHost<TestJobs>();
+            var concurrencyManager = host.GetServiceOrNull<ConcurrencyManager>();
+            int initialConcurrency = 15;
+            ApplyTestSnapshot(concurrencyManager, highMemoryConcurrency: initialConcurrency);
+
+            host.Start();
+
+            await TestHelpers.Await(() =>
+            {
+                // wait until we've processed some events and we've throttled down
+                // in high memory situations, we see CPU/ThreadStarvation throttles fire
+                var logs = GetConcurrencyLogs(host);
+                var concurrencyDecreaseLogs = logs.Where(p => p.FormattedMessage.Contains($"{functionId} Decreasing concurrency")).ToArray();
+                var throttleLogs = logs.Where(p => p.Level == LogLevel.Warning).ToArray();
+                bool complete = throttleLogs.Length > 5 && concurrencyDecreaseLogs.Length > 0;
+
+                return Task.FromResult(complete);
+            });
+
+            await host.StopAsync();
+
+            host.Dispose();
+        }
+
+        [Fact]
+        public async Task DynamicConcurrencyEnabled_Lightweight_NoThrottling()
+        {
+            FunctionConcurrencySnapshot snapshot = null;
+            string functionName = nameof(TestJobs.ConcurrencyTest_Lightweight);
+            int targetConcurrency = 10;
+            int eventCount = 3000;
+            AddTestEvents("concurrency-work-items-2", eventCount);
+
+            IHost host = CreateTestJobHost<TestJobs>();
+            await WaitForQuietHostAsync(host);
+
+            host.Start();
+
+            var concurrencyManager = host.GetServiceOrNull<ConcurrencyManager>();
+
+            await TestHelpers.Await(() =>
+            {
+                // wait until we've increased concurrency several times and we've processed
+                // many events
+                snapshot = GetFunctionSnapshotOrNull(concurrencyManager, functionName);
+                return Task.FromResult(snapshot?.Concurrency >= targetConcurrency && TestJobs.InvokeCount > 250);
+            });
+
+            await host.StopAsync();
+
+            var logs = GetConcurrencyLogs(host);
+            var warningsOrErrors = logs.Where(p => p.Level > LogLevel.Information).ToArray();
+
+            var functionSnapshot = GetFunctionSnapshotOrNull(concurrencyManager, functionName);
+
+            // verify no warnings/errors and also that we've increased concurrency
+            Assert.True(warningsOrErrors.Length == 0, string.Join(Environment.NewLine, warningsOrErrors.Take(5).Select(p => p.FormattedMessage)));
+            Assert.True(functionSnapshot.Concurrency >= targetConcurrency);
+
+            host.Dispose();
+        }
+
+        [Fact]
+        public async Task MultipleFunctionsEnabled_Succeeds()
+        {
+            // add events for the lightweight and high cpu functions
+            AddTestEvents("concurrency-work-items-1", 500);
+            AddTestEvents("concurrency-work-items-2", 5000);
+
+            // force an initial concurrency to ensure throttles are hit relatively quickly
+            IHost host = CreateTestJobHost<TestJobs>();
+            var concurrencyManager = host.GetServiceOrNull<ConcurrencyManager>();
+            ApplyTestSnapshot(concurrencyManager, highCpuConcurrency: 10);
+
+            host.Start();
+
+            // just run for a bit to exercise ConcurrencyManager 
+            await Task.Delay(TimeSpan.FromSeconds(10));
+
+            await host.StopAsync();
+
+            // verify no errors, and that we do have some throttle warnings
+            var logs = GetConcurrencyLogs(host);
+            Assert.Empty(logs.Where(p => p.Level == LogLevel.Error));
+            Assert.NotEmpty(logs.Where(p => p.Level == LogLevel.Warning));
+
+            // When run for a longer period of time, concurrency for the lightweight function can
+            // break away while the heavier function stays limited.
+            // That can take longer than we want to allow this test to run for so we can't perform
+            // those verification checks here.
+            host.Dispose();
+        }
+
+        [Fact]
+        public async Task SnapshotsEnabled_AppliesSnapshotOnStartup()
+        {
+            int eventCount = 500;
+            AddTestEvents("concurrency-work-items-2", eventCount);
+
+            IHost host = CreateTestJobHost<TestJobs>(snapshotPersistenceEnabled: true);
+            await WaitForQuietHostAsync(host);
+
+            var concurrencyManager = host.GetServiceOrNull<ConcurrencyManager>();
+            var repository = host.Services.GetServices<IConcurrencyStatusRepository>().Last();
+            int initialConcurrency = 50;
+            var snapshot = CreateTestSnapshot(lightweightConcurrency: initialConcurrency);
+            await repository.WriteAsync(snapshot, CancellationToken.None);
+
+            host.Start();
+
+            await TestHelpers.Await(() =>
+            {
+                // wait for all events to be processed
+                return Task.FromResult(TestJobs.InvokeCount >= eventCount);
+            });
+
+            await host.StopAsync();
+
+            // ensure no warnings or errors
+            var logs = GetConcurrencyLogs(host);
+            var warningsOrErrors = logs.Where(p => p.Level > LogLevel.Information).ToArray();
+            Assert.Empty(warningsOrErrors);
+
+            // ensure the snapshot was applied on startup
+            string functionId = GetFunctionId(nameof(TestJobs.ConcurrencyTest_Lightweight));
+            var log = logs.SingleOrDefault(p => p.FormattedMessage == $"Applying status snapshot for function {functionId} (Concurrency: {initialConcurrency})");
+            Assert.NotNull(log);
+
+            host.Dispose();
+        }
+
+        private void AddTestEvents(string source, int count)
+        {
+            List<TestEvent> events = new List<TestEvent>();
+            for (int i = 0; i < count; i++)
+            {
+                events.Add(new TestEvent { ID = Guid.NewGuid(), Data = $"TestEvent{i}" });
+            }
+            TestEventSource.AddEvents(source, events);
+        }
+
+        private async Task WaitForQuietHostAsync(IHost host)
+        {
+            // some tests are expecting no throttle warnings to happen
+            // to make tests more stable, wait until all throttles are
+            // disabled before starting the test
+            // e.g. a previous test may have driven CPU up so we need to
+            // wait for it to drop back down
+            var throttleManager = host.GetServiceOrNull<IConcurrencyThrottleManager>();
+            await TestHelpers.Await(() =>
+            {
+                return throttleManager.GetStatus().State == ThrottleState.Disabled;
+            });
+            host.GetTestLoggerProvider().ClearAllLogMessages();
+        }
+        private IHost CreateTestJobHost<TProg>(int totalAvailableMemoryBytes = -1, bool snapshotPersistenceEnabled = false, Action<IHostBuilder> extraConfig = null)
+        {
+            var hostBuilder = new HostBuilder()
+                .ConfigureDefaultTestHost<TProg>(b =>
+                {
+                    b.UseHostId(TestHostId)
+                        .AddAzureStorage()
+                        .AddExtension<TestTriggerAttributeBindingProvider>();
+
+                    RuntimeStorageWebJobsBuilderExtensions.AddAzureStorageCoreServices(b);
+
+                    b.Services.AddOptions<ConcurrencyOptions>().Configure(options =>
+                    {
+                        options.DynamicConcurrencyEnabled = true;
+                        options.SnapshotPersistenceEnabled = snapshotPersistenceEnabled;
+                        options.MaximumFunctionConcurrency = 500;
+
+                        // configure memory limit if specified
+                        // memory throttle is disabled by default unless a value > 0 is specified here
+                        if (totalAvailableMemoryBytes > 0)
+                        {
+                            options.TotalAvailableMemoryBytes = (long)totalAvailableMemoryBytes;
+                        }
+                    });
+                })
+                .ConfigureServices(services =>
+                {
+                })
+                .ConfigureLogging((context, b) =>
+                {
+                    b.SetMinimumLevel(LogLevel.Information);
+
+                    b.AddFilter((category, level) =>
+                    {
+                        if (category == LogCategories.Concurrency)
+                        {
+                            return true;
+                        }
+                        return level >= LogLevel.Information;
+                    });
+                });
+
+            extraConfig?.Invoke(hostBuilder);
+
+            IHost host = hostBuilder.Build();
+
+            return host;
+        }
+
+        private FunctionConcurrencySnapshot GetFunctionSnapshotOrNull(ConcurrencyManager concurrencyManager, string functionName)
+        {
+            var snapshot = concurrencyManager.GetSnapshot();
+
+            string functionId = GetFunctionId(functionName);
+            snapshot.FunctionSnapshots.TryGetValue(functionId, out FunctionConcurrencySnapshot functionSnapshot);
+
+            return functionSnapshot;
+        }
+
+        private HostConcurrencySnapshot CreateTestSnapshot(int lightweightConcurrency = 1, int highCpuConcurrency = 1, int highMemoryConcurrency = 1)
+        {
+            var snapshot = new HostConcurrencySnapshot
+            {
+                NumberOfCores = Utility.GetEffectiveCoresCount(),
+                FunctionSnapshots = new Dictionary<string, FunctionConcurrencySnapshot>()
+            };
+
+            snapshot.FunctionSnapshots.Add(GetFunctionId(nameof(TestJobs.ConcurrencyTest_Lightweight)), new FunctionConcurrencySnapshot
+            {
+                Concurrency = lightweightConcurrency
+            });
+            snapshot.FunctionSnapshots.Add(GetFunctionId(nameof(TestJobs.ConcurrencyTest_HighCpu)), new FunctionConcurrencySnapshot
+            {
+                Concurrency = highCpuConcurrency
+            });
+            snapshot.FunctionSnapshots.Add(GetFunctionId(nameof(TestJobs.ConcurrencyTest_HighMemory)), new FunctionConcurrencySnapshot
+            {
+                Concurrency = highMemoryConcurrency
+            });
+
+            return snapshot;
+        }
+
+        private void ApplyTestSnapshot(ConcurrencyManager concurrencyManager, int lightweightConcurrency = 1, int highCpuConcurrency = 1, int highMemoryConcurrency = 1)
+        {
+            var snapshot = CreateTestSnapshot(lightweightConcurrency, highCpuConcurrency, highMemoryConcurrency);
+            concurrencyManager.ApplySnapshot(snapshot);
+        }
+
+        private string GetFunctionId(string methodName)
+        {
+            MethodInfo methodInfo = typeof(TestJobs).GetMethod(methodName, BindingFlags.Public | BindingFlags.Static);
+            return $"{methodInfo.DeclaringType.FullName}.{methodInfo.Name}";
+        }
+
+        private static LogMessage[] GetConcurrencyLogs(IHost host)
+        {
+            var logProvider = host.GetTestLoggerProvider();
+            var logs = logProvider.GetAllLogMessages().Where(p => p.Category == LogCategories.Concurrency).ToArray();
+
+            return logs;
+        }
+
+        /// <summary>
+        /// Test in memory event source.
+        /// </summary>
+        public static class TestEventSource
+        {
+            public static ConcurrentDictionary<string, ConcurrentQueue<TestEvent>> Events = new ConcurrentDictionary<string, ConcurrentQueue<TestEvent>>(StringComparer.OrdinalIgnoreCase);
+            private static object _syncLock = new object();
+
+            public static void AddEvents(string source, List<TestEvent> events)
+            {
+                var existingEvents = Events.GetOrAdd(source, s =>
+                {
+                    return new ConcurrentQueue<TestEvent>();
+                });
+
+                foreach (var evt in events)
+                {
+                    existingEvents.Enqueue(evt);
+                }
+            }
+
+            public static List<TestEvent> GetEvents(string source, int count)
+            {
+                List<TestEvent> result = new List<TestEvent>();
+
+                if (Events.TryGetValue(source, out ConcurrentQueue<TestEvent> events))
+                {
+                    for (int i = 0; i < count; i++)
+                    {
+                        if (events.TryDequeue(out TestEvent evt))
+                        {
+                            result.Add(evt);
+                        }
+                        else
+                        {
+                            break;
+                        }
+                    }
+                }
+
+                return result;
+            }
+
+            public static void Reset()
+            {
+                Events.Clear();
+            }
+        }
+
+        public class TestEvent
+        {
+            public Guid ID { get; set; }
+
+            public string Data { get; set; }
+        }
+
+        /// <summary>
+        /// Set of test functions that exercise the various host throttles.
+        /// </summary>
+        public class TestJobs
+        {
+            private static readonly Random _rand = new Random();
+            public static int InvokeCount = 0;
+
+            public static async Task ConcurrencyTest_HighCpu([TestTrigger("concurrency-work-items-1")] TestEvent evt, ILogger log)
+            {
+                log.LogInformation($"C# Test trigger function processed: {evt.Data}");
+
+                await GenerateLoadAllCoresAsync();
+
+                Interlocked.Increment(ref InvokeCount);
+            }
+
+            public static async Task ConcurrencyTest_Lightweight([TestTrigger("concurrency-work-items-2")] TestEvent evt, ILogger log)
+            {
+                log.LogInformation($"C# Test trigger function processed: {evt.Data}");
+
+                await Task.Delay(50);
+
+                Interlocked.Increment(ref InvokeCount);
+            }
+
+            public static async Task ConcurrencyTest_HighMemory([TestTrigger("concurrency-work-items-3")] TestEvent evt, ILogger log)
+            {
+                log.LogInformation($"C# Test trigger function processed: {evt.Data}");
+
+                // allocate a large chunk of memory
+                int numMBs = _rand.Next(100, 250);
+                int numBytes = numMBs * 1024 * 1024;
+                byte[] bytes = new byte[numBytes];
+
+                // write to that memory
+                _rand.NextBytes(bytes);
+
+                await Task.Delay(_rand.Next(50, 250));
+
+                Interlocked.Increment(ref InvokeCount);
+            }
+
+            public static async Task GenerateLoadAllCoresAsync()
+            {
+                int cores = Utility.GetEffectiveCoresCount();
+                List<Task> tasks = new List<Task>();
+                for (int i = 0; i < cores; i++)
+                {
+                    var task = Task.Run(() => GenerateLoad());
+                    tasks.Add(task);
+                }
+
+                await Task.WhenAll(tasks);
+            }
+
+            public static void GenerateLoad()
+            {
+                int start = 2000;
+                int numPrimes = 200;
+
+                for (int i = start; i < start + numPrimes; i++)
+                {
+                    FindPrimeNumber(i);
+                }
+            }
+
+            public static long FindPrimeNumber(int n)
+            {
+                int count = 0;
+                long a = 2;
+                while (count < n)
+                {
+                    long b = 2;
+                    int prime = 1; // to check if found a prime
+                    while (b * b <= a)
+                    {
+                        if (a % b == 0)
+                        {
+                            prime = 0;
+                            break;
+                        }
+                        b++;
+                    }
+                    if (prime > 0)
+                    {
+                        count++;
+                    }
+                    a++;
+                }
+                return (--a);
+            }
+        }
+
+        [Binding]
+        [AttributeUsage(AttributeTargets.Parameter)]
+        public class TestTriggerAttribute : Attribute
+        {
+            public TestTriggerAttribute(string source)
+            {
+                Source = source;
+            }
+
+            public string Source { get; }
+        }
+
+        /// <summary>
+        /// Test trigger binding that is Dynamic Concurrency enabled.
+        /// </summary>
+        public class TestTriggerAttributeBindingProvider : ITriggerBindingProvider, IExtensionConfigProvider
+        {
+            private readonly ConcurrencyManager _concurrencyManager;
+
+            public TestTriggerAttributeBindingProvider(ConcurrencyManager concurrencyManager)
+            {
+                _concurrencyManager = concurrencyManager;
+            }
+
+            public void Initialize(ExtensionConfigContext context)
+            {
+                context
+                    .AddBindingRule<TestTriggerAttribute>()
+                    .BindToTrigger<TestEvent>(this);
+            }
+
+            public Task<ITriggerBinding> TryCreateAsync(TriggerBindingProviderContext context)
+            {
+                TestTriggerAttribute attribute = context.Parameter.GetCustomAttributes<TestTriggerAttribute>().SingleOrDefault();
+                ITriggerBinding binding = null;
+
+                if (attribute != null)
+                {
+                    binding = new TestTriggerBinding(attribute.Source, _concurrencyManager);
+                }
+
+                return Task.FromResult(binding);
+            }
+
+            public class TestTriggerBinding : ITriggerBinding
+            {
+                private readonly string _source;
+                private readonly ConcurrencyManager _concurrencyManager;
+
+                public TestTriggerBinding(string source, ConcurrencyManager concurrencyManager)
+                {
+                    _source = source;
+                    _concurrencyManager = concurrencyManager;
+                }
+
+                public Type TriggerValueType
+                {
+                    get { return typeof(TestEvent); }
+                }
+
+                public IReadOnlyDictionary<string, Type> BindingDataContract
+                {
+                    get { return new Dictionary<string, Type>(); }
+                }
+
+                public Task<ITriggerData> BindAsync(object value, ValueBindingContext context)
+                {
+                    TestEvent evt = (TestEvent)value;
+
+                    return Task.FromResult<ITriggerData>(new TestTriggerData(evt));
+                }
+
+                public Task<IListener> CreateListenerAsync(ListenerFactoryContext context)
+                {
+                    return Task.FromResult<IListener>(new TestTriggerListener(_concurrencyManager, context.Executor, context.Descriptor, _source));
+                }
+
+                public ParameterDescriptor ToParameterDescriptor()
+                {
+                    return new ParameterDescriptor();
+                }
+
+                public class TestTriggerListener : IListener
+                {
+                    private const int _intervalMS = 50;
+                    private readonly Timer _timer;
+                    private readonly ITriggeredFunctionExecutor _executor;
+                    private readonly ConcurrencyManager _concurrencyManager;
+                    private readonly FunctionDescriptor _descriptor;
+                    private readonly string _source;
+                    private bool _disposed;
+
+                    public TestTriggerListener(ConcurrencyManager concurrencyManager, ITriggeredFunctionExecutor executor, FunctionDescriptor descriptor, string source)
+                    {
+                        _concurrencyManager = concurrencyManager;
+                        _executor = executor;
+                        _descriptor = descriptor;
+                        _source = source;
+                        _timer = new Timer(OnTimer);
+                    }
+
+                    public Task StartAsync(CancellationToken cancellationToken)
+                    {
+                        _timer.Change(_intervalMS, Timeout.Infinite);
+                        return Task.CompletedTask;
+                    }
+
+                    public Task StopAsync(CancellationToken cancellationToken)
+                    {
+                        _timer.Change(Timeout.Infinite, Timeout.Infinite);
+                        return Task.CompletedTask;
+                    }
+
+                    public void Cancel()
+                    {
+                        _timer.Change(Timeout.Infinite, Timeout.Infinite);
+                    }
+
+                    public void Dispose()
+                    {
+                        _disposed = true;
+                        _timer?.Dispose();
+                    }
+
+                    public void OnTimer(object state)
+                    {
+                        try
+                        {
+                            if (_disposed)
+                            {
+                                return;
+                            }
+
+                            int currentBatchSize = 32;
+
+                            if (_concurrencyManager.Enabled)
+                            {
+                                // Demonstrates how a listener integrates with Dynamic Concurrency querying
+                                // ConcurrencyManager and limiting the amount of new invocations started.
+                                var concurrencyStatus = _concurrencyManager.GetStatus(_descriptor.Id);
+                                currentBatchSize = Math.Min(concurrencyStatus.AvailableInvocationCount, 32);
+                                if (currentBatchSize == 0)
+                                {
+                                    // if we're not healthy or we're at our limit, we'll wait
+                                    // a bit before checking again
+                                    _timer.Change((int)concurrencyStatus.NextStatusDelay.TotalMilliseconds, Timeout.Infinite);
+                                    return;
+                                }
+                            }
+
+                            // fetch new work up to the recommended batch size and dispatch invocations
+                            var events = TestEventSource.GetEvents(_source, currentBatchSize);
+                            bool foundEvent = false;
+                            foreach (var evt in events)
+                            {
+                                foundEvent = true;
+                                _executor.TryExecuteAsync(new TriggeredFunctionData { TriggerValue = evt }, CancellationToken.None);
+                            }
+
+                            if (foundEvent)
+                            {
+                                // we want the next invocation to run right away to ensure we quickly fetch
+                                // to our max degree of concurrency (we know there were messages).
+                                _timer.Change(0, Timeout.Infinite);
+                            }
+                            else
+                            {
+                                // when no events were present we delay before checking again
+                                _timer.Change(250, Timeout.Infinite);
+                            }
+                        }
+                        catch
+                        {
+                            // don't let background exceptions propagate
+                        }
+                    }
+                }
+
+                private class TestTriggerData : ITriggerData
+                {
+                    private TestEvent _evt;
+
+                    public TestTriggerData(TestEvent evt)
+                    {
+                        _evt = evt;
+                    }
+
+                    public IValueProvider ValueProvider
+                    {
+                        get { return new TestValueProvider(_evt); }
+                    }
+
+                    public IReadOnlyDictionary<string, object> BindingData
+                    {
+                        get { return new Dictionary<string, object>(); }
+                    }
+
+                    private class TestValueProvider : IValueProvider
+                    {
+                        private TestEvent _evt;
+
+                        public TestValueProvider(TestEvent evt)
+                        {
+                            _evt = evt;
+                        }
+
+                        public Type Type
+                        {
+                            get { return typeof(TestEvent); }
+                        }
+
+                        public Task<object> GetValueAsync()
+                        {
+                            return Task.FromResult<object>(_evt);
+                        }
+
+                        public string ToInvokeString()
+                        {
+                            return _evt.ID.ToString();
+                        }
+                    }
+                }
+            }
+        }
+    }
+}

test/Microsoft.Azure.WebJobs.Host.EndToEndTests/WebJobs.Host.EndToEndTests.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
   <Import Project="..\..\build\common.props" />
   <PropertyGroup>
-    <TargetFramework>netcoreapp2.1</TargetFramework>
+    <TargetFramework>netcoreapp3.1</TargetFramework>
     <IsPackable>false</IsPackable>
     <AssemblyName>Microsoft.Azure.WebJobs.Host.EndToEndTests</AssemblyName>
     <RootNamespace>Microsoft.Azure.WebJobs.Host.EndToEndTests</RootNamespace>

test/Microsoft.Azure.WebJobs.Host.FunctionalTests/ILoggerTests.cs

@@ -36,7 +36,7 @@ namespace Microsoft.Azure.WebJobs.Host.FunctionalTests
 
             // Six loggers are the startup, singleton, results, function and function.user
             // Note: We currently have 3 additional Logger<T> categories that need to be renamed
-            Assert.Equal(8, loggerProvider.CreatedLoggers.Count); // $$$ was 9?
+            Assert.Equal(9, loggerProvider.CreatedLoggers.Count); // $$$ was 9?
 
             var functionLogger = loggerProvider.CreatedLoggers.Where(l => l.Category == LogCategories.CreateFunctionUserCategory(functionName)).Single();
             var resultsLogger = loggerProvider.CreatedLoggers.Where(l => l.Category == LogCategories.Results).Single();
@@ -69,7 +69,7 @@ namespace Microsoft.Azure.WebJobs.Host.FunctionalTests
 
             // Six loggers are the startup, singleton, results, function and function.user
             // Note: We currently have 3 additional Logger<T> categories that need to be renamed
-            Assert.Equal(8, loggerProvider.CreatedLoggers.Count); // $$$ was 9?
+            Assert.Equal(9, loggerProvider.CreatedLoggers.Count); // $$$ was 9?
 
             var functionLogger = loggerProvider.CreatedLoggers.Where(l => l.Category == LogCategories.CreateFunctionUserCategory(functionName)).Single();
             var resultsLogger = loggerProvider.CreatedLoggers.Where(l => l.Category == LogCategories.Results).Single();
@@ -106,7 +106,7 @@ namespace Microsoft.Azure.WebJobs.Host.FunctionalTests
             }
 
             // Five loggers are the startup, singleton, results, function and function.user
-            Assert.Equal(8, loggerProvider.CreatedLoggers.Count); // $$$ was 9? 
+            Assert.Equal(9, loggerProvider.CreatedLoggers.Count); // $$$ was 9? 
             var functionLogger = loggerProvider.CreatedLoggers.Where(l => l.Category == LogCategories.CreateFunctionUserCategory(functionName)).Single();
             Assert.Equal(2, functionLogger.GetLogMessages().Count);
             var infoMessage = functionLogger.GetLogMessages()[0];

test/Microsoft.Azure.WebJobs.Host.FunctionalTests/JobHostTests.cs

@@ -650,25 +650,25 @@ namespace Microsoft.Azure.WebJobs.Host.UnitTests
                 Assert.Equal("BindingErrorsProgram.Invalid", fex.MethodName);
 
                 // verify that the binding error was logged
-                Assert.Equal(10, errorLogger.GetLogMessages().Count);
+                Assert.Equal(11, errorLogger.GetLogMessages().Count);
 
                 // Skip validating the initial 'Starting JobHost' message and the OptionsFormatters
                 
-                LogMessage logMessage = errorLogger.GetLogMessages()[6];
+                LogMessage logMessage = errorLogger.GetLogMessages()[7];
                 Assert.Equal("Error indexing method 'BindingErrorsProgram.Invalid'", logMessage.FormattedMessage);
                 Assert.Same(fex, logMessage.Exception);
                 Assert.Equal("Invalid container name: invalid$=+1", logMessage.Exception.InnerException.Message);
 
-                logMessage = errorLogger.GetLogMessages()[7];
+                logMessage = errorLogger.GetLogMessages()[8];
                 Assert.Equal("Function 'BindingErrorsProgram.Invalid' failed indexing and will be disabled.", logMessage.FormattedMessage);
 
                 // verify that the valid function was still indexed
-                logMessage = errorLogger.GetLogMessages()[8];
+                logMessage = errorLogger.GetLogMessages()[9];
                 Assert.True(logMessage.FormattedMessage.Contains("Found the following functions"));
                 Assert.True(logMessage.FormattedMessage.Contains("BindingErrorsProgram.Valid"));
 
                 // verify that the job host was started successfully
-                logMessage = errorLogger.GetLogMessages()[9];
+                logMessage = errorLogger.GetLogMessages()[10];
                 Assert.Equal("Job host started", logMessage.FormattedMessage);
 
                 await host.StopAsync();

test/Microsoft.Azure.WebJobs.Host.FunctionalTests/PrimaryHostCoordinatorTests.cs

@@ -0,0 +1,415 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT License. See License.txt in the project root for license information.
+
+using System;
+using System.IO;
+using System.Linq;
+using System.Text.RegularExpressions;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.Azure.Storage;
+using Microsoft.Azure.Storage.Blob;
+using Microsoft.Azure.WebJobs.Host.Executors;
+using Microsoft.Azure.WebJobs.Host.TestCommon;
+using Microsoft.Azure.WebJobs.Hosting;
+using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Hosting;
+using Microsoft.Extensions.Logging;
+using Moq;
+using Xunit;
+
+namespace Microsoft.Azure.WebJobs.Host.FunctionalTests
+{
+    [Trait(TestTraits.CategoryTraitName, TestTraits.DynamicConcurrency)]
+    public class PrimaryHostCoordinatorTests
+    {
+        private TestLoggerProvider _loggerProvider = new TestLoggerProvider();
+        private bool _enabled;
+
+        public PrimaryHostCoordinatorTests()
+        {
+            _enabled = true;
+        }
+
+        private IHost CreateHost(Action<IServiceCollection> configure = null)
+        {
+            IHost host = new HostBuilder()
+                .ConfigureDefaultTestHost<Program>(b =>
+                {
+                    b.AddAzureStorageCoreServices();
+                })
+                .ConfigureServices(s =>
+                {
+                    s.AddOptions<PrimaryHostCoordinatorOptions>().Configure(o =>
+                    {
+                        o.Enabled = _enabled;
+                    });
+
+                    configure?.Invoke(s);
+                })
+                .ConfigureLogging(b =>
+                {
+                    b.AddProvider(_loggerProvider);
+                })
+                .Build();
+
+            return host;
+        }
+
+        [Theory]
+        [InlineData(14.99)]
+        [InlineData(60.01)]
+        public void RejectsInvalidLeaseTimeout(double leaseTimeoutSeconds)
+        {
+            var leaseTimeout = TimeSpan.FromSeconds(leaseTimeoutSeconds);
+            Assert.Throws<ArgumentOutOfRangeException>(() => new PrimaryHostCoordinatorOptions { LeaseTimeout = leaseTimeout });
+        }
+
+        [Fact]
+        public async Task Disabled_DoesNotStartTimer()
+        {
+            // disable the host coordinator
+            _enabled = false;
+
+            var host = CreateHost();
+            using (host)
+            {
+                await host.StartAsync();
+
+                // verify the lease timer isn't running
+                var hostedServices = host.Services.GetServices<IHostedService>();
+                var coordinator = (PrimaryHostCoordinator)hostedServices.Single(p => p.GetType() == typeof(PrimaryHostCoordinator));
+                Assert.False(coordinator.LeaseTimerRunning);
+
+                await host.StopAsync();
+            }
+        }
+
+        [Fact]
+        public async Task HasLease_WhenLeaseIsAcquired_ReturnsTrue()
+        {
+            var host = CreateHost();
+
+            string connectionString = GetStorageConnectionString(host);
+            string hostId = GetHostId(host);
+
+            using (host)
+            {
+                await host.StartAsync();
+
+                var primaryState = host.Services.GetService<IPrimaryHostStateProvider>();
+                await TestHelpers.Await(() => primaryState.IsPrimary);
+
+                await host.StopAsync();
+            }
+
+            await ClearLeaseBlob(connectionString, hostId);
+        }
+
+        [Fact]
+        public async Task HasLeaseChanged_WhenLeaseIsAcquired()
+        {
+            var host = CreateHost();
+
+            string connectionString = GetStorageConnectionString(host);
+            string hostId = GetHostId(host);
+
+            ICloudBlob blob = await GetLockBlobAsync(connectionString, hostId);
+
+            // Acquire a lease on the host lock blob
+            string leaseId = await blob.AcquireLeaseAsync(TimeSpan.FromMinutes(1));
+
+            using (host)
+            {
+                await host.StartAsync();
+
+                var primaryState = host.Services.GetService<IPrimaryHostStateProvider>();
+
+                // The test owns the lease, so the host doesn't have it.
+                Assert.False(primaryState.IsPrimary);
+
+                // Now release it, and we should reclaim it.
+                await blob.ReleaseLeaseAsync(new AccessCondition { LeaseId = leaseId });
+
+                await TestHelpers.Await(() => primaryState.IsPrimary,
+                    userMessageCallback: () => $"{nameof(IPrimaryHostStateProvider.IsPrimary)} was not correctly set to 'true' when lease was acquired.");
+
+                await host.StopAsync();
+            }
+
+            await ClearLeaseBlob(connectionString, hostId);
+        }
+
+        [Fact]
+        public async Task HasLeaseChanged_WhenLeaseIsLost()
+        {
+            var host = CreateHost();
+
+            string connectionString = GetStorageConnectionString(host);
+            string hostId = GetHostId(host);
+
+            using (host)
+            {
+                ICloudBlob blob = await GetLockBlobAsync(connectionString, hostId);
+                var primaryState = host.Services.GetService<IPrimaryHostStateProvider>();
+                var manager = host.Services.GetServices<IHostedService>().OfType<PrimaryHostCoordinator>().Single();
+                var lockManager = host.Services.GetService<IDistributedLockManager>();
+                string tempLeaseId = null;
+
+                await host.StartAsync();
+
+                try
+                {
+                    await TestHelpers.Await(() => primaryState.IsPrimary);
+
+                    // Release the manager's lease and acquire one with a different id
+                    await lockManager.ReleaseLockAsync(manager.LockHandle, CancellationToken.None);
+                    tempLeaseId = await blob.AcquireLeaseAsync(TimeSpan.FromSeconds(30), Guid.NewGuid().ToString());
+
+                    await TestHelpers.Await(() => !primaryState.IsPrimary,
+                        userMessageCallback: () => $"{nameof(IPrimaryHostStateProvider.IsPrimary)} was not correctly set to 'false' when lease lost.");
+                }
+                finally
+                {
+                    if (tempLeaseId != null)
+                    {
+                        await blob.ReleaseLeaseAsync(new AccessCondition { LeaseId = tempLeaseId });
+                    }
+                }
+
+                await host.StopAsync();
+            }
+
+            await ClearLeaseBlob(connectionString, hostId);
+        }
+
+        [Fact]
+        public async Task Dispose_ReleasesBlobLease()
+        {
+            var host = CreateHost();
+
+            string connectionString = GetStorageConnectionString(host);
+            string hostId = GetHostId(host);
+            var primaryHostCoordinator = host.Services.GetServices<IHostedService>().OfType<PrimaryHostCoordinator>().Single();
+
+            using (host)
+            {
+                await host.StartAsync();
+
+                var primaryState = host.Services.GetService<IPrimaryHostStateProvider>();
+                await TestHelpers.Await(() => primaryState.IsPrimary);
+
+                await host.StopAsync();
+            }
+
+            // Container disposal is a fire-and-forget so this service disposal could be delayed. This will force it.
+            primaryHostCoordinator.Dispose();
+
+            ICloudBlob blob = await GetLockBlobAsync(connectionString, hostId);
+
+            string leaseId = null;
+            try
+            {
+                // Acquire a lease on the host lock blob
+                leaseId = await blob.AcquireLeaseAsync(TimeSpan.FromSeconds(15));
+
+                await blob.ReleaseLeaseAsync(new AccessCondition { LeaseId = leaseId });
+            }
+            catch (StorageException exc) when (exc.RequestInformation.HttpStatusCode == 409)
+            {
+            }
+
+            Assert.False(string.IsNullOrEmpty(leaseId), "Failed to acquire a blob lease. The lease was not properly released.");
+
+            await ClearLeaseBlob(connectionString, hostId);
+        }
+
+        [Fact]
+        public async Task TraceOutputsMessagesWhenLeaseIsAcquired()
+        {
+            var blobMock = new Mock<IDistributedLockManager>();
+            blobMock.Setup(b => b.TryLockAsync(It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string>(),
+                It.IsAny<TimeSpan>(), It.IsAny<CancellationToken>()))
+                .Returns(() => Task.FromResult<IDistributedLock>(new FakeLock()));
+
+            var host = CreateHost(s =>
+            {
+                s.AddSingleton<IDistributedLockManager>(_ => blobMock.Object);
+            });
+
+            string connectionString = GetStorageConnectionString(host);
+            string hostId = GetHostId(host);
+            string instanceId = Microsoft.Azure.WebJobs.Utility.GetInstanceId();
+
+            using (host)
+            {
+                await host.StartAsync();
+
+                // Make sure we have enough time to trace the renewal
+                await TestHelpers.Await(() => _loggerProvider.GetAllLogMessages().Any(m => m.FormattedMessage.StartsWith("Host lock lease acquired by instance ID ")), 10000, 500);
+
+                LogMessage acquisitionEvent = _loggerProvider.GetAllLogMessages().Last();
+                Assert.Contains($"Host lock lease acquired by instance ID '{instanceId}'.", acquisitionEvent.FormattedMessage);
+                Assert.Equal(Microsoft.Extensions.Logging.LogLevel.Information, acquisitionEvent.Level);
+
+                await host.StopAsync();
+            }
+        }
+
+        [Fact]
+        public async Task TraceOutputsMessagesWhenLeaseRenewalFails()
+        {
+            var renewResetEvent = new ManualResetEventSlim();
+
+            var blobMock = new Mock<IDistributedLockManager>();
+            blobMock.Setup(b => b.TryLockAsync(It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string>(),
+                It.IsAny<TimeSpan>(), It.IsAny<CancellationToken>()))
+                .Returns(() => Task.FromResult<IDistributedLock>(new FakeLock()));
+
+            blobMock.Setup(b => b.RenewAsync(It.IsAny<IDistributedLock>(), It.IsAny<CancellationToken>()))
+                .Returns(() => Task.FromException<bool>(new StorageException(new RequestResult { HttpStatusCode = 409 }, "test", null)))
+                .Callback(() => renewResetEvent.Set());
+
+            var host = CreateHost(s =>
+            {
+                s.AddSingleton<IDistributedLockManager>(_ => blobMock.Object);
+            });
+
+            string hostId = GetHostId(host);
+            string instanceId = Microsoft.Azure.WebJobs.Utility.GetInstanceId();
+
+            using (host)
+            {
+                await host.StartAsync();
+
+                renewResetEvent.Wait(TimeSpan.FromSeconds(10));
+                await TestHelpers.Await(() => _loggerProvider.GetAllLogMessages().Any(m => m.FormattedMessage.StartsWith("Failed to renew host lock lease: ")), 10000, 500);
+
+                await host.StopAsync();
+            }
+
+            LogMessage acquisitionEvent = _loggerProvider.GetAllLogMessages().Single(m => m.FormattedMessage.Contains($"Host lock lease acquired by instance ID '{instanceId}'."));
+            Assert.Equal(Microsoft.Extensions.Logging.LogLevel.Information, acquisitionEvent.Level);
+
+            LogMessage renewalEvent = _loggerProvider.GetAllLogMessages().Single(m => m.FormattedMessage.Contains(@"Failed to renew host lock lease: Another host has acquired the lease."));
+            string pattern = @"Failed to renew host lock lease: Another host has acquired the lease. The last successful renewal completed at (.+) \([0-9]+ milliseconds ago\) with a duration of [0-9]+ milliseconds.";
+            Assert.True(Regex.IsMatch(renewalEvent.FormattedMessage, pattern), $"Expected trace event {pattern} not found.");
+            Assert.Equal(Microsoft.Extensions.Logging.LogLevel.Information, renewalEvent.Level);
+        }
+
+        [Fact]
+        public async Task DifferentHosts_UsingSameStorageAccount_CanObtainLease()
+        {
+            string hostId1 = Guid.NewGuid().ToString();
+            string hostId2 = Guid.NewGuid().ToString();
+
+            var host1 = CreateHost(s =>
+            {
+                s.AddSingleton<IHostIdProvider>(_ => new FixedHostIdProvider(hostId1));
+            });
+
+            var host2 = CreateHost(s =>
+            {
+                s.AddSingleton<IHostIdProvider>(_ => new FixedHostIdProvider(hostId2));
+            });
+
+            string host1ConnectionString = GetStorageConnectionString(host1);
+            string host2ConnectionString = GetStorageConnectionString(host2);
+
+            using (host1)
+            using (host2)
+            {
+                await host1.StartAsync();
+                await host2.StartAsync();
+
+                var primaryState1 = host1.Services.GetService<IPrimaryHostStateProvider>();
+                var primaryState2 = host2.Services.GetService<IPrimaryHostStateProvider>();
+
+                Task manager1Check = TestHelpers.Await(() => primaryState1.IsPrimary);
+                Task manager2Check = TestHelpers.Await(() => primaryState2.IsPrimary);
+
+                await Task.WhenAll(manager1Check, manager2Check);
+
+                await host1.StopAsync();
+                await host2.StopAsync();
+            }
+
+            await Task.WhenAll(ClearLeaseBlob(host1ConnectionString, hostId1), ClearLeaseBlob(host2ConnectionString, hostId2));
+        }
+
+        private static async Task<ICloudBlob> GetLockBlobAsync(string accountConnectionString, string hostId)
+        {
+            CloudStorageAccount account = CloudStorageAccount.Parse(accountConnectionString);
+            CloudBlobClient client = account.CreateCloudBlobClient();
+
+            var container = client.GetContainerReference(HostContainerNames.Hosts);
+
+            await container.CreateIfNotExistsAsync();
+
+            // the StorageDistributedLockManager puts things under the /locks path by default
+            CloudBlockBlob blob = container.GetBlockBlobReference("locks/" + PrimaryHostCoordinator.GetBlobName(hostId));
+            if (!await blob.ExistsAsync())
+            {
+                await blob.UploadFromStreamAsync(new MemoryStream());
+            }
+
+            return blob;
+        }
+
+        private async Task ClearLeaseBlob(string connectionString, string hostId)
+        {
+            ICloudBlob blob = await GetLockBlobAsync(connectionString, hostId);
+
+            try
+            {
+                await blob.BreakLeaseAsync(TimeSpan.Zero);
+            }
+            catch
+            {
+            }
+
+            await blob.DeleteIfExistsAsync();
+        }
+
+        private class FakeLock : IDistributedLock
+        {
+            public string LockId => "lockid";
+
+            public Task LeaseLost => throw new NotImplementedException();
+        }
+
+        private class FixedHostIdProvider : IHostIdProvider
+        {
+            private readonly string _hostId;
+
+            public FixedHostIdProvider(string hostId)
+            {
+                _hostId = hostId;
+            }
+
+            public Task<string> GetHostIdAsync(CancellationToken cancellationToken)
+            {
+                return Task.FromResult(_hostId);
+            }
+        }
+
+        private static string GetStorageConnectionString(IHost host)
+        {
+            return host.Services.GetService<IConfiguration>().GetWebJobsConnectionString("Storage");
+        }
+
+        public static string GetHostId(IHost host)
+        {
+            return host.Services.GetService<IHostIdProvider>().GetHostIdAsync(CancellationToken.None).GetAwaiter().GetResult();
+        }
+
+        public class Program
+        {
+            [NoAutomaticTrigger]
+            public void TestFunction()
+            {
+            }
+        }
+    }
+}

test/Microsoft.Azure.WebJobs.Host.FunctionalTests/WebJobs.Host.FunctionalTests.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
   <Import Project="..\..\build\common.props" />
   <PropertyGroup>
-    <TargetFramework>netcoreapp2.1</TargetFramework>
+    <TargetFramework>netcoreapp3.1</TargetFramework>
     <IsPackable>false</IsPackable>
     <AssemblyName>Microsoft.Azure.WebJobs.Host.FunctionalTests</AssemblyName>
     <RootNamespace>Microsoft.Azure.WebJobs.Host.FunctionalTests</RootNamespace>

test/Microsoft.Azure.WebJobs.Host.TestCommon/ExceptionAssert.cs

@@ -71,7 +71,7 @@ namespace Microsoft.Azure.WebJobs.Host.TestCommon
                 return message;
             }
 
-            return String.Format("{0}{1}Parameter name: {2}", message, Environment.NewLine, parameterName);
+            return String.Format("{0} (Parameter '{2}')", message, Environment.NewLine, parameterName);
         }
     }
 }

test/Microsoft.Azure.WebJobs.Host.TestCommon/TestHelpers.cs

@@ -243,6 +243,11 @@ namespace Microsoft.Azure.WebJobs.Host.TestCommon
             return host.Services.GetServices<ILoggerProvider>().OfType<TestLoggerProvider>().Single();
         }
 
+        public static TService GetServiceOrNull<TService>(this IHost host)
+        {
+            return host.Services.GetServices<TService>().SingleOrDefault();
+        }
+
         public static TExtension GetExtension<TExtension>(this IHost host)
         {
             return host.Services.GetServices<IExtensionConfigProvider>().OfType<TExtension>().SingleOrDefault();
@@ -336,6 +341,15 @@ namespace Microsoft.Azure.WebJobs.Host.TestCommon
         {
             return new ConfigurationBuilder().AddInMemoryCollection(dict).Build();
         }
+
+        public static void SetupStopwatch(Stopwatch sw, TimeSpan elapsed)
+        {
+            sw.Restart();
+
+            // set elapsed so to simulate time having passed
+            FieldInfo elapsedFieldInfo = typeof(Stopwatch).GetField("_elapsed", BindingFlags.NonPublic | BindingFlags.Instance);
+            elapsedFieldInfo.SetValue(sw, elapsed.Ticks);
+        }
     }
 
     public class TestProgram

test/Microsoft.Azure.WebJobs.Host.TestCommon/TestTraits.cs

@@ -0,0 +1,12 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT License. See License.txt in the project root for license information.
+
+namespace Microsoft.Azure.WebJobs.Host.TestCommon
+{
+    public static class TestTraits
+    {
+        public const string CategoryTraitName = "Category";
+
+        public const string DynamicConcurrency = "DynamicConcurrency";
+    }
+}

test/Microsoft.Azure.WebJobs.Host.TestCommon/WebJobs.Host.TestCommon.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
   <Import Project="..\..\build\common.props" />
   <PropertyGroup>
-    <TargetFramework>netcoreapp2.1</TargetFramework>
+    <TargetFramework>netcoreapp3.1</TargetFramework>
     <IsPackable>true</IsPackable>
     <AssemblyName>Microsoft.Azure.WebJobs.Host.TestCommon</AssemblyName>
     <RootNamespace>Microsoft.Azure.WebJobs.Host.TestCommon</RootNamespace>

test/Microsoft.Azure.WebJobs.Host.UnitTests/Bindings/Path/BindingParameterResolverTests.cs

@@ -61,7 +61,7 @@ namespace Microsoft.Azure.WebJobs.Host.UnitTests.Bindings.Path
                 resolver.Resolve("datetime:mm-dd-yyyy");
             });
 
-            Assert.Equal($"The value specified is not a 'rand-guid' binding parameter.{Environment.NewLine}Parameter name: value", ex.Message);
+            Assert.Equal($"The value specified is not a 'rand-guid' binding parameter. (Parameter 'value')", ex.Message);
         }
 
         [Fact]

test/Microsoft.Azure.WebJobs.Host.UnitTests/Configuration/PrimaryHostCoordinatorOptionsSetupTests.cs

@@ -0,0 +1,49 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT License. See License.txt in the project root for license information.
+
+using System;
+using Microsoft.Azure.WebJobs.Host.Config;
+using Microsoft.Azure.WebJobs.Host.Scale;
+using Microsoft.Azure.WebJobs.Host.TestCommon;
+using Microsoft.Azure.WebJobs.Hosting;
+using Microsoft.Extensions.Options;
+using Xunit;
+
+namespace Microsoft.Azure.WebJobs.Host.UnitTests.Configuration
+{
+    [Trait(TestTraits.CategoryTraitName, TestTraits.DynamicConcurrency)]
+    public class PrimaryHostCoordinatorOptionsSetupTests
+    {
+        private readonly ConcurrencyOptions _concurrencyOptions;
+        private readonly PrimaryHostCoordinatorOptionsSetup _setup;
+
+        public PrimaryHostCoordinatorOptionsSetupTests()
+        {
+            _concurrencyOptions = new ConcurrencyOptions();
+            var optionsWrapper = new OptionsWrapper<ConcurrencyOptions>(_concurrencyOptions);
+
+            _setup = new PrimaryHostCoordinatorOptionsSetup(optionsWrapper);
+        }
+
+        [Fact]
+        public void OptionsConstructor_ConfiguresExpectedDefaults()
+        {
+            var options = new PrimaryHostCoordinatorOptions();
+            Assert.False(options.Enabled);
+            Assert.Equal(TimeSpan.FromSeconds(15), options.LeaseTimeout);
+        }
+
+        [Fact]
+        public void Configure_ConfiguresExpectedValues()
+        {
+            _concurrencyOptions.DynamicConcurrencyEnabled = false;
+            var options = new PrimaryHostCoordinatorOptions();
+            _setup.Configure(options);
+            Assert.False(options.Enabled);
+
+            _concurrencyOptions.DynamicConcurrencyEnabled = true;
+            _setup.Configure(options);
+            Assert.True(options.Enabled);
+        }
+    }
+}

test/Microsoft.Azure.WebJobs.Host.UnitTests/Converters/StringToTConverterFactoryTests.cs

@@ -426,7 +426,7 @@ namespace Microsoft.Azure.WebJobs.Host.UnitTests.Converters
             // Assert
             Assert.NotNull(converter);
             ExceptionAssert.ThrowsFormat(() => converter.Convert("{00000000-0000-0000-0000-000000000000}"),
-                "Unrecognized Guid format.");
+                "Guid should contain 32 digits with 4 dashes (xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx).");
         }
 
         [Fact]
@@ -539,7 +539,7 @@ namespace Microsoft.Azure.WebJobs.Host.UnitTests.Converters
                 // Assert
                 Assert.NotNull(converter);
                 ExceptionAssert.ThrowsFormat(() => converter.Convert("-10:00:00,123"),
-                    "String was not recognized as a valid TimeSpan.");
+                    "String '-10:00:00,123' was not recognized as a valid TimeSpan.");
             }
         }
 

test/Microsoft.Azure.WebJobs.Host.UnitTests/Executors/FunctionExecutorTests.cs

@@ -13,6 +13,7 @@ using Microsoft.Azure.WebJobs.Host.Executors;
 using Microsoft.Azure.WebJobs.Host.Indexers;
 using Microsoft.Azure.WebJobs.Host.Loggers;
 using Microsoft.Azure.WebJobs.Host.Protocols;
+using Microsoft.Azure.WebJobs.Host.Scale;
 using Microsoft.Azure.WebJobs.Host.TestCommon;
 using Microsoft.Azure.WebJobs.Host.Timers;
 using Microsoft.Extensions.Configuration;
@@ -312,12 +313,14 @@ namespace Microsoft.Azure.WebJobs.Host.UnitTests.Executors
             var mockFunctionOutputLogger = new NullFunctionOutputLogger();
             var mockExceptionHandler = new Mock<IWebJobsExceptionHandler>();
             var mockFunctionEventCollector = new Mock<IAsyncCollector<FunctionInstanceLogEntry>>();
+            var mockConcurrencyManager = new Mock<ConcurrencyManager>();
 
             var functionExecutor = new FunctionExecutor(
                 mockFunctionInstanceLogger.Object,
                 mockFunctionOutputLogger,
                 mockExceptionHandler.Object,
                 mockFunctionEventCollector.Object,
+                mockConcurrencyManager.Object,
                 NullLoggerFactory.Instance,
                 null,
                 drainModeManager);

test/Microsoft.Azure.WebJobs.Host.UnitTests/Executors/TriggeredFunctionExecutorTests.cs

@@ -13,6 +13,7 @@ using Microsoft.Azure.WebJobs.Host.Executors;
 using Microsoft.Azure.WebJobs.Host.Indexers;
 using Microsoft.Azure.WebJobs.Host.Loggers;
 using Microsoft.Azure.WebJobs.Host.Protocols;
+using Microsoft.Azure.WebJobs.Host.Scale;
 using Microsoft.Azure.WebJobs.Host.TestCommon;
 using Microsoft.Azure.WebJobs.Host.Timers;
 using Microsoft.Azure.WebJobs.Host.Triggers;
@@ -166,12 +167,14 @@ namespace Microsoft.Azure.WebJobs.Host.UnitTests.Executors
             var mockFunctionOutputLogger = new NullFunctionOutputLogger();
             var mockExceptionHandler = new Mock<IWebJobsExceptionHandler>();
             var mockFunctionEventCollector = new Mock<IAsyncCollector<FunctionInstanceLogEntry>>();
+            var mockConcurrencyManager = new Mock<ConcurrencyManager>();
 
             var functionExecutor = new FunctionExecutor(
                 mockFunctionInstanceLogger.Object,
                 mockFunctionOutputLogger,
                 mockExceptionHandler.Object,
                 mockFunctionEventCollector.Object,
+                mockConcurrencyManager.Object,
                 NullLoggerFactory.Instance,
                 null,
                 drainModeManager);

test/Microsoft.Azure.WebJobs.Host.UnitTests/PublicSurfaceTests.cs

@@ -269,7 +269,23 @@ namespace Microsoft.Azure.WebJobs.Host.UnitTests
                 "RetryAttribute",
                 "FixedDelayRetryAttribute",
                 "ExponentialBackoffRetryAttribute",
-                "RetryContext"
+                "RetryContext",
+                "ConcurrencyManager",
+                "ConcurrencyOptions",
+                "ConcurrencyStatus",
+                "HostConcurrencySnapshot",
+                "ConcurrencyThrottleStatus",
+                "ConcurrencyThrottleAggregateStatus",
+                "FunctionConcurrencySnapshot",
+                "HostHealthState",
+                "HostProcessStatus",
+                "IConcurrencyStatusRepository",
+                "IConcurrencyThrottleManager",
+                "IConcurrencyThrottleProvider",
+                "IHostProcessMonitor",
+                "IPrimaryHostStateProvider",
+                "PrimaryHostCoordinatorOptions",
+                "ThrottleState"
             };
 
             TestHelpers.AssertPublicTypes(expected, assembly);

test/Microsoft.Azure.WebJobs.Host.UnitTests/Scale/ConcurrencyManagerServiceTests.cs

@@ -0,0 +1,229 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT License. See License.txt in the project root for license information.
+
+using System;
+using System.Collections.Generic;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.Azure.WebJobs.Host.Indexers;
+using Microsoft.Azure.WebJobs.Host.Scale;
+using Microsoft.Azure.WebJobs.Host.TestCommon;
+using Microsoft.Azure.WebJobs.Hosting;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
+using Moq;
+using System.Linq;
+using Xunit;
+using System.Timers;
+using Microsoft.Azure.WebJobs.Host.Protocols;
+
+namespace Microsoft.Azure.WebJobs.Host.UnitTests.Scale
+{
+    [Trait(TestTraits.CategoryTraitName, TestTraits.DynamicConcurrency)]
+    public class ConcurrencyManagerServiceTests
+    {
+        private readonly LoggerFactory _loggerFactory;
+        private readonly TestLoggerProvider _loggerProvider;
+        private readonly Mock<ConcurrencyManager> _concurrencyManagerMock;
+        private readonly Mock<IConcurrencyStatusRepository> _repositoryMock;
+        private readonly Mock<IPrimaryHostStateProvider> _primaryHostStateProviderMock;
+        private readonly ConcurrencyManagerService _concurrencyManagerService;
+        private readonly ConcurrencyOptions _concurrencyOptions;
+        private readonly HostConcurrencySnapshot _snapshot;
+
+        private bool _isPrimaryHost;
+
+        public ConcurrencyManagerServiceTests()
+        {
+            _isPrimaryHost = false;
+            _concurrencyOptions = new ConcurrencyOptions
+            {
+                DynamicConcurrencyEnabled = true
+            };
+            var optionsWrapper = new OptionsWrapper<ConcurrencyOptions>(_concurrencyOptions);
+            _loggerFactory = new LoggerFactory();
+            _loggerProvider = new TestLoggerProvider();
+            _loggerFactory.AddProvider(_loggerProvider);
+
+            _concurrencyManagerMock = new Mock<ConcurrencyManager>(MockBehavior.Strict);
+            _repositoryMock = new Mock<IConcurrencyStatusRepository>(MockBehavior.Strict);
+
+            _snapshot = new HostConcurrencySnapshot
+            {
+                NumberOfCores = 4,
+                Timestamp = DateTime.UtcNow
+            };
+            _snapshot.FunctionSnapshots = new Dictionary<string, FunctionConcurrencySnapshot>
+            {
+                { "function0", new FunctionConcurrencySnapshot { Concurrency = 5 } },
+                { "function1", new FunctionConcurrencySnapshot { Concurrency = 10 } },
+                { "function2", new FunctionConcurrencySnapshot { Concurrency = 15 } }
+            };
+
+            List<IFunctionDefinition> functions = new List<IFunctionDefinition>();
+            for (int i = 0; i < 3; i++)
+            {
+                var functionDefinitionMock = new Mock<IFunctionDefinition>(MockBehavior.Strict);
+                functionDefinitionMock.Setup(p => p.Descriptor).Returns(new FunctionDescriptor { Id = $"function{i}" });
+                functions.Add(functionDefinitionMock.Object); 
+            }
+
+            var functionIndexMock = new Mock<IFunctionIndex>(MockBehavior.Strict);
+            functionIndexMock.Setup(p => p.ReadAll()).Returns(functions);
+
+            var functionIndexProviderMock = new Mock<IFunctionIndexProvider>(MockBehavior.Strict);
+            functionIndexProviderMock.Setup(p => p.GetAsync(CancellationToken.None)).ReturnsAsync(functionIndexMock.Object);
+
+            _primaryHostStateProviderMock = new Mock<IPrimaryHostStateProvider>(MockBehavior.Strict);
+            _primaryHostStateProviderMock.SetupGet(p => p.IsPrimary).Returns(() => _isPrimaryHost);
+
+            _concurrencyManagerService = new ConcurrencyManagerService(optionsWrapper, _loggerFactory, _concurrencyManagerMock.Object, _repositoryMock.Object, functionIndexProviderMock.Object, _primaryHostStateProviderMock.Object);
+        }
+
+        [Fact]
+        public async Task StartAsync_DynamicConcurrencyDisabled_DoesNotStart()
+        {
+            _concurrencyOptions.DynamicConcurrencyEnabled = false;
+
+            await _concurrencyManagerService.StartAsync(CancellationToken.None);
+
+            Assert.False(_concurrencyManagerService.StatusPersistenceTimer.Enabled);
+        }
+
+        [Fact]
+        public async Task StartAsync_SnapshotPersistenceDisabled_DoesNotStart()
+        {
+            _concurrencyOptions.DynamicConcurrencyEnabled = true;
+            _concurrencyOptions.SnapshotPersistenceEnabled = false;
+
+            await _concurrencyManagerService.StartAsync(CancellationToken.None);
+
+            Assert.False(_concurrencyManagerService.StatusPersistenceTimer.Enabled);
+        }
+
+        [Fact]
+        public async Task StartAsync_SnapshotPersistenceEnabled_AppliesSnapshotAndStarts()
+        {
+            _concurrencyOptions.DynamicConcurrencyEnabled = true;
+            _concurrencyOptions.SnapshotPersistenceEnabled = true;
+
+            var snapshot = new HostConcurrencySnapshot();
+            _repositoryMock.Setup(p => p.ReadAsync(CancellationToken.None)).ReturnsAsync(snapshot);
+
+            _concurrencyManagerMock.Setup(p => p.ApplySnapshot(snapshot));
+
+            await _concurrencyManagerService.StartAsync(CancellationToken.None);
+
+            _repositoryMock.VerifyAll();
+            _concurrencyManagerMock.VerifyAll();
+
+            Assert.True(_concurrencyManagerService.StatusPersistenceTimer.Enabled);
+        }
+
+        [Fact]
+        public async Task StartAsync_Throws_SnapshotNotApplied_Starts()
+        {
+            _concurrencyOptions.DynamicConcurrencyEnabled = true;
+            _concurrencyOptions.SnapshotPersistenceEnabled = true;
+
+            _repositoryMock.Setup(p => p.ReadAsync(CancellationToken.None)).ThrowsAsync(new Exception("Kaboom!"));
+
+            await _concurrencyManagerService.StartAsync(CancellationToken.None);
+
+            Assert.True(_concurrencyManagerService.StatusPersistenceTimer.Enabled);
+
+            var log = _loggerProvider.GetAllLogMessages().Single();
+            Assert.Equal("Error applying concurrency snapshot.", log.FormattedMessage);
+        }
+
+        [Fact]
+        public async Task StopAsync_Stops()
+        {
+            _concurrencyOptions.DynamicConcurrencyEnabled = true;
+            _concurrencyOptions.SnapshotPersistenceEnabled = true;
+
+            var snapshot = new HostConcurrencySnapshot();
+            _repositoryMock.Setup(p => p.ReadAsync(CancellationToken.None)).ReturnsAsync(snapshot);
+
+            _concurrencyManagerMock.Setup(p => p.ApplySnapshot(snapshot));
+
+            await _concurrencyManagerService.StartAsync(CancellationToken.None);
+
+            Assert.True(_concurrencyManagerService.StatusPersistenceTimer.Enabled);
+
+            await _concurrencyManagerService.StopAsync(CancellationToken.None);
+
+            Assert.False(_concurrencyManagerService.StatusPersistenceTimer.Enabled);
+        }
+
+        [Fact]
+        public async Task OnPersistenceTimer_NonPrimary_DoesNotWriteSnapshot()
+        {
+            _isPrimaryHost = false;
+
+            await _concurrencyManagerService.OnPersistenceTimer();
+
+            Assert.True(_concurrencyManagerService.StatusPersistenceTimer.Enabled);
+        }
+
+        [Fact]
+        public async Task OnPersistenceTimer_Primary_WriteSnapshot()
+        {
+            _isPrimaryHost = true;
+
+            _concurrencyManagerMock.Setup(p => p.GetSnapshot()).Returns(_snapshot);
+
+            _repositoryMock.Setup(p => p.WriteAsync(_snapshot, CancellationToken.None)).Returns(Task.CompletedTask);
+
+            await _concurrencyManagerService.OnPersistenceTimer();
+
+            _repositoryMock.VerifyAll();
+            _concurrencyManagerMock.VerifyAll();
+
+            Assert.True(_concurrencyManagerService.StatusPersistenceTimer.Enabled);
+            Assert.Equal(3, _snapshot.FunctionSnapshots.Count);
+        }
+
+        [Fact]
+        public async Task WriteSnapshotAsync_RemovesStaleFunctions()
+        {
+            _isPrimaryHost = true;
+
+            // snapshot includes one function that isn't in the current index
+            _snapshot.FunctionSnapshots["function3"] = new FunctionConcurrencySnapshot { Concurrency = 15 };
+
+            _concurrencyManagerMock.Setup(p => p.GetSnapshot()).Returns(_snapshot);
+
+            _repositoryMock.Setup(p => p.WriteAsync(_snapshot, CancellationToken.None)).Returns(Task.CompletedTask);
+
+            await _concurrencyManagerService.OnPersistenceTimer();
+
+            _repositoryMock.VerifyAll();
+            _concurrencyManagerMock.VerifyAll();
+
+            Assert.True(_concurrencyManagerService.StatusPersistenceTimer.Enabled);
+            Assert.Equal(3, _snapshot.FunctionSnapshots.Count);
+            Assert.Collection(_snapshot.FunctionSnapshots,
+                p => Assert.Equal("function0", p.Key),
+                p => Assert.Equal("function1", p.Key),
+                p => Assert.Equal("function2", p.Key));
+        }
+
+        [Fact]
+        public async Task WriteSnapshotAsync_NoChanges_DoesNotWriteSnapshot()
+        {
+            _isPrimaryHost = true;
+
+            _concurrencyManagerMock.Setup(p => p.GetSnapshot()).Returns(_snapshot);
+
+            _repositoryMock.Setup(p => p.WriteAsync(_snapshot, CancellationToken.None)).Returns(Task.CompletedTask);
+
+            await _concurrencyManagerService.OnPersistenceTimer();
+            await _concurrencyManagerService.OnPersistenceTimer();
+
+            _repositoryMock.VerifyAll();
+            _repositoryMock.Verify(p => p.WriteAsync(_snapshot, CancellationToken.None), Times.Once);
+            _concurrencyManagerMock.VerifyAll();
+        }
+    }
+}

test/Microsoft.Azure.WebJobs.Host.UnitTests/Scale/ConcurrencyManagerTests.cs

@@ -0,0 +1,409 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT License. See License.txt in the project root for license information.
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using Microsoft.Azure.WebJobs.Host.Scale;
+using Microsoft.Azure.WebJobs.Host.TestCommon;
+using Microsoft.Azure.WebJobs.Logging;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
+using Moq;
+using Xunit;
+
+namespace Microsoft.Azure.WebJobs.Host.UnitTests.Scale
+{
+    [Trait(TestTraits.CategoryTraitName, TestTraits.DynamicConcurrency)]
+    public class ConcurrencyManagerTests
+    {
+        private const string TestFunctionId = "testfunction";
+
+        private readonly ConcurrencyManager _concurrencyManager;
+        private readonly LoggerFactory _loggerFactory;
+        private readonly TestLoggerProvider _loggerProvider;
+        private readonly Mock<IConcurrencyThrottleManager> _mockThrottleManager;
+        private readonly ConcurrencyStatus _testFunctionConcurrencyStatus;
+
+        private ConcurrencyThrottleAggregateStatus _throttleStatus;
+
+        public ConcurrencyManagerTests()
+        {
+            var options = new ConcurrencyOptions
+            {
+                DynamicConcurrencyEnabled = true
+            };
+            var optionsWrapper = new OptionsWrapper<ConcurrencyOptions>(options);
+            _loggerFactory = new LoggerFactory();
+            _loggerProvider = new TestLoggerProvider();
+            _loggerFactory.AddProvider(_loggerProvider);
+
+            _mockThrottleManager = new Mock<IConcurrencyThrottleManager>(MockBehavior.Strict);
+            _mockThrottleManager.Setup(p => p.GetStatus()).Returns(() => _throttleStatus);
+
+            _concurrencyManager = new ConcurrencyManager(optionsWrapper, _loggerFactory, _mockThrottleManager.Object);
+
+            _testFunctionConcurrencyStatus = new ConcurrencyStatus(TestFunctionId, _concurrencyManager);
+            _concurrencyManager.ConcurrencyStatuses[TestFunctionId] = _testFunctionConcurrencyStatus;
+        }
+
+        [Theory]
+        [InlineData(true)]
+        [InlineData(false)]
+        public void Enabled_ReturnsExpectedResult(bool enabled)
+        {
+            var options = new ConcurrencyOptions
+            {
+                DynamicConcurrencyEnabled = enabled
+            };
+            var optionsWrapper = new OptionsWrapper<ConcurrencyOptions>(options);
+            var concurrencyManager = new ConcurrencyManager(optionsWrapper, _loggerFactory, _mockThrottleManager.Object);
+
+            Assert.Equal(concurrencyManager.Enabled, enabled);
+        }
+
+        [Theory]
+        [InlineData(ThrottleState.Enabled)]
+        [InlineData(ThrottleState.Disabled)]
+        [InlineData(ThrottleState.Unknown)]
+        public void ThrottleEnabled_ReturnsExpectedResult(ThrottleState throttleState)
+        {
+            _throttleStatus = new ConcurrencyThrottleAggregateStatus { State = throttleState };
+
+            // set the last adjustment outside of the throttle window so GetStatus
+            // won't short circuit
+            TestHelpers.SetupStopwatch(_testFunctionConcurrencyStatus.LastConcurrencyAdjustmentStopwatch, TimeSpan.FromSeconds(ConcurrencyStatus.DefaultMinAdjustmentFrequencySeconds + 1));
+
+            _concurrencyManager.GetStatus(TestFunctionId);
+
+            Assert.Equal(_concurrencyManager.ThrottleEnabled, throttleState == ThrottleState.Enabled);
+        }
+
+        [Theory]
+        [InlineData(null)]
+        [InlineData("")]
+        public void GetStatus_InvalidFunction_Throws(string functionId)
+        {
+            var ex = Assert.Throws<ArgumentNullException>(() => _concurrencyManager.GetStatus(functionId));
+            Assert.Equal(nameof(functionId), ex.ParamName);
+        }
+
+        [Fact]
+        public void GetStatus_ThrottleDisabled_IncreasesConcurrency()
+        {
+            _throttleStatus = new ConcurrencyThrottleAggregateStatus { State = ThrottleState.Disabled, ConsecutiveCount = ConcurrencyManager.MinConsecutiveIncreaseLimit };
+
+            TestHelpers.SetupStopwatch(_testFunctionConcurrencyStatus.LastConcurrencyAdjustmentStopwatch, TimeSpan.FromSeconds(ConcurrencyStatus.DefaultMinAdjustmentFrequencySeconds + 1));
+
+            int prevConcurrency = _testFunctionConcurrencyStatus.CurrentConcurrency;
+            SimulateFunctionInvocations(TestFunctionId, 3);
+
+            var status = _concurrencyManager.GetStatus(TestFunctionId);
+
+            Assert.Equal(prevConcurrency + 1, status.CurrentConcurrency);
+
+            var logs = _loggerProvider.GetAllLogMessages().ToArray();
+            Assert.Equal(2, logs.Length);
+
+            var log = logs[0];
+            Assert.Equal(LogLevel.Debug, log.Level);
+            Assert.Equal(LogCategories.Concurrency, log.Category);
+            Assert.Equal("testfunction Increasing concurrency", log.FormattedMessage);
+
+            log = logs[1];
+            Assert.Equal(LogLevel.Debug, log.Level);
+            Assert.Equal(LogCategories.Concurrency, log.Category);
+            Assert.Equal("testfunction Concurrency: 2, OutstandingInvocations: 0", log.FormattedMessage);
+        }
+
+        [Fact]
+        public void GetStatus_ThrottleEnabled_DecreasesConcurrency()
+        {
+            _throttleStatus = new ConcurrencyThrottleAggregateStatus 
+            { 
+                State = ThrottleState.Enabled, 
+                ConsecutiveCount = ConcurrencyManager.MinConsecutiveDecreaseLimit,
+                EnabledThrottles = new List<string> { DefaultHostProcessMonitor.CpuLimitName, ThreadPoolStarvationThrottleProvider.ThreadPoolStarvationThrottleName }
+            };
+
+            TestHelpers.SetupStopwatch(_testFunctionConcurrencyStatus.LastConcurrencyAdjustmentStopwatch, TimeSpan.FromSeconds(ConcurrencyStatus.DefaultMinAdjustmentFrequencySeconds));
+
+            _testFunctionConcurrencyStatus.CurrentConcurrency = 5;
+            SimulateFunctionInvocations(TestFunctionId, 3);
+
+            var status = _concurrencyManager.GetStatus(TestFunctionId);
+
+            Assert.Equal(4, status.CurrentConcurrency);
+
+            var logs = _loggerProvider.GetAllLogMessages().ToArray();
+            Assert.Equal(2, logs.Length);
+
+            var log = logs[0];
+            Assert.Equal(LogLevel.Debug, log.Level);
+            Assert.Equal(LogCategories.Concurrency, log.Category);
+            Assert.Equal("testfunction Decreasing concurrency (Enabled throttles: CPU,ThreadPoolStarvation)", log.FormattedMessage);
+
+            log = logs[1];
+            Assert.Equal(LogLevel.Debug, log.Level);
+            Assert.Equal(LogCategories.Concurrency, log.Category);
+            Assert.Equal("testfunction Concurrency: 4, OutstandingInvocations: 0", log.FormattedMessage);
+        }
+
+        [Fact]
+        public void GetStatus_ThrottleUnknown_ReturnsCurrentStatus()
+        {
+            _throttleStatus = new ConcurrencyThrottleAggregateStatus { State = ThrottleState.Unknown };
+
+            TestHelpers.SetupStopwatch(_testFunctionConcurrencyStatus.LastConcurrencyAdjustmentStopwatch, TimeSpan.FromSeconds(ConcurrencyStatus.DefaultMinAdjustmentFrequencySeconds));
+
+            _testFunctionConcurrencyStatus.CurrentConcurrency = 1;
+
+            var status = _concurrencyManager.GetStatus(TestFunctionId);
+
+            Assert.Equal(1, status.CurrentConcurrency);
+        }
+
+        [Fact]
+        public void GetStatus_FunctionRecentlyAdjusted_ReturnsCurrentStatus()
+        {
+            _testFunctionConcurrencyStatus.LastConcurrencyAdjustmentStopwatch.Restart();
+
+            var status = _concurrencyManager.GetStatus(TestFunctionId);
+
+            _mockThrottleManager.Verify(p => p.GetStatus(), Times.Never);
+        }
+
+        [Theory]
+        [InlineData(50, true)]
+        [InlineData(1, false)]
+        public void CanDecreaseConcurrency_ReturnsExpectedValue(int concurrency, bool expected)
+        {
+            _testFunctionConcurrencyStatus.CurrentConcurrency = concurrency;
+            Assert.Equal(expected, _testFunctionConcurrencyStatus.CanDecreaseConcurrency());
+        }
+
+        [Fact]
+        public void FunctionInvocationTracking_MaintainsExpectedCounts()
+        {
+            // FunctionStarted/FunctionCompleted only operates on functions
+            // that are DC enabled, so we must prime the pump here
+            _concurrencyManager.GetStatus("testfunction1");
+            _concurrencyManager.GetStatus("testfunction2");
+            _concurrencyManager.GetStatus("testfunction3");
+
+            // simulate some function invocations and verify bookkeeping
+            _concurrencyManager.FunctionStarted("testfunction1");
+
+            _concurrencyManager.FunctionStarted("testfunction1");
+            _concurrencyManager.FunctionCompleted("testfunction1", TimeSpan.FromMilliseconds(110));
+
+            _concurrencyManager.FunctionStarted("testfunction1");
+            _concurrencyManager.FunctionStarted("testfunction1");
+
+            var status = _concurrencyManager.ConcurrencyStatuses["testfunction1"];
+            Assert.Equal(3, status.OutstandingInvocations);
+            Assert.Equal(3, status.MaxConcurrentExecutionsSinceLastAdjustment);
+
+            _concurrencyManager.FunctionStarted("testfunction2");
+            _concurrencyManager.FunctionStarted("testfunction2");
+
+            status = _concurrencyManager.ConcurrencyStatuses["testfunction2"];
+            Assert.Equal(2, status.OutstandingInvocations);
+            Assert.Equal(2, status.MaxConcurrentExecutionsSinceLastAdjustment);
+
+            _concurrencyManager.FunctionStarted("testfunction3");
+
+            status = _concurrencyManager.ConcurrencyStatuses["testfunction3"];
+            Assert.Equal(1, status.OutstandingInvocations);
+            Assert.Equal(1, status.MaxConcurrentExecutionsSinceLastAdjustment);
+
+            // complete the invocations and verify bookkeeping
+            _concurrencyManager.FunctionCompleted("testfunction1", TimeSpan.FromMilliseconds(100));
+            _concurrencyManager.FunctionCompleted("testfunction1", TimeSpan.FromMilliseconds(150));
+            _concurrencyManager.FunctionCompleted("testfunction1", TimeSpan.FromMilliseconds(90));
+
+            status = _concurrencyManager.ConcurrencyStatuses["testfunction1"];
+            Assert.Equal(0, status.OutstandingInvocations);
+            Assert.Equal(3, status.MaxConcurrentExecutionsSinceLastAdjustment);
+            Assert.Equal(4, status.InvocationsSinceLastAdjustment);
+            Assert.Equal(450, status.TotalInvocationTimeSinceLastAdjustmentMs);
+
+            _concurrencyManager.FunctionCompleted("testfunction2", TimeSpan.FromMilliseconds(1000));
+            _concurrencyManager.FunctionCompleted("testfunction2", TimeSpan.FromMilliseconds(1500));
+
+            status = _concurrencyManager.ConcurrencyStatuses["testfunction2"];
+            Assert.Equal(0, status.OutstandingInvocations);
+            Assert.Equal(2, status.MaxConcurrentExecutionsSinceLastAdjustment);
+            Assert.Equal(2, status.InvocationsSinceLastAdjustment);
+            Assert.Equal(2500, status.TotalInvocationTimeSinceLastAdjustmentMs);
+
+            _concurrencyManager.FunctionCompleted("testfunction3", TimeSpan.FromMilliseconds(25));
+
+            status = _concurrencyManager.ConcurrencyStatuses["testfunction3"];
+            Assert.Equal(0, status.OutstandingInvocations);
+            Assert.Equal(1, status.MaxConcurrentExecutionsSinceLastAdjustment);
+            Assert.Equal(1, status.InvocationsSinceLastAdjustment);
+            Assert.Equal(25, status.TotalInvocationTimeSinceLastAdjustmentMs);
+        }
+
+        [Theory]
+        [InlineData(null)]
+        [InlineData("")]
+        public void FunctionStarted_InvalidFunction_Throws(string functionId)
+        {
+            var ex = Assert.Throws<ArgumentNullException>(() => _concurrencyManager.FunctionStarted(functionId));
+            Assert.Equal(nameof(functionId), ex.ParamName);
+        }
+
+        [Fact]
+        public void FunctionStarted_DoesNotCreateStatus()
+        {
+            _concurrencyManager.ConcurrencyStatuses.Clear();
+            Assert.Empty(_concurrencyManager.ConcurrencyStatuses);
+
+            _concurrencyManager.FunctionStarted(TestFunctionId);
+
+            Assert.Empty(_concurrencyManager.ConcurrencyStatuses);
+
+            var status = _concurrencyManager.GetStatus(TestFunctionId);
+            _concurrencyManager.FunctionStarted(TestFunctionId);
+            Assert.Single(_concurrencyManager.ConcurrencyStatuses);
+            Assert.Equal(1, status.OutstandingInvocations);
+        }
+
+        [Theory]
+        [InlineData(null)]
+        [InlineData("")]
+        public void FunctionCompleted_InvalidFunction_Throws(string functionId)
+        {
+            var ex = Assert.Throws<ArgumentNullException>(() => _concurrencyManager.FunctionCompleted(functionId, TimeSpan.FromMilliseconds(50)));
+            Assert.Equal(nameof(functionId), ex.ParamName);
+        }
+
+        [Fact]
+        public void FunctionCompleted_DoesNotCreateStatus()
+        {
+            _concurrencyManager.ConcurrencyStatuses.Clear();
+            Assert.Empty(_concurrencyManager.ConcurrencyStatuses);
+
+            _concurrencyManager.FunctionCompleted(TestFunctionId, TimeSpan.FromMilliseconds(10));
+
+            Assert.Empty(_concurrencyManager.ConcurrencyStatuses);
+
+            var status = _concurrencyManager.GetStatus(TestFunctionId);
+            _concurrencyManager.FunctionCompleted(TestFunctionId, TimeSpan.FromMilliseconds(10));
+            Assert.Single(_concurrencyManager.ConcurrencyStatuses);
+            Assert.Equal(0, status.OutstandingInvocations);
+        }
+
+        [Fact]
+        public void GetSnapshot_ReturnsExpectedResult()
+        {
+            _concurrencyManager.ConcurrencyStatuses["testfunction1"] = new ConcurrencyStatus("testfunction1", _concurrencyManager)
+            {
+                CurrentConcurrency = 5
+            };
+            _concurrencyManager.ConcurrencyStatuses["testfunction2"] = new ConcurrencyStatus("testfunction2", _concurrencyManager)
+            {
+                CurrentConcurrency = 10
+            };
+            _concurrencyManager.ConcurrencyStatuses["testfunction3"] = new ConcurrencyStatus("testfunction3", _concurrencyManager)
+            {
+                CurrentConcurrency = 15
+            };
+
+            var snapshot = _concurrencyManager.GetSnapshot();
+
+            Assert.Equal(Utility.GetEffectiveCoresCount(), snapshot.NumberOfCores);
+
+            var functionSnapshot = snapshot.FunctionSnapshots["testfunction1"];
+            Assert.Equal(5, functionSnapshot.Concurrency);
+
+            functionSnapshot = snapshot.FunctionSnapshots["testfunction2"];
+            Assert.Equal(10, functionSnapshot.Concurrency);
+
+            functionSnapshot = snapshot.FunctionSnapshots["testfunction3"];
+            Assert.Equal(15, functionSnapshot.Concurrency);
+        }
+
+        [Fact]
+        public void ApplySnapshot_PerformsExpectedUpdates()
+        {
+            var snapshot = new HostConcurrencySnapshot
+            {
+                Timestamp = DateTime.UtcNow,
+                NumberOfCores = Utility.GetEffectiveCoresCount(),
+                FunctionSnapshots = new Dictionary<string, FunctionConcurrencySnapshot>
+                {
+                    { "testfunction1", new FunctionConcurrencySnapshot { Concurrency = 5 } },
+                    { "testfunction2", new FunctionConcurrencySnapshot { Concurrency = 10 } },
+                    { "testfunction3", new FunctionConcurrencySnapshot { Concurrency = 15 } }
+                }
+            };
+
+            Assert.Single(_concurrencyManager.ConcurrencyStatuses);
+            Assert.Equal(1, _concurrencyManager.ConcurrencyStatuses["testfunction"].CurrentConcurrency);
+
+            _concurrencyManager.ApplySnapshot(snapshot);
+
+            Assert.Equal(4, _concurrencyManager.ConcurrencyStatuses.Count);
+
+            Assert.Equal(1, _concurrencyManager.ConcurrencyStatuses["testfunction"].CurrentConcurrency);
+            Assert.Equal(5, _concurrencyManager.ConcurrencyStatuses["testfunction1"].CurrentConcurrency);
+            Assert.Equal(10, _concurrencyManager.ConcurrencyStatuses["testfunction2"].CurrentConcurrency);
+            Assert.Equal(15, _concurrencyManager.ConcurrencyStatuses["testfunction3"].CurrentConcurrency);
+        }
+
+        [Theory]
+        [InlineData(100, 8, 4, 50)]
+        [InlineData(100, 4, 8, 200)]
+        [InlineData(1, 8, 4, 1)]
+        [InlineData(1, 4, 8, 2)]
+        public void GetCoreAdjustedConcurrency_ReturnsExpectedValue(int concurrency, int otherCores, int cores, int expectedConcurrency)
+        {
+            Assert.Equal(expectedConcurrency, ConcurrencyManager.GetCoreAdjustedConcurrency(concurrency, otherCores, cores));
+        }
+
+        [Fact]
+        public void ApplySnapshot_DifferentCoreCount_PerformsExpectedUpdates()
+        {
+            _concurrencyManager.EffectiveCoresCount = 4;
+            int snapshotCoreCount = 8;
+
+            var snapshot = new HostConcurrencySnapshot
+            {
+                Timestamp = DateTime.UtcNow,
+                NumberOfCores = snapshotCoreCount,
+                FunctionSnapshots = new Dictionary<string, FunctionConcurrencySnapshot>
+                {
+                    { "testfunction1", new FunctionConcurrencySnapshot { Concurrency = 50 } },
+                    { "testfunction2", new FunctionConcurrencySnapshot { Concurrency = 100 } },
+                    { "testfunction3", new FunctionConcurrencySnapshot { Concurrency = 150 } }
+                }
+            };
+
+            Assert.Single(_concurrencyManager.ConcurrencyStatuses);
+            Assert.Equal(1, _concurrencyManager.ConcurrencyStatuses["testfunction"].CurrentConcurrency);
+
+            _concurrencyManager.ApplySnapshot(snapshot);
+
+            Assert.Equal(4, _concurrencyManager.ConcurrencyStatuses.Count);
+
+            // since our core count is half that of the snapshot, we expect the applied concurrency levels
+            // to be halved
+            Assert.Equal(1, _concurrencyManager.ConcurrencyStatuses["testfunction"].CurrentConcurrency);
+            Assert.Equal(25, _concurrencyManager.ConcurrencyStatuses["testfunction1"].CurrentConcurrency);
+            Assert.Equal(50, _concurrencyManager.ConcurrencyStatuses["testfunction2"].CurrentConcurrency);
+            Assert.Equal(75, _concurrencyManager.ConcurrencyStatuses["testfunction3"].CurrentConcurrency);
+        }
+
+        private void SimulateFunctionInvocations(string functionId, int count)
+        {
+            for (int i = 0; i < count; i++)
+            {
+                _concurrencyManager.FunctionStarted(functionId);
+                _concurrencyManager.FunctionCompleted(functionId, TimeSpan.FromMilliseconds(10));
+            }
+        }
+    }
+}

test/Microsoft.Azure.WebJobs.Host.UnitTests/Scale/ConcurrencyOptionSetupTests.cs

@@ -0,0 +1,29 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT License. See License.txt in the project root for license information.
+
+using System;
+using Microsoft.Azure.WebJobs.Host.Config;
+using Microsoft.Azure.WebJobs.Host.Scale;
+using Microsoft.Azure.WebJobs.Host.TestCommon;
+using Xunit;
+
+namespace Microsoft.Azure.WebJobs.Host.UnitTests.Scale
+{
+    [Trait(TestTraits.CategoryTraitName, TestTraits.DynamicConcurrency)]
+    public class ConcurrencyOptionSetupTests
+    {
+        [Theory]
+        [InlineData("standard", 4, 7516192768)]
+        [InlineData("dynamic", 1, 1610612736)]
+        [InlineData(null, 4, -1)]
+        [InlineData("", 4, -1)]
+        [InlineData("invalidsku", 4, -1)]
+        public void Configure_Memory_SetsExpectedValues(string sku, int numCores, long expectedMemory)
+        {
+            var options = new ConcurrencyOptions();
+            ConcurrencyOptionsSetup.ConfigureMemoryOptions(options, sku, numCores);
+
+            Assert.Equal(expectedMemory, options.TotalAvailableMemoryBytes);
+        }
+    }
+}

test/Microsoft.Azure.WebJobs.Host.UnitTests/Scale/ConcurrencyOptionsTests.cs

@@ -0,0 +1,137 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT License. See License.txt in the project root for license information.
+
+using System;
+using System.Text.RegularExpressions;
+using Microsoft.Azure.WebJobs.Host.Scale;
+using Microsoft.Azure.WebJobs.Host.TestCommon;
+using Xunit;
+
+namespace Microsoft.Azure.WebJobs.Host.UnitTests.Scale
+{
+    [Trait(TestTraits.CategoryTraitName, TestTraits.DynamicConcurrency)]
+    public class ConcurrencyOptionsTests
+    {
+        [Fact]
+        public void Constructor_Defaults()
+        {
+            var options = new ConcurrencyOptions();
+
+            Assert.Equal(true, options.SnapshotPersistenceEnabled);
+            Assert.Equal(500, options.MaximumFunctionConcurrency);
+            Assert.Equal(-1, options.TotalAvailableMemoryBytes);
+            Assert.Equal(0.80F, options.CPUThreshold);
+            Assert.Equal(0.80F, options.MemoryThreshold);
+
+            Assert.False(options.MemoryThrottleEnabled);
+        }
+
+        [Theory]
+        [InlineData(100000000, 0.8F, true)]
+        [InlineData(100000000, -1, false)]
+        [InlineData(-1, 0.8F, false)]
+        public void MemoryThrottleEnabled_ReturnsExpectedValue(int availableMemoryBytes, float memoryThreshold, bool expected)
+        {
+            var options = new ConcurrencyOptions
+            {
+                TotalAvailableMemoryBytes = availableMemoryBytes,
+                MemoryThreshold = memoryThreshold
+            };
+
+            Assert.Equal(expected, options.MemoryThrottleEnabled);
+        }
+
+        [Fact]
+        public void MaximumFunctionConcurrency_Validation()
+        {
+            var options = new ConcurrencyOptions();
+
+            var ex = Assert.Throws<ArgumentOutOfRangeException>(() => options.MaximumFunctionConcurrency = 0);
+            Assert.Equal(nameof(ConcurrencyOptions.MaximumFunctionConcurrency), ex.ParamName);
+
+            options.MaximumFunctionConcurrency = -1;
+            Assert.Equal(-1, options.MaximumFunctionConcurrency);
+
+            options.MaximumFunctionConcurrency = 100;
+            Assert.Equal(100, options.MaximumFunctionConcurrency);
+        }
+
+        [Fact]
+        public void TotalAvailableMemoryBytes_Validation()
+        {
+            var options = new ConcurrencyOptions();
+
+            var ex = Assert.Throws<ArgumentOutOfRangeException>(() => options.TotalAvailableMemoryBytes = 0);
+            Assert.Equal(nameof(ConcurrencyOptions.TotalAvailableMemoryBytes), ex.ParamName);
+
+            options.TotalAvailableMemoryBytes = -1;
+            Assert.Equal(-1, options.TotalAvailableMemoryBytes);
+
+            options.TotalAvailableMemoryBytes = 3000000000;
+            Assert.Equal(3000000000, options.TotalAvailableMemoryBytes);
+        }
+
+        [Fact]
+        public void MemoryThreshold_Validation()
+        {
+            var options = new ConcurrencyOptions();
+
+            var ex = Assert.Throws<ArgumentOutOfRangeException>(() => options.MemoryThreshold = 0);
+            Assert.Equal(nameof(ConcurrencyOptions.MemoryThreshold), ex.ParamName);
+
+            ex = Assert.Throws<ArgumentOutOfRangeException>(() => options.MemoryThreshold = 1);
+            Assert.Equal(nameof(ConcurrencyOptions.MemoryThreshold), ex.ParamName);
+
+            ex = Assert.Throws<ArgumentOutOfRangeException>(() => options.MemoryThreshold = 1.1F);
+            Assert.Equal(nameof(ConcurrencyOptions.MemoryThreshold), ex.ParamName);
+
+            options.MemoryThreshold = -1;
+            Assert.Equal(-1, options.MemoryThreshold);
+
+            options.MemoryThreshold = 0.75F;
+            Assert.Equal(0.75, options.MemoryThreshold);
+        }
+
+        [Fact]
+        public void CPUThreshold_Validation()
+        {
+            var options = new ConcurrencyOptions();
+
+            var ex = Assert.Throws<ArgumentOutOfRangeException>(() => options.CPUThreshold = 0);
+            Assert.Equal(nameof(ConcurrencyOptions.CPUThreshold), ex.ParamName);
+
+            ex = Assert.Throws<ArgumentOutOfRangeException>(() => options.CPUThreshold = 1);
+            Assert.Equal(nameof(ConcurrencyOptions.CPUThreshold), ex.ParamName);
+
+            ex = Assert.Throws<ArgumentOutOfRangeException>(() => options.CPUThreshold = 1.1F);
+            Assert.Equal(nameof(ConcurrencyOptions.CPUThreshold), ex.ParamName);
+
+            options.CPUThreshold = 0.75F;
+            Assert.Equal(0.75, options.CPUThreshold);
+        }
+
+        [Fact]
+        public void Format_ReturnsExpectedResult()
+        {
+            var options = new ConcurrencyOptions
+            {
+                DynamicConcurrencyEnabled = true,
+                TotalAvailableMemoryBytes = 3000000,
+                CPUThreshold = 0.85F,
+                MemoryThreshold = 0.85F,
+                SnapshotPersistenceEnabled = true,
+                MaximumFunctionConcurrency = 1000
+            };
+
+            string result = options.Format();
+            string expected = @"{
+  ""DynamicConcurrencyEnabled"": true,
+  ""MaximumFunctionConcurrency"": 1000,
+  ""TotalAvailableMemoryBytes"": 3000000,
+  ""CPUThreshold"": 0.85,
+  ""SnapshotPersistenceEnabled"": true
+}";
+            Assert.Equal(Regex.Replace(expected, @"\s+", ""), Regex.Replace(result, @"\s+", ""));
+        }
+    }
+}

test/Microsoft.Azure.WebJobs.Host.UnitTests/Scale/ConcurrencyStatusSnapshotTests.cs

@@ -0,0 +1,77 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT License. See License.txt in the project root for license information.
+
+using System;
+using System.Collections.Generic;
+using Microsoft.Azure.WebJobs.Host.Scale;
+using Microsoft.Azure.WebJobs.Host.TestCommon;
+using Xunit;
+
+namespace Microsoft.Azure.WebJobs.Host.UnitTests.Scale
+{
+    [Trait(TestTraits.CategoryTraitName, TestTraits.DynamicConcurrency)]
+    public class ConcurrencyStatusSnapshotTests
+    {
+        [Fact]
+        public void Equals_ReturnsExpectedResult()
+        {
+            var snapshot1 = new HostConcurrencySnapshot();
+            var snapshot2 = new HostConcurrencySnapshot();
+            Assert.True(snapshot1.Equals(snapshot2));
+
+            // timestamp not included in equality check
+            snapshot1.Timestamp = DateTime.UtcNow.Subtract(TimeSpan.FromSeconds(15));
+            snapshot2.Timestamp = DateTime.UtcNow;
+            Assert.True(snapshot1.Equals(snapshot2));
+
+            Assert.False(snapshot1.Equals(null));
+
+            // differing top level properties
+            snapshot1 = new HostConcurrencySnapshot
+            {
+                NumberOfCores = 1
+            };
+            snapshot2 = new HostConcurrencySnapshot
+            {
+                NumberOfCores = 4
+            };
+            Assert.False(snapshot1.Equals(snapshot2));
+
+            snapshot1.NumberOfCores = snapshot2.NumberOfCores = 4;
+            snapshot1.FunctionSnapshots = new Dictionary<string, FunctionConcurrencySnapshot>
+            {
+                { "function0", new FunctionConcurrencySnapshot { Concurrency = 5 } },
+                { "function1", new FunctionConcurrencySnapshot { Concurrency = 10 } },
+                { "function2", new FunctionConcurrencySnapshot { Concurrency = 15 } }
+            };
+            Assert.False(snapshot1.Equals(snapshot2));
+
+            // different functions
+            snapshot2.FunctionSnapshots = new Dictionary<string, FunctionConcurrencySnapshot>
+            {
+                { "function0", new FunctionConcurrencySnapshot { Concurrency = 5 } },
+                { "function1", new FunctionConcurrencySnapshot { Concurrency = 10 } },
+                { "function5", new FunctionConcurrencySnapshot { Concurrency = 15 } }
+            };
+            Assert.False(snapshot1.Equals(snapshot2));
+
+            // same functions, but differences in function snapshot properties
+            snapshot2.FunctionSnapshots = new Dictionary<string, FunctionConcurrencySnapshot>
+            {
+                { "function0", new FunctionConcurrencySnapshot { Concurrency = 1 } },
+                { "function1", new FunctionConcurrencySnapshot { Concurrency = 10 } },
+                { "function2", new FunctionConcurrencySnapshot { Concurrency = 50 } }
+            };
+            Assert.False(snapshot1.Equals(snapshot2));
+
+            // everything equal
+            snapshot2.FunctionSnapshots = new Dictionary<string, FunctionConcurrencySnapshot>
+            {
+                { "function0", new FunctionConcurrencySnapshot { Concurrency = 5 } },
+                { "function1", new FunctionConcurrencySnapshot { Concurrency = 10 } },
+                { "function2", new FunctionConcurrencySnapshot { Concurrency = 15 } }
+            };
+            Assert.True(snapshot1.Equals(snapshot2));
+        }
+    }
+}

test/Microsoft.Azure.WebJobs.Host.UnitTests/Scale/ConcurrencyStatusTests.cs

@@ -0,0 +1,426 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT License. See License.txt in the project root for license information.
+
+using System;
+using System.Collections.Concurrent;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading.Tasks;
+using Microsoft.Azure.WebJobs.Host.Scale;
+using Microsoft.Azure.WebJobs.Host.TestCommon;
+using Moq;
+using Xunit;
+
+namespace Microsoft.Azure.WebJobs.Host.UnitTests.Scale
+{
+    [Trait(TestTraits.CategoryTraitName, TestTraits.DynamicConcurrency)]
+    public class ConcurrencyStatusTests
+    {
+        private const string TestFunctionId = "testfunction";
+        private const int TestMaxConcurrency = 500;
+
+        private readonly Random _rand = new Random();
+        private readonly Mock<ConcurrencyManager> _concurrencyManagerMock;
+        private readonly ConcurrencyStatus _testConcurrencyStatus;
+
+        private bool _throttleEnabled;
+
+        public ConcurrencyStatusTests()
+        {
+            _concurrencyManagerMock = new Mock<ConcurrencyManager>(MockBehavior.Strict);
+            _concurrencyManagerMock.SetupGet(p => p.ThrottleEnabled).Returns(() => _throttleEnabled);
+
+            _testConcurrencyStatus = new ConcurrencyStatus(TestFunctionId, _concurrencyManagerMock.Object);
+        }
+
+        [Fact]
+        public void FunctionId_ReturnsExpectedValue()
+        {
+            Assert.Equal(TestFunctionId, _testConcurrencyStatus.FunctionId);
+        }
+
+        [Fact]
+        public void AvailableInvocationCount_ReturnsAvailableBuffer()
+        {
+            _testConcurrencyStatus.OutstandingInvocations = 25;
+            _testConcurrencyStatus.CurrentConcurrency = 100;
+
+            Assert.Equal(75, _testConcurrencyStatus.AvailableInvocationCount);
+        }
+
+        [Fact]
+        public void AvailableInvocationCount_OverLimit_ReturnsZero()
+        {
+            _testConcurrencyStatus.OutstandingInvocations = 105;
+            _testConcurrencyStatus.CurrentConcurrency = 100;
+
+            Assert.Equal(0, _testConcurrencyStatus.AvailableInvocationCount);
+        }
+
+        [Fact]
+        public void AvailableInvocationCount_ThrottleEnabled_ReturnsZero()
+        {
+            _testConcurrencyStatus.OutstandingInvocations = 25;
+            _testConcurrencyStatus.CurrentConcurrency = 100;
+            _throttleEnabled = true;
+
+            Assert.Equal(0, _testConcurrencyStatus.AvailableInvocationCount);
+        }
+
+        [Fact]
+        public void NextStatusDelay_ReturnsExpectedValue()
+        {
+            // AvailableInvocationCount > 0
+            _testConcurrencyStatus.OutstandingInvocations = 25;
+            _testConcurrencyStatus.CurrentConcurrency = 100;
+            _throttleEnabled = false;
+            Assert.True(_testConcurrencyStatus.AvailableInvocationCount > 0);
+            Assert.Equal(TimeSpan.Zero, _testConcurrencyStatus.NextStatusDelay);
+
+            // AvailableInvocationCount == 0
+            _testConcurrencyStatus.OutstandingInvocations = 125;
+            _testConcurrencyStatus.CurrentConcurrency = 100;
+            _throttleEnabled = false;
+            Assert.Equal(0, _testConcurrencyStatus.AvailableInvocationCount);
+            Assert.Equal(TimeSpan.FromSeconds(ConcurrencyStatus.NextStatusDelayDefaultSeconds), _testConcurrencyStatus.NextStatusDelay);
+
+            // AvailableInvocationCount == 0 (Throttling)
+            _testConcurrencyStatus.OutstandingInvocations = 125;
+            _testConcurrencyStatus.CurrentConcurrency = 100;
+            _throttleEnabled = true;
+            Assert.Equal(0, _testConcurrencyStatus.AvailableInvocationCount);
+            Assert.Equal(TimeSpan.FromSeconds(ConcurrencyStatus.NextStatusDelayDefaultSeconds), _testConcurrencyStatus.NextStatusDelay);
+        }
+
+        [Fact]
+        public void ApplySnapshot_PerformsExpectedUpdates()
+        {
+            var snapshot = new FunctionConcurrencySnapshot
+            {
+                Concurrency = 50
+            };
+            Assert.Equal(1, _testConcurrencyStatus.CurrentConcurrency);
+
+            _testConcurrencyStatus.ApplySnapshot(snapshot);
+            Assert.Equal(50, _testConcurrencyStatus.CurrentConcurrency);
+
+            // a snapshot concurrency whose value is lower is not applied
+            snapshot.Concurrency = 10;
+            _testConcurrencyStatus.ApplySnapshot(snapshot);
+            Assert.Equal(50, _testConcurrencyStatus.CurrentConcurrency);
+        }
+
+        [Fact]
+        public void GetLatencyAdjustedInterval_ReturnsExpectedValue()
+        {
+            // faster functions should have shorter adjustment intervals
+            // this function averages 100ms per invocation
+            _testConcurrencyStatus.TotalInvocationTimeSinceLastAdjustmentMs = 2000;
+            _testConcurrencyStatus.InvocationsSinceLastAdjustment = 20;
+            TimeSpan interval = _testConcurrencyStatus.GetLatencyAdjustedInterval(TimeSpan.FromMilliseconds(2000), TimeSpan.FromMilliseconds(5000), 1);
+            Assert.Equal(2100, interval.TotalMilliseconds);
+
+            // if we haven't had any invocations to compute a latency, we
+            // return the default interval
+            _testConcurrencyStatus.InvocationsSinceLastAdjustment = 0;
+            interval = _testConcurrencyStatus.GetLatencyAdjustedInterval(TimeSpan.FromMilliseconds(2000), TimeSpan.FromMilliseconds(5000), 1);
+            Assert.Equal(5000, interval.TotalMilliseconds);
+
+            // a longer running function that exceeds the max interval is
+            // capped to max
+            _testConcurrencyStatus.TotalInvocationTimeSinceLastAdjustmentMs = 30000;
+            _testConcurrencyStatus.InvocationsSinceLastAdjustment = 5;
+            interval = _testConcurrencyStatus.GetLatencyAdjustedInterval(TimeSpan.FromMilliseconds(2000), TimeSpan.FromMilliseconds(5000), 1);
+            Assert.Equal(5000, interval.TotalMilliseconds);
+        }
+
+        [Fact]
+        public void CanAdjustConcurrency_ReturnsExpectedValue()
+        {
+            TestHelpers.SetupStopwatch(_testConcurrencyStatus.LastConcurrencyAdjustmentStopwatch, TimeSpan.FromMilliseconds(3000));
+            _testConcurrencyStatus.TotalInvocationTimeSinceLastAdjustmentMs = 100;
+            _testConcurrencyStatus.InvocationsSinceLastAdjustment = 1;
+            bool canAdjust = _testConcurrencyStatus.CanAdjustConcurrency();
+            Assert.True(canAdjust);
+
+            TestHelpers.SetupStopwatch(_testConcurrencyStatus.LastConcurrencyAdjustmentStopwatch, TimeSpan.FromMilliseconds(500));
+            canAdjust = _testConcurrencyStatus.CanAdjustConcurrency();
+            Assert.False(canAdjust);
+
+            // no invocations to compute a latency based interval, so the default is used
+            TestHelpers.SetupStopwatch(_testConcurrencyStatus.LastConcurrencyAdjustmentStopwatch, TimeSpan.FromMilliseconds(5001));
+            _testConcurrencyStatus.TotalInvocationTimeSinceLastAdjustmentMs = 0;
+            _testConcurrencyStatus.InvocationsSinceLastAdjustment = 0;
+            canAdjust = _testConcurrencyStatus.CanAdjustConcurrency();
+            Assert.True(canAdjust);
+
+            TestHelpers.SetupStopwatch(_testConcurrencyStatus.LastConcurrencyAdjustmentStopwatch, TimeSpan.FromMilliseconds(4000));
+            canAdjust = _testConcurrencyStatus.CanAdjustConcurrency();
+            Assert.False(canAdjust);
+        }
+
+        [Theory]
+        [InlineData(50, true)]
+        [InlineData(1, false)]
+        public void CanDecreaseConcurrency_ReturnsExpectedValue(int concurrency, bool expected)
+        {
+            _testConcurrencyStatus.CurrentConcurrency = concurrency;
+            Assert.Equal(expected, _testConcurrencyStatus.CanDecreaseConcurrency());
+        }
+
+        [Fact]
+        public void CanIncreaseConcurrency_ReturnsExpectedValue()
+        {
+            _testConcurrencyStatus.CurrentConcurrency = 500;
+
+            // we've had to decrease concurrency recently, so we expect false
+            TestHelpers.SetupStopwatch(_testConcurrencyStatus.LastConcurrencyDecreaseStopwatch, TimeSpan.FromSeconds(20));
+            _testConcurrencyStatus.MaxConcurrentExecutionsSinceLastAdjustment = 0;
+            _testConcurrencyStatus.TotalInvocationTimeSinceLastAdjustmentMs = 0;
+            _testConcurrencyStatus.InvocationsSinceLastAdjustment = 0;
+            bool canAdjust = _testConcurrencyStatus.CanIncreaseConcurrency(TestMaxConcurrency);
+            Assert.False(canAdjust);
+
+            // we've had invocations, but our latency adjusted window is still too small
+            TestHelpers.SetupStopwatch(_testConcurrencyStatus.LastConcurrencyDecreaseStopwatch, TimeSpan.FromSeconds(14));
+            _testConcurrencyStatus.MaxConcurrentExecutionsSinceLastAdjustment = 1;
+            _testConcurrencyStatus.TotalInvocationTimeSinceLastAdjustmentMs = 500;
+            _testConcurrencyStatus.InvocationsSinceLastAdjustment = 1;
+            canAdjust = _testConcurrencyStatus.CanIncreaseConcurrency(TestMaxConcurrency);
+            Assert.False(canAdjust);
+
+            // we satisfy the quiet period window, but we're not utilizing our current concurrency level
+            TestHelpers.SetupStopwatch(_testConcurrencyStatus.LastConcurrencyDecreaseStopwatch, TimeSpan.FromSeconds(16));
+            canAdjust = _testConcurrencyStatus.CanIncreaseConcurrency(TestMaxConcurrency);
+            Assert.False(canAdjust);
+
+            // we satisfy the quiet period window, but we're not utilizing our current concurrency level
+            TestHelpers.SetupStopwatch(_testConcurrencyStatus.LastConcurrencyDecreaseStopwatch, TimeSpan.FromSeconds(16));
+            canAdjust = _testConcurrencyStatus.CanIncreaseConcurrency(TestMaxConcurrency);
+            Assert.False(canAdjust);
+
+            // we've utilized the full current concurrency level, but can't increase because we're at the
+            // max concurrency level
+            TestHelpers.SetupStopwatch(_testConcurrencyStatus.LastConcurrencyDecreaseStopwatch, TimeSpan.FromSeconds(16));
+            _testConcurrencyStatus.MaxConcurrentExecutionsSinceLastAdjustment = 500;
+            canAdjust = _testConcurrencyStatus.CanIncreaseConcurrency(TestMaxConcurrency);
+            Assert.False(canAdjust);
+
+            // all conditions satisfied so we can increase
+            TestHelpers.SetupStopwatch(_testConcurrencyStatus.LastConcurrencyDecreaseStopwatch, TimeSpan.FromSeconds(16));
+            _testConcurrencyStatus.CurrentConcurrency = 10;
+            _testConcurrencyStatus.MaxConcurrentExecutionsSinceLastAdjustment = 10;
+            canAdjust = _testConcurrencyStatus.CanIncreaseConcurrency(TestMaxConcurrency);
+            Assert.True(canAdjust);
+        }
+
+        [Theory]
+        [InlineData(1)]
+        [InlineData(-1)]
+        public void GetNextAdjustment_RunInSameDirection_ReturnsExpectedValues(int direction)
+        {
+            List<int> adjustments = new List<int>();
+
+            int delta;
+            for (int i = 0; i < 10; i++)
+            {
+                _testConcurrencyStatus.LastConcurrencyAdjustmentStopwatch.Restart();
+                delta = _testConcurrencyStatus.GetNextAdjustment(direction);
+                adjustments.Add(delta);
+            }
+
+            Assert.Equal(new int[] { 1, 2, 3, 4, 5, 6, 6, 6, 6, 6 }.Select(p => direction * p), adjustments);
+
+            // if too much time passes, the run window ends and we start back at 1
+            TestHelpers.SetupStopwatch(_testConcurrencyStatus.LastConcurrencyAdjustmentStopwatch, TimeSpan.FromSeconds(11));
+            delta = _testConcurrencyStatus.GetNextAdjustment(1);
+            Assert.Equal(1, delta);
+        }
+
+        [Fact]
+        public void GetNextAdjustment_DirectionChange_EndsRun()
+        {
+            List<int> adjustments = new List<int>();
+
+            int delta;
+            for (int i = 0; i < 5; i++)
+            {
+                _testConcurrencyStatus.LastConcurrencyAdjustmentStopwatch.Restart();
+                delta = _testConcurrencyStatus.GetNextAdjustment(1);
+                adjustments.Add(delta);
+            }
+
+            Assert.Equal(adjustments, new int[] { 1, 2, 3, 4, 5 });
+
+            // if we switch directions, the run ends
+            delta = _testConcurrencyStatus.GetNextAdjustment(-1);
+            Assert.Equal(-1, delta);
+        }
+
+        [Fact]
+        public void IncreaseConcurrency_PerformsExpectedAdjustments()
+        {
+            // initialize some values - we expect these to be reset below
+            _testConcurrencyStatus.MaxConcurrentExecutionsSinceLastAdjustment = 25;
+            _testConcurrencyStatus.InvocationsSinceLastAdjustment = 50;
+            _testConcurrencyStatus.TotalInvocationTimeSinceLastAdjustmentMs = 30000;
+
+            Assert.False(_testConcurrencyStatus.LastConcurrencyDecreaseStopwatch.IsRunning);
+            Assert.True(_testConcurrencyStatus.LastConcurrencyAdjustmentStopwatch.IsRunning);
+
+            var lastAdjustment = _testConcurrencyStatus.LastConcurrencyAdjustmentStopwatch.Elapsed;
+
+            Assert.Equal(1, _testConcurrencyStatus.CurrentConcurrency);
+
+            _testConcurrencyStatus.IncreaseConcurrency();
+            Assert.Equal(2, _testConcurrencyStatus.CurrentConcurrency);
+
+            Assert.Equal(0, _testConcurrencyStatus.MaxConcurrentExecutionsSinceLastAdjustment);
+            Assert.Equal(0, _testConcurrencyStatus.InvocationsSinceLastAdjustment);
+            Assert.Equal(0, _testConcurrencyStatus.TotalInvocationTimeSinceLastAdjustmentMs);
+
+            // expect the adjustment stopwatch to restart
+            Assert.False(_testConcurrencyStatus.LastConcurrencyDecreaseStopwatch.IsRunning);
+            Assert.True(_testConcurrencyStatus.LastConcurrencyAdjustmentStopwatch.IsRunning);
+            Assert.True(_testConcurrencyStatus.LastConcurrencyAdjustmentStopwatch.Elapsed < lastAdjustment);
+        }
+
+        [Fact]
+        public void DecreaseConcurrency_PerformsExpectedAdjustments()
+        {
+            // initialize some values - we expect these to be reset below
+            _testConcurrencyStatus.MaxConcurrentExecutionsSinceLastAdjustment = 25;
+            _testConcurrencyStatus.InvocationsSinceLastAdjustment = 50;
+            _testConcurrencyStatus.TotalInvocationTimeSinceLastAdjustmentMs = 30000;
+            _testConcurrencyStatus.CurrentConcurrency = 5;
+
+            Assert.False(_testConcurrencyStatus.LastConcurrencyDecreaseStopwatch.IsRunning);
+            Assert.True(_testConcurrencyStatus.LastConcurrencyAdjustmentStopwatch.IsRunning);
+
+            TimeSpan lastDecrease = _testConcurrencyStatus.LastConcurrencyDecreaseStopwatch.Elapsed;
+            TimeSpan lastAdjustment = _testConcurrencyStatus.LastConcurrencyAdjustmentStopwatch.Elapsed;
+
+            _testConcurrencyStatus.DecreaseConcurrency();
+            Assert.Equal(4, _testConcurrencyStatus.CurrentConcurrency);
+
+            Assert.Equal(0, _testConcurrencyStatus.MaxConcurrentExecutionsSinceLastAdjustment);
+            Assert.Equal(0, _testConcurrencyStatus.InvocationsSinceLastAdjustment);
+            Assert.Equal(0, _testConcurrencyStatus.TotalInvocationTimeSinceLastAdjustmentMs);
+
+            // expect both stopwatches to restart
+            Assert.True(_testConcurrencyStatus.LastConcurrencyDecreaseStopwatch.IsRunning);
+            Assert.True(_testConcurrencyStatus.LastConcurrencyAdjustmentStopwatch.IsRunning);
+            Assert.True(_testConcurrencyStatus.LastConcurrencyDecreaseStopwatch.Elapsed > lastDecrease);
+            Assert.True(_testConcurrencyStatus.LastConcurrencyAdjustmentStopwatch.Elapsed < lastAdjustment);
+        }
+
+        [Fact]
+        public void IncreaseConcurrency_Run_EndsWithExpectedResult()
+        {
+            Assert.Equal(1, _testConcurrencyStatus.CurrentConcurrency);
+
+            for (int i = 0; i < 10; i++)
+            {
+                _testConcurrencyStatus.IncreaseConcurrency();
+            }
+
+            Assert.Equal(46, _testConcurrencyStatus.CurrentConcurrency);
+        }
+
+        [Fact]
+        public void DecreaseConcurrency_Run_EndsWithExpectedResult()
+        {
+            _testConcurrencyStatus.CurrentConcurrency = 50;
+
+            for (int i = 0; i < 5; i++)
+            {
+                _testConcurrencyStatus.DecreaseConcurrency();
+            }
+
+            Assert.Equal(35, _testConcurrencyStatus.CurrentConcurrency);
+        }
+
+        [Fact]
+        public void FunctionInvocationTracking_MaintainsExpectedCounts()
+        {
+            Assert.Equal(0, _testConcurrencyStatus.OutstandingInvocations);
+            Assert.Equal(0, _testConcurrencyStatus.InvocationsSinceLastAdjustment);
+            Assert.Equal(0, _testConcurrencyStatus.MaxConcurrentExecutionsSinceLastAdjustment);
+
+            _testConcurrencyStatus.FunctionStarted();
+            Assert.Equal(1, _testConcurrencyStatus.OutstandingInvocations);
+            Assert.Equal(0, _testConcurrencyStatus.InvocationsSinceLastAdjustment);
+            Assert.Equal(1, _testConcurrencyStatus.MaxConcurrentExecutionsSinceLastAdjustment);
+
+            _testConcurrencyStatus.FunctionStarted();
+            Assert.Equal(2, _testConcurrencyStatus.OutstandingInvocations);
+            Assert.Equal(0, _testConcurrencyStatus.InvocationsSinceLastAdjustment);
+            Assert.Equal(2, _testConcurrencyStatus.MaxConcurrentExecutionsSinceLastAdjustment);
+
+            _testConcurrencyStatus.FunctionCompleted(TimeSpan.FromMilliseconds(25));
+            Assert.Equal(1, _testConcurrencyStatus.OutstandingInvocations);
+            Assert.Equal(1, _testConcurrencyStatus.InvocationsSinceLastAdjustment);
+            Assert.Equal(2, _testConcurrencyStatus.MaxConcurrentExecutionsSinceLastAdjustment);
+            Assert.Equal(25, _testConcurrencyStatus.TotalInvocationTimeSinceLastAdjustmentMs);
+
+            _testConcurrencyStatus.FunctionCompleted(TimeSpan.FromMilliseconds(25));
+            Assert.Equal(0, _testConcurrencyStatus.OutstandingInvocations);
+            Assert.Equal(2, _testConcurrencyStatus.InvocationsSinceLastAdjustment);
+            Assert.Equal(2, _testConcurrencyStatus.MaxConcurrentExecutionsSinceLastAdjustment);
+            Assert.Equal(50, _testConcurrencyStatus.TotalInvocationTimeSinceLastAdjustmentMs);
+        }
+
+        [Fact]
+        public async Task FunctionExecutionEvents_Synchronization()
+        {
+            int numExecutionsPerThread = 250;
+            int numThreads = 20;
+            int expectedInvocationCount = numExecutionsPerThread * numThreads;
+
+            List<Task> tasks = new List<Task>();
+            ConcurrentBag<int> latencies = new ConcurrentBag<int>();
+            object syncLock = new object();
+            int maxConcurrentExecutions = 0;
+            int outstandingExecutions = 0;
+            for (int i = 0; i < numThreads; i++)
+            {
+                var task = Task.Run(async () =>
+                {
+                    for (int j = 0; j < numExecutionsPerThread; j++)
+                    {
+                        lock (syncLock)
+                        {
+                            outstandingExecutions++;
+
+                            if (outstandingExecutions > maxConcurrentExecutions)
+                            {
+                                maxConcurrentExecutions = outstandingExecutions;
+                            }
+                        }
+
+                        _testConcurrencyStatus.FunctionStarted();
+
+                        int latency = _rand.Next(5, 25);
+                        latencies.Add(latency);
+                        await Task.Delay(latency);
+
+                        lock (syncLock)
+                        {
+                            outstandingExecutions--;
+                        }
+
+                        _testConcurrencyStatus.FunctionCompleted(TimeSpan.FromMilliseconds(latency));
+                    }
+                });
+                tasks.Add(task);
+            }
+
+            await Task.WhenAll(tasks);
+
+            Assert.Equal(0, _testConcurrencyStatus.OutstandingInvocations);
+            Assert.Equal(expectedInvocationCount, _testConcurrencyStatus.InvocationsSinceLastAdjustment);
+            Assert.Equal(latencies.Sum(), _testConcurrencyStatus.TotalInvocationTimeSinceLastAdjustmentMs);
+            Assert.Equal(numThreads, _testConcurrencyStatus.MaxConcurrentExecutionsSinceLastAdjustment);
+        }
+    }
+}

test/Microsoft.Azure.WebJobs.Host.UnitTests/Scale/DefaultConcurrencyThrottleManagerTests.cs

@@ -0,0 +1,205 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT License. See License.txt in the project root for license information.
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading.Tasks;
+using Microsoft.Azure.WebJobs.Host.Scale;
+using Microsoft.Azure.WebJobs.Host.TestCommon;
+using Microsoft.Azure.WebJobs.Logging;
+using Microsoft.Extensions.Logging;
+using Moq;
+using Xunit;
+
+namespace Microsoft.Azure.WebJobs.Host.UnitTests.Scale
+{
+    [Trait(TestTraits.CategoryTraitName, TestTraits.DynamicConcurrency)]
+    public class DefaultConcurrencyThrottleManagerTests
+    {
+        private readonly LoggerFactory _loggerFactory;
+        private readonly TestLoggerProvider _loggerProvider;
+        private readonly DefaultConcurrencyThrottleManager _throttleManager;
+        private ConcurrencyThrottleStatus _throttleProvider1Status;
+        private ConcurrencyThrottleStatus _throttleProvider2Status;
+        private ConcurrencyThrottleStatus _throttleProvider3Status;
+
+        public DefaultConcurrencyThrottleManagerTests()
+        {
+            _loggerFactory = new LoggerFactory();
+            _loggerProvider = new TestLoggerProvider();
+            _loggerFactory.AddProvider(_loggerProvider);
+            var logger = _loggerFactory.CreateLogger(LogCategories.Concurrency);
+
+            var mockThrottleProvider1 = new Mock<IConcurrencyThrottleProvider>(MockBehavior.Strict);
+            mockThrottleProvider1.Setup(p => p.GetStatus(logger)).Returns(() => _throttleProvider1Status);
+
+            var mockThrottleProvider2 = new Mock<IConcurrencyThrottleProvider>(MockBehavior.Strict);
+            mockThrottleProvider2.Setup(p => p.GetStatus(logger)).Returns(() => _throttleProvider2Status);
+
+            var mockThrottleProvider3 = new Mock<IConcurrencyThrottleProvider>(MockBehavior.Strict);
+            mockThrottleProvider3.Setup(p => p.GetStatus(logger)).Returns(() => _throttleProvider3Status);
+
+            var throttleProviders = new IConcurrencyThrottleProvider[] { mockThrottleProvider1.Object, mockThrottleProvider2.Object, mockThrottleProvider3.Object };
+            _throttleManager = new DefaultConcurrencyThrottleManager(throttleProviders, _loggerFactory);
+        }
+
+        [Fact]
+        public void GetStatus_ThrottleEnabled_ReturnsExpectedResult()
+        {
+            TestHelpers.SetupStopwatch(_throttleManager.LastThrottleCheckStopwatch, TimeSpan.FromSeconds(5));
+            var lastThrottleCheck = _throttleManager.LastThrottleCheckStopwatch.Elapsed;
+
+            _throttleProvider1Status = new ConcurrencyThrottleStatus { State = ThrottleState.Disabled };
+            _throttleProvider2Status = new ConcurrencyThrottleStatus { State = ThrottleState.Enabled, EnabledThrottles = new List<string> { "Test" }.AsReadOnly() };
+            _throttleProvider3Status = new ConcurrencyThrottleStatus { State = ThrottleState.Disabled };
+
+            var status = _throttleManager.GetStatus();
+            Assert.Equal(ThrottleState.Enabled, status.State);
+            Assert.Equal(0, status.ConsecutiveCount);
+            Assert.Single(status.EnabledThrottles, p => p == "Test");
+            Assert.True(_throttleManager.LastThrottleCheckStopwatch.Elapsed < lastThrottleCheck);
+        }
+
+        [Fact]
+        public void GetStatus_MultipleThrottlesEnabled_ReturnsExpectedResult()
+        {
+            TestHelpers.SetupStopwatch(_throttleManager.LastThrottleCheckStopwatch, TimeSpan.FromSeconds(5));
+            var lastThrottleCheck = _throttleManager.LastThrottleCheckStopwatch.Elapsed;
+
+            _throttleProvider1Status = new ConcurrencyThrottleStatus { State = ThrottleState.Disabled };
+            _throttleProvider2Status = new ConcurrencyThrottleStatus { State = ThrottleState.Enabled, EnabledThrottles = new List<string> { "Throttle1" }.AsReadOnly() };
+            _throttleProvider3Status = new ConcurrencyThrottleStatus { State = ThrottleState.Enabled, EnabledThrottles = new List<string> { "Throttle2", "Throttle3" }.AsReadOnly() };
+
+            var status = _throttleManager.GetStatus();
+            Assert.Equal(ThrottleState.Enabled, status.State);
+            Assert.Equal(0, status.ConsecutiveCount);
+            Assert.Equal(3, status.EnabledThrottles.Count);
+            Assert.Collection(status.EnabledThrottles,
+                p => Assert.Equal(p, "Throttle1"),
+                p => Assert.Equal(p, "Throttle2"),
+                p => Assert.Equal(p, "Throttle3"));
+            Assert.True(_throttleManager.LastThrottleCheckStopwatch.Elapsed < lastThrottleCheck);
+        }
+
+        [Fact]
+        public void GetStatus_ThrottleEnabled_ThrottleUnknown_ReturnsExpectedResult()
+        {
+            TestHelpers.SetupStopwatch(_throttleManager.LastThrottleCheckStopwatch, TimeSpan.FromSeconds(5));
+            var lastThrottleCheck = _throttleManager.LastThrottleCheckStopwatch.Elapsed;
+
+            _throttleProvider1Status = new ConcurrencyThrottleStatus { State = ThrottleState.Disabled };
+            _throttleProvider2Status = new ConcurrencyThrottleStatus { State = ThrottleState.Enabled, EnabledThrottles = new List<string> { "Test" }.AsReadOnly() };
+            _throttleProvider3Status = new ConcurrencyThrottleStatus { State = ThrottleState.Unknown };
+
+            var status = _throttleManager.GetStatus();
+            Assert.Equal(ThrottleState.Enabled, status.State);
+            Assert.Equal(0, status.ConsecutiveCount);
+            Assert.Single(status.EnabledThrottles, p => p == "Test");
+            Assert.True(_throttleManager.LastThrottleCheckStopwatch.Elapsed < lastThrottleCheck);
+        }
+
+        [Fact]
+        public void GetStatus_ThrottleDisabled_ReturnsExpectedResult()
+        {
+            TestHelpers.SetupStopwatch(_throttleManager.LastThrottleCheckStopwatch, TimeSpan.FromSeconds(5));
+            var lastThrottleCheck = _throttleManager.LastThrottleCheckStopwatch.Elapsed;
+
+            _throttleProvider1Status = new ConcurrencyThrottleStatus { State = ThrottleState.Disabled };
+            _throttleProvider2Status = new ConcurrencyThrottleStatus { State = ThrottleState.Disabled };
+            _throttleProvider3Status = new ConcurrencyThrottleStatus { State = ThrottleState.Disabled };
+
+            var status = _throttleManager.GetStatus();
+            Assert.Equal(ThrottleState.Disabled, status.State);
+            Assert.Equal(0, status.ConsecutiveCount);
+            Assert.Null(status.EnabledThrottles);
+            Assert.True(_throttleManager.LastThrottleCheckStopwatch.Elapsed < lastThrottleCheck);
+        }
+
+        [Fact]
+        public void GetStatus_ThrottleUnknown_ReturnsExpectedResult()
+        {
+            TestHelpers.SetupStopwatch(_throttleManager.LastThrottleCheckStopwatch, TimeSpan.FromSeconds(5));
+            var lastThrottleCheck = _throttleManager.LastThrottleCheckStopwatch.Elapsed;
+
+            _throttleProvider1Status = new ConcurrencyThrottleStatus { State = ThrottleState.Disabled };
+            _throttleProvider2Status = new ConcurrencyThrottleStatus { State = ThrottleState.Unknown };
+            _throttleProvider3Status = new ConcurrencyThrottleStatus { State = ThrottleState.Disabled };
+
+            var status = _throttleManager.GetStatus();
+            Assert.Equal(ThrottleState.Unknown, status.State);
+            Assert.Equal(1, status.ConsecutiveCount);
+            Assert.Null(status.EnabledThrottles);
+            Assert.True(_throttleManager.LastThrottleCheckStopwatch.Elapsed < lastThrottleCheck);
+        }
+
+        [Fact]
+        public async Task GetStatus_ThrottlesUpdates()
+        {
+            TestHelpers.SetupStopwatch(_throttleManager.LastThrottleCheckStopwatch, TimeSpan.FromSeconds(5));
+            TimeSpan lastThrottleCheck = _throttleManager.LastThrottleCheckStopwatch.Elapsed;
+
+            _throttleProvider1Status = new ConcurrencyThrottleStatus { State = ThrottleState.Disabled };
+            _throttleProvider2Status = new ConcurrencyThrottleStatus { State = ThrottleState.Enabled, EnabledThrottles = new List<string> { "Test" }.AsReadOnly() };
+            _throttleProvider3Status = new ConcurrencyThrottleStatus { State = ThrottleState.Disabled };
+
+            var status = _throttleManager.GetStatus();
+            Assert.Equal(ThrottleState.Enabled, status.State);
+            Assert.Equal(0, status.ConsecutiveCount);
+            Assert.Single(status.EnabledThrottles, p => p == "Test");
+            Assert.True(_throttleManager.LastThrottleCheckStopwatch.Elapsed < lastThrottleCheck);
+
+            lastThrottleCheck = _throttleManager.LastThrottleCheckStopwatch.Elapsed;
+
+            // now make a bunch of rapid requests - we shouldn't query providers
+            // we should return the last result
+            for (int i = 0; i < 5; i++)
+            {
+                status = _throttleManager.GetStatus();
+                Assert.Equal(ThrottleState.Enabled, status.State);
+                Assert.Equal(0, status.ConsecutiveCount);
+                Assert.Single(status.EnabledThrottles, p => p == "Test");
+                Assert.True(_throttleManager.LastThrottleCheckStopwatch.Elapsed > lastThrottleCheck);
+
+                await Task.Delay(100);
+            }
+
+            // simulate time moving forward
+            TestHelpers.SetupStopwatch(_throttleManager.LastThrottleCheckStopwatch, TimeSpan.FromSeconds(5));
+            lastThrottleCheck = _throttleManager.LastThrottleCheckStopwatch.Elapsed;
+
+            status = _throttleManager.GetStatus();
+            Assert.Equal(ThrottleState.Enabled, status.State);
+            Assert.Equal(1, status.ConsecutiveCount);
+            Assert.Single(status.EnabledThrottles, p => p == "Test");
+            Assert.True(_throttleManager.LastThrottleCheckStopwatch.Elapsed < lastThrottleCheck);
+        }
+
+        [Fact]
+        public void GetStatus_ThrottleStateRun()
+        {
+            _throttleProvider1Status = new ConcurrencyThrottleStatus { State = ThrottleState.Disabled };
+            _throttleProvider2Status = new ConcurrencyThrottleStatus { State = ThrottleState.Enabled, EnabledThrottles = new List<string> { "Test" }.AsReadOnly() };
+            _throttleProvider3Status = new ConcurrencyThrottleStatus { State = ThrottleState.Disabled };
+
+            ConcurrencyThrottleAggregateStatus status;
+            for (int i = 0; i < 5; i++)
+            {
+                TestHelpers.SetupStopwatch(_throttleManager.LastThrottleCheckStopwatch, TimeSpan.FromMilliseconds(1100));
+                status = _throttleManager.GetStatus();
+                Assert.Equal(ThrottleState.Enabled, status.State);
+                Assert.Single(status.EnabledThrottles, p => p == "Test");
+                Assert.Equal(i, status.ConsecutiveCount);
+            }
+
+            // now break the run
+            _throttleProvider2Status = new ConcurrencyThrottleStatus { State = ThrottleState.Disabled };
+
+            TestHelpers.SetupStopwatch(_throttleManager.LastThrottleCheckStopwatch, TimeSpan.FromMilliseconds(1100));
+            status = _throttleManager.GetStatus();
+            Assert.Equal(ThrottleState.Disabled, status.State);
+            Assert.Null(status.EnabledThrottles);
+            Assert.Equal(0, status.ConsecutiveCount);
+        }
+    }
+}

test/Microsoft.Azure.WebJobs.Host.UnitTests/Scale/DefaultHostProcessMonitorTests.cs

@@ -0,0 +1,388 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT License. See License.txt in the project root for license information.
+
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Linq;
+using System.Threading.Tasks;
+using Microsoft.Azure.WebJobs.Host.Scale;
+using Microsoft.Azure.WebJobs.Host.TestCommon;
+using Microsoft.Azure.WebJobs.Logging;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
+using Moq;
+using Xunit;
+
+namespace Microsoft.Azure.WebJobs.Host.UnitTests.Scale
+{
+    [Trait(TestTraits.CategoryTraitName, TestTraits.DynamicConcurrency)]
+    public class DefaultHostProcessMonitorTests
+    {
+        private readonly DefaultHostProcessMonitor _hostProcessMonitor;
+        private readonly LoggerFactory _loggerFactory;
+        private readonly TestLoggerProvider _loggerProvider;
+        private readonly ILogger _logger;
+
+        private ProcessStats _testHostProcessStats;
+        private ProcessStats _testChildProcessStats;
+
+        public DefaultHostProcessMonitorTests()
+        {
+            _loggerFactory = new LoggerFactory();
+            _loggerProvider = new TestLoggerProvider();
+            _loggerFactory.AddProvider(_loggerProvider);
+            _logger = _loggerFactory.CreateLogger(LogCategories.Concurrency);
+
+            var mockProcessMonitor = new Mock<ProcessMonitor>(MockBehavior.Strict);
+            mockProcessMonitor.Setup(p => p.GetStats()).Returns(() => _testHostProcessStats);
+
+            var options = new ConcurrencyOptions
+            {
+                DynamicConcurrencyEnabled = true,
+                TotalAvailableMemoryBytes = 10000
+            };
+            var optionsWrapper = new OptionsWrapper<ConcurrencyOptions>(options);
+            _hostProcessMonitor = new DefaultHostProcessMonitor(optionsWrapper, mockProcessMonitor.Object);
+
+            // add a child process monitor
+            mockProcessMonitor = new Mock<ProcessMonitor>(MockBehavior.Strict);
+            mockProcessMonitor.Setup(p => p.GetStats()).Returns(() => _testChildProcessStats);
+            mockProcessMonitor.Setup(p => p.Process).Returns(Process.GetCurrentProcess());
+
+            _hostProcessMonitor.RegisterChildProcessMonitor(mockProcessMonitor.Object);
+
+            _testHostProcessStats = new ProcessStats
+            {
+                ProcessId = 1,
+                CpuLoadHistory = new double[0],
+                MemoryUsageHistory = new long[0]
+            };
+
+            _testChildProcessStats = new ProcessStats
+            {
+                ProcessId = 2,
+                CpuLoadHistory = new double[0],
+                MemoryUsageHistory = new long[0]
+            };
+        }
+
+        [Fact]
+        public void GetStatus_SingleProcess_LowCpu_ReturnsOk()
+        {
+            _testHostProcessStats = new ProcessStats
+            {
+                ProcessId = 1,
+                CpuLoadHistory = new List<double> { 50, 45, 40, 30, 35, 40, 40, 30, 35, 25 },
+                MemoryUsageHistory = new long[0]
+            };
+
+            var status = _hostProcessMonitor.GetStatus(_logger);
+
+            Assert.Equal(HostHealthState.Ok, status.State);
+            var logs = _loggerProvider.GetAllLogMessages().ToArray();
+            Assert.Equal(2, logs.Length);
+
+            var log = logs[0];
+            Assert.Equal(LogLevel.Debug, log.Level);
+            Assert.Equal("[HostMonitor] Host process CPU stats (PID 1): History=(40,40,30,35,25), AvgCpuLoad=34, MaxCpuLoad=40", log.FormattedMessage);
+
+            log = logs[1];
+            Assert.Equal(LogLevel.Debug, log.Level);
+            Assert.Equal("[HostMonitor] Host aggregate CPU load 34", log.FormattedMessage);
+        }
+
+        [Fact]
+        public void GetStatus_SingleProcess_LowMemory_ReturnsOk()
+        {
+            _testHostProcessStats = new ProcessStats
+            {
+                ProcessId = 1,
+                CpuLoadHistory = new double[0],
+                MemoryUsageHistory = new List<long> { 2000, 2100, 2500, 2500, 3000, 2700, 2200, 1500, 2000, 2500 }
+            };
+
+            var status = _hostProcessMonitor.GetStatus(_logger);
+
+            Assert.Equal(HostHealthState.Ok, status.State);
+            var logs = _loggerProvider.GetAllLogMessages().ToArray();
+            Assert.Equal(2, logs.Length);
+
+            var log = logs[0];
+            Assert.Equal(LogLevel.Debug, log.Level);
+            Assert.Equal("[HostMonitor] Host process memory usage (PID 1): History=(2700,2200,1500,2000,2500), AvgUsage=2180, MaxUsage=2700", log.FormattedMessage);
+
+            log = logs[1];
+            Assert.Equal(LogLevel.Debug, log.Level);
+            Assert.Equal("[HostMonitor] Host aggregate memory usage 2500 (31% of threshold)", log.FormattedMessage);
+        }
+
+        [Fact]
+        public void GetStatus_SingleProcess_HighCpu_ReturnsOverloaded()
+        {
+            _testHostProcessStats = new ProcessStats
+            {
+                ProcessId = 1,
+                CpuLoadHistory = new List<double> { 50, 100, 85, 95, 80, 70, 80, 90, 95, 100 },
+                MemoryUsageHistory = new long[0]
+            };
+
+            var status = _hostProcessMonitor.GetStatus(_logger);
+
+            Assert.Equal(HostHealthState.Overloaded, status.State);
+            var logs = _loggerProvider.GetAllLogMessages().ToArray();
+            Assert.Equal(3, logs.Length);
+
+            var log = logs[0];
+            Assert.Equal(LogLevel.Debug, log.Level);
+            Assert.Equal("[HostMonitor] Host process CPU stats (PID 1): History=(70,80,90,95,100), AvgCpuLoad=87, MaxCpuLoad=100", log.FormattedMessage);
+
+            log = logs[1];
+            Assert.Equal(LogLevel.Debug, log.Level);
+            Assert.Equal("[HostMonitor] Host aggregate CPU load 87", log.FormattedMessage);
+
+            log = logs[2];
+            Assert.Equal(LogLevel.Warning, log.Level);
+            Assert.Equal("[HostMonitor] Host CPU threshold exceeded (87 >= 80)", log.FormattedMessage);
+        }
+
+        [Fact]
+        public void GetStatus_SingleProcess_HighMemory_ReturnsOverloaded()
+        {
+            _testHostProcessStats = new ProcessStats
+            {
+                ProcessId = 1,
+                CpuLoadHistory = new double[0],
+                MemoryUsageHistory = new List<long> { 2000, 2100, 2500, 4000, 7000, 8000, 8500, 8600, 8750, 8800 }
+            };
+
+            var status = _hostProcessMonitor.GetStatus(_logger);
+
+            Assert.Equal(HostHealthState.Overloaded, status.State);
+            var logs = _loggerProvider.GetAllLogMessages().ToArray();
+            Assert.Equal(3, logs.Length);
+
+            var log = logs[0];
+            Assert.Equal(LogLevel.Debug, log.Level);
+            Assert.Equal("[HostMonitor] Host process memory usage (PID 1): History=(8000,8500,8600,8750,8800), AvgUsage=8530, MaxUsage=8800", log.FormattedMessage);
+
+            log = logs[1];
+            Assert.Equal(LogLevel.Debug, log.Level);
+            Assert.Equal("[HostMonitor] Host aggregate memory usage 8800 (110% of threshold)", log.FormattedMessage);
+
+            log = logs[2];
+            Assert.Equal(LogLevel.Warning, log.Level);
+            Assert.Equal("[HostMonitor] Host memory threshold exceeded (8800 >= 8000)", log.FormattedMessage);
+        }
+
+        [Fact]
+        public void GetStatus_MultiProcess_LowCpu_ReturnsOk()
+        {
+            _testHostProcessStats = new ProcessStats
+            {
+                ProcessId = 1,
+                CpuLoadHistory = new List<double> { 0, 5, 10, 30, 35, 40, 30, 20, 10, 0 },
+                MemoryUsageHistory = new long[0]
+            };
+
+            _testChildProcessStats = new ProcessStats
+            {
+                ProcessId = 2,
+                CpuLoadHistory = new List<double> { 10, 15, 10, 25, 15, 20, 10, 15, 10, 5 },
+                MemoryUsageHistory = new long[0]
+            };
+
+            var status = _hostProcessMonitor.GetStatus(_logger);
+
+            Assert.Equal(HostHealthState.Ok, status.State);
+            var logs = _loggerProvider.GetAllLogMessages().ToArray();
+            Assert.Equal(3, logs.Length);
+
+            var log = logs[0];
+            Assert.Equal(LogLevel.Debug, log.Level);
+            Assert.Equal("[HostMonitor] Host process CPU stats (PID 2): History=(20,10,15,10,5), AvgCpuLoad=12, MaxCpuLoad=20", log.FormattedMessage);
+
+            log = logs[1];
+            Assert.Equal(LogLevel.Debug, log.Level);
+            Assert.Equal("[HostMonitor] Host process CPU stats (PID 1): History=(40,30,20,10,0), AvgCpuLoad=20, MaxCpuLoad=40", log.FormattedMessage);
+
+            log = logs[2];
+            Assert.Equal(LogLevel.Debug, log.Level);
+            Assert.Equal("[HostMonitor] Host aggregate CPU load 32", log.FormattedMessage);
+        }
+
+        [Fact]
+        public void GetStatus_MultiProcess_LowMemory_ReturnsOk()
+        {
+            _testHostProcessStats = new ProcessStats
+            {
+                ProcessId = 1,
+                CpuLoadHistory = new double[0],
+                MemoryUsageHistory = new List<long> { 2000, 2100, 2500, 2500, 3000, 2700, 2200, 1500, 2000, 2500 }
+            };
+
+            _testChildProcessStats = new ProcessStats
+            {
+                ProcessId = 2,
+                CpuLoadHistory = new double[0],
+                MemoryUsageHistory = new List<long> { 800, 900, 1000, 950, 900, 800, 800, 850, 900, 1000 }
+            };
+
+            var status = _hostProcessMonitor.GetStatus(_logger);
+
+            Assert.Equal(HostHealthState.Ok, status.State);
+            var logs = _loggerProvider.GetAllLogMessages().ToArray();
+            Assert.Equal(3, logs.Length);
+
+            var log = logs[0];
+            Assert.Equal(LogLevel.Debug, log.Level);
+            Assert.Equal("[HostMonitor] Host process memory usage (PID 2): History=(800,800,850,900,1000), AvgUsage=870, MaxUsage=1000", log.FormattedMessage);
+
+            log = logs[1];
+            Assert.Equal(LogLevel.Debug, log.Level);
+            Assert.Equal("[HostMonitor] Host process memory usage (PID 1): History=(2700,2200,1500,2000,2500), AvgUsage=2180, MaxUsage=2700", log.FormattedMessage);
+
+            log = logs[2];
+            Assert.Equal(LogLevel.Debug, log.Level);
+            Assert.Equal("[HostMonitor] Host aggregate memory usage 3500 (43% of threshold)", log.FormattedMessage);
+        }
+
+        [Fact]
+        public void GetStatus_MultiProcess_HighCpu_ReturnsOverloaded()
+        {
+            _testHostProcessStats = new ProcessStats
+            {
+                ProcessId = 1,
+                CpuLoadHistory = new List<double> { 20, 10, 30, 50, 50, 60, 50, 40, 45, 50 },
+                MemoryUsageHistory = new long[0]
+            };
+
+            _testChildProcessStats = new ProcessStats
+            {
+                ProcessId = 2,
+                CpuLoadHistory = new List<double> { 10, 15, 10, 25, 30, 30, 35, 40, 45, 45 },
+                MemoryUsageHistory = new long[0]
+            };
+
+            var status = _hostProcessMonitor.GetStatus(_logger);
+
+            Assert.Equal(HostHealthState.Overloaded, status.State);
+            var logs = _loggerProvider.GetAllLogMessages().ToArray();
+            Assert.Equal(4, logs.Length);
+
+            var log = logs[0];
+            Assert.Equal(LogLevel.Debug, log.Level);
+            Assert.Equal("[HostMonitor] Host process CPU stats (PID 2): History=(30,35,40,45,45), AvgCpuLoad=39, MaxCpuLoad=45", log.FormattedMessage);
+
+            log = logs[1];
+            Assert.Equal(LogLevel.Debug, log.Level);
+            Assert.Equal("[HostMonitor] Host process CPU stats (PID 1): History=(60,50,40,45,50), AvgCpuLoad=49, MaxCpuLoad=60", log.FormattedMessage);
+
+            log = logs[2];
+            Assert.Equal(LogLevel.Debug, log.Level);
+            Assert.Equal("[HostMonitor] Host aggregate CPU load 88", log.FormattedMessage);
+
+            log = logs[3];
+            Assert.Equal(LogLevel.Warning, log.Level);
+            Assert.Equal("[HostMonitor] Host CPU threshold exceeded (88 >= 80)", log.FormattedMessage);
+        }
+
+        [Fact]
+        public void GetStatus_MultiProcess_HighMemory_ReturnsOverloaded()
+        {
+            _testHostProcessStats = new ProcessStats
+            {
+                ProcessId = 1,
+                CpuLoadHistory = new double[0],
+                MemoryUsageHistory = new List<long> { 2000, 2100, 2500, 4000, 4000, 4200, 4300, 4300, 4000, 4500 }
+            };
+
+            _testChildProcessStats = new ProcessStats
+            {
+                ProcessId = 2,
+                CpuLoadHistory = new double[0],
+                MemoryUsageHistory = new List<long> { 800, 900, 1000, 950, 1000, 1500, 2000, 3000, 3100, 4000 }
+            };
+
+            var status = _hostProcessMonitor.GetStatus(_logger);
+
+            Assert.Equal(HostHealthState.Overloaded, status.State);
+            var logs = _loggerProvider.GetAllLogMessages().ToArray();
+            Assert.Equal(4, logs.Length);
+
+            var log = logs[0];
+            Assert.Equal(LogLevel.Debug, log.Level);
+            Assert.Equal("[HostMonitor] Host process memory usage (PID 2): History=(1500,2000,3000,3100,4000), AvgUsage=2720, MaxUsage=4000", log.FormattedMessage);
+
+            log = logs[1];
+            Assert.Equal(LogLevel.Debug, log.Level);
+            Assert.Equal("[HostMonitor] Host process memory usage (PID 1): History=(4200,4300,4300,4000,4500), AvgUsage=4260, MaxUsage=4500", log.FormattedMessage);
+
+            log = logs[2];
+            Assert.Equal(LogLevel.Debug, log.Level);
+            Assert.Equal("[HostMonitor] Host aggregate memory usage 8500 (106% of threshold)", log.FormattedMessage);
+
+            log = logs[3];
+            Assert.Equal(LogLevel.Warning, log.Level);
+            Assert.Equal("[HostMonitor] Host memory threshold exceeded (8500 >= 8000)", log.FormattedMessage);
+        }
+
+        [Fact(Skip = "Failing on CI. Need to investigate.")]
+        public async Task ChildProcessLifetimeManagement_ExitedProcessesAreRemoved()
+        {
+            var options = new ConcurrencyOptions();
+            var localProcessMonitor = new DefaultHostProcessMonitor(new OptionsWrapper<ConcurrencyOptions>(options));
+            var hostProcess = Process.GetCurrentProcess();
+
+            int numChildProcesses = 3;
+            List<Process> childProcesses = new List<Process>();
+            for (int i = 0; i < numChildProcesses; i++)
+            {
+                var childProcess = Process.Start("TestChildProcess.exe");
+                childProcesses.Add(childProcess);
+            }
+
+            Assert.True(childProcesses.All(p => !p.HasExited));
+
+            foreach (var currProcess in childProcesses)
+            {
+                localProcessMonitor.RegisterChildProcess(currProcess);
+            }
+
+            // initial call to get status which will start the monitoring
+            localProcessMonitor.GetStatus(_logger);
+
+            // wait for enough samples to accululate
+            await Task.Delay(TimeSpan.FromSeconds(ProcessMonitor.DefaultSampleIntervalSeconds * 2 * DefaultHostProcessMonitor.MinSampleCount));
+
+            // verify all child processes are being monitored
+            localProcessMonitor.GetStatus(_logger);
+            var logs = _loggerProvider.GetAllLogMessages().ToArray();
+            Assert.Equal(2 + numChildProcesses, logs.Length);
+            for (int i = 0; i < numChildProcesses; i++)
+            {
+                Assert.True(logs[i].FormattedMessage.StartsWith($"[HostMonitor] Host process CPU stats (PID {childProcesses[i].Id})"));
+            }
+            Assert.True(logs[numChildProcesses].FormattedMessage.StartsWith($"[HostMonitor] Host process CPU stats (PID {hostProcess.Id})"));
+            Assert.True(logs[numChildProcesses + 1].FormattedMessage.StartsWith("[HostMonitor] Host aggregate CPU load"));
+
+            // now kill one of the child processes
+            var killedProcess = childProcesses[1];
+            killedProcess.Kill();
+
+            // wait for the process to exit fully and give time for a couple more samples to be taken
+            await TestHelpers.Await(() =>
+            {
+                return killedProcess.HasExited;
+            });
+            await Task.Delay(TimeSpan.FromSeconds(ProcessMonitor.DefaultSampleIntervalSeconds + 1));
+
+            // verify the killed process is no longer being monitored
+            _loggerProvider.ClearAllLogMessages();
+            localProcessMonitor.GetStatus(_logger);
+            logs = _loggerProvider.GetAllLogMessages().ToArray();
+            Assert.Equal(2 + numChildProcesses - 1, logs.Length);
+            Assert.Empty(logs.Where(p => p.FormattedMessage.Contains(killedProcess.Id.ToString())));
+        }
+    }
+}

test/Microsoft.Azure.WebJobs.Host.UnitTests/Scale/DefaultProcessMetricsProviderTests.cs

@@ -0,0 +1,42 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT License. See License.txt in the project root for license information.
+
+using System.Diagnostics;
+using Microsoft.Azure.WebJobs.Host.Scale;
+using Microsoft.Azure.WebJobs.Host.TestCommon;
+using Xunit;
+
+namespace Microsoft.Azure.WebJobs.Host.UnitTests.Scale
+{
+    [Trait(TestTraits.CategoryTraitName, TestTraits.DynamicConcurrency)]
+    public class DefaultProcessMetricsProviderTests
+    {
+        [Fact]  
+        public void TotalProcessorTime_ReturnsExpectedResult()
+        {
+            var process = Process.GetCurrentProcess();
+            var provider = new DefaultProcessMetricsProvider(process);
+
+            Assert.Equal(process.TotalProcessorTime, provider.TotalProcessorTime);
+            Assert.Equal(process.TotalProcessorTime, provider.TotalProcessorTime);
+
+            process.Refresh();
+
+            Assert.Equal(process.TotalProcessorTime, provider.TotalProcessorTime);
+        }
+
+        [Fact]
+        public void PrivateMemoryBytes_ReturnsExpectedResult()
+        {
+            var process = Process.GetCurrentProcess();
+            var provider = new DefaultProcessMetricsProvider(process);
+
+            Assert.Equal(process.PrivateMemorySize64, provider.PrivateMemoryBytes);
+            Assert.Equal(process.PrivateMemorySize64, provider.PrivateMemoryBytes);
+
+            process.Refresh();
+
+            Assert.Equal(process.PrivateMemorySize64, provider.PrivateMemoryBytes);
+        }
+    }
+}

test/Microsoft.Azure.WebJobs.Host.UnitTests/Scale/HostHealthThrottleProviderTests.cs

@@ -0,0 +1,52 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT License. See License.txt in the project root for license information.
+
+using System;
+using System.Collections.Generic;
+using System.Threading.Tasks;
+using Microsoft.Azure.WebJobs.Host.Scale;
+using Microsoft.Azure.WebJobs.Host.TestCommon;
+using Microsoft.Azure.WebJobs.Logging;
+using Microsoft.Extensions.Logging;
+using Moq;
+using Xunit;
+
+namespace Microsoft.Azure.WebJobs.Host.UnitTests.Scale
+{
+    [Trait(TestTraits.CategoryTraitName, TestTraits.DynamicConcurrency)]
+    public class HostHealthThrottleProviderTests
+    {
+        private readonly HostHealthThrottleProvider _throttleProvider;
+        private readonly ILogger _testLogger;
+
+        private HostProcessStatus _status;
+
+        public HostHealthThrottleProviderTests()
+        {
+            _testLogger = new TestLogger("Test");
+
+            var processMonitorMock = new Mock<IHostProcessMonitor>(MockBehavior.Strict);
+            processMonitorMock.Setup(p => p.GetStatus(_testLogger)).Returns(() => _status);
+
+            _throttleProvider = new HostHealthThrottleProvider(processMonitorMock.Object);
+        }
+
+        [Theory]
+        [InlineData(HostHealthState.Ok, null, ThrottleState.Disabled)]
+        [InlineData(HostHealthState.Overloaded, new string[] { "CPU" }, ThrottleState.Enabled)]
+        [InlineData(HostHealthState.Overloaded, new string[] { "CPU", "Memory" }, ThrottleState.Enabled)]
+        [InlineData(HostHealthState.Unknown, null, ThrottleState.Unknown)]
+        public void GetStatus_ReturnsExpectedResult(HostHealthState state, string[] expectedEnabledThrottles, ThrottleState expectedThrottleState)
+        {
+            _status = new HostProcessStatus
+            {
+                State = state,
+                ExceededLimits = expectedEnabledThrottles
+            };
+
+            var status = _throttleProvider.GetStatus(_testLogger);
+            Assert.Equal(expectedThrottleState, status.State);
+            Assert.Equal(expectedEnabledThrottles, status.EnabledThrottles);
+        }
+    }
+}

test/Microsoft.Azure.WebJobs.Host.UnitTests/Scale/ProcessMonitorTests.cs

@@ -0,0 +1,166 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT License. See License.txt in the project root for license information.
+
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Linq;
+using System.Threading.Tasks;
+using Microsoft.Azure.WebJobs.Host.Scale;
+using Microsoft.Azure.WebJobs.Host.TestCommon;
+using Xunit;
+
+namespace Microsoft.Azure.WebJobs.Host.UnitTests.Scale
+{
+    [Trait(TestTraits.CategoryTraitName, TestTraits.DynamicConcurrency)]
+    public class ProcessMonitorTests
+    {
+        private readonly ProcessMonitor _monitor;
+        private List<TimeSpan> _testProcessorTimeValues;
+        private List<long> _testPrivateMemoryValues;
+
+        public ProcessMonitorTests()
+        {
+            _testProcessorTimeValues = new List<TimeSpan>();
+            _testPrivateMemoryValues = new List<long>();
+            int effectiveCores = 1;
+            _monitor = new ProcessMonitor(Process.GetCurrentProcess(), new TestProcessMetricsProvider(_testProcessorTimeValues, _testPrivateMemoryValues), effectiveCores: effectiveCores, autoStart: false);
+        }
+
+        public static IEnumerable<object[]> CPULoadTestData =>
+            new List<object[]>
+            {
+                new object[]
+                {
+                    new List<double> { 0, 250, 750, 1500, 2500, 3300, 3900, 4600, 5100, 5500, 5800 },
+                    new List<double> { 25, 50, 75, 100, 80, 60, 70, 50, 40, 30 }
+                },
+                new object[]
+                {
+                    new List<double> { 100, 300, 500, 700, 1000, 1500, 2000, 3000, 3850, 4800, 5600, 6300, 7000, 7500, 7800, 8000 },
+                    new List<double> { 50, 100, 85, 95, 80, 70, 70, 50, 30, 20 }
+                }
+            };
+
+        public static IEnumerable<object[]> MemoryUsageTestData =>
+            new List<object[]>
+            {
+                new object[]
+                {
+                    new List<long> { 524288000, 524690234, 525598012, 528710023, 538765843, 560987645, 598712101, 628928343, 645986754, 645985532, 656887110, 667853423 }
+                }
+            };
+
+        [Fact]
+        public async Task GetStats_StartsSampleTimer()
+        {
+            int intervalMS = 10;
+            var localMonitor = new ProcessMonitor(Process.GetCurrentProcess(), TimeSpan.FromMilliseconds(intervalMS));
+
+            var stats = localMonitor.GetStats();
+            Assert.Empty(stats.CpuLoadHistory);
+            Assert.Empty(stats.MemoryUsageHistory);
+
+            localMonitor.GetStats();
+
+            // wait long enough for enough samples to be taken to verify sample history rolling
+            await Task.Delay(2 * ProcessMonitor.SampleHistorySize * intervalMS);
+
+            stats = localMonitor.GetStats();
+            Assert.Equal(ProcessMonitor.SampleHistorySize, stats.CpuLoadHistory.Count());
+            Assert.Equal(ProcessMonitor.SampleHistorySize, stats.MemoryUsageHistory.Count());
+        }
+
+        [Theory]
+        [MemberData(nameof(CPULoadTestData))]
+        public void SampleCPULoad_AccumulatesSamples(List<double> testProcessorTimeValues, List<double> expectedLoadValues)
+        {
+            _testProcessorTimeValues.AddRange(testProcessorTimeValues.Select(p => TimeSpan.FromMilliseconds(p)));
+
+            var stats = _monitor.GetStats();
+            Assert.Empty(stats.CpuLoadHistory);
+
+            // start taking samples, using a constant duration so our expected
+            // calculations are deterministic
+            var sampleDuration = TimeSpan.FromSeconds(1);
+            for (int i = 0; i < _testProcessorTimeValues.Count; i++)
+            {
+                _monitor.SampleCPULoad(sampleDuration);
+            }
+
+            stats = _monitor.GetStats();
+
+            // expect a max of 10 - old samples are removed
+            var cpuLoadResults = stats.CpuLoadHistory.ToList();
+            Assert.Equal(Math.Min(cpuLoadResults.Count, ProcessMonitor.SampleHistorySize), cpuLoadResults.Count);
+
+            Assert.Equal(expectedLoadValues, cpuLoadResults);
+        }
+
+        [Theory]
+        [MemberData(nameof(MemoryUsageTestData))]
+        public void SampleMemoryUsage_AccumulatesSamples(List<long> testMemoryValues)
+        {
+            _testPrivateMemoryValues.AddRange(testMemoryValues);
+
+            var stats = _monitor.GetStats();
+            Assert.Empty(stats.MemoryUsageHistory);
+
+            // start taking samples
+            for (int i = 0; i < _testPrivateMemoryValues.Count; i++)
+            {
+                _monitor.SampleMemoryUsage();
+            }
+
+            stats = _monitor.GetStats();
+
+            // expect a max of 10 - old samples are removed
+            var memoryUsageResults = stats.MemoryUsageHistory.ToList();
+            Assert.Equal(Math.Min(memoryUsageResults.Count, ProcessMonitor.SampleHistorySize), memoryUsageResults.Count);
+
+            int skip = testMemoryValues.Count - 10;
+            var expectedMemoryValues = testMemoryValues.Skip(skip).ToList();
+            Assert.Equal(expectedMemoryValues, memoryUsageResults);
+        }
+
+        [Theory]
+        [InlineData(1000, 3500, 3000, 1, 50)]
+        [InlineData(1000, 3400, 3000, 4, 10)]
+        [InlineData(1000, 3500, 3250, 1, 25)]
+        [InlineData(1000, 2500, 1000, 1, 100)]
+        public void CalculateCpuLoad(int sampleDurationMS, int currProcessorTimeMS, int lastProcessorTimeMS, int coreCount, double expected)
+        {
+            double cpuLoad = ProcessMonitor.CalculateCpuLoad(TimeSpan.FromMilliseconds(sampleDurationMS), TimeSpan.FromMilliseconds(currProcessorTimeMS), TimeSpan.FromMilliseconds(lastProcessorTimeMS), coreCount);
+            Assert.Equal(expected, cpuLoad);
+        }
+
+        private class TestProcessMetricsProvider : IProcessMetricsProvider
+        {
+            private int _idx = 0;
+            private List<TimeSpan> _processorTimeValues;
+            private List<long> _privateMemoryValues;
+
+            public TestProcessMetricsProvider(List<TimeSpan> processorTimeValues, List<long> privateMemoryValues)
+            {
+                _processorTimeValues = processorTimeValues;
+                _privateMemoryValues = privateMemoryValues;
+            }
+
+            public TimeSpan TotalProcessorTime
+            {
+                get
+                {
+                    return _processorTimeValues[_idx++];
+                }
+            }
+
+            public long PrivateMemoryBytes
+            {
+                get
+                {
+                    return _privateMemoryValues[_idx++];
+                }
+            }
+        }
+    }
+}

test/Microsoft.Azure.WebJobs.Host.UnitTests/Scale/ThreadPoolStarvationThrottleProviderTests.cs

@@ -0,0 +1,65 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT License. See License.txt in the project root for license information.
+
+using System;
+using System.Linq;
+using System.Threading.Tasks;
+using Microsoft.Azure.WebJobs.Host.Scale;
+using Microsoft.Azure.WebJobs.Host.TestCommon;
+using Microsoft.Azure.WebJobs.Logging;
+using Microsoft.Extensions.Logging;
+using Moq;
+using Xunit;
+
+namespace Microsoft.Azure.WebJobs.Host.UnitTests.Scale
+{
+    [Trait(TestTraits.CategoryTraitName, TestTraits.DynamicConcurrency)]
+    public class ThreadPoolStarvationThrottleProviderTests
+    {
+        private readonly ThreadPoolStarvationThrottleProvider _throttleProvider;
+        private readonly LoggerFactory _loggerFactory;
+        private readonly TestLoggerProvider _loggerProvider;
+        private readonly ILogger _logger;
+
+        public ThreadPoolStarvationThrottleProviderTests()
+        {
+            _throttleProvider = new ThreadPoolStarvationThrottleProvider();
+            _loggerFactory = new LoggerFactory();
+            _loggerProvider = new TestLoggerProvider();
+            _loggerFactory.AddProvider(_loggerProvider);
+            _logger = _loggerFactory.CreateLogger(LogCategories.Concurrency);
+        }
+
+        [Fact]
+        public async Task GetState_Healthy_ThrottleDisabled()
+        {
+            for (int i = 0; i < 3; i++)
+            {
+                await Task.Delay(500);
+
+                var status = _throttleProvider.GetStatus(_logger);
+                Assert.Equal(ThrottleState.Disabled, status.State);
+                Assert.Null(status.EnabledThrottles);
+            }
+        }
+
+        [Fact]
+        public async Task GetState_Unhealthy_ThrottleEnabled()
+        {
+            var status = _throttleProvider.GetStatus(_logger);
+            Assert.Equal(ThrottleState.Disabled, status.State);
+            Assert.Null(status.EnabledThrottles);
+
+            await Task.Delay(500);
+
+            _throttleProvider.ResetInvocations();
+            status = _throttleProvider.GetStatus(_logger);
+            Assert.Equal(ThrottleState.Enabled, status.State);
+            Assert.Single(status.EnabledThrottles, p => p == ThreadPoolStarvationThrottleProvider.ThreadPoolStarvationThrottleName);
+
+            var log = _loggerProvider.GetAllLogMessages().Single();
+            Assert.Equal(LogLevel.Warning, log.Level);
+            Assert.Equal("Possible thread pool starvation detected.", log.FormattedMessage);
+        }
+    }
+}

test/Microsoft.Azure.WebJobs.Host.UnitTests/UtilityTests.cs

@@ -0,0 +1,131 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT License. See License.txt in the project root for license information.
+
+using System;
+using System.Collections.Generic;
+using System.Text;
+using Microsoft.Azure.WebJobs.Host.TestCommon;
+using Xunit;
+
+namespace Microsoft.Azure.WebJobs.Host.UnitTests
+{
+    public class UtilityTests
+    {
+        [Fact]
+        [Trait(TestTraits.CategoryTraitName, TestTraits.DynamicConcurrency)]
+        public void TakeLastN_ReturnsExpectedValues()
+        {
+            int[] values = new int[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 };
+
+            Assert.Throws<ArgumentOutOfRangeException>(() =>
+            {
+                values.TakeLastN(-1);
+            });
+
+            var result = values.TakeLastN(0);
+            Assert.Empty(result);
+
+            result = values.TakeLastN(1);
+            Assert.Collection(result,
+                p => Assert.Equal(10, p));
+
+            result = values.TakeLastN(5);
+            Assert.Collection(result,
+                p => Assert.Equal(6, p),
+                p => Assert.Equal(7, p),
+                p => Assert.Equal(8, p),
+                p => Assert.Equal(9, p),
+                p => Assert.Equal(10, p));
+
+            result = values.TakeLastN(10);
+            Assert.Collection(result,
+                p => Assert.Equal(1, p),
+                p => Assert.Equal(2, p),
+                p => Assert.Equal(3, p),
+                p => Assert.Equal(4, p),
+                p => Assert.Equal(5, p),
+                p => Assert.Equal(6, p),
+                p => Assert.Equal(7, p),
+                p => Assert.Equal(8, p),
+                p => Assert.Equal(9, p),
+                p => Assert.Equal(10, p));
+
+            result = values.TakeLastN(15);
+            Assert.Collection(result,
+                p => Assert.Equal(1, p),
+                p => Assert.Equal(2, p),
+                p => Assert.Equal(3, p),
+                p => Assert.Equal(4, p),
+                p => Assert.Equal(5, p),
+                p => Assert.Equal(6, p),
+                p => Assert.Equal(7, p),
+                p => Assert.Equal(8, p),
+                p => Assert.Equal(9, p),
+                p => Assert.Equal(10, p));
+        }
+
+        [Fact]
+        public void FlattenException_AggregateException_ReturnsExpectedResult()
+        {
+            ApplicationException ex1 = new ApplicationException("Incorrectly configured setting 'Foo'");
+            ex1.Source = "Acme.CloudSystem";
+
+            // a dupe of the first
+            ApplicationException ex2 = new ApplicationException("Incorrectly configured setting 'Foo'");
+            ex1.Source = "Acme.CloudSystem";
+
+            AggregateException aex = new AggregateException("One or more errors occurred.", ex1, ex2);
+
+            string formattedResult = Utility.FlattenException(aex);
+            Assert.Equal("Acme.CloudSystem: Incorrectly configured setting 'Foo'.", formattedResult);
+        }
+
+        [Fact]
+        public void FlattenException_SingleException_ReturnsExpectedResult()
+        {
+            ApplicationException ex = new ApplicationException("Incorrectly configured setting 'Foo'");
+            ex.Source = "Acme.CloudSystem";
+
+            string formattedResult = Utility.FlattenException(ex);
+            Assert.Equal("Acme.CloudSystem: Incorrectly configured setting 'Foo'.", formattedResult);
+        }
+
+        [Fact]
+        public void FlattenException_MultipleInnerExceptions_ReturnsExpectedResult()
+        {
+            ApplicationException ex1 = new ApplicationException("Exception message 1");
+            ex1.Source = "Source1";
+
+            ApplicationException ex2 = new ApplicationException("Exception message 2.", ex1);
+            ex2.Source = "Source2";
+
+            ApplicationException ex3 = new ApplicationException("Exception message 3", ex2);
+
+            string formattedResult = Utility.FlattenException(ex3);
+            Assert.Equal("Exception message 3. Source2: Exception message 2. Source1: Exception message 1.", formattedResult);
+        }
+
+        [Fact]
+        public void GetEffectiveCoresCount_ReturnsExpectedResult()
+        {
+            string prevSku = Environment.GetEnvironmentVariable(Constants.AzureWebsiteSku);
+            string prevRoleInstanceId = Environment.GetEnvironmentVariable("RoleInstanceId");
+
+            try
+            {
+                Assert.Equal(Environment.ProcessorCount, Utility.GetEffectiveCoresCount());
+
+                Environment.SetEnvironmentVariable(Constants.AzureWebsiteSku, Constants.DynamicSku);
+                Assert.Equal(1, Utility.GetEffectiveCoresCount());
+
+                Environment.SetEnvironmentVariable("RoleInstanceId", "dw0SmallDedicatedWebWorkerRole_hr0HostRole -0-VM-1");
+                Assert.Equal(Environment.ProcessorCount, Utility.GetEffectiveCoresCount());
+            }
+            finally
+            {
+                Environment.SetEnvironmentVariable(Constants.AzureWebsiteSku, prevSku);
+                Environment.SetEnvironmentVariable("RoleInstanceId", prevRoleInstanceId);
+            }
+        }
+    }
+}

test/Microsoft.Azure.WebJobs.Host.UnitTests/WebJobs.Host.UnitTests.csproj

@@ -3,7 +3,7 @@
   <PropertyGroup>
     <!-- This project needs a static version number for the HostId checks, which are based on the version -->
     <Version>3.0.0$(VersionSuffix)</Version>
-    <TargetFramework>netcoreapp2.1</TargetFramework>
+    <TargetFramework>netcoreapp3.1</TargetFramework>
     <IsPackable>false</IsPackable>
     <AssemblyName>Microsoft.Azure.WebJobs.Host.UnitTests</AssemblyName>
     <RootNamespace>Microsoft.Azure.WebJobs.Host.UnitTests</RootNamespace>
@@ -46,6 +46,7 @@
     <ProjectReference Include="..\..\src\Microsoft.Azure.WebJobs.Host\WebJobs.Host.csproj" />
     <ProjectReference Include="..\..\src\Microsoft.Azure.WebJobs.Logging.ApplicationInsights\WebJobs.Logging.ApplicationInsights.csproj" />
     <ProjectReference Include="..\..\src\Microsoft.Azure.WebJobs.Logging\WebJobs.Logging.csproj" />
+    <ProjectReference Include="..\..\TestChildProcess\TestChildProcess.csproj" />
     <ProjectReference Include="..\Microsoft.Azure.WebJobs.Host.TestCommon\WebJobs.Host.TestCommon.csproj" />
   </ItemGroup>
 

test/Microsoft.Azure.WebJobs.Logging.FunctionalTests/WebJobs.Logging.FunctionalTests.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
   <Import Project="..\..\build\common.props" />
   <PropertyGroup>
-    <TargetFramework>netcoreapp2.1</TargetFramework>
+    <TargetFramework>netcoreapp3.1</TargetFramework>
     <IsPackable>false</IsPackable>
     <AssemblyName>Microsoft.Azure.WebJobs.Logging.FunctionalTests</AssemblyName>
     <RootNamespace>Microsoft.Azure.WebJobs.Logging.FunctionalTests</RootNamespace>

test/Microsoft.Azure.Webjobs.Extensions.Storage.UnitTests/Queues/QueueTests.cs

@@ -82,8 +82,7 @@ namespace Microsoft.Azure.WebJobs.Host.FunctionalTests
 
         private static string GetErrorMessageForBadQueueName(string value, string parameterName)
         {
-            return "A queue name can contain only letters, numbers, and dash(-) characters - \"" + value + "\"" +
-                "\r\nParameter name: " + parameterName; // from ArgumentException 
+            return $"A queue name can contain only letters, numbers, and dash(-) characters - \"{value}\" (Parameter '{parameterName}')"; // from ArgumentException 
         }
 
         // Program with variable queue name containing both %% and { }.

test/Microsoft.Azure.Webjobs.Extensions.Storage.UnitTests/Queues/QueueTriggerTests.cs

@@ -109,7 +109,7 @@ namespace Microsoft.Azure.WebJobs.Host.FunctionalTests
             Assert.Equal("Exception binding parameter 'message'", exception.Message);
             Exception innerException = exception.InnerException;
             Assert.IsType<DecoderFallbackException>(innerException);
-            Assert.Equal("Unable to translate bytes [FF] at index -1 from specified code page to Unicode.",
+            Assert.Equal("Unable to translate bytes [FF] at index 0 from specified code page to Unicode.",
                 innerException.Message);
         }
 

test/Microsoft.Azure.Webjobs.Extensions.Storage.UnitTests/WebJobs.Extensions.Storage.UnitTests.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
   <Import Project="..\..\build\common.props" />
   <PropertyGroup>
-    <TargetFramework>netcoreapp2.1</TargetFramework>
+    <TargetFramework>netcoreapp3.1</TargetFramework>
     <IsPackable>false</IsPackable>
     <AssemblyName>Microsoft.Azure.WebJobs.Extensions.Storage.UnitTests</AssemblyName>
     <RootNamespace>Microsoft.Azure.WebJobs.Extensions.Storage.UnitTests</RootNamespace>