Merged PR 640762: Support out-of-proc mode for cache service

This PR adds a new mode of running cache out-of-proc with the current cache bits.

I'll check this change in CI and will create a CloudBuild PR with the required changes before merging the change in.

Related work items: #1856822

Public/Src/Cache/ContentStore/DistributedTest/ContentLocation/LocalLocationStoreDistributedContentTestsBase.cs

@@ -27,6 +27,7 @@ using BuildXL.Cache.ContentStore.Interfaces.Tracing;
 using BuildXL.Cache.ContentStore.InterfacesTest.Results;
 using BuildXL.Cache.ContentStore.Service;
 using BuildXL.Cache.ContentStore.Tracing;
+using BuildXL.Cache.ContentStore.Tracing.Internal;
 using BuildXL.Cache.Host.Configuration;
 using BuildXL.Cache.Host.Service;
 using BuildXL.Cache.Host.Service.Internal;
@@ -307,7 +308,7 @@ namespace ContentStoreTest.Distributed.Sessions
 
             if (UseGrpcServer)
             {
-                var server = (ILocalContentServer<TStore>)new CacheServerFactory(arguments).Create();
+                var server = (ILocalContentServer<TStore>)new CacheServerFactory(arguments).CreateAsync(new OperationContext(context)).GetAwaiter().GetResult();
                 TStore store = server.StoresByName["Default"];
                 //if (store is MultiplexedContentStore multiplexedStore)
                 //{
@@ -398,9 +399,9 @@ namespace ContentStoreTest.Distributed.Sessions
                 return _secrets[key];
             }
 
-            public Task<Dictionary<string, Secret>> RetrieveSecretsAsync(List<RetrieveSecretsRequest> requests, CancellationToken token)
+            public Task<RetrievedSecrets> RetrieveSecretsAsync(List<RetrieveSecretsRequest> requests, CancellationToken token)
             {
-                return Task.FromResult(requests.ToDictionary(r => r.Name, r => (Secret)new PlainTextSecret(_secrets[r.Name])));
+                return Task.FromResult(new RetrievedSecrets(requests.ToDictionary(r => r.Name, r => (Secret)new PlainTextSecret(_secrets[r.Name]))));
             }
 
             public void OnStartedService() { }

Public/Src/Cache/ContentStore/DistributedTest/DeploymentIngesterTests.cs

@@ -305,7 +305,7 @@ namespace BuildXL.Cache.ContentStore.Distributed.Test
 
         private class TestSecretsProvider : ISecretsProvider
         {
-            public Task<Dictionary<string, Secret>> RetrieveSecretsAsync(List<RetrieveSecretsRequest> requests, CancellationToken token)
+            public Task<RetrievedSecrets> RetrieveSecretsAsync(List<RetrieveSecretsRequest> requests, CancellationToken token)
             {
                 var secrets = new Dictionary<string, Secret>();
 
@@ -318,16 +318,17 @@ namespace BuildXL.Cache.ContentStore.Distributed.Test
                     else
                     {
                         request.Kind.Should().Be(SecretKind.SasToken);
-                        secrets.Add(request.Name, new UpdatingSasToken(new SasToken()
+                        secrets.Add(
-                        {
+                            request.Name, 
-                            StorageAccount = $"https://{request.Name}.azure.blob.com/",
+                            new UpdatingSasToken(
-                            ResourcePath = "ResourcePath",
+                                new SasToken(
-                            Token = Guid.NewGuid().ToString()
+                                    storageAccount: $"https://{request.Name}.azure.blob.com/",
-                        }));
+                                    resourcePath: "ResourcePath",
+                                    token: Guid.NewGuid().ToString())));
                     }
                 }
 
-                return Task.FromResult(secrets);
+                return Task.FromResult(new RetrievedSecrets(secrets));
             }
         }
     }

Public/Src/Cache/ContentStore/Interfaces/Secrets/Secrets.cs

@@ -4,6 +4,8 @@
 using System;
 using System.Diagnostics.ContractsLight;
 
+#nullable enable
+
 namespace BuildXL.Cache.ContentStore.Interfaces.Secrets
 {
     /// <nodoc />
@@ -17,8 +19,35 @@ namespace BuildXL.Cache.ContentStore.Interfaces.Secrets
     }
 
     /// <nodoc />
-    public abstract class Secret
+    public abstract class Secret : IEquatable<Secret>
     {
+        /// <inheritdoc />
+        public abstract bool Equals(Secret? other);
+
+        /// <nodoc />
+        public abstract override int GetHashCode();
+
+        /// <inheritdoc />
+        public override bool Equals(object? obj)
+        {
+            if (obj is null)
+            {
+                return false;
+            }
+
+            if (ReferenceEquals(this, obj))
+            {
+                return true;
+            }
+
+            if (obj.GetType() != GetType())
+            {
+                return false;
+            }
+
+            return Equals((Secret)obj);
+        }
+
     }
 
     /// <nodoc />
@@ -33,19 +62,67 @@ namespace BuildXL.Cache.ContentStore.Interfaces.Secrets
             Contract.Requires(!string.IsNullOrEmpty(secret));
             Secret = secret;
         }
+
+        /// <inheritdoc />
+        public override bool Equals(Secret? other)
+        {
+            if (other is null)
+            {
+                return false;
+            }
+
+            return Secret == ((PlainTextSecret)other).Secret;
+        }
+
+        /// <inheritdoc />
+        public override int GetHashCode()
+        {
+            return Secret.GetHashCode();
+        }
     }
 
     /// <nodoc />
-    public class SasToken
+    public sealed class SasToken : IEquatable<SasToken>
     {
         /// <nodoc />
-        public string? Token { get; set; }
+        public string Token { get; }
 
         /// <nodoc />
-        public string? StorageAccount { get; set; }
+        public string StorageAccount { get; }
 
         /// <nodoc />
-        public string? ResourcePath { get; set; }
+        public string? ResourcePath { get; init; }
+
+        /// <nodoc />
+        public SasToken(string token, string storageAccount, string? resourcePath = null)
+        {
+            Token = token;
+            StorageAccount = storageAccount;
+            ResourcePath = resourcePath;
+        }
+
+        /// <inheritdoc />
+        public bool Equals(SasToken? other)
+        {
+            if (other is null)
+            {
+                return false;
+            }
+
+            return Token == other.Token && StorageAccount == other.StorageAccount && ResourcePath == other.ResourcePath;
+        }
+
+        /// <inheritdoc />
+        public override bool Equals(object? obj)
+        {
+            return Equals(obj as SasToken);
+        }
+
+        /// <inheritdoc />
+        public override int GetHashCode()
+        {
+            return (Token, StorageAccount, ResourcePath ?? string.Empty).GetHashCode();
+        }
     }
 
     /// <nodoc />
@@ -66,9 +143,27 @@ namespace BuildXL.Cache.ContentStore.Interfaces.Secrets
         /// <nodoc />
         public void UpdateToken(SasToken token)
         {
-            Contract.RequiresNotNull(token);
+            Contract.Requires(token != null);
+
             Token = token;
             TokenUpdated?.Invoke(this, token);
         }
+
+        /// <inheritdoc />
+        public override bool Equals(Secret? other)
+        {
+            if (other is not UpdatingSasToken otherToken)
+            {
+                return false;
+            }
+
+            return Token.Equals(otherToken.Token);
+        }
+
+        /// <inheritdoc />
+        public override int GetHashCode()
+        {
+            return Token.GetHashCode();
+        }
     }
 }

Public/Src/Cache/ContentStore/Library/Service/ServiceLifetimeManager.cs

@@ -67,7 +67,7 @@ namespace BuildXL.Cache.ContentStore.Service
         }
 
         /// <summary>
-        /// Gets set of environment variables used to launch service in child process using <see cref="RunDeployedInterruptableServiceAsync"/>
+        /// Gets set of environment variables used to launch service in child process using <see cref="RunDeployedInterruptableServiceAsync{T}"/>
         /// </summary>
         public IDictionary<string, string> GetDeployedInterruptableServiceVariables(string serviceId)
         {
@@ -95,7 +95,7 @@ namespace BuildXL.Cache.ContentStore.Service
         private string PreventStartupFile(string serviceId) => Path.Combine(SignalFileRoot.Path, $"{serviceId}.preventstartup");
 
         /// <summary>
-        /// Run a service which can be interrupted by another service (via <see cref="RunInterrupterServiceAsync"/>)
+        /// Run a service which can be interrupted by another service (via <see cref="RunInterrupterServiceAsync{T}"/>)
         /// or shutdown (via <see cref="ShutdownServiceAsync"/>).
         /// </summary>
         public async Task<T> RunInterruptableServiceAsync<T>(OperationContext context, string serviceId, Func<CancellationToken, Task<T>> runAsync)
@@ -117,7 +117,7 @@ namespace BuildXL.Cache.ContentStore.Service
         }
 
         /// <summary>
-        /// Runs a service which can interrupt another service started with <see cref="RunInterruptableServiceAsync"/>
+        /// Runs a service which can interrupt another service started with <see cref="RunInterruptableServiceAsync{T}"/>
         /// </summary>
         public async Task<T> RunInterrupterServiceAsync<T>(OperationContext context, string serviceId, string serviceToInterruptId, Func<CancellationToken, Task<T>> runAsync)
         {
@@ -131,7 +131,7 @@ namespace BuildXL.Cache.ContentStore.Service
         }
 
         /// <summary>
-        /// Signals and waits for shutdown a service run under <see cref="RunInterruptableServiceAsync"/>
+        /// Signals and waits for shutdown a service run under <see cref="RunInterruptableServiceAsync{T}"/>
         /// </summary>
         public Task ShutdownServiceAsync(OperationContext context, string serviceId)
         {

Public/Src/Cache/ContentStore/Library/Tracing/LifetimeTrackerTracer.cs

@@ -6,12 +6,13 @@ using System.Diagnostics;
 using System.Diagnostics.ContractsLight;
 using System.Runtime.CompilerServices;
 using BuildXL.Cache.ContentStore.FileSystem;
-using BuildXL.Cache.ContentStore.Interfaces.FileSystem;
 using BuildXL.Cache.ContentStore.Interfaces.Logging;
 using BuildXL.Cache.ContentStore.Interfaces.Results;
 using BuildXL.Cache.ContentStore.Interfaces.Time;
 using BuildXL.Cache.ContentStore.Interfaces.Tracing;
 using BuildXL.Cache.ContentStore.Tracing.Internal;
+using BuildXL.Utilities;
+using AbsolutePath = BuildXL.Cache.ContentStore.Interfaces.FileSystem.AbsolutePath;
 
 namespace BuildXL.Cache.ContentStore.Tracing
 {
@@ -187,7 +188,8 @@ namespace BuildXL.Cache.ContentStore.Tracing
 
             LifetimeTrackerHelper = Tracing.LifetimeTrackerHelper.Starting(clock.GetUtcNow(), processStartupTime ?? GetProcessStartupTimeUtc(), offlineTime.Then(v => Result.Success(v.lastServiceHeartbeatTime)));
 
-            Trace(context, $"Starting CaSaaS instance{offlineTime.ToStringSelect(r => $". LastHeartBeatTime={r.lastServiceHeartbeatTime}, ShutdownCorrectly={r.shutdownCorrectly}")}");
+            var runtime = OperatingSystemHelper.GetRuntimeFrameworkNameAndVersion();
+            Trace(context, $"Starting CaSaaS instance. Runtime={runtime}{offlineTime.ToStringSelect(r => $". LastHeartBeatTime={r.lastServiceHeartbeatTime}, ShutdownCorrectly={r.shutdownCorrectly}")}");
         }
 
         /// <summary>

Public/Src/Cache/ContentStore/Library/Tracing/OperationContextExtensions.cs

@@ -34,6 +34,11 @@ namespace BuildXL.Cache.ContentStore.Tracing.Internal
     /// </summary>
     public static class OperationContextExtensions
     {
+        /// <summary>
+        /// Detaches a <see cref="CancellationToken"/> instance from <paramref name="context"/>.
+        /// </summary>
+        public static OperationContext WithoutCancellationToken(this OperationContext context) => new OperationContext(context.TracingContext, token: CancellationToken.None);
+
         /// <nodoc />
         public static PerformAsyncOperationBuilder<TResult> CreateOperation<TResult>(this OperationContext context, Tracer tracer, Func<Task<TResult>> operation) where TResult : ResultBase
         {

Public/Src/Cache/ContentStore/UtilitiesCore/CollectionUtilities.cs

@@ -3,7 +3,6 @@
 
 using System;
 using System.Collections.Generic;
-using System.Collections.ObjectModel;
 
 namespace BuildXL.Cache.ContentStore.UtilitiesCore.Internal
 {
@@ -29,6 +28,16 @@ namespace BuildXL.Cache.ContentStore.UtilitiesCore.Internal
             public static readonly T[] EmptyArray = new T[] { };
         }
 
+        private static class Empty<TKey, TValue>
+        {
+            public static readonly Dictionary<TKey, TValue> EmptyDictionary = new Dictionary<TKey, TValue>();
+        }
+
+        /// <summary>
+        /// Returns an empty instance of <see cref="IReadOnlyDictionary{TKey,TValue}"/>
+        /// </summary>
+        public static IReadOnlyDictionary<TKey, TValue> EmptyDictionary<TKey, TValue>() => Empty<TKey, TValue>.EmptyDictionary;
+
         /// <summary>
         /// Allows deconstructing a key value pair to a tuple
         /// </summary>

Public/Src/Cache/DistributedCache.Host/Configuration/DeploymentConfiguration.cs

@@ -144,9 +144,9 @@ namespace BuildXL.Cache.Host.Configuration
         public string Name { get; set; }
 
         /// <summary>
-        /// The amount of time the secret can be cached before needing to be requeried
+        /// The amount of time the secret can be cached before needing to be re-queried.
         /// </summary>
-        public TimeSpan TimeToLive { get; set; }
+        public TimeSpan TimeToLive { get; set; } = TimeSpan.FromHours(1);
 
         /// <summary>
         /// Overrides the key vault uri used to retrieve this secret

Public/Src/Cache/DistributedCache.Host/Configuration/DeploymentParameters.cs

@@ -3,6 +3,8 @@
 
 using System;
 using System.Collections.Generic;
+using System.Diagnostics.ContractsLight;
+using BuildXL.Cache.ContentStore.Interfaces.Logging;
 
 #nullable disable
 
@@ -68,6 +70,30 @@ namespace BuildXL.Cache.Host.Configuration
         {
             return $"Machine={Machine} Stamp={Stamp}";
         }
+
+        public void ApplyFromTelemetryProviderIfNeeded(ITelemetryFieldsProvider telemetryProvider)
+        {
+            if (telemetryProvider is null)
+            {
+                return;
+            }
+
+            Ring ??= telemetryProvider.Ring;
+            Stamp ??= telemetryProvider.Stamp;
+            Machine ??= telemetryProvider.MachineName;
+            MachineFunction ??= telemetryProvider.MachineName;
+            Environment ??= telemetryProvider.APEnvironment;
+        }
+
+        public static HostParameters FromTelemetryProvider(ITelemetryFieldsProvider telemetryProvider)
+        {
+            Contract.Requires(telemetryProvider is not null);
+
+            var result = new HostParameters();
+            result.ApplyFromTelemetryProviderIfNeeded(telemetryProvider);
+
+            return result;
+        }
     }
 
     public class DeploymentParameters : HostParameters

Public/Src/Cache/DistributedCache.Host/Configuration/DistributedContentSettings.cs

@@ -12,7 +12,9 @@ using BuildXL.Cache.ContentStore.Interfaces.Distributed;
 using BuildXL.Cache.ContentStore.Interfaces.Logging;
 using BuildXL.Cache.ContentStore.Interfaces.Utils;
 using ContentStore.Grpc;
+
 #nullable disable
+
 namespace BuildXL.Cache.Host.Configuration
 {
     /// <summary>
@@ -63,6 +65,22 @@ namespace BuildXL.Cache.Host.Configuration
         [DataMember]
         public LauncherSettings LauncherSettings { get; set; } = null;
 
+        /// <summary>
+        /// Settings for running cache out of proc as a .net core process.
+        /// </summary>
+        [DataMember]
+        public OutOfProcCacheSettings OutOfProcCacheSettings { get; set; }
+
+        /// <summary>
+        /// If true the cache should run out of proc as a .net core process.
+        /// </summary>
+        /// <remarks>
+        /// If this property is true and <see cref="OutOfProcCacheSettings"/> is null, that property should be created
+        /// by the host and set <see cref="BuildXL.Cache.Host.Configuration.OutOfProcCacheSettings.CacheConfigPath"/> property.
+        /// </remarks>
+        [DataMember]
+        public bool? RunCacheOutOfProc { get; set; }
+
         [DataMember]
         public LogManagerConfiguration LogManager { get; set; } = null;
 

Public/Src/Cache/DistributedCache.Host/Configuration/OutOfProcCacheSettings.cs

@@ -0,0 +1,27 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+namespace BuildXL.Cache.Host.Configuration
+{
+#nullable enable
+
+    public class OutOfProcCacheSettings
+    {
+        /// <summary>
+        /// A path to the cache configuration that the launched process will use.
+        /// </summary>
+        /// <remarks>
+        /// This property needs to be set in CloudBuild in order to use 'out-of-proc' cache.
+        /// </remarks>
+        public string? CacheConfigPath { get; set; }
+
+        /// <summary>
+        /// A relative path from the current executing assembly to the stand-alone cache service that will be launched in a separate process.
+        /// </summary>
+        public string? Executable { get; set; }
+
+        public int? ServiceLifetimePollingIntervalSeconds { get; set; }
+
+        public int? ShutdownTimeoutSeconds { get; set; }
+    }
+}

Public/Src/Cache/DistributedCache.Host/LauncherServer/BuildXL.Launcher.Server.dsc

@@ -3,7 +3,6 @@
 
 import * as Managed from "Sdk.Managed";
 import * as BuildXLSdk from "Sdk.BuildXL";
-import { NetFx } from "Sdk.BuildXL";
 
 namespace LauncherServer {
 
@@ -30,7 +29,6 @@ namespace LauncherServer {
             importFrom("Azure.Core").pkg,
             importFrom("Microsoft.Identity.Client").pkg,
 
-
             // AspNetCore assemblies
             Managed.Factory.filterRuntimeSpecificBinaries(BuildXLSdk.WebFramework.getFrameworkPackage(), [
                 importFrom("System.IO.Pipelines").pkg

Public/Src/Cache/DistributedCache.Host/LauncherServer/CacheServiceStartup.cs

@@ -1,13 +1,8 @@
-using System;
 using System.Collections.Generic;
 using System.Diagnostics.ContractsLight;
-using System.Linq;
-using System.Reflection;
 using System.Threading;
 using System.Threading.Tasks;
-using BuildXL.Cache.ContentStore.Interfaces.FileSystem;
 using BuildXL.Cache.ContentStore.Interfaces.Results;
-using BuildXL.Cache.ContentStore.Interfaces.Time;
 using BuildXL.Cache.ContentStore.Interfaces.Tracing;
 using BuildXL.Cache.ContentStore.Logging;
 using BuildXL.Cache.ContentStore.Tracing;
@@ -16,15 +11,10 @@ using BuildXL.Cache.Host.Configuration;
 using BuildXL.Cache.Host.Service;
 using BuildXL.Launcher.Server.Controllers;
 using BuildXL.Utilities.Collections;
-using Microsoft.AspNetCore.Builder;
 using Microsoft.AspNetCore.Hosting;
-using Microsoft.AspNetCore.HttpsPolicy;
-using Microsoft.AspNetCore.Mvc;
-using Microsoft.AspNetCore.Mvc.Controllers;
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Hosting;
-using Microsoft.Extensions.Primitives;
 using ILogger = BuildXL.Cache.ContentStore.Interfaces.Logging.ILogger;
 
 namespace BuildXL.Launcher.Server
@@ -64,13 +54,24 @@ namespace BuildXL.Launcher.Server
             var logger = new Logger(consoleLog);
 
             var cacheConfigurationPath = configuration["CacheConfigurationPath"];
-            var standalone = configuration.GetValue<bool>("standalone", true);
+            var standalone = configuration.GetValue("standalone", true);
+            var secretsProviderKind = configuration.GetValue("secretsProviderKind", CrossProcessSecretsCommunicationKind.Environment);
+            
             return CacheServiceRunner.RunCacheServiceAsync(
                 new OperationContext(new Context(logger), token),
                 cacheConfigurationPath,
-                (hostParameters, config, token) =>
+                createHost: (hostParameters, config, token) =>
                 {
-                    var serviceHost = new ServiceHost(commandLineArgs, config, hostParameters);
+                    // If this process was started as a standalone cache service, we need to change the mode
+                    // this time to avoid trying to start the cache service process again.
+                    config.DistributedContentSettings.RunCacheOutOfProc = false;
+                    if (config.DataRootPath is null)
+                    {
+                        // The required property is not set, so it should be passed through the command line options by the parent process.
+                        config.DataRootPath = configuration.GetValue("DataRootPath", "Unknown DataRootPath");
+                    }
+
+                    var serviceHost = new ServiceHost(commandLineArgs, config, hostParameters, retrieveAllSecretsFromSingleEnvironmentVariable: secretsProviderKind == CrossProcessSecretsCommunicationKind.EnvironmentSingleEntry);
                     return serviceHost;
                 },
                 requireServiceInterruptable: !standalone);
@@ -120,11 +121,16 @@ namespace BuildXL.Launcher.Server
                 services.AddSingleton<HostParameters>(hostParameters);
             }
 
+            // Only the launcher-based invocation would have 'ProxyConfigurationPath' and
+            // the out-of-proc case would not.
+            var configurationPath = Configuration.GetValue<string>("ProxyConfigurationPath", null);
+
+            if (configurationPath is not null)
+            {
                 // Add ProxyServiceConfiguration as a singleton in service provider
                 services.AddSingleton(sp =>
                 {
                     var context = sp.GetRequiredService<BoxRef<OperationContext>>().Value;
-                var configurationPath = Configuration["ProxyConfigurationPath"];
                     var hostParameters = sp.GetService<HostParameters>();
 
                     return context.PerformOperation(
@@ -143,6 +149,7 @@ namespace BuildXL.Launcher.Server
                         messageFactory: r => $"ConfigurationPath=[{configurationPath}], Port={r.GetValueOrDefault()?.Port}",
                         caller: "LoadConfiguration").ThrowIfFailure();
                 });
+            }
 
             // Add DeploymentProxyService as a singleton in service provider
             services.AddSingleton(sp =>
@@ -188,7 +195,8 @@ namespace BuildXL.Launcher.Server
             /// Constructs the service host and takes command line arguments because
             /// ASP.Net core application host is used to parse command line.
             /// </summary>
-            public ServiceHost(string[] commandLineArgs, DistributedCacheServiceConfiguration configuration, HostParameters hostParameters)
+            public ServiceHost(string[] commandLineArgs, DistributedCacheServiceConfiguration configuration, HostParameters hostParameters, bool retrieveAllSecretsFromSingleEnvironmentVariable)
+                : base(retrieveAllSecretsFromSingleEnvironmentVariable)
             {
                 HostParameters = hostParameters;
                 ServiceConfiguration = configuration;

Public/Src/Cache/DistributedCache.Host/LauncherServer/DeploymentServiceStartup.cs

@@ -1,21 +1,11 @@
 using System;
-using System.Collections.Generic;
 using System.IO;
-using System.Linq;
-using System.Reflection;
 using System.Text.Json;
-using System.Threading.Tasks;
 using BuildXL.Cache.ContentStore.Interfaces.FileSystem;
 using BuildXL.Cache.ContentStore.Interfaces.Time;
 using BuildXL.Cache.ContentStore.Logging;
 using BuildXL.Cache.Host.Configuration;
 using BuildXL.Cache.Host.Service;
-using BuildXL.Launcher.Server.Controllers;
-using Microsoft.AspNetCore.Builder;
-using Microsoft.AspNetCore.Hosting;
-using Microsoft.AspNetCore.HttpsPolicy;
-using Microsoft.AspNetCore.Mvc;
-using Microsoft.AspNetCore.Mvc.Controllers;
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Hosting;

Public/Src/Cache/DistributedCache.Host/LauncherServer/KeyVaultSecretsProvider.cs

@@ -24,7 +24,7 @@ namespace BuildXL.Launcher.Server
             _client = new SecretClient(new Uri(keyVaultUri), new DefaultAzureCredential());
         }
 
-        public async Task<Dictionary<string, Secret>> RetrieveSecretsAsync(
+        public async Task<RetrievedSecrets> RetrieveSecretsAsync(
             List<RetrieveSecretsRequest> requests, 
             CancellationToken token)
         {
@@ -38,7 +38,7 @@ namespace BuildXL.Launcher.Server
                 secrets[request.Name] = new PlainTextSecret(secret);
             }
 
-            return secrets;
+            return new RetrievedSecrets(secrets);
         }
     }
 }

Public/Src/Cache/DistributedCache.Host/LauncherServer/Program.cs

@@ -1,12 +1,9 @@
 using System;
-using System.Collections.Generic;
 using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
 using Microsoft.AspNetCore.Hosting;
-using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.Hosting;
-using Microsoft.Extensions.Logging;
 
 namespace BuildXL.Launcher.Server
 {

Public/Src/Cache/DistributedCache.Host/LauncherServer/StartupBase.cs

@@ -1,21 +1,12 @@
 using System;
-using System.Collections.Generic;
-using System.Linq;
 using System.Reflection;
-using System.Threading.Tasks;
-using BuildXL.Cache.ContentStore.Interfaces.FileSystem;
-using BuildXL.Cache.ContentStore.Interfaces.Time;
-using BuildXL.Cache.ContentStore.Logging;
 using BuildXL.Launcher.Server.Controllers;
 using Microsoft.AspNetCore.Builder;
 using Microsoft.AspNetCore.Hosting;
-using Microsoft.AspNetCore.HttpsPolicy;
-using Microsoft.AspNetCore.Mvc;
 using Microsoft.AspNetCore.Mvc.Controllers;
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Hosting;
-using ILogger = BuildXL.Cache.ContentStore.Interfaces.Logging.ILogger;
 
 namespace BuildXL.Launcher.Server
 {

Public/Src/Cache/DistributedCache.Host/Service/Application/CacheServiceRunner.cs

@@ -100,7 +100,7 @@ namespace BuildXL.Cache.Host.Service
         public static async Task RunCacheServiceAsync(
             OperationContext context,
             string configurationPath,
-            Func<HostParameters, DistributedCacheServiceConfiguration, CancellationToken, IDistributedCacheServiceHost> createhost,
+            Func<HostParameters, DistributedCacheServiceConfiguration, CancellationToken, IDistributedCacheServiceHost> createHost,
             HostParameters hostParameters = null,
             bool requireServiceInterruptable = true)
         {
@@ -122,7 +122,7 @@ namespace BuildXL.Cache.Host.Service
                 {
                     var hostInfo = new HostInfo(hostParameters.Stamp, hostParameters.Ring, new List<string>());
 
-                    var host = createhost(hostParameters, config, token);
+                    var host = createHost(hostParameters, config, token);
 
                     await DistributedCacheServiceFacade.RunWithConfigurationAsync(
                         logger: context.TracingContext.Logger,

Public/Src/Cache/DistributedCache.Host/Service/Application/EnvironmentVariableHost.cs

@@ -6,44 +6,83 @@ using System.Collections.Generic;
 using System.Diagnostics.ContractsLight;
 using System.Threading;
 using System.Threading.Tasks;
+using BuildXL.Cache.ContentStore.Interfaces.Results;
 using BuildXL.Cache.ContentStore.Interfaces.Secrets;
-using BuildXL.Cache.Host.Service;
+using BuildXL.Cache.Host.Service.Internal;
 using Microsoft.WindowsAzure.Storage;
 
-// ReSharper disable once UnusedMember.Global
 namespace BuildXL.Cache.Host.Service
 {
     /// <summary>
-    /// Host where secrets are derived from environment variables
+    /// Host where secrets are derived from environment variables.
     /// </summary>
     public class EnvironmentVariableHost : IDistributedCacheServiceHost
     {
+        private readonly bool _retrieveAllSecretsFromSingleEnvironmentVariable;
+        private Result<RetrievedSecrets> _secrets;
+
         public CancellationTokenSource TeardownCancellationTokenSource { get; } = new CancellationTokenSource();
 
+        public EnvironmentVariableHost(bool retrieveAllSecretsFromSingleEnvironmentVariable = false)
+        {
+            _retrieveAllSecretsFromSingleEnvironmentVariable = retrieveAllSecretsFromSingleEnvironmentVariable;
+        }
+
+        /// <inheritdoc />
         public virtual void RequestTeardown(string reason)
         {
             TeardownCancellationTokenSource.Cancel();
         }
 
-        public string GetSecretStoreValue(string key)
+        private string GetSecretStoreValue(string key)
         {
             return Environment.GetEnvironmentVariable(key);
         }
 
+        /// <inheritdoc />
         public virtual void OnStartedService()
         {
         }
 
+        /// <inheritdoc />
         public virtual Task OnStartingServiceAsync()
         {
             return Task.CompletedTask;
         }
 
+        /// <inheritdoc />
         public virtual void OnTeardownCompleted()
         {
         }
         
-        public Task<Dictionary<string, Secret>> RetrieveSecretsAsync(List<RetrieveSecretsRequest> requests, CancellationToken token)
+        /// <inheritdoc />
+        public Task<RetrievedSecrets> RetrieveSecretsAsync(List<RetrieveSecretsRequest> requests, CancellationToken token)
+        {
+            // Checking the mode first: out-of-proc cache service passes all secrets via a single environment variable.
+            if (_retrieveAllSecretsFromSingleEnvironmentVariable)
+            {
+                var secretsResult = LazyInitializer.EnsureInitialized(ref _secrets, () => DeserializeFromEnvironmentVariable());
+
+                secretsResult.ThrowIfFailure();
+                return Task.FromResult(secretsResult.Value);
+            }
+
+            return RetrieveSecretsCoreAsync(requests, token);
+        }
+
+        private static Result<RetrievedSecrets> DeserializeFromEnvironmentVariable()
+        {
+            var variableName = RetrievedSecretsSerializer.SerializedSecretsKeyName;
+            var variable = Environment.GetEnvironmentVariable(variableName);
+            if (string.IsNullOrEmpty(variable))
+            {
+                return Result.FromErrorMessage<RetrievedSecrets>($"Environment variable '{variableName}' is null or empty.");
+            }
+
+            return RetrievedSecretsSerializer.Deserialize(variable);
+        }
+
+        private Task<RetrievedSecrets> RetrieveSecretsCoreAsync(List<RetrieveSecretsRequest> requests, CancellationToken token)
         {
             var secrets = new Dictionary<string, Secret>();
 
@@ -75,7 +114,7 @@ namespace BuildXL.Cache.Host.Service
                 secrets[request.Name] = secret;
             }
 
-            return Task.FromResult(secrets);
+            return Task.FromResult(new RetrievedSecrets(secrets));
         }
 
         private Secret CreateSasTokenSecret(RetrieveSecretsRequest request, string secretValue)
@@ -112,11 +151,10 @@ namespace BuildXL.Cache.Host.Service
                 IPAddressOrRange = null,
             });
 
-            var internalSasToken = new SasToken()
+            var internalSasToken = new SasToken(
-            {
+                                       token: sasToken,
-                Token = sasToken,
+                                       storageAccount: cloudStorageAccount.Credentials.AccountName,
-                StorageAccount = cloudStorageAccount.Credentials.AccountName,
+                                       resourcePath: null);
-            };
             return new UpdatingSasToken(internalSasToken);
         }
     }

Public/Src/Cache/DistributedCache.Host/Service/ContentCacheService.cs

@@ -2,7 +2,6 @@
 // Licensed under the MIT License.
 
 using System;
-using System.IO;
 using System.Threading.Tasks;
 using BuildXL.Cache.ContentStore.Distributed.NuCache;
 using BuildXL.Cache.ContentStore.Hashing;

Public/Src/Cache/DistributedCache.Host/Service/Deployment/CrossProcessSecretsCommunicationKind.cs

@@ -0,0 +1,28 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+using BuildXL.Cache.Host.Service.OutOfProc;
+
+namespace BuildXL.Cache.Host.Service
+{
+    /// <summary>
+    /// Defines which communication process is used for passing secrets between the current and the launched processes.
+    /// </summary>
+    public enum CrossProcessSecretsCommunicationKind
+    {
+        /// <summary>
+        /// The mode used by the launcher via <see cref="EnvironmentVariableHost"/> when all the secrets serialized through environment variables one by one.
+        /// </summary>
+        Environment,
+
+        /// <summary>
+        /// The mode used by <see cref="CacheServiceWrapper"/> when all the <see cref="RetrievedSecrets"/> serialized in a single environment variable.
+        /// </summary>
+        EnvironmentSingleEntry,
+
+        /// <summary>
+        /// Not implemented yet: will be used by <see cref="CacheServiceWrapper"/> when the secrets will be communicated via memory mapped file that will also support updates.
+        /// </summary>
+        MemoryMappedFile,
+    }
+}

Public/Src/Cache/DistributedCache.Host/Service/Deployment/DeploymentIngester.cs

@@ -8,17 +8,13 @@ using System.Diagnostics.ContractsLight;
 using System.IO;
 using System.Linq;
 using System.Text.Json;
-using System.Threading;
 using System.Threading.Tasks;
 using System.Web;
-using BuildXL.Cache.ContentStore.Distributed;
 using BuildXL.Cache.ContentStore.Hashing;
 using BuildXL.Cache.ContentStore.Interfaces.FileSystem;
-using BuildXL.Cache.ContentStore.Interfaces.Logging;
 using BuildXL.Cache.ContentStore.Interfaces.Results;
 using BuildXL.Cache.ContentStore.Interfaces.Sessions;
 using BuildXL.Cache.ContentStore.Interfaces.Time;
-using BuildXL.Cache.ContentStore.Interfaces.Tracing;
 using BuildXL.Cache.ContentStore.Stores;
 using BuildXL.Cache.ContentStore.Tracing;
 using BuildXL.Cache.ContentStore.Tracing.Internal;

Public/Src/Cache/DistributedCache.Host/Service/Deployment/DeploymentLauncher.cs

@@ -4,23 +4,17 @@
 using System;
 using System.Collections.Generic;
 using System.Diagnostics;
-using System.Diagnostics.ContractsLight;
 using System.IO;
 using System.Linq;
-using System.Net.Http;
 using System.Text.Json;
 using System.Text.RegularExpressions;
 using System.Threading;
 using System.Threading.Tasks;
-using System.Web;
-using BuildXL.Cache.ContentStore.Distributed;
 using BuildXL.Cache.ContentStore.Hashing;
 using BuildXL.Cache.ContentStore.Interfaces.FileSystem;
-using BuildXL.Cache.ContentStore.Interfaces.Logging;
 using BuildXL.Cache.ContentStore.Interfaces.Results;
 using BuildXL.Cache.ContentStore.Interfaces.Sessions;
 using BuildXL.Cache.ContentStore.Interfaces.Time;
-using BuildXL.Cache.ContentStore.Interfaces.Tracing;
 using BuildXL.Cache.ContentStore.Service;
 using BuildXL.Cache.ContentStore.Stores;
 using BuildXL.Cache.ContentStore.Tracing;
@@ -28,12 +22,10 @@ using BuildXL.Cache.ContentStore.Tracing.Internal;
 using BuildXL.Cache.ContentStore.UtilitiesCore.Internal;
 using BuildXL.Cache.ContentStore.Utils;
 using BuildXL.Cache.Host.Configuration;
-using BuildXL.Launcher.Server;
 using BuildXL.Native.IO;
 using BuildXL.Processes;
 using BuildXL.Utilities.ParallelAlgorithms;
 using BuildXL.Utilities.Tasks;
-using Microsoft.Win32.SafeHandles;
 using static BuildXL.Cache.Host.Configuration.DeploymentManifest;
 
 namespace BuildXL.Cache.Host.Service
@@ -407,12 +399,12 @@ namespace BuildXL.Cache.Host.Service
         {
             private readonly SemaphoreSlim _mutex = TaskUtilities.CreateMutex();
 
+            private LauncherManagedProcess _runningProcess;
+
             /// <summary>
             /// The active process for the tool
             /// </summary>
-            public ILauncherProcess RunningProcess { get; private set; }
+            public ILauncherProcess RunningProcess => _runningProcess?.Process;
-
-            private TaskSourceSlim<bool> ProcessExitSource { get; set; }
 
             /// <summary>
             /// Gets whether the tool process is running
@@ -430,6 +422,7 @@ namespace BuildXL.Cache.Host.Service
             public DisposableDirectory Directory { get; }
 
             private DeploymentLauncher Launcher { get; }
+
             public PinRequest PinRequest { get; set; }
 
             public AbsolutePath DirectoryPath => Directory.Path;
@@ -511,6 +504,7 @@ namespace BuildXL.Cache.Host.Service
             protected override Task<BoolResult> StartupCoreAsync(OperationContext context)
             {
                 int? processId = null;
+                var tool = Manifest.Tool;
                 return context.PerformOperationAsync(
                     Tracer,
                     async () =>
@@ -519,9 +513,9 @@ namespace BuildXL.Cache.Host.Service
                         //       Or maybe process should terminate itself if its not healthy?
                         using (await _mutex.AcquireAsync(context.Token))
                         {
-                            if (RunningProcess == null)
+                            if (_runningProcess == null)
                             {
-                                var executablePath = (Directory.Path / Manifest.Tool.Executable).Path;
+                                var executablePath = (Directory.Path / tool.Executable).Path;
                                 if (!File.Exists(executablePath))
                                 {
                                     return new BoolResult($"Executable '{executablePath}' does not exist.");
@@ -532,26 +526,22 @@ namespace BuildXL.Cache.Host.Service
                                     return new BoolResult($"Executable permissions could not be set on '{executablePath}'.");
                                 }
 
-                                RunningProcess = Launcher._host.CreateProcess(new ProcessStartInfo()
+                                var process = Launcher._host.CreateProcess(
+                                    new ProcessStartInfo()
                                     {
                                         UseShellExecute = false,
                                         FileName = executablePath,
-                                    Arguments = string.Join(" ", Manifest.Tool.Arguments.Select(arg => QuoteArgumentIfNecessary(ExpandTokens(arg)))),
+                                        Arguments = string.Join(" ", tool.Arguments.Select(arg => QuoteArgumentIfNecessary(ExpandTokens(arg)))),
                                         Environment =
                                         {
                                             Launcher.Settings.DeploymentParameters.ToEnvironment(),
-                                        Manifest.Tool.EnvironmentVariables.ToDictionary(kvp => kvp.Key, kvp => ExpandTokens(kvp.Value)),
+                                            tool.EnvironmentVariables.ToDictionary(kvp => kvp.Key, kvp => ExpandTokens(kvp.Value)),
-                                        Launcher.LifetimeManager.GetDeployedInterruptableServiceVariables(Manifest.Tool.ServiceId)
+                                            Launcher.LifetimeManager.GetDeployedInterruptableServiceVariables(tool.ServiceId)
                                         }
                                     });
+                                _runningProcess = new LauncherManagedProcess(process, tool.ServiceId, Launcher.LifetimeManager);
 
-                                ProcessExitSource = TaskSourceSlim.Create<bool>();
+                                _runningProcess.Start(context).ThrowIfFailure();
-                                RunningProcess.Exited += () =>
-                                {
-                                    OnExited(context, "ProcessExitedEvent");
-                                };
-
-                                RunningProcess.Start(context);
                                 processId = RunningProcess.Id;
                             }
 
@@ -559,35 +549,22 @@ namespace BuildXL.Cache.Host.Service
                         }
                     },
                     traceOperationStarted: true,
-                    extraStartMessage: $"ServiceId={Manifest.Tool.ServiceId}",
+                    extraStartMessage: $"ServiceId={tool.ServiceId}",
-                    extraEndMessage: r => $"ProcessId={processId ?? -1}, ServiceId={Manifest.Tool.ServiceId}");
+                    extraEndMessage: r => $"ProcessId={processId ?? -1}, ServiceId={tool.ServiceId}");
             }
 
-            private void OnExited(OperationContext context, string trigger)
+            public static string QuoteArgumentIfNecessary(string arg)
-            {
-                context.PerformOperation(
-                    Launcher.Tracer,
-                    () =>
-                    {
-                        ProcessExitSource.TrySetResult(true);
-                        return Result.Success(RunningProcess.ExitCode.ToString());
-                    },
-                    caller: "ServiceExited",
-                    messageFactory: r => $"ProcessId={RunningProcess.Id}, ServiceId={Manifest.Tool.ServiceId}, ExitCode={r.GetValueOrDefault(string.Empty)} Trigger={trigger}").IgnoreFailure();
-            }
-
-            private static string QuoteArgumentIfNecessary(string arg)
             {
                 return arg.Contains(" ") ? $"\"{arg}\"" : arg;
             }
 
-            private string ExpandTokens(string value)
+            public string ExpandTokens(string value)
             {
                 value = ExpandToken(value, "ServiceDir", DirectoryPath.Path);
                 return value;
             }
 
-            private static string ExpandToken(string value, string tokenName, string tokenValue)
+            public static string ExpandToken(string value, string tokenName, string tokenValue)
             {
                 if (!string.IsNullOrEmpty(tokenValue))
                 {
@@ -600,41 +577,19 @@ namespace BuildXL.Cache.Host.Service
             /// <summary>
             /// Terminates the tool process
             /// </summary>
-            protected override Task<BoolResult> ShutdownCoreAsync(OperationContext context)
+            protected override async Task<BoolResult> ShutdownCoreAsync(OperationContext context)
             {
                 var tool = Manifest.Tool;
 
-                return context.PerformOperationWithTimeoutAsync<BoolResult>(
-                    Tracer,
-                    async nestedContext =>
-                    {
                 using (await _mutex.AcquireAsync(context.Token))
                 {
                     try
                     {
-                                if (RunningProcess != null && !RunningProcess.HasExited)
+                        if (_runningProcess != null)
                         {
-                                    using var registration = nestedContext.Token.Register(() =>
+                            return await _runningProcess
-                                    {
+                                .StopAsync(context, TimeSpan.FromSeconds(tool.ShutdownTimeoutSeconds))
-                                        TerminateService(context);
+                                .ThrowIfFailure();
-                                        ProcessExitSource.TrySetCanceled();
-                                    });
-
-                                    await context.PerformOperationAsync(
-                                        Tracer,
-                                        async () =>
-                                        {
-                                            await Launcher.LifetimeManager.ShutdownServiceAsync(nestedContext, tool.ServiceId);
-                                            return BoolResult.Success;
-                                        },
-                                        caller: "GracefulShutdownService").IgnoreFailure();
-
-                                    if (RunningProcess.HasExited)
-                                    {
-                                        OnExited(context, "ShutdownAlreadyExited");
-                                    }
-
-                                    await ProcessExitSource.Task;
                         }
                     }
                     finally
@@ -646,34 +601,6 @@ namespace BuildXL.Cache.Host.Service
 
                     return BoolResult.Success;
                 }
-                    },
-                    timeout: TimeSpan.FromSeconds(tool.ShutdownTimeoutSeconds),
-                    extraStartMessage: $"ProcessId={RunningProcess?.Id}, ServiceId={Manifest.Tool.ServiceId}",
-                    extraEndMessage: r => $"ProcessId={RunningProcess?.Id}, ServiceId={Manifest.Tool.ServiceId}");
-            }
-
-            /// <summary>
-            /// Terminates the service by killing the process
-            /// </summary>
-            private void TerminateService(OperationContext context)
-            {
-                context.PerformOperation(
-                    Tracer,
-                    () =>
-                    {
-                        if (RunningProcess.HasExited)
-                        {
-                            OnExited(context, "TerminateServiceAlreadyExited");
-                        }
-                        else
-                        {
-                            RunningProcess.Kill(context);
-                        }
-
-                        return BoolResult.Success;
-                    },
-                    extraStartMessage: $"ProcessId={RunningProcess?.Id}, ServiceId={Manifest.Tool.ServiceId}",
-                    messageFactory: r => $"ProcessId={RunningProcess?.Id}, ServiceId={Manifest.Tool.ServiceId}").IgnoreFailure();
             }
         }
     }

Public/Src/Cache/DistributedCache.Host/Service/Deployment/DeploymentLauncherHost.cs

@@ -2,24 +2,13 @@
 // Licensed under the MIT License.
 
 using System;
-using System.Collections.Generic;
 using System.Diagnostics;
 using System.IO;
 using System.Net.Http;
-using System.Net.Http.Headers;
-using System.Text;
 using System.Text.Json;
-using System.Threading;
 using System.Threading.Tasks;
-using BuildXL.Cache.ContentStore.Hashing;
-using BuildXL.Cache.ContentStore.Interfaces.FileSystem;
-using BuildXL.Cache.ContentStore.Interfaces.Secrets;
-using BuildXL.Cache.ContentStore.Stores;
-using BuildXL.Cache.ContentStore.Tracing;
 using BuildXL.Cache.ContentStore.Tracing.Internal;
 using BuildXL.Cache.Host.Configuration;
-using BuildXL.Utilities.Collections;
-using static BuildXL.Cache.Host.Configuration.DeploymentManifest;
 
 namespace BuildXL.Cache.Host.Service
 {
@@ -117,87 +106,5 @@ namespace BuildXL.Cache.Host.Service
                 // Do nothing. This instance is reused
             }
         }
-
-        private class LauncherProcess : ILauncherProcess
-        {
-            private static readonly Tracer _tracer = new Tracer(nameof(LauncherProcess));
-
-            private readonly Process _process;
-
-            public LauncherProcess(ProcessStartInfo info)
-            {
-                info.RedirectStandardOutput = true;
-                info.RedirectStandardError = true;
-
-                _process = new Process()
-                {
-                    StartInfo = info,
-                    EnableRaisingEvents = true
-                };
-
-                _process.Exited += (sender, e) => Exited?.Invoke();
-            }
-
-            public int ExitCode => _process.ExitCode;
-
-            public int Id => _process.Id;
-
-            public bool HasExited => _process.HasExited;
-
-            public event Action Exited;
-
-            public void Kill(OperationContext context)
-            {
-                _process.Kill();
-            }
-
-            public void Start(OperationContext context)
-            {
-                // Using nagle queues to "batch" messages together and to avoid writing them to the logs one by one.
-                var outputMessagesNagleQueue = NagleQueue<string>.Create(
-                    messages =>
-                        {
-                            _tracer.Debug(context, $"Service Output: {string.Join(Environment.NewLine, messages)}");
-                            return Task.CompletedTask;
-                        },
-                    maxDegreeOfParallelism: 1, interval: TimeSpan.FromSeconds(10), batchSize: 1024);
-
-                var errorMessagesNagleQueue = NagleQueue<string>.Create(
-                    messages =>
-                    {
-                        _tracer.Error(context, $"Service Error: {string.Join(Environment.NewLine, messages)}");
-                        return Task.CompletedTask;
-                    },
-                    maxDegreeOfParallelism: 1, interval: TimeSpan.FromSeconds(10), batchSize: 1024);
-
-                _process.OutputDataReceived += (s, e) =>
-                {
-                    if (!string.IsNullOrEmpty(e.Data))
-                    {
-                        outputMessagesNagleQueue.Enqueue(e.Data);
-                    }
-                };
-
-                _process.ErrorDataReceived += (s, e) =>
-                {
-                    if (!string.IsNullOrEmpty(e.Data))
-                    {
-                        errorMessagesNagleQueue.Enqueue(e.Data);
-                    }
-                };
-
-                _process.Exited += (sender, args) =>
-                                   {
-                                       // Dispose will drain all the existing items from the message queues.
-                                       outputMessagesNagleQueue.Dispose();
-                                       errorMessagesNagleQueue.Dispose();
-                                   };
-
-                _process.Start();
-
-                _process.BeginOutputReadLine();
-                _process.BeginErrorReadLine();
-            }
-        }
     }
 }

Public/Src/Cache/DistributedCache.Host/Service/Deployment/DeploymentProxyService.cs

@@ -1,34 +1,26 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
 using System;
-using System.Collections.Concurrent;
-using System.Collections.Generic;
 using System.IO;
-using System.Linq;
-using System.Text.Json;
 using System.Threading.Tasks;
-using BuildXL.Cache.ContentStore.Distributed;
 using BuildXL.Cache.ContentStore.Distributed.NuCache;
+using BuildXL.Cache.ContentStore.FileSystem;
 using BuildXL.Cache.ContentStore.Hashing;
 using BuildXL.Cache.ContentStore.Interfaces.FileSystem;
 using BuildXL.Cache.ContentStore.Interfaces.Results;
-using BuildXL.Cache.ContentStore.Interfaces.Secrets;
 using BuildXL.Cache.ContentStore.Interfaces.Time;
-using BuildXL.Cache.ContentStore.Tracing.Internal;
-using BuildXL.Cache.Host.Configuration;
-using BuildXL.Cache.Host.Service;
-using BuildXL.Utilities;
-using BuildXL.Cache.ContentStore.Interfaces.Extensions;
-using static BuildXL.Cache.Host.Configuration.DeploymentManifest;
-using AbsolutePath = BuildXL.Cache.ContentStore.Interfaces.FileSystem.AbsolutePath;
-using BuildXL.Utilities.ParallelAlgorithms;
-using BuildXL.Cache.ContentStore.Utils;
-using BuildXL.Cache.ContentStore.Tracing;
-using System.Threading;
-using System.Text;
-using BuildXL.Cache.ContentStore.Stores;
 using BuildXL.Cache.ContentStore.Interfaces.Tracing;
-using BuildXL.Cache.ContentStore.FileSystem;
+using BuildXL.Cache.ContentStore.Stores;
+using BuildXL.Cache.ContentStore.Tracing;
+using BuildXL.Cache.ContentStore.Tracing.Internal;
+using BuildXL.Cache.ContentStore.Utils;
+using BuildXL.Cache.Host.Configuration;
+using BuildXL.Utilities;
+using BuildXL.Utilities.ParallelAlgorithms;
+using AbsolutePath = BuildXL.Cache.ContentStore.Interfaces.FileSystem.AbsolutePath;
 
-namespace BuildXL.Launcher.Server
+namespace BuildXL.Cache.Host.Service
 {
     /// <summary>
     /// Service used ensure deployments are uploaded to target storage accounts and provide manifest for with download urls and tools to launch

Public/Src/Cache/DistributedCache.Host/Service/Deployment/DeploymentService.cs

@@ -1,6 +1,9 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
 using System;
-using System.Collections.Concurrent;
 using System.Collections.Generic;
+using System.Diagnostics.ContractsLight;
 using System.IO;
 using System.Linq;
 using System.Text.Json;
@@ -8,30 +11,25 @@ using System.Threading.Tasks;
 using BuildXL.Cache.ContentStore.Distributed;
 using BuildXL.Cache.ContentStore.Distributed.NuCache;
 using BuildXL.Cache.ContentStore.Hashing;
-using BuildXL.Cache.ContentStore.Interfaces.FileSystem;
+using BuildXL.Cache.ContentStore.Interfaces.Extensions;
 using BuildXL.Cache.ContentStore.Interfaces.Results;
 using BuildXL.Cache.ContentStore.Interfaces.Secrets;
 using BuildXL.Cache.ContentStore.Interfaces.Time;
+using BuildXL.Cache.ContentStore.Interfaces.Tracing;
+using BuildXL.Cache.ContentStore.Tracing;
 using BuildXL.Cache.ContentStore.Tracing.Internal;
+using BuildXL.Cache.ContentStore.UtilitiesCore;
+using BuildXL.Cache.ContentStore.Utils;
 using BuildXL.Cache.Host.Configuration;
-using BuildXL.Cache.Host.Service;
 using BuildXL.Utilities;
-using BuildXL.Cache.ContentStore.Interfaces.Extensions;
+using BuildXL.Utilities.Collections;
+using BuildXL.Utilities.ParallelAlgorithms;
+using JetBrains.Annotations;
 using static BuildXL.Cache.Host.Configuration.DeploymentManifest;
 using static BuildXL.Cache.Host.Service.DeploymentUtilities;
 using AbsolutePath = BuildXL.Cache.ContentStore.Interfaces.FileSystem.AbsolutePath;
-using BuildXL.Utilities.ParallelAlgorithms;
-using BuildXL.Cache.ContentStore.Utils;
-using BuildXL.Cache.ContentStore.Tracing;
-using System.Threading;
-using System.Text;
-using BuildXL.Cache.ContentStore.UtilitiesCore;
-using BuildXL.Cache.ContentStore.Interfaces.Tracing;
-using JetBrains.Annotations;
-using BuildXL.Utilities.Collections;
-using System.Diagnostics.ContractsLight;
 
-namespace BuildXL.Launcher.Server
+namespace BuildXL.Cache.Host.Service
 {
     /// <summary>
     /// Service used ensure deployments are uploaded to target storage accounts and provide manifest for with download urls and tools to launch

Public/Src/Cache/DistributedCache.Host/Service/Deployment/DeploymentUtilities.cs

@@ -6,16 +6,13 @@ using System.Collections.Generic;
 using System.Linq;
 using System.Text;
 using System.Text.Json;
-using System.Text.Json.Serialization;
 using System.Threading;
 using System.Threading.Tasks;
 using BuildXL.Cache.ContentStore.Distributed.NuCache;
 using BuildXL.Cache.ContentStore.Distributed.Utilities;
 using BuildXL.Cache.ContentStore.Hashing;
 using BuildXL.Cache.ContentStore.Interfaces.FileSystem;
-using BuildXL.Cache.ContentStore.Interfaces.Secrets;
 using BuildXL.Cache.ContentStore.Stores;
-using BuildXL.Cache.ContentStore.Utils;
 using BuildXL.Cache.Host.Configuration;
 using static BuildXL.Cache.Host.Configuration.DeploymentManifest;
 

Public/Src/Cache/DistributedCache.Host/Service/Deployment/IDeploymentLauncherHost.cs

@@ -2,19 +2,12 @@
 // Licensed under the MIT License.
 
 using System;
-using System.Collections.Generic;
 using System.Diagnostics;
 using System.IO;
-using System.Text.Json;
-using System.Threading;
 using System.Threading.Tasks;
-using BuildXL.Cache.ContentStore.Hashing;
 using BuildXL.Cache.ContentStore.Interfaces.FileSystem;
-using BuildXL.Cache.ContentStore.Interfaces.Secrets;
-using BuildXL.Cache.ContentStore.Stores;
 using BuildXL.Cache.ContentStore.Tracing.Internal;
 using BuildXL.Cache.Host.Configuration;
-using static BuildXL.Cache.Host.Configuration.DeploymentManifest;
 
 namespace BuildXL.Cache.Host.Service
 {
@@ -24,7 +17,7 @@ namespace BuildXL.Cache.Host.Service
     public interface IDeploymentLauncherHost
     {
         /// <summary>
-        /// Creates an unstarted process using the given start info
+        /// Creates an unstarted process using the given start info.
         /// </summary>
         ILauncherProcess CreateProcess(ProcessStartInfo info);
 
@@ -56,7 +49,7 @@ namespace BuildXL.Cache.Host.Service
     }
 
     /// <summary>
-    /// Represents a launched system process
+    /// Represents a light-weight wrapper around launched system process.
     /// </summary>
     public interface ILauncherProcess
     {
@@ -91,6 +84,40 @@ namespace BuildXL.Cache.Host.Service
         bool HasExited { get; }
     }
 
+    ///// <summary>
+    ///// Represents a launched system process
+    ///// </summary>
+    //public interface ILauncherProcess
+    //{
+    //    /// <summary>
+    //    /// Starts the process.
+    //    /// </summary>
+    //    BoolResult Start(OperationContext context);
+
+    //    /// <summary>
+    //    /// Stop the service gracefully and kill it if it won't shutdown on time.
+    //    /// </summary>
+    //    /// <remarks>
+    //    /// If the shutdown is successful the result contains an exit code.
+    //    /// </remarks>
+    //    Task<Result<int>> StopAsync(OperationContext context, TimeSpan shutdownTimeout);
+
+    //    /// <summary>
+    //    /// The id of the process.
+    //    /// </summary>
+    //    int Id { get; }
+
+    //    /// <summary>
+    //    /// The id of the service that this process represents.
+    //    /// </summary>
+    //    string ServiceId { get; }
+
+    //    /// <summary>
+    //    /// Indicates if the process has exited.
+    //    /// </summary>
+    //    bool HasExited { get; }
+    //}
+
     /// <summary>
     /// Represents a tool deployed and launched by the <see cref="DeploymentLauncher"/>
     /// </summary>

Public/Src/Cache/DistributedCache.Host/Service/Deployment/JsonMerger.cs

@@ -1,12 +1,10 @@
-// Copyright (c) Microsoft Corporation. All rights reserved.
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
 
-using System;
 using System.Collections.Generic;
 using System.IO;
-using System.Linq;
 using System.Text;
 using System.Text.Json;
-using System.Text.RegularExpressions;
 
 namespace BuildXL.Cache.Host.Service
 {

Public/Src/Cache/DistributedCache.Host/Service/Deployment/JsonPreprocessor.cs

@@ -1,4 +1,5 @@
-// Copyright (c) Microsoft Corporation. All rights reserved.
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
 
 using System;
 using System.Collections.Generic;

Public/Src/Cache/DistributedCache.Host/Service/Deployment/LauncherManagedProcess.cs

@@ -0,0 +1,148 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+using System;
+using System.Threading.Tasks;
+using BuildXL.Cache.ContentStore.Interfaces.Results;
+using BuildXL.Cache.ContentStore.Service;
+using BuildXL.Cache.ContentStore.Tracing;
+using BuildXL.Cache.ContentStore.Tracing.Internal;
+using BuildXL.Utilities.Tasks;
+
+namespace BuildXL.Cache.Host.Service
+{
+    /// <summary>
+    /// A wrapper around launched processed managed by <see cref="ServiceLifetimeManager"/>.
+    /// </summary>
+    internal sealed class LauncherManagedProcess 
+    {
+        private static readonly Tracer Tracer = new Tracer(nameof(LauncherManagedProcess));
+
+        private readonly ILauncherProcess _process;
+        private readonly ServiceLifetimeManager _lifetimeManager;
+        private readonly TaskSourceSlim<int> _processExitSource = TaskSourceSlim.Create<int>();
+
+        public LauncherManagedProcess(ILauncherProcess process, string serviceId, ServiceLifetimeManager lifetimeManager)
+        {
+            _process = process;
+            _lifetimeManager = lifetimeManager;
+            ServiceId = serviceId;
+        }
+
+        /// <nodoc />
+        public string ServiceId { get; }
+
+        /// <summary>
+        /// Returns an underlying process that this class manages lifetime of.
+        /// </summary>
+        public ILauncherProcess Process => _process;
+
+        /// <nodoc />
+        public int ProcessId => _process.Id;
+
+        /// <nodoc />
+        public bool HasExited => _process.HasExited;
+
+        /// <nodoc />
+        public BoolResult Start(OperationContext context)
+        {
+            return context.PerformOperation(
+                Tracer,
+                () =>
+                {
+                    _process.Start(context);
+                    return Result.Success(_process.Id);
+                },
+                traceOperationStarted: true,
+                extraStartMessage: $"ServiceId={ServiceId}",
+                messageFactory: r => $"ProcessId={r.GetValueOrDefault(defaultValue: -1)}, ServiceId={ServiceId}"
+            );
+        }
+
+        /// <nodoc />
+        public Task<Result<int>> StopAsync(OperationContext context, TimeSpan shutdownTimeout)
+        {
+            bool alreadyExited = false;
+            return context.PerformOperationWithTimeoutAsync(
+                Tracer,
+                async nestedContext =>
+                {
+                    if (HasExited)
+                    {
+                        alreadyExited = true;
+                        return Result.Success(_process.ExitCode);
+                    }
+
+                    // Terminating the process after timeout if it won't shutdown gracefully.
+                    using var registration = nestedContext.Token.Register(
+                        () =>
+                        {
+                            // It is important to pass 'context' and not 'nestedContext',
+                            // because 'nestedContext' will be canceled at a time we call TerminateService.
+                            Kill(context);
+                        });
+
+                    await context.PerformOperationAsync(
+                        Tracer,
+                        async () =>
+                        {
+                            await _lifetimeManager.ShutdownServiceAsync(nestedContext, ServiceId);
+                            return BoolResult.Success;
+                        },
+                        caller: "GracefulShutdownService").IgnoreFailure();
+
+                    return await _processExitSource.Task;
+                },
+                timeout: shutdownTimeout,
+                extraStartMessage: $"ProcessId={ProcessId}, ServiceId={ServiceId}",
+                extraEndMessage: r => $"ProcessId={ProcessId}, ServiceId={ServiceId}, ExitCode={r.GetValueOrDefault(-1)}, AlreadyExited={alreadyExited}");
+        }
+
+        private void Kill(OperationContext context)
+        {
+            context
+                .WithoutCancellationToken() // Not using the cancellation token from the context.
+                .PerformOperation(
+                    Tracer,
+                    () =>
+                    {
+                        // Using Result<string> for tracing purposes.
+                        if (HasExited)
+                        {
+                            OnExited(context, "TerminateServiceAlreadyExited");
+                            return Result.Success("AlreadyExited");
+                        }
+
+                        _process.Kill(context);
+                        return Result.Success("ProcessKilled");
+
+                    },
+                    extraStartMessage: $"ProcessId={ProcessId}, ServiceId={ServiceId}",
+                    messageFactory: r => $"ProcessId={ProcessId}, ServiceId={ServiceId}, {r}")
+                .IgnoreFailure();
+
+            // Intentionally trying to set the result that indicates the cancellation after PerformOperation call that will never throw.
+            _processExitSource.TrySetResult(-1);
+        }
+
+        private void OnExited(OperationContext context, string trigger)
+        {
+            // It is important to disable the cancellation here because in some cases the token associated
+            // with the context can be set.
+            // But the operation that we do here is very fast and we use context for tracing purposes only.
+            context
+                .WithoutCancellationToken()
+                .PerformOperation(
+                    Tracer,
+                    () =>
+                    {
+                        _processExitSource.TrySetResult(_process.ExitCode);
+                        return Result.Success(_process.ExitCode.ToString());
+                    },
+                    caller: "ServiceExited",
+                    messageFactory: r => $"ProcessId={ProcessId}, ServiceId={ServiceId}, ExitCode={r.GetValueOrDefault(string.Empty)} Trigger={trigger}")
+                .IgnoreFailure();
+        }
+    }
+}
+

Public/Src/Cache/DistributedCache.Host/Service/Deployment/LauncherProcess.cs

@@ -0,0 +1,105 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+using System;
+using System.Diagnostics;
+using System.Threading.Tasks;
+using BuildXL.Cache.ContentStore.Tracing;
+using BuildXL.Cache.ContentStore.Tracing.Internal;
+using BuildXL.Utilities.Collections;
+
+namespace BuildXL.Cache.Host.Service
+{
+    /// <summary>
+    /// A lightweight wrapper around launched process.
+    /// </summary>
+    internal sealed class LauncherProcess : ILauncherProcess
+    {
+        private static readonly Tracer _tracer = new Tracer(nameof(LauncherProcess));
+
+        private bool _started;
+        private readonly Process _process;
+
+        public LauncherProcess(ProcessStartInfo info)
+        {
+            info.RedirectStandardOutput = true;
+            info.RedirectStandardError = true;
+
+            _process = new Process()
+                       {
+                           StartInfo = info,
+                           EnableRaisingEvents = true
+                       };
+
+            _process.Exited += (sender, e) => Exited?.Invoke();
+        }
+
+        /// <inheritdoc />
+        public int ExitCode => _process.ExitCode;
+
+        /// <inheritdoc />
+        public int Id => _started ? _process.Id : -1;
+
+        /// <inheritdoc />
+        public bool HasExited => _process.HasExited;
+
+        /// <inheritdoc />
+        public event Action Exited;
+
+        /// <inheritdoc />
+        public void Kill(OperationContext context)
+        {
+            _process.Kill();
+        }
+
+        /// <inheritdoc />
+        public void Start(OperationContext context)
+        {
+            // Using nagle queues to "batch" messages together and to avoid writing them to the logs one by one.
+            var outputMessagesNagleQueue = NagleQueue<string>.Create(
+                messages =>
+                {
+                    _tracer.Debug(context, $"Service Output: {string.Join(Environment.NewLine, messages)}");
+                    return Task.CompletedTask;
+                },
+                maxDegreeOfParallelism: 1, interval: TimeSpan.FromSeconds(1), batchSize: 1024);
+
+            var errorMessagesNagleQueue = NagleQueue<string>.Create(
+                messages =>
+                {
+                    _tracer.Error(context, $"Service Error: {string.Join(Environment.NewLine, messages)}");
+                    return Task.CompletedTask;
+                },
+                maxDegreeOfParallelism: 1, interval: TimeSpan.FromSeconds(1), batchSize: 1024);
+
+            _process.OutputDataReceived += (s, e) =>
+                                           {
+                                               if (!string.IsNullOrEmpty(e.Data))
+                                               {
+                                                   outputMessagesNagleQueue.Enqueue(e.Data);
+                                               }
+                                           };
+
+            _process.ErrorDataReceived += (s, e) =>
+                                          {
+                                              if (!string.IsNullOrEmpty(e.Data))
+                                              {
+                                                  errorMessagesNagleQueue.Enqueue(e.Data);
+                                              }
+                                          };
+
+            _process.Exited += (sender, args) =>
+                               {
+                                   // Dispose will drain all the existing items from the message queues.
+                                   outputMessagesNagleQueue.Dispose();
+                                   errorMessagesNagleQueue.Dispose();
+                               };
+            
+            _process.Start();
+
+            _process.BeginOutputReadLine();
+            _process.BeginErrorReadLine();
+            _started = true;
+        }
+    }
+}

Public/Src/Cache/DistributedCache.Host/Service/Deployment/OutOfProc/CacheServiceWrapper.cs

@@ -0,0 +1,206 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.IO;
+using System.Reflection;
+using System.Threading.Tasks;
+using BuildXL.Cache.ContentStore.Interfaces.FileSystem;
+using BuildXL.Cache.ContentStore.Interfaces.Results;
+using BuildXL.Cache.ContentStore.Service;
+using BuildXL.Cache.ContentStore.Tracing;
+using BuildXL.Cache.ContentStore.Tracing.Internal;
+using BuildXL.Cache.ContentStore.UtilitiesCore.Internal;
+using BuildXL.Cache.ContentStore.Utils;
+using BuildXL.Cache.Host.Configuration;
+using BuildXL.Cache.Host.Service.Internal;
+using BuildXL.Utilities.ConfigurationHelpers;
+// The next using is needed in order to create ProcessStartInfo.EnvironmentVariables with collection initialization syntax.
+
+#nullable enable
+
+namespace BuildXL.Cache.Host.Service.OutOfProc
+{
+    /// <summary>
+    /// A helper class that "wraps" an out-of-proc cache service.
+    /// </summary>
+    public class CacheServiceWrapper : StartupShutdownBase
+    {
+        private readonly CacheServiceWrapperConfiguration _configuration;
+        private readonly ServiceLifetimeManager _serviceLifetimeManager;
+        private readonly RetrievedSecrets _secrets;
+
+        /// <inheritdoc />
+        protected override Tracer Tracer { get; } = new Tracer(nameof(CacheServiceWrapper));
+
+        private LauncherManagedProcess? _runningProcess;
+
+        public CacheServiceWrapper(CacheServiceWrapperConfiguration configuration, ServiceLifetimeManager serviceLifetimeManager, RetrievedSecrets secrets)
+        {
+            _configuration = configuration;
+            _serviceLifetimeManager = serviceLifetimeManager;
+            _secrets = secrets;
+        }
+
+        /// <summary>
+        /// Creates <see cref="CacheServiceWrapper"/> from <paramref name="configuration"/>.
+        /// </summary>
+        public static async Task<Result<CacheServiceWrapper>> CreateAsync(DistributedCacheServiceArguments configuration)
+        {
+            // Validating the cache configuration
+            
+            var wrapperConfiguration = tryCreateConfiguration(configuration);
+            if (!wrapperConfiguration.Succeeded)
+            {
+                const string BaseError = "Can't start cache service as a separate process because";
+                return Result.FromErrorMessage<CacheServiceWrapper>($"{BaseError} {wrapperConfiguration.ErrorMessage}");
+            }
+
+            // Obtaining the secrets and creating a wrapper.
+            var serviceLifetimeManager = new ServiceLifetimeManager(wrapperConfiguration.Value.WorkingDirectory, wrapperConfiguration.Value.ServiceLifetimePollingInterval);
+            var secretsRetriever = new DistributedCacheSecretRetriever(configuration);
+
+            var secrets = await secretsRetriever.TryRetrieveSecretsAsync();
+            if (!secrets.Succeeded)
+            {
+                return new Result<CacheServiceWrapper>(secrets);
+            }
+
+            return Result.Success(new CacheServiceWrapper(wrapperConfiguration.Value, serviceLifetimeManager, secrets.Value));
+
+            // Creating final configuration based on provided settings and by using reasonable defaults.
+            static Result<CacheServiceWrapperConfiguration> tryCreateConfiguration(DistributedCacheServiceArguments configuration)
+            {
+                var outOfProcSettings = configuration.Configuration.DistributedContentSettings.OutOfProcCacheSettings;
+
+                if (outOfProcSettings is null)
+                {
+                    return Result.FromErrorMessage<CacheServiceWrapperConfiguration>($"{nameof(configuration.Configuration.DistributedContentSettings.OutOfProcCacheSettings)} should not be null.");
+                }
+
+                if (outOfProcSettings.Executable is null)
+                {
+                    return Result.FromErrorMessage<CacheServiceWrapperConfiguration>($"{nameof(outOfProcSettings.Executable)} is null.");
+                }
+
+                if (!File.Exists(outOfProcSettings.Executable))
+                {
+                    // This is not a bullet proof check, but if the executable is not found we should not even trying to create an out of proc cache service.
+                    return Result.FromErrorMessage<CacheServiceWrapperConfiguration>($"the executable is not found at '{outOfProcSettings.Executable}'.");
+                }
+
+                if (outOfProcSettings.CacheConfigPath is null)
+                {
+                    return Result.FromErrorMessage<CacheServiceWrapperConfiguration>($"{nameof(outOfProcSettings.CacheConfigPath)} is null.");
+                }
+
+                if (!File.Exists(outOfProcSettings.CacheConfigPath))
+                {
+                    // This is not a bullet proof check, but if the executable is not found we should not even trying to create an out of proc cache service.
+                    return Result.FromErrorMessage<CacheServiceWrapperConfiguration>($"the cache configuration is not found at '{outOfProcSettings.CacheConfigPath}'.");
+                }
+
+                // The next layout should be in sync with CloudBuild.
+                AbsolutePath executable = getExecutingPath() / outOfProcSettings.Executable;
+                var workingDirectory = getRootPath(configuration.Configuration);
+
+                var hostParameters = HostParameters.FromTelemetryProvider(configuration.TelemetryFieldsProvider);
+
+                var resultingConfiguration = new CacheServiceWrapperConfiguration(
+                    serviceId: "OutOfProcCache",
+                    executable: executable,
+                    workingDirectory: workingDirectory,
+                    hostParameters: hostParameters,
+                    cacheConfigPath: new AbsolutePath(outOfProcSettings.CacheConfigPath),
+                    // DataRootPath is set in CloudBuild and we need to propagate this configuration to the launched process.
+                    dataRootPath: new AbsolutePath(configuration.Configuration.DataRootPath));
+
+                outOfProcSettings.ServiceLifetimePollingIntervalSeconds.ApplyIfNotNull(v => resultingConfiguration.ServiceLifetimePollingInterval = TimeSpan.FromSeconds(v));
+                outOfProcSettings.ShutdownTimeoutSeconds.ApplyIfNotNull(v => resultingConfiguration.ShutdownTimeout = TimeSpan.FromSeconds(v));
+
+                return resultingConfiguration;
+            }
+
+            static AbsolutePath getRootPath(DistributedCacheServiceConfiguration configuration) => configuration.LocalCasSettings.GetCacheRootPathWithScenario(LocalCasServiceSettings.DefaultCacheName);
+
+            static AbsolutePath getExecutingPath() => new AbsolutePath(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location)!);
+        }
+
+        /// <inheritdoc />
+        protected override Task<BoolResult> StartupCoreAsync(OperationContext context)
+        {
+            var executablePath = _configuration.Executable.Path;
+            if (!File.Exists(executablePath))
+            {
+                return Task.FromResult(new BoolResult($"Executable '{executablePath}' does not exist."));
+            }
+
+            // Need to specify through the arguments what type of secrets provider to use.
+            // Currently we serialize all the secrets as a single string.
+            var secretsProviderKind = CrossProcessSecretsCommunicationKind.EnvironmentSingleEntry;
+
+            var argumentsList = new []
+                                {
+                                    "CacheService",
+                                    // If cacheConfigPath is null the validation will fail and the process won't be started.
+                                    "--cacheConfigurationPath", _configuration.CacheConfigPath?.Path ?? string.Empty,
+                                    // This is not a standalone cache service, it is controlled by ServiceLifetimeManager.
+                                    "--standalone", "false",
+                                    "--secretsProviderKind", secretsProviderKind.ToString(),
+                                    "--dataRootPath", _configuration.DataRootPath.ToString(),
+                                };
+
+            var environment = new Dictionary<string, string>
+                              {
+                                  _configuration.HostParameters.ToEnvironment(),
+                                  _serviceLifetimeManager.GetDeployedInterruptableServiceVariables(_configuration.ServiceId),
+
+                                  // Passing the secrets via environment variable in a single value.
+                                  // This may be problematic if the serialized size will exceed some size (like 32K), but
+                                  // it should not be the case for now.
+                                  { RetrievedSecretsSerializer.SerializedSecretsKeyName, RetrievedSecretsSerializer.Serialize(_secrets) },
+                                  getDotNetEnvironmentVariables()
+                              };
+
+            var process = new LauncherProcess(
+                new ProcessStartInfo()
+                {
+                    UseShellExecute = false,
+                    FileName = executablePath,
+                    Arguments = string.Join(" ", argumentsList),
+                    // A strange cast to a nullable dictionary is needed to avoid warnings from the C# compiler.
+                    Environment = { (IDictionary<string, string?>)environment },
+                });
+
+            _runningProcess = new LauncherManagedProcess(process, _configuration.ServiceId, _serviceLifetimeManager);
+            Tracer.Info(context, "Starting out-of-proc cache process.");
+            var result = _runningProcess.Start(context);
+            Tracer.Info(context, $"Started out-of-proc cache process (Id={process.Id}). Result: {result}.");
+            return Task.FromResult(result);
+
+            static IDictionary<string, string> getDotNetEnvironmentVariables()
+            {
+                return new Dictionary<string, string>
+                       {
+                           ["COMPlus_GCCpuGroup"] = "1",
+                           ["DOTNET_GCCpuGroup"] = "1", // This is the same option that is used by .net6+
+                           ["COMPlus_Thread_UseAllCpuGroups"] = "1",
+                           ["DOTNET_Thread_UseAllCpuGroups"] = "1", // This is the same option that is used by .net6+
+                };
+            }
+        }
+
+        /// <inheritdoc />
+        protected override async Task<BoolResult> ShutdownCoreAsync(OperationContext context)
+        {
+            if (_runningProcess != null)
+            {
+                return await _runningProcess.StopAsync(context, _configuration.ShutdownTimeout);
+            }
+
+            return BoolResult.Success;
+        }
+    }
+}

Public/Src/Cache/DistributedCache.Host/Service/Deployment/OutOfProc/CacheServiceWrapperConfiguration.cs

@@ -0,0 +1,75 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+using System;
+using BuildXL.Cache.ContentStore.Interfaces.FileSystem;
+using BuildXL.Cache.ContentStore.Service;
+using BuildXL.Cache.Host.Configuration;
+
+#nullable enable
+
+namespace BuildXL.Cache.Host.Service.OutOfProc
+{
+    /// <summary>
+    /// Configuration class used by <see cref="CacheServiceWrapper"/>.
+    /// </summary>
+    public record class CacheServiceWrapperConfiguration
+    {
+        /// <nodoc />
+        public CacheServiceWrapperConfiguration(
+            string serviceId,
+            AbsolutePath executable,
+            AbsolutePath workingDirectory,
+            HostParameters hostParameters,
+            AbsolutePath cacheConfigPath,
+            AbsolutePath dataRootPath)
+        {
+            ServiceId = serviceId;
+            Executable = executable;
+            WorkingDirectory = workingDirectory;
+            HostParameters = hostParameters;
+            CacheConfigPath = cacheConfigPath;
+            DataRootPath = dataRootPath;
+        }
+
+        /// <summary>
+        /// The identifier used to identify the service for service lifetime management and interruption
+        /// </summary>
+        public string ServiceId { get; }
+
+        /// <summary>
+        /// Path to the executable used when launching the tool relative to the layout root
+        /// </summary>
+        public AbsolutePath Executable { get; }
+
+        /// <summary>
+        /// A working directory used for a process's lifetime tracking and other.
+        /// </summary>
+        public AbsolutePath WorkingDirectory { get; }
+
+        /// <summary>
+        /// Parameters of the running machine like Stamp, Region etc.
+        /// </summary>
+        public HostParameters HostParameters { get; }
+
+        /// <summary>
+        /// A path to the cache configuration (CacheConfiguration.json) file that the child process will use.
+        /// </summary>
+        public AbsolutePath CacheConfigPath { get; }
+
+        /// <summary>
+        /// A root of the data directory (CloudBuild sets this property for the in-proc-mode).
+        /// </summary>
+        public AbsolutePath DataRootPath { get; }
+
+        /// <summary>
+        /// The polling interval of the <see cref="ServiceLifetimeManager"/> used for lifetime tracking of a launched process.
+        /// </summary>
+        public TimeSpan ServiceLifetimePollingInterval { get; set; } = TimeSpan.FromSeconds(1);
+
+        /// <summary>
+        /// The time to wait for service to shutdown before terminating the process
+        /// </summary>
+        public TimeSpan ShutdownTimeout { get; set; } = TimeSpan.FromSeconds(60);
+    }
+}

Public/Src/Cache/DistributedCache.Host/Service/DistributedCacheServiceArguments.cs

@@ -29,7 +29,7 @@ namespace BuildXL.Cache.Host.Service
 
         /// <summary>
         ///     When this functor is present, and assuming the cache replaces the host's logger with its own, it is
-        ///     expected to buid <see cref="Copier"/> and <see cref="CopyRequester"/>.
+        ///     expected to build <see cref="Copier"/> and <see cref="CopyRequester"/>.
         ///     
         ///     This is done this way because constructing those elements requires access to an <see cref="ILogger"/>,
         ///     which will be replaced cache-side.

Public/Src/Cache/DistributedCache.Host/Service/DistributedCacheServiceFacade.cs

@@ -109,7 +109,8 @@ namespace BuildXL.Cache.Host.Service
             var context = new Context(arguments.Logger);
             var operationContext = new OperationContext(context, arguments.Cancellation);
 
-            InitializeActivityTrackerIfNeeded(context, arguments.Configuration.DistributedContentSettings);
+            var distributedSettings = arguments.Configuration.DistributedContentSettings;
+            InitializeActivityTrackerIfNeeded(context, distributedSettings);
 
             AdjustCopyInfrastructure(arguments);
 
@@ -120,7 +121,7 @@ namespace BuildXL.Cache.Host.Service
             // Technically, this method doesn't own the file copier, but no one actually owns it.
             // So to clean up the resources (and print some stats) we dispose it here.
             using (arguments.Copier as IDisposable)
-            using (var server = factory.Create())
+            using (var server = await factory.CreateAsync(operationContext))
             {
                 try
                 {
@@ -130,7 +131,7 @@ namespace BuildXL.Cache.Host.Service
                         throw new CacheException(startupResult.ToString());
                     }
 
-                    await ReportServiceStartedAsync(operationContext, server, host);
+                    await ReportServiceStartedAsync(operationContext, server, host, distributedSettings);
                     using var cancellationAwaiter = arguments.Cancellation.ToAwaitable();
                     await cancellationAwaiter.CompletionTask;
                     await ReportShuttingDownServiceAsync(operationContext, host);
@@ -142,7 +143,7 @@ namespace BuildXL.Cache.Host.Service
                 }
                 finally
                 {
-                    var timeoutInMinutes = arguments.Configuration?.DistributedContentSettings?.MaxShutdownDurationInMinutes ?? 5;
+                    var timeoutInMinutes = distributedSettings?.MaxShutdownDurationInMinutes ?? 5;
                     var result = await server
                         .ShutdownAsync(context)
                         .WithTimeoutAsync("Server shutdown", TimeSpan.FromMinutes(timeoutInMinutes));
@@ -187,12 +188,16 @@ namespace BuildXL.Cache.Host.Service
         private static async Task ReportServiceStartedAsync(
             OperationContext context,
             StartupShutdownSlimBase server,
-            IDistributedCacheServiceHost host)
+            IDistributedCacheServiceHost host,
+            DistributedContentSettings distributedContentSettings)
         {
             LifetimeTracker.ServiceStarted(context);
             host.OnStartedService();
 
-            if (host is IDistributedCacheServiceHostInternal hostInternal
+            if (
+                // Don't need to call the following callback for out-of-proc cache
+                distributedContentSettings.OutOfProcCacheSettings is null &&
+                host is IDistributedCacheServiceHostInternal hostInternal
                 && server is IServicesProvider sp
                 && sp.TryGetService<ICacheServerServices>(out var services))
             {

Public/Src/Cache/DistributedCache.Host/Service/HostInfo.cs

@@ -1,7 +1,6 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT License.
 
-using System;
 using System.Collections.Generic;
 using System.Linq;
 

Public/Src/Cache/DistributedCache.Host/Service/IDistributedCacheServiceHost.cs

@@ -1,12 +1,7 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT License.
 
-using System.Collections.Generic;
-using System.Threading;
 using System.Threading.Tasks;
-using BuildXL.Cache.ContentStore.Interfaces.FileSystem;
-using BuildXL.Cache.ContentStore.Interfaces.Secrets;
-using BuildXL.Cache.ContentStore.Stores;
 using BuildXL.Cache.ContentStore.Tracing.Internal;
 
 namespace BuildXL.Cache.Host.Service

Public/Src/Cache/DistributedCache.Host/Service/ISecretsProvider.cs

@@ -8,14 +8,19 @@ using BuildXL.Cache.ContentStore.Interfaces.Secrets;
 
 namespace BuildXL.Cache.Host.Service
 {
+    /// <summary>
+    /// Contains all the secrets returned by <see cref="ISecretsProvider.RetrieveSecretsAsync"/>
+    /// </summary>
+    public record RetrievedSecrets(IReadOnlyDictionary<string, Secret> Secrets);
+
     /// <summary>
     /// Used to provide secrets
     /// </summary>
     public interface ISecretsProvider
     {
         /// <summary>
-        /// Retrieves secrets from key vault
+        /// Retrieves secrets from key vault or some other provider
         /// </summary>
-        Task<Dictionary<string, Secret>> RetrieveSecretsAsync(List<RetrieveSecretsRequest> requests, CancellationToken token);
+        Task<RetrievedSecrets> RetrieveSecretsAsync(List<RetrieveSecretsRequest> requests, CancellationToken token);
     }
 }

Public/Src/Cache/DistributedCache.Host/Service/Internal/CacheServerFactory.cs

@@ -3,9 +3,10 @@
 
 using System;
 using System.Collections.Generic;
+using System.Diagnostics.ContractsLight;
 using System.Linq;
+using System.Threading.Tasks;
 using BuildXL.Cache.ContentStore.Distributed.NuCache;
-using BuildXL.Cache.ContentStore.Distributed.Stores;
 using BuildXL.Cache.ContentStore.FileSystem;
 using BuildXL.Cache.ContentStore.Interfaces.FileSystem;
 using BuildXL.Cache.ContentStore.Interfaces.Logging;
@@ -25,8 +26,11 @@ using BuildXL.Cache.MemoizationStore.Vsts;
 using BuildXL.Cache.MemoizationStore.Service;
 using BuildXL.Cache.MemoizationStore.Sessions;
 using BuildXL.Cache.MemoizationStore.Stores;
-using static BuildXL.Utilities.ConfigurationHelper;
 using BuildXL.Cache.ContentStore.Tracing;
+using BuildXL.Cache.ContentStore.Tracing.Internal;
+using BuildXL.Cache.Host.Service.OutOfProc;
+using BuildXL.Utilities.ConfigurationHelpers;
+using AbsolutePath = BuildXL.Cache.ContentStore.Interfaces.FileSystem.AbsolutePath;
 
 namespace BuildXL.Cache.Host.Service.Internal
 {
@@ -36,6 +40,7 @@ namespace BuildXL.Cache.Host.Service.Internal
     /// <remarks>Marked as public because it is used externally.</remarks>
     public class CacheServerFactory
     {
+        private static readonly Tracer _tracer = new Tracer(nameof(CacheServerFactory));
         private readonly IAbsFileSystem _fileSystem;
         private readonly ILogger _logger;
         private readonly DistributedCacheServiceArguments _arguments;
@@ -50,20 +55,52 @@ namespace BuildXL.Cache.Host.Service.Internal
             _fileSystem = new PassThroughFileSystem(_logger);
         }
 
-        public StartupShutdownBase Create()
+        /// <summary>
+        /// Creates a cache server.
+        /// </summary>
+        /// <remarks>
+        /// Currently it can be one of the following:
+        /// * Launcher that will download configured bits and start them.
+        /// * Out-of-proc launcher that will start the current bits in a separate process.
+        /// * In-proc distributed cache service.
+        /// * In-proc local cache service.
+        /// </remarks>
+        public async Task<StartupShutdownBase> CreateAsync(OperationContext operationContext)
         {
             var cacheConfig = _arguments.Configuration;
-            if (TryCreateLauncherIfSpecified(cacheConfig, out var launcher))
+
+            if (IsLauncherEnabled(cacheConfig))
             {
-                return launcher;
+                _tracer.Debug(operationContext, $"Creating a launcher.");
+                return await CreateLauncherAsync(cacheConfig);
             }
 
+            var distributedSettings = cacheConfig.DistributedContentSettings;
+
+            if (IsOutOfProcCacheEnabled(cacheConfig))
+            {
+                _tracer.Debug(operationContext, $"Creating an out-of-proc cache service.");
+                var outOfProcCache = await CacheServiceWrapper.CreateAsync(_arguments);
+
+                if (outOfProcCache.Succeeded)
+                {
+                    return outOfProcCache.Value;
+                }
+
+                // Tracing and falling back to the in-proc cache
+                _tracer.Error(operationContext, $"Failed to create out of proc cache: {outOfProcCache}. Using in-proc cache instead.");
+            }
+
+            _tracer.Debug(operationContext, "Creating an in-proc cache service.");
             cacheConfig.LocalCasSettings = cacheConfig.LocalCasSettings.FilterUnsupportedNamedCaches(_arguments.HostInfo.Capabilities, _logger);
             
-            var distributedSettings = cacheConfig.DistributedContentSettings;
             var isLocal = distributedSettings == null || !distributedSettings.IsDistributedContentEnabled;
 
+            if (distributedSettings is not null)
+            {
                 LogManager.Update(distributedSettings.LogManager);
+            }
+
             var serviceConfiguration = CreateServiceConfiguration(
                 _logger,
                 _fileSystem,
@@ -93,29 +130,22 @@ namespace BuildXL.Cache.Host.Service.Internal
             }
         }
 
-        private bool TryCreateLauncherIfSpecified(DistributedCacheServiceConfiguration cacheConfig, out DeploymentLauncher launcher)
+        private bool IsLauncherEnabled(DistributedCacheServiceConfiguration cacheConfig) =>
+            cacheConfig.DistributedContentSettings.LauncherSettings != null;
+
+        private bool IsOutOfProcCacheEnabled(DistributedCacheServiceConfiguration cacheConfig) =>
+            cacheConfig.DistributedContentSettings.RunCacheOutOfProc == true;
+
+        private async Task<DeploymentLauncher> CreateLauncherAsync(DistributedCacheServiceConfiguration cacheConfig)
         {
             var launcherSettings = cacheConfig.DistributedContentSettings.LauncherSettings;
-            if (launcherSettings != null)
+            Contract.Assert(launcherSettings is not null);
-            {
+
             var deploymentParams = launcherSettings.DeploymentParameters;
-                deploymentParams.Stamp ??= _arguments.TelemetryFieldsProvider?.Stamp;
+            deploymentParams.ApplyFromTelemetryProviderIfNeeded(_arguments.TelemetryFieldsProvider);
-                deploymentParams.Machine ??= Environment.MachineName;
+            deploymentParams.AuthorizationSecret ??= await _arguments.Host.GetPlainSecretAsync(deploymentParams.AuthorizationSecretName, _arguments.Cancellation);
-                deploymentParams.MachineFunction ??= _arguments.TelemetryFieldsProvider?.APMachineFunction;
-                deploymentParams.Ring ??= _arguments.TelemetryFieldsProvider?.Ring;
 
-                deploymentParams.AuthorizationSecret ??= _arguments.Host.GetPlainSecretAsync(deploymentParams.AuthorizationSecretName, _arguments.Cancellation).GetAwaiter().GetResult();
+            return new DeploymentLauncher(launcherSettings, _fileSystem);
-
-                launcher = new DeploymentLauncher(
-                    launcherSettings,
-                    _fileSystem);
-                return true;
-            }
-            else
-            {
-                launcher = null;
-                return false;
-            }
         }
 
         private StartupShutdownBase CreateLocalServer(LocalServerConfiguration localServerConfiguration, DistributedContentSettings distributedSettings = null)
@@ -302,18 +332,19 @@ namespace BuildXL.Cache.Host.Service.Internal
 
             var localContentServerConfiguration = new LocalServerConfiguration(serviceConfiguration);
             
-            ApplyIfNotNull(localCasServiceSettings.UnusedSessionTimeoutMinutes, value => localContentServerConfiguration.UnusedSessionTimeout = TimeSpan.FromMinutes(value));
+            localCasServiceSettings.UnusedSessionTimeoutMinutes.ApplyIfNotNull(value => localContentServerConfiguration.UnusedSessionTimeout = TimeSpan.FromMinutes(value));
-            ApplyIfNotNull(localCasServiceSettings.UnusedSessionHeartbeatTimeoutMinutes, value => localContentServerConfiguration.UnusedSessionHeartbeatTimeout = TimeSpan.FromMinutes(value));
+            localCasServiceSettings.UnusedSessionHeartbeatTimeoutMinutes.ApplyIfNotNull(value => localContentServerConfiguration.UnusedSessionHeartbeatTimeout = TimeSpan.FromMinutes(value));
-            ApplyIfNotNull(localCasServiceSettings.GrpcCoreServerOptions, value => localContentServerConfiguration.GrpcCoreServerOptions = value);
+            localCasServiceSettings.GrpcCoreServerOptions.ApplyIfNotNull(value => localContentServerConfiguration.GrpcCoreServerOptions = value);
-            ApplyIfNotNull(localCasServiceSettings.GrpcEnvironmentOptions, value => localContentServerConfiguration.GrpcEnvironmentOptions = value);
+            localCasServiceSettings.GrpcEnvironmentOptions.ApplyIfNotNull(value => localContentServerConfiguration.GrpcEnvironmentOptions = value);
-            ApplyIfNotNull(localCasServiceSettings.DoNotShutdownSessionsInUse, value => localContentServerConfiguration.DoNotShutdownSessionsInUse = value);
+            localCasServiceSettings.DoNotShutdownSessionsInUse.ApplyIfNotNull(value => localContentServerConfiguration.DoNotShutdownSessionsInUse = value);
 
-            ApplyIfNotNull(distributedSettings?.UseUnsafeByteStringConstruction, value =>
+            (distributedSettings?.UseUnsafeByteStringConstruction).ApplyIfNotNull(
+                value =>
             {
                 GrpcExtensions.UnsafeByteStringOptimizations = value;
             });
 
-            ApplyIfNotNull(distributedSettings?.ShutdownEvictionBeforeHibernation, value => localContentServerConfiguration.ShutdownEvictionBeforeHibernation = value);
+            (distributedSettings?.ShutdownEvictionBeforeHibernation).ApplyIfNotNull(value => localContentServerConfiguration.ShutdownEvictionBeforeHibernation = value);
 
             return localContentServerConfiguration;
         }
@@ -364,7 +395,7 @@ namespace BuildXL.Cache.Host.Service.Internal
                 logIncrementalStatsCounterNames: distributedSettings?.IncrementalStatisticsCounterNames,
                 asyncSessionShutdownTimeout: distributedSettings?.AsyncSessionShutdownTimeout);
 
-            ApplyIfNotNull(distributedSettings?.TraceServiceGrpcOperations, v => result.TraceGrpcOperation = v);
+            distributedSettings?.TraceServiceGrpcOperations.ApplyIfNotNull(v => result.TraceGrpcOperation = v);
             return result;
         }
 

Public/Src/Cache/DistributedCache.Host/Service/Internal/ContentStoreFactory.cs

@@ -5,7 +5,9 @@ using BuildXL.Cache.ContentStore.Interfaces.FileSystem;
 using BuildXL.Cache.ContentStore.Interfaces.Stores;
 using BuildXL.Cache.ContentStore.Interfaces.Time;
 using BuildXL.Cache.ContentStore.Stores;
+
 #nullable enable
+
 namespace BuildXL.Cache.Host.Service.Internal
 {
     public class ContentStoreFactory

Public/Src/Cache/DistributedCache.Host/Service/Internal/DistributedCacheSecretRetriever.cs

@@ -8,12 +8,14 @@ using System.Security;
 using System.Text;
 using System.Threading;
 using System.Threading.Tasks;
-using BuildXL.Cache.ContentStore.Distributed;
 using BuildXL.Cache.ContentStore.Interfaces.Logging;
+using BuildXL.Cache.ContentStore.Interfaces.Results;
 using BuildXL.Cache.ContentStore.Interfaces.Secrets;
 using BuildXL.Cache.ContentStore.Utils;
 using BuildXL.Cache.Host.Configuration;
 
+#nullable enable
+
 namespace BuildXL.Cache.Host.Service.Internal
 {
     /// <summary>
@@ -22,35 +24,43 @@ namespace BuildXL.Cache.Host.Service.Internal
     public class DistributedCacheSecretRetriever
     {
         private readonly DistributedContentSettings _distributedSettings;
+        private readonly AzureBlobStorageLogPublicConfiguration? _loggingConfiguration;
+
         private readonly ILogger _logger;
         private readonly IDistributedCacheServiceHost _host;
 
-        private readonly Lazy<Task<(Dictionary<string, Secret>, string)>> _secrets;
+        private readonly Lazy<Task<Result<RetrievedSecrets>>> _secrets;
 
         /// <nodoc />
         public DistributedCacheSecretRetriever(DistributedCacheServiceArguments arguments)
         {
             _distributedSettings = arguments.Configuration.DistributedContentSettings;
+            _loggingConfiguration = arguments.LoggingSettings?.Configuration;
             _logger = arguments.Logger;
             _host = arguments.Host;
 
-            _secrets = new Lazy<Task<(Dictionary<string, Secret>, string)>>(TryGetSecretsAsync);
+            _secrets = new Lazy<Task<Result<RetrievedSecrets>>>(TryGetSecretsAsync);
         }
 
         /// <summary>
         /// Retrieves the secrets. Results will be cached so secrets are only computed the first time this is called.
         /// </summary>
-        public Task<(Dictionary<string, Secret>, string)> TryRetrieveSecretsAsync() => _secrets.Value;
+        public Task<Result<RetrievedSecrets>> TryRetrieveSecretsAsync() => _secrets.Value;
 
-        private async Task<(Dictionary<string, Secret>, string errors)> TryGetSecretsAsync()
+        private async Task<Result<RetrievedSecrets>> TryGetSecretsAsync()
         {
             var errorBuilder = new StringBuilder();
 
             var result = await impl();
 
-            return (result, errorBuilder.ToString());
+            if (result is null)
+            {
+                return Result.FromErrorMessage<RetrievedSecrets>(errorBuilder.ToString());
+            }
 
-            async Task<Dictionary<string, Secret>> impl()
+            return Result.Success(result);
+
+            async Task<RetrievedSecrets?> impl()
             {
                 _logger.Debug(
                     $"{nameof(_distributedSettings.EventHubSecretName)}: {_distributedSettings.EventHubSecretName}, " +
@@ -78,8 +88,9 @@ namespace BuildXL.Cache.Host.Service.Internal
                     return null;
                 }
 
-                var azureBlobStorageCredentialsKind = _distributedSettings.AzureBlobStorageUseSasTokens ? SecretKind.SasToken : SecretKind.PlainText;
+                retrieveSecretsRequests.AddRange(
-                retrieveSecretsRequests.AddRange(storageSecretNames.Select(secretName => new RetrieveSecretsRequest(secretName, azureBlobStorageCredentialsKind)));
+                    storageSecretNames
+                        .Select(tpl => new RetrieveSecretsRequest(tpl.secretName, tpl.useSasTokens ? SecretKind.SasToken : SecretKind.PlainText)));
 
                 if (string.IsNullOrEmpty(_distributedSettings.GlobalRedisSecretName))
                 {
@@ -103,6 +114,8 @@ namespace BuildXL.Cache.Host.Service.Internal
 
                 addOptionalSecret(_distributedSettings.SecondaryGlobalRedisSecretName);
                 addOptionalSecret(_distributedSettings.ContentMetadataRedisSecretName);
+
+                var azureBlobStorageCredentialsKind = _distributedSettings.AzureBlobStorageUseSasTokens ? SecretKind.SasToken : SecretKind.PlainText;
                 addOptionalSecret(_distributedSettings.ContentMetadataBlobSecretName, azureBlobStorageCredentialsKind);
 
                 // Ask the host for credentials
@@ -118,7 +131,7 @@ namespace BuildXL.Cache.Host.Service.Internal
                 // Validate requests match as expected
                 foreach (var request in retrieveSecretsRequests)
                 {
-                    if (secrets.TryGetValue(request.Name, out var secret))
+                    if (secrets.Secrets.TryGetValue(request.Name, out var secret))
                     {
                         bool typeMatch = true;
                         switch (request.Kind)
@@ -143,7 +156,7 @@ namespace BuildXL.Cache.Host.Service.Internal
                 return secrets;
             }
 
-            bool appendIfNull(object value, string propertyName)
+            bool appendIfNull(object? value, string propertyName)
             {
                 if (value is null)
                 {
@@ -165,17 +178,23 @@ namespace BuildXL.Cache.Host.Service.Internal
                 deltaBackoff: TimeSpan.FromSeconds(settings.SecretsRetrievalDeltaBackoffSeconds));
         }
 
-        private List<string> GetAzureStorageSecretNames(StringBuilder errorBuilder)
+        private List<(string secretName, bool useSasTokens)>? GetAzureStorageSecretNames(StringBuilder errorBuilder)
         {
-            var secretNames = new List<string>();
+            bool useSasToken = _distributedSettings.AzureBlobStorageUseSasTokens;
+            var secretNames = new List<(string secretName, bool useSasTokens)>();
             if (_distributedSettings.AzureStorageSecretName != null && !string.IsNullOrEmpty(_distributedSettings.AzureStorageSecretName))
             {
-                secretNames.Add(_distributedSettings.AzureStorageSecretName);
+                secretNames.Add((_distributedSettings.AzureStorageSecretName, useSasToken));
             }
 
             if (_distributedSettings.AzureStorageSecretNames != null && !_distributedSettings.AzureStorageSecretNames.Any(string.IsNullOrEmpty))
             {
-                secretNames.AddRange(_distributedSettings.AzureStorageSecretNames);
+                secretNames.AddRange(_distributedSettings.AzureStorageSecretNames.Select(n => (n, useSasToken)));
+            }
+
+            if (_loggingConfiguration?.SecretName != null)
+            {
+                secretNames.Add((_loggingConfiguration.SecretName, _loggingConfiguration.UseSasTokens));
             }
 
             if (secretNames.Count > 0)

Public/Src/Cache/DistributedCache.Host/Service/Internal/DistributedContentStoreFactory.cs

@@ -15,7 +15,6 @@ using BuildXL.Cache.ContentStore.Distributed.Redis;
 using BuildXL.Cache.ContentStore.Distributed.Sessions;
 using BuildXL.Cache.ContentStore.Distributed.Stores;
 using BuildXL.Cache.ContentStore.Interfaces.Distributed;
-using BuildXL.Cache.ContentStore.Interfaces.Extensions;
 using BuildXL.Cache.ContentStore.Interfaces.FileSystem;
 using BuildXL.Cache.ContentStore.Interfaces.Logging;
 using BuildXL.Cache.ContentStore.Interfaces.Results;
@@ -29,8 +28,6 @@ using BuildXL.Cache.ContentStore.UtilitiesCore;
 using BuildXL.Cache.Host.Configuration;
 using BuildXL.Cache.MemoizationStore.Distributed.Stores;
 using BuildXL.Cache.MemoizationStore.Interfaces.Stores;
-using BandwidthConfiguration = BuildXL.Cache.ContentStore.Distributed.BandwidthConfiguration;
-using static BuildXL.Utilities.ConfigurationHelper;
 using BuildXL.Cache.ContentStore.Utils;
 using BuildXL.Cache.ContentStore.Distributed.NuCache.CopyScheduling;
 using ContentStore.Grpc;
@@ -41,6 +38,10 @@ using BuildXL.Cache.ContentStore.FileSystem;
 using BuildXL.Cache.ContentStore.Tracing;
 using BuildXL.Cache.ContentStore.Distributed.Services;
 
+using AbsolutePath = BuildXL.Cache.ContentStore.Interfaces.FileSystem.AbsolutePath;
+using BandwidthConfiguration = BuildXL.Cache.ContentStore.Distributed.BandwidthConfiguration;
+using static BuildXL.Utilities.ConfigurationHelper;
+
 namespace BuildXL.Cache.Host.Service.Internal
 {
     public sealed class DistributedContentStoreFactory : IDistributedServicesSecrets
@@ -67,7 +68,7 @@ namespace BuildXL.Cache.Host.Service.Internal
 
         public IReadOnlyList<ResolvedNamedCacheSettings> OrderedResolvedCacheSettings => _orderedResolvedCacheSettings;
         private readonly List<ResolvedNamedCacheSettings> _orderedResolvedCacheSettings;
-        private readonly Dictionary<string, Secret> _secrets;
+        private readonly RetrievedSecrets _secrets;
 
         public RedisContentLocationStoreConfiguration RedisContentLocationStoreConfiguration { get; }
 
@@ -82,14 +83,14 @@ namespace BuildXL.Cache.Host.Service.Internal
             _fileSystem = arguments.FileSystem;
             _secretRetriever = new DistributedCacheSecretRetriever(arguments);
 
-            (var secrets, var errors) = _secretRetriever.TryRetrieveSecretsAsync().GetAwaiter().GetResult();
+            var secretsResult = _secretRetriever.TryRetrieveSecretsAsync().GetAwaiter().GetResult();
-            if (secrets == null)
+            if (!secretsResult.Succeeded)
             {
-                _logger.Error($"Unable to retrieve secrets. {errors}");
+                _logger.Error($"Unable to retrieve secrets. {secretsResult}");
-                secrets = new Dictionary<string, Secret>();
+                _secrets = new RetrievedSecrets(new Dictionary<string, Secret>());
             }
 
-            _secrets = secrets;
+            _secrets = secretsResult.Value;
 
             _orderedResolvedCacheSettings = ResolveCacheSettingsInPrecedenceOrder(arguments);
             Contract.Assert(_orderedResolvedCacheSettings.Count != 0);
@@ -294,12 +295,51 @@ namespace BuildXL.Cache.Host.Service.Internal
                 yield break;
             }
 
+            var primaryCacheRoot = OrderedResolvedCacheSettings[0].ResolvedCacheRootPath;
+
+            var configuration = new GlobalCacheServiceConfiguration()
+            {
+                MaxEventParallelism = RedisContentLocationStoreConfiguration.EventStore.MaxEventProcessingConcurrency,
+                MasterLeaseStaleThreshold = RedisContentLocationStoreConfiguration.Checkpoint.MasterLeaseExpiryTime.Multiply(0.5),
+                VolatileEventStorage = new RedisVolatileEventStorageConfiguration()
+                {
+                    ConnectionString = (GetRequiredSecret(_distributedSettings.ContentMetadataRedisSecretName) as PlainTextSecret).Secret,
+                    KeyPrefix = _distributedSettings.RedisWriteAheadKeyPrefix,
+                    MaximumKeyLifetime = _distributedSettings.ContentMetadataRedisMaximumKeyLifetime,
+                },
+                PersistentEventStorage = new BlobEventStorageConfiguration()
+                {
+                    Credentials = GetStorageCredentials(new[] { _distributedSettings.ContentMetadataBlobSecretName }).First(),
+                    FolderName = "events" + _distributedSettings.KeySpacePrefix,
+                    ContainerName = _distributedSettings.ContentMetadataLogBlobContainerName,
+                },
+                CentralStorage = RedisContentLocationStoreConfiguration.CentralStore with
+                {
+                    ContainerName = _distributedSettings.ContentMetadataCentralStorageContainerName
+                },
+                EventStream = new ContentMetadataEventStreamConfiguration()
+                {
+                    BatchWriteAheadWrites = _distributedSettings.ContentMetadataBatchVolatileWrites,
+                    ShutdownTimeout = _distributedSettings.ContentMetadataShutdownTimeout,
+                    LogBlockRefreshInterval = _distributedSettings.ContentMetadataPersistInterval
+                },
+                Checkpoint = RedisContentLocationStoreConfiguration.Checkpoint with
+                {
+                    WorkingDirectory = primaryCacheRoot / "cmschkpt"
+                },
+                ClusterManagement = new ClusterManagementConfiguration()
+                {
+                    MachineExpiryInterval = RedisContentLocationStoreConfiguration.MachineActiveToExpiredInterval,
+                }
+            };
+
+            CentralStreamStorage centralStreamStorage = configuration.CentralStorage.CreateCentralStorage();
+
             if (_distributedSettings.IsMasterEligible)
             {
                 var service = Services.GlobalCacheService.Instance;
                 yield return new ProtobufNetGrpcServiceEndpoint<IGlobalCacheService, GlobalCacheService>(nameof(GlobalCacheService), service);
             }
-
         }
 
         public IGrpcServiceEndpoint[] GetAdditionalEndpoints()
@@ -567,7 +607,7 @@ namespace BuildXL.Cache.Host.Service.Internal
             AbsolutePath localCacheRoot,
             RocksDbContentLocationDatabaseConfiguration dbConfig)
         {
-            if (_secrets.Count == 0)
+            if (_secrets.Secrets.Count == 0)
             {
                 return;
             }
@@ -823,7 +863,7 @@ namespace BuildXL.Cache.Host.Service.Internal
 
         public Secret GetRequiredSecret(string secretName)
         {
-            if (!_secrets.TryGetValue(secretName, out var value))
+            if (!_secrets.Secrets.TryGetValue(secretName, out var value))
             {
                 throw new KeyNotFoundException($"Missing secret: {secretName}");
             }

Public/Src/Cache/DistributedCache.Host/Service/Internal/MultiLevelContentStore.cs

@@ -2,20 +2,13 @@
 // Licensed under the MIT License.
 
 using System;
-using System.Collections.Generic;
 using System.Diagnostics.ContractsLight;
-using System.Linq;
-using System.Text;
-using System.Threading;
 using System.Threading.Tasks;
-using BuildXL.Cache.ContentStore.Extensions;
 using BuildXL.Cache.ContentStore.Hashing;
-using BuildXL.Cache.ContentStore.Interfaces.FileSystem;
 using BuildXL.Cache.ContentStore.Interfaces.Results;
 using BuildXL.Cache.ContentStore.Interfaces.Sessions;
 using BuildXL.Cache.ContentStore.Interfaces.Stores;
 using BuildXL.Cache.ContentStore.Interfaces.Tracing;
-using BuildXL.Cache.ContentStore.Stores;
 using BuildXL.Cache.ContentStore.Tracing;
 using BuildXL.Cache.ContentStore.Tracing.Internal;
 using BuildXL.Cache.ContentStore.UtilitiesCore;

Public/Src/Cache/DistributedCache.Host/Service/Internal/MultiLevelReadOnlyContentSession.cs

@@ -39,7 +39,7 @@ namespace BuildXL.Cache.Host.Service.Internal
         protected override Tracer Tracer { get; } = new Tracer(nameof(MultiLevelContentStore));
 
         /// <summary>
-        /// Initializes a new instance of the <see cref="MultiLevelReadOnlyContentSession"/> class.
+        /// Initializes a new instance of the <see cref="MultiLevelReadOnlyContentSession{TSession}"/> class.
         /// </summary>
         public MultiLevelReadOnlyContentSession(
             string name,

Public/Src/Cache/DistributedCache.Host/Service/Internal/MultiplexedContentSession.cs

@@ -6,15 +6,12 @@ using System.Collections.Generic;
 using System.IO;
 using System.Threading;
 using System.Threading.Tasks;
-using BuildXL.Cache.ContentStore.Distributed.Sessions;
 using BuildXL.Cache.ContentStore.Hashing;
 using BuildXL.Cache.ContentStore.Interfaces.FileSystem;
 using BuildXL.Cache.ContentStore.Interfaces.Results;
 using BuildXL.Cache.ContentStore.Interfaces.Sessions;
 using BuildXL.Cache.ContentStore.Interfaces.Tracing;
 using BuildXL.Cache.ContentStore.Sessions.Internal;
-using BuildXL.Cache.ContentStore.Tracing;
-using BuildXL.Cache.ContentStore.Tracing.Internal;
 
 namespace BuildXL.Cache.Host.Service.Internal
 {

Public/Src/Cache/DistributedCache.Host/Service/Internal/MultiplexedContentStore.cs

@@ -34,7 +34,7 @@ namespace BuildXL.Cache.Host.Service.Internal
         /// <summary>
         /// Indicates whether to use all sessions rather than only the primary for session read operations
         /// </summary>
-        public bool TryAllSesssions { get; }
+        public bool TryAllSessions { get; }
 
         private ContentStoreTracer StoreTracer { get; } = new ContentStoreTracer(nameof(MultiplexedContentStore));
 
@@ -51,7 +51,7 @@ namespace BuildXL.Cache.Host.Service.Internal
             DrivesWithContentStore = drivesWithContentStore;
             PreferredCacheDrive = preferredCacheDrive;
             PreferredContentStore = drivesWithContentStore[preferredCacheDrive];
-            TryAllSesssions = tryAllSessions;
+            TryAllSessions = tryAllSessions;
         }
 
         /// <inheritdoc />

Public/Src/Cache/DistributedCache.Host/Service/Internal/MultiplexedReadOnlyContentSession.cs

@@ -303,7 +303,7 @@ namespace BuildXL.Cache.Host.Service.Internal
                 yield return session;
             }
 
-            if (!Store.TryAllSesssions)
+            if (!Store.TryAllSessions)
             {
                 yield break;
             }

Public/Src/Cache/DistributedCache.Host/Service/Internal/ResolvedNamedCacheSettings.cs

@@ -1,20 +1,14 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT License.
 
-using System.IO;
 using BuildXL.Cache.ContentStore.Distributed;
-using BuildXL.Cache.ContentStore.Hashing;
 using BuildXL.Cache.ContentStore.Interfaces.FileSystem;
-using BuildXL.Cache.ContentStore.Interfaces.Stores;
-using BuildXL.Cache.ContentStore.Interfaces.Time;
-using BuildXL.Cache.ContentStore.Stores;
-using BuildXL.Cache.ContentStore.Utils;
 using BuildXL.Cache.Host.Configuration;
 
 namespace BuildXL.Cache.Host.Service.Internal
 {
     /// <summary>
-    /// Final settings object used for intializing a distributed cache instance
+    /// Final settings object used for initializing a distributed cache instance
     /// </summary>
     public class ResolvedNamedCacheSettings
     {

Public/Src/Cache/DistributedCache.Host/Service/Internal/RetrievedSecretsSerializer.cs

@@ -0,0 +1,81 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+using System;
+using System.Collections.Generic;
+using System.Diagnostics.ContractsLight;
+using System.Linq;
+using System.Text.Json;
+using BuildXL.Cache.ContentStore.Interfaces.Results;
+using BuildXL.Cache.ContentStore.Interfaces.Secrets;
+
+namespace BuildXL.Cache.Host.Service.Internal
+{
+    /// <summary>
+    /// A helper type used for serializing <see cref="RetrievedSecrets"/> to and from a string.
+    /// </summary>
+    internal static class RetrievedSecretsSerializer
+    {
+        public const string SerializedSecretsKeyName = "SerializedSecretsKey";
+
+        public static string Serialize(RetrievedSecrets secrets)
+        {
+            var secretsList = secrets.Secrets
+                .Select(kvp => SecretData.FromSecret(kvp.Key, kvp.Value))
+                .OrderBy(s => s.Name)
+                .ToList();
+
+            return JsonSerializer.Serialize(secretsList);
+        }
+
+        public static Result<RetrievedSecrets> Deserialize(string content)
+        {
+            Contract.Requires(!string.IsNullOrEmpty(content));
+
+            List<SecretData> secrets = JsonSerializer.Deserialize<List<SecretData>>(content);
+            return Result.Success(
+                new RetrievedSecrets(
+                    secrets.ToDictionary(s => s.Name, s => s.ToSecret())));
+
+        }
+
+        internal class SecretData
+        {
+            public string Name { get; set; }
+            public SecretKind Kind { get; set; }
+
+            public string SecretOrToken { get; set; }
+
+            public string StorageAccount { get; set; }
+
+            public string ResourcePath { get; set; }
+
+            public Secret ToSecret()
+                => Kind switch
+                {
+                    SecretKind.PlainText => new PlainTextSecret(SecretOrToken),
+                    SecretKind.SasToken => new UpdatingSasToken(new SasToken(SecretOrToken, StorageAccount, ResourcePath)),
+                    _ => throw new ArgumentOutOfRangeException(nameof(Kind))
+                };
+
+            public static SecretData FromSecret(string name, Secret secret)
+                => secret switch
+                {
+                    PlainTextSecret plainTextSecret
+                        => new SecretData { Name = name, Kind = SecretKind.PlainText, SecretOrToken = plainTextSecret.Secret },
+
+                    UpdatingSasToken updatingSasToken
+                        => new SecretData
+                           {
+                               Name = name,
+                               Kind = SecretKind.SasToken,
+                               SecretOrToken = updatingSasToken.Token.Token,
+                               ResourcePath = updatingSasToken.Token.ResourcePath,
+                               StorageAccount = updatingSasToken.Token.StorageAccount
+                           },
+
+                    _ => throw new ArgumentOutOfRangeException(nameof(secret))
+                };
+        }
+    }
+}

Public/Src/Cache/DistributedCache.Host/Service/LoggerFactory.cs

@@ -7,7 +7,6 @@ using System.Diagnostics.ContractsLight;
 using System.IO;
 using System.Linq;
 using System.Reflection;
-using System.Security.AccessControl;
 using System.Threading.Tasks;
 using System.Xml;
 using BuildXL.Cache.ContentStore.FileSystem;

Public/Src/Cache/DistributedCache.Host/Service/LoggerFactoryArguments.cs

@@ -1,7 +1,6 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT License.
 
-using System.Diagnostics.ContractsLight;
 using BuildXL.Cache.ContentStore.Interfaces.Logging;
 using BuildXL.Cache.Host.Configuration;
 

Public/Src/Cache/DistributedCache.Host/Service/RetrieveSecretsRequest.cs

@@ -9,7 +9,7 @@ namespace BuildXL.Cache.Host.Service
     /// <summary>
     /// Type to signal the host which kind of secret is expected to be returned
     /// </summary>
-    public struct RetrieveSecretsRequest
+    public readonly struct RetrieveSecretsRequest
     {
         public string Name { get; }
 

Public/Src/Cache/DistributedCache.Host/Service/SecretsProviderExtensions.cs

@@ -23,7 +23,7 @@ namespace BuildXL.Cache.Host.Service
                     new RetrieveSecretsRequest(secretName, SecretKind.PlainText)
                 }, token);
 
-            return ((PlainTextSecret)secrets[secretName]).Secret;
+            return ((PlainTextSecret)secrets.Secrets[secretName]).Secret;
         }
 
         /// <summary>
@@ -42,7 +42,7 @@ namespace BuildXL.Cache.Host.Service
                     new RetrieveSecretsRequest(secretName, SecretKind.SasToken)
                 }, token);
 
-                return new AzureBlobStorageCredentials((UpdatingSasToken)secrets[secretName]);
+                return new AzureBlobStorageCredentials((UpdatingSasToken)secrets.Secrets[secretName]);
             }
             else
             {
@@ -51,7 +51,7 @@ namespace BuildXL.Cache.Host.Service
                     new RetrieveSecretsRequest(secretName, SecretKind.PlainText)
                 }, token);
 
-                return new AzureBlobStorageCredentials((PlainTextSecret)secrets[secretName]);
+                return new AzureBlobStorageCredentials((PlainTextSecret)secrets.Secrets[secretName]);
             }
         }
     }

Public/Src/Cache/DistributedCache.Host/Test/LauncherIntegrationTests.cs

@@ -16,12 +16,14 @@ using System.Text;
 using FluentAssertions;
 using System.Collections.Generic;
 using System.Diagnostics;
+using BuildXL.Cache.ContentStore.Interfaces.Results;
+using BuildXL.Cache.Host.Configuration;
 using BuildXL.Cache.Host.Service;
 using BuildXL.Cache.ContentStore.Tracing.Internal;
 using BuildXL.Cache.ContentStore.Service;
 using BuildXL.Utilities.CLI;
 using BuildXL.Utilities;
-using BuildXL.Cache.Host.Configuration;
+using BuildXL.Utilities.Tasks;
 
 namespace BuildXL.Cache.Host.Test
 {

Public/Src/Cache/DistributedCache.Host/Test/RetrievedSecretsSerializerTests.cs

@@ -0,0 +1,45 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+using System.Collections.Generic;
+using BuildXL.Cache.ContentStore.Interfaces.Secrets;
+using BuildXL.Cache.ContentStore.InterfacesTest.Results;
+using BuildXL.Cache.Host.Service;
+using BuildXL.Cache.Host.Service.Internal;
+using Xunit;
+
+namespace BuildXL.Cache.Host.Test;
+
+public class RetrievedSecretsSerializerTests
+{
+    [Fact]
+    public void TestSerialization()
+    {
+        var secretsMap = new Dictionary<string, Secret>
+                               {
+                                   ["cbcache-test-redis-dm_s1"] =
+                                       new PlainTextSecret("Fake secret that is quite long to emulate the size of the serialized entry."),
+                                   ["cbcache-test-redis-secondary-dm_s1"] =
+                                       new PlainTextSecret("Fake secret that is quite long to emulate the size of the serialized entry."),
+                                   ["cbcache-test-event-hub-dm_s1"] =
+                                       new PlainTextSecret("Fake secret that is quite long to emulate the size of the serialized entry."),
+                                   ["cbcacheteststorage-dm_s1-sas"] =
+                                       new UpdatingSasToken(new SasToken("token_name", "storage_account", "resource_path")),
+                                   ["ContentMetadataBlobSecretName-dm_s1"] = new PlainTextSecret(
+                                       "Fake secret that is quite long to emulate the size of the serialized entry.")
+                               };
+        var secrets = new RetrievedSecrets(secretsMap);
+
+        var text = RetrievedSecretsSerializer.Serialize(secrets);
+
+        var deserializedSecretsMap = RetrievedSecretsSerializer.Deserialize(text).ShouldBeSuccess().Value.Secrets;
+
+        Assert.Equal(secretsMap.Count, deserializedSecretsMap.Count);
+
+        foreach (var kvp in secretsMap)
+        {
+            Assert.Equal(kvp.Value, deserializedSecretsMap[kvp.Key]);
+            Assert.Equal(kvp.Value, deserializedSecretsMap[kvp.Key]);
+        }
+    }
+}

Public/Src/Utilities/Utilities/ConfigurationHelperExtensions.cs

@@ -0,0 +1,29 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+using System;
+
+namespace BuildXL.Utilities.ConfigurationHelpers
+{
+    /// <summary>
+    /// A special class that defines some methods from <see cref="ConfigurationHelper"/> as extension methods.
+    /// </summary>
+    /// <remarks>
+    /// <code>using static</code> does not allow using methods defined as extension methods.
+    /// It means that the code should decide if to use such helpers as extension methods or via 'using static',
+    /// defining a separate class solves this ambiguity.
+    /// This type is moved intentionally into a sub-namespace of 'BuildXL.Utilities' to name conflicts that may occur
+    /// if the client code have 'using BuildXL.Utilities'.
+    /// </remarks>
+    public static class ConfigurationHelperExtensions
+    {
+        /// <nodoc />
+        public static void ApplyIfNotNull<T>(this T value, Action<T> apply)
+            where T : class => ConfigurationHelper.ApplyIfNotNull(value, apply);
+
+        /// <nodoc />
+        public static void ApplyIfNotNull<T>(this T? value, Action<T> apply)
+            where T : struct
+            => ConfigurationHelper.ApplyIfNotNull(value, apply);
+    }
+}