diff --git a/Public/Src/Cache/ContentStore/Distributed/Blob/AzureBlobStorageContentStore.cs b/Public/Src/Cache/ContentStore/Distributed/Blob/AzureBlobStorageContentStore.cs index 37dbacfbe..2d52fd2cd 100644 --- a/Public/Src/Cache/ContentStore/Distributed/Blob/AzureBlobStorageContentStore.cs +++ b/Public/Src/Cache/ContentStore/Distributed/Blob/AzureBlobStorageContentStore.cs @@ -107,20 +107,6 @@ namespace BuildXL.Cache.ContentStore.Distributed.Blobs timeout: _configuration.StorageInteractionTimeout); } - /// - public CreateSessionResult CreateReadOnlySession(Context context, string name, ImplicitPin implicitPin) - { - using var guard = TrackShutdown(context, default); - var operationContext = guard.Context; - - return operationContext.PerformOperation(Tracer, () => - { - return new CreateSessionResult(CreateSessionCore(name, implicitPin)); - }, - traceOperationStarted: false, - messageFactory: _ => $"Name=[{name}] ImplicitPin=[{implicitPin}]"); - } - /// public CreateSessionResult CreateSession(Context context, string name, ImplicitPin implicitPin) { diff --git a/Public/Src/Cache/ContentStore/Distributed/Sessions/DistributedContentSession.cs b/Public/Src/Cache/ContentStore/Distributed/Sessions/DistributedContentSession.cs index 00ef4d39c..64a04a87d 100644 --- a/Public/Src/Cache/ContentStore/Distributed/Sessions/DistributedContentSession.cs +++ b/Public/Src/Cache/ContentStore/Distributed/Sessions/DistributedContentSession.cs @@ -2,9 +2,16 @@ // Licensed under the MIT License. using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.Diagnostics.ContractsLight; using System.IO; +using System.Linq; +using System.Threading; using System.Threading.Tasks; using BuildXL.Cache.ContentStore.Distributed.Stores; +using BuildXL.Cache.ContentStore.Distributed.Utilities; +using BuildXL.Cache.ContentStore.Extensions; using BuildXL.Cache.ContentStore.Hashing; using BuildXL.Cache.ContentStore.Interfaces.Distributed; using BuildXL.Cache.ContentStore.Interfaces.Extensions; @@ -12,41 +19,1743 @@ using BuildXL.Cache.ContentStore.Interfaces.FileSystem; using BuildXL.Cache.ContentStore.Interfaces.Logging; using BuildXL.Cache.ContentStore.Interfaces.Results; using BuildXL.Cache.ContentStore.Interfaces.Sessions; +using BuildXL.Cache.ContentStore.Interfaces.Stores; +using BuildXL.Cache.ContentStore.Interfaces.Time; +using BuildXL.Cache.ContentStore.Interfaces.Tracing; +using BuildXL.Cache.ContentStore.Interfaces.Utils; +using BuildXL.Cache.ContentStore.Service.Grpc; +using BuildXL.Cache.ContentStore.Sessions; using BuildXL.Cache.ContentStore.Sessions.Internal; -using BuildXL.Cache.ContentStore.Stores; +using BuildXL.Cache.ContentStore.Tracing; using BuildXL.Cache.ContentStore.Tracing.Internal; using BuildXL.Cache.ContentStore.UtilitiesCore; +using BuildXL.Cache.ContentStore.Utils; +using BuildXL.Utilities.Collections; +using BuildXL.Utilities.Core.Tasks; +using BuildXL.Utilities.ParallelAlgorithms; using BuildXL.Utilities.Tracing; +using ContentStore.Grpc; +using ResultsExtensions = BuildXL.Cache.ContentStore.Interfaces.Results.ResultsExtensions; namespace BuildXL.Cache.ContentStore.Distributed.Sessions { - /// - /// A content location based content session with an inner content session for storage. - /// - public class DistributedContentSession : ReadOnlyDistributedContentSession, IContentSession + public class DistributedContentSession : ContentSessionBase, IContentSession, IHibernateContentSession, IConfigurablePin { + internal enum Counters + { + GetLocationsSatisfiedFromLocal, + GetLocationsSatisfiedFromRemote, + PinUnverifiedCountSatisfied, + StartCopyForPinWhenUnverifiedCountSatisfied, + ProactiveCopiesSkipped, + ProactiveCopy_OutsideRingFromPreferredLocations, + ProactiveCopy_OutsideRingCopies, + ProactiveCopyRetries, + ProactiveCopyInsideRingRetries, + ProactiveCopyOutsideRingRetries, + ProactiveCopy_InsideRingCopies, + ProactiveCopy_InsideRingFullyReplicated, + } + + internal CounterCollection SessionCounters { get; } = new CounterCollection(); + + private string _buildId = null; + private ContentHash? _buildIdHash = null; + private readonly ExpiringValue _buildRingMachinesCache; + + private MachineLocation[] BuildRingMachines => _buildRingMachinesCache.GetValueOrDefault() ?? Array.Empty(); + + private readonly ConcurrentBigSet _pendingProactivePuts = new ConcurrentBigSet(); + private ResultNagleQueue _proactiveCopyGetBulkNagleQueue; + + // The method used for remote pins depends on which pin configuration is enabled. + private readonly RemotePinAsync _remotePinner; + + /// + /// The store that persists content locations to a persistent store. + /// + internal readonly IContentLocationStore ContentLocationStore; + + private readonly ColdStorage _coldStorage; + + /// + /// The machine location for the current cache. + /// + protected readonly MachineLocation LocalCacheRootMachineLocation; + + /// + /// The content session that actually stores content. + /// + public IContentSession Inner { get; } + + /// + protected override Tracer Tracer { get; } = new Tracer(nameof(DistributedContentSession)); + + /// + protected readonly DistributedContentCopier DistributedCopier; + + /// + /// Settings for the session. + /// + protected readonly DistributedContentStoreSettings Settings; + + /// + /// Trace only stops and errors to reduce the Kusto traffic. + /// + protected override bool TraceOperationStarted => false; + + /// + /// Semaphore that limits the maximum number of concurrent put and place operations + /// + protected readonly SemaphoreSlim PutAndPlaceFileGate; + + private readonly DistributedContentStore _contentStore; + /// public DistributedContentSession( string name, IContentSession inner, IContentLocationStore contentLocationStore, DistributedContentCopier contentCopier, - DistributedContentStore distributedStore, + DistributedContentStore distributedContentStore, MachineLocation localMachineLocation, ColdStorage coldStorage, DistributedContentStoreSettings settings = default) - : base( - name, - inner, - contentLocationStore, - contentCopier, - distributedStore, - localMachineLocation, - coldStorage, - settings) + : base(name) { + Contract.Requires(name != null); + Contract.Requires(inner != null); + Contract.Requires(contentLocationStore != null); + Contract.Requires(contentLocationStore is StartupShutdownSlimBase, "The store must derive from StartupShutdownSlimBase"); + Contract.Requires(localMachineLocation.IsValid); + + Inner = inner; + ContentLocationStore = contentLocationStore; + LocalCacheRootMachineLocation = localMachineLocation; + Settings = settings ?? DistributedContentStoreSettings.DefaultSettings; + _contentStore = distributedContentStore; + _remotePinner = PinFromMultiLevelContentLocationStore; + DistributedCopier = contentCopier; + PutAndPlaceFileGate = new SemaphoreSlim(Settings.MaximumConcurrentPutAndPlaceFileOperations); + + _coldStorage = coldStorage; + + _buildRingMachinesCache = new ExpiringValue( + Settings.ProactiveCopyInRingMachineLocationsExpiryCache, + SystemClock.Instance, + originalValue: Array.Empty()); } + /// + protected override async Task StartupCoreAsync(OperationContext context) + { + var canHibernate = Inner is IHibernateContentSession ? "can" : "cannot"; + Tracer.Debug(context, $"Session {Name} {canHibernate} hibernate"); + await Inner.StartupAsync(context).ThrowIfFailure(); + + _proactiveCopyGetBulkNagleQueue = new ResultNagleQueue( + execute: hashes => GetLocationsForProactiveCopyAsync(context.CreateNested(Tracer.Name), hashes), + maxDegreeOfParallelism: 1, + interval: Settings.ProactiveCopyGetBulkInterval, + batchSize: Settings.ProactiveCopyGetBulkBatchSize); + _proactiveCopyGetBulkNagleQueue.Start(); + + TryRegisterMachineWithBuildId(context); + + return BoolResult.Success; + } + + private void TryRegisterMachineWithBuildId(OperationContext context) + { + if (Constants.TryExtractBuildId(Name, out _buildId) && Guid.TryParse(_buildId, out var buildIdGuid)) + { + // Generate a fake hash for the build and register a content entry in the location store to represent + // machines in the build ring + var buildIdContent = buildIdGuid.ToByteArray(); + + var buildIdHash = GetBuildIdHash(buildIdContent); + + var arguments = $"Build={_buildId}, BuildIdHash={buildIdHash.ToShortString()}"; + + context.PerformOperationAsync( + Tracer, + async () => + { + // Storing the build id in the cache to prevent this data to be removed during reconciliation. + using var stream = new MemoryStream(buildIdContent); + var result = await Inner.PutStreamAsync(context, buildIdHash, stream, CancellationToken.None, UrgencyHint.Nominal); + result.ThrowIfFailure(); + + // Even if 'StoreBuildIdInCache' is true we need to register the content manually, + // because Inner.PutStreamAsync will just add the content to the cache but won't send a registration message. + await ContentLocationStore.RegisterLocalLocationAsync( + context, + new[] { new ContentHashWithSize(buildIdHash, buildIdContent.Length) }, + context.Token, + UrgencyHint.Nominal).ThrowIfFailure(); + + // Updating the build id field only if the registration or the put operation succeeded. + _buildIdHash = buildIdHash; + + return BoolResult.Success; + }, + extraStartMessage: arguments, + extraEndMessage: r => arguments).FireAndForget(context); + } + } + + private static ContentHash GetBuildIdHash(byte[] buildId) => buildId.CalculateHash(HashType.MD5); + + /// + protected override async Task ShutdownCoreAsync(OperationContext context) + { + var counterSet = new CounterSet(); + counterSet.Merge(GetCounters(), $"{Tracer.Name}."); + + // Unregister from build machine location set + if (_buildIdHash.HasValue) + { + Guid.TryParse(_buildId, out var buildIdGuid); + var buildIdHash = GetBuildIdHash(buildIdGuid.ToByteArray()); + Tracer.Debug(context, $"Deleting in-ring mapping from cache: Build={_buildId}, BuildIdHash={buildIdHash.ToShortString()}"); + + // DeleteAsync will unregister the content as well. No need for calling 'TrimBulkAsync'. + await TaskUtilities.IgnoreErrorsAndReturnCompletion( + _contentStore.DeleteAsync(context, _buildIdHash.Value, new DeleteContentOptions() { DeleteLocalOnly = true })); + } + + await Inner.ShutdownAsync(context).ThrowIfFailure(); + + _proactiveCopyGetBulkNagleQueue?.Dispose(); + Tracer.TraceStatisticsAtShutdown(context, counterSet, prefix: "DistributedContentSessionStats"); + + return BoolResult.Success; + } + + /// + protected override void DisposeCore() + { + base.DisposeCore(); + + Inner.Dispose(); + } + + /// + protected override async Task PinCoreAsync( + OperationContext operationContext, + ContentHash contentHash, + UrgencyHint urgencyHint, + Counter retryCounter) + { + // Call bulk API + var result = await PinHelperAsync(operationContext, new[] { contentHash }, urgencyHint, PinOperationConfiguration.Default()); + return (await result.First()).Item; + } + + /// + protected override async Task OpenStreamCoreAsync( + OperationContext operationContext, + ContentHash contentHash, + UrgencyHint urgencyHint, + Counter retryCounter) + { + OpenStreamResult streamResult = + await Inner.OpenStreamAsync(operationContext, contentHash, operationContext.Token, urgencyHint); + if (streamResult.Code == OpenStreamResult.ResultCode.Success) + { + return streamResult; + } + + var contentRegistrationUrgencyHint = urgencyHint; + + long? size = null; + GetBulkLocationsResult localGetBulkResult = null; + + // First try to fetch file based on locally stored locations for the hash + // Then fallback to fetching file based on global locations minus the locally stored locations which were already checked + foreach (var getBulkTask in ContentLocationStoreExtensions.MultiLevelGetLocations( + ContentLocationStore, + operationContext, + new[] { contentHash }, + operationContext.Token, + urgencyHint, + subtractLocalResults: true)) + { + var getBulkResult = await getBulkTask; + // There is an issue with GetBulkLocationsResult construction from Exception that may loose the information about the origin. + // So we rely on the result order of MultiLevelGetLocations method: the first result is always local and the second one is global. + + GetBulkOrigin origin = localGetBulkResult == null ? GetBulkOrigin.Local : GetBulkOrigin.Global; + if (origin == GetBulkOrigin.Local) + { + localGetBulkResult = getBulkResult; + } + + // Local function: Use content locations for GetBulk to copy file locally + async Task tryCopyContentLocalAsync() + { + if (!getBulkResult || !getBulkResult.ContentHashesInfo.Any()) + { + return new BoolResult($"Metadata records for hash {contentHash.ToShortString()} not found in content location store."); + } + + // Don't reconsider locally stored results that were checked in prior iteration + getBulkResult = getBulkResult.Subtract(localGetBulkResult); + + var hashInfo = getBulkResult.ContentHashesInfo.Single(); + + if (!CanCopyContentHash( + operationContext, + hashInfo, + isGlobal: getBulkResult.Origin == GetBulkOrigin.Global, + out var useInRingLocations, + out var errorMessage)) + { + return new BoolResult(errorMessage); + } + + // Using in-ring machines if configured + var copyResult = await TryCopyAndPutAsync( + operationContext, + hashInfo, + urgencyHint, + CopyReason.OpenStream, + trace: false, + useInRingLocations); + if (!copyResult) + { + return new BoolResult(copyResult); + } + + size = copyResult.ContentSize; + + // If configured, register the content eagerly on put to make sure the content is discoverable by the other builders, + // even if the current machine used to have the content and evicted it recently. + if (Settings.RegisterEagerlyOnPut && !copyResult.ContentAlreadyExistsInCache) + { + contentRegistrationUrgencyHint = UrgencyHint.RegisterEagerly; + } + + return BoolResult.Success; + } + + var copyLocalResult = await tryCopyContentLocalAsync(); + + // Throw operation canceled to avoid operations below which are not value for canceled case. + operationContext.Token.ThrowIfCancellationRequested(); + + if (copyLocalResult.Succeeded) + { + // Succeeded in copying content locally. No need to try with more content locations + break; + } + else if (origin == GetBulkOrigin.Global) + { + return new OpenStreamResult(copyLocalResult, OpenStreamResult.ResultCode.ContentNotFound); + } + } + + Contract.Assert(size != null, "Size should be set if operation succeeded"); + + var updateResult = await UpdateContentTrackerWithNewReplicaAsync( + operationContext, + new[] { new ContentHashWithSize(contentHash, size.Value) }, + contentRegistrationUrgencyHint); + if (!updateResult.Succeeded) + { + return new OpenStreamResult(updateResult); + } + + return await Inner.OpenStreamAsync(operationContext, contentHash, operationContext.Token, urgencyHint); + } + + /// + Task>>> IConfigurablePin.PinAsync( + Context context, + IReadOnlyList contentHashes, + PinOperationConfiguration pinOperationConfiguration) + { + // The lifetime of this operation should be detached from the lifetime of the session. + // But still tracking the lifetime of the store. + return WithStoreCancellationAsync( + context, + opContext => PinHelperAsync(opContext, contentHashes, pinOperationConfiguration.UrgencyHint, pinOperationConfiguration), + pinOperationConfiguration.CancellationToken); + } + + /// + protected override Task>>> PinCoreAsync( + OperationContext operationContext, + IReadOnlyList contentHashes, + UrgencyHint urgencyHint, + Counter retryCounter, + Counter fileCounter) + { + return PinHelperAsync(operationContext, contentHashes, urgencyHint, PinOperationConfiguration.Default()); + } + + private async Task>>> PinHelperAsync( + OperationContext operationContext, + IReadOnlyList contentHashes, + UrgencyHint urgencyHint, + PinOperationConfiguration pinOperationConfiguration) + { + Contract.Requires(contentHashes != null); + + IEnumerable>> pinResults = null; + + IEnumerable>> intermediateResult = null; + if (pinOperationConfiguration.ReturnGlobalExistenceFast) + { + Tracer.Debug(operationContext.TracingContext, $"Detected {nameof(PinOperationConfiguration.ReturnGlobalExistenceFast)}"); + + // Check globally for existence, but do not copy locally and do not update content tracker. + pinResults = await Workflows.RunWithFallback( + contentHashes, + async hashes => + { + intermediateResult = await Inner.PinAsync(operationContext, hashes, operationContext.Token, urgencyHint); + return intermediateResult; + }, + hashes => _remotePinner(operationContext, hashes, succeedWithOneLocation: true, urgencyHint), + result => result.Succeeded); + + // Replace operation context with a new cancellation token so it can outlast this call. + // Using a cancellation token from the store avoid the operations lifetime to be greater than the lifetime of the outer store. + operationContext = new OperationContext(operationContext.TracingContext, token: StoreShutdownStartedCancellationToken); + } + + // Default pin action + var pinTask = Workflows.RunWithFallback( + contentHashes, + hashes => intermediateResult == null + ? Inner.PinAsync(operationContext, hashes, operationContext.Token, urgencyHint) + : Task.FromResult(intermediateResult), + hashes => _remotePinner(operationContext, hashes, succeedWithOneLocation: false, urgencyHint), + result => result.Succeeded, + // Exclude the empty hash because it is a special case which is hard coded for place/openstream/pin. + async hits => await UpdateContentTrackerWithLocalHitsAsync( + operationContext, + hits.Where(x => !contentHashes[x.Index].IsEmptyHash()).Select( + x => new ContentHashWithSizeAndLastAccessTime(contentHashes[x.Index], x.Item.ContentSize, x.Item.LastAccessTime)).ToList(), + urgencyHint)); + + // Initiate a proactive copy if just pinned content is under-replicated + if (Settings.ProactiveCopyOnPin && Settings.ProactiveCopyMode != ProactiveCopyMode.Disabled) + { + pinTask = ProactiveCopyOnPinAsync(operationContext, contentHashes, pinTask); + } + + if (pinOperationConfiguration.ReturnGlobalExistenceFast) + { + // Fire off the default pin action, but do not await the result. + // + // Creating a new OperationContext instance without existing 'CancellationToken', + // because the operation we triggered that stored in 'pinTask' can outlive the lifetime of the current instance. + // And we don't want the PerformNonResultOperationAsync to fail because the current instance is shut down (or disposed). + new OperationContext(operationContext.TracingContext).PerformNonResultOperationAsync( + Tracer, + () => pinTask, + extraEndMessage: results => + { + var resultString = string.Join( + ",", + results.Select( + async task => + { + // Since all bulk operations are constructed with Task.FromResult, it is safe to just access the result; + Indexed result = await task; + return result != null + ? $"{contentHashes[result.Index].ToShortString()}:{result.Item}" + : string.Empty; + })); + + return $"ConfigurablePin Count={contentHashes.Count}, Hashes=[{resultString}]"; + }, + traceErrorsOnly: TraceErrorsOnly, + traceOperationStarted: TraceOperationStarted, + traceOperationFinished: true, + isCritical: false).FireAndForget(operationContext); + } + else + { + pinResults = await pinTask; + } + + Contract.Assert(pinResults != null); + return pinResults; + } + + private async Task>>> ProactiveCopyOnPinAsync( + OperationContext context, + IReadOnlyList contentHashes, + Task>>> pinTask) + { + var results = await pinTask; + + // Since the rest of the operation is done asynchronously, using a cancellation context from the outer store. + var proactiveCopyTask = WithStoreCancellationAsync( + context, + async opContext => + { + var proactiveTasks = results.Select(resultTask => proactiveCopyOnSinglePinAsync(opContext, resultTask)).ToList(); + + // Ensure all tasks are completed, after awaiting outer task + await Task.WhenAll(proactiveTasks); + + return proactiveTasks; + }); + + if (Settings.InlineOperationsForTests) + { + return await proactiveCopyTask; + } + else + { + proactiveCopyTask.FireAndForget(context); + return results; + } + + // Proactive copy an individual pin + async Task> proactiveCopyOnSinglePinAsync(OperationContext opContext, Task> resultTask) + { + Indexed indexedPinResult = await resultTask; + var pinResult = indexedPinResult.Item; + + // Local pins and distributed pins which are copied locally allow proactive copy + if (pinResult.Succeeded && (!(pinResult is DistributedPinResult distributedPinResult) || distributedPinResult.CopyLocally)) + { + var proactiveCopyResult = await ProactiveCopyIfNeededAsync( + opContext, + contentHashes[indexedPinResult.Index], + tryBuildRing: true, + CopyReason.ProactiveCopyOnPin); + + // Only fail if all copies failed. + if (!proactiveCopyResult.Succeeded) + { + return new PinResult(proactiveCopyResult).WithIndex(indexedPinResult.Index); + } + } + + return indexedPinResult; + } + } + + /// + protected override async Task PlaceFileCoreAsync( + OperationContext operationContext, + ContentHash contentHash, + AbsolutePath path, + FileAccessMode accessMode, + FileReplacementMode replacementMode, + FileRealizationMode realizationMode, + UrgencyHint urgencyHint, + Counter retryCounter) + { + var resultWithData = await PerformPlaceFileGatedOperationAsync( + operationContext, + () => PlaceHelperAsync( + operationContext, + new[] { new ContentHashWithPath(contentHash, path) }, + accessMode, + replacementMode, + realizationMode, + urgencyHint + )); + + var result = await resultWithData.Result.SingleAwaitIndexed(); + result.Metadata = resultWithData.Metadata; + return result; + } + + /// + protected override async Task>>> PlaceFileCoreAsync( + OperationContext operationContext, + IReadOnlyList hashesWithPaths, + FileAccessMode accessMode, + FileReplacementMode replacementMode, + FileRealizationMode realizationMode, + UrgencyHint urgencyHint, + Counter retryCounter) + { + // The fallback is invoked for cache misses only. This preserves existing behavior of + // bubbling up errors with Inner store instead of trying remote. + var resultWithData = await PerformPlaceFileGatedOperationAsync( + operationContext, + () => PlaceHelperAsync( + operationContext, + hashesWithPaths, + accessMode, + replacementMode, + realizationMode, + urgencyHint + )); + + // We are tracing here because we did not want to change the signature for PlaceFileCoreAsync, which is implemented in multiple locations + operationContext.TracingContext.Debug( + $"PlaceFileBulk, Gate.OccupiedCount={resultWithData.Metadata.GateOccupiedCount} Gate.Wait={resultWithData.Metadata.GateWaitTime.TotalMilliseconds}ms Hashes.Count={hashesWithPaths.Count}", + component: nameof(DistributedContentSession), + operation: "PlaceFileBulk"); + + return resultWithData.Result; + } + + private Task>>> PlaceHelperAsync( + OperationContext operationContext, + IReadOnlyList hashesWithPaths, + FileAccessMode accessMode, + FileReplacementMode replacementMode, + FileRealizationMode realizationMode, + UrgencyHint urgencyHint) + { + return Workflows.RunWithFallback( + hashesWithPaths, + args => Inner.PlaceFileAsync( + operationContext, + args, + accessMode, + replacementMode, + realizationMode, + operationContext.Token, + urgencyHint), + args => fetchFromMultiLevelContentLocationStoreThenPlaceFileAsync(args), + result => IsPlaceFileSuccess(result), + async hits => await UpdateContentTrackerWithLocalHitsAsync( + operationContext, + hits.Select( + x => new ContentHashWithSizeAndLastAccessTime( + hashesWithPaths[x.Index].Hash, + x.Item.FileSize, + x.Item.LastAccessTime)) + .ToList(), + urgencyHint)); + + Task>>> fetchFromMultiLevelContentLocationStoreThenPlaceFileAsync( + IReadOnlyList fetchedContentInfo) + { + return MultiLevelUtilities.RunMultiLevelAsync( + fetchedContentInfo, + runFirstLevelAsync: args => FetchFromMultiLevelContentLocationStoreThenPutAsync( + operationContext, + args, + urgencyHint, + CopyReason.Place), + runSecondLevelAsync: args => innerPlaceAsync(args), + // NOTE: We just use the first level result if the fetch using content location store fails because the place cannot succeed since the + // content will not have been put into the local CAS + useFirstLevelResult: result => !IsPlaceFileSuccess(result)); + } + + async Task>>> innerPlaceAsync(IReadOnlyList input) + { + // When the content is obtained from remote we still have to place it from the local. + // So in order to return the right source of the file placement + // we have to place locally but change the source for each result. + var results = await Inner.PlaceFileAsync( + operationContext, + input, + accessMode, + replacementMode, + realizationMode, + operationContext.Token, + urgencyHint); + var updatedResults = new List>(); + foreach (var resultTask in results) + { + var result = await resultTask; + updatedResults.Add( + IndexedExtensions.WithIndex( + result.Item + .WithMaterializationSource(PlaceFileResult.Source.DatacenterCache), + result.Index)); + } + + return updatedResults.AsTasks(); + } + } + + private Task>>>> PerformPlaceFileGatedOperationAsync( + OperationContext operationContext, + Func>>>> func) + { + return GateExtensions.GatedOperationAsync( + PutAndPlaceFileGate, + async (timeWaiting, currentCount) => + { + var gateOccupiedCount = Settings.MaximumConcurrentPutAndPlaceFileOperations - currentCount; + + var result = await func(); + + return new ResultWithMetaData>>>( + new ResultMetaData(timeWaiting, gateOccupiedCount), + result); + }, + operationContext.Token); + } + + private static bool IsPlaceFileSuccess(PlaceFileResult result) + { + return result.Code != PlaceFileResult.ResultCode.Error && result.Code != PlaceFileResult.ResultCode.NotPlacedContentNotFound; + } + + /// + public IEnumerable EnumeratePinnedContentHashes() + { + return Inner is IHibernateContentSession session + ? session.EnumeratePinnedContentHashes() + : Enumerable.Empty(); + } + + /// + public Task PinBulkAsync(Context context, IEnumerable contentHashes) + { + // TODO: Replace PinBulkAsync in hibernate with PinAsync bulk call (bug 1365340) + return Inner is IHibernateContentSession session + ? session.PinBulkAsync(context, contentHashes) + : Task.FromResult(0); + } + + /// + public Task ShutdownEvictionAsync(Context context) + { + return Inner is IHibernateContentSession session + ? session.ShutdownEvictionAsync(context) + : BoolResult.SuccessTask; + } + + private Task>>> FetchFromMultiLevelContentLocationStoreThenPutAsync( + OperationContext context, + IReadOnlyList hashesWithPaths, + UrgencyHint urgencyHint, + CopyReason reason) + { + // First try to place file by fetching files based on locally stored locations for the hash + // Then fallback to fetching file based on global locations minus the locally stored locations which were already checked + + var localGetBulkResult = new BuildXL.Utilities.AsyncOut(); + + GetIndexedResults initialFunc = async args => + { + var contentHashes = args.Select(p => p.Hash).ToList(); + localGetBulkResult.Value = + await ContentLocationStore.GetBulkAsync( + context, + contentHashes, + context.Token, + urgencyHint, + GetBulkOrigin.Local); + return await FetchFromContentLocationStoreThenPutAsync( + context, + args, + GetBulkOrigin.Local, + urgencyHint, + localGetBulkResult.Value, + reason); + }; + + GetIndexedResults fallbackFunc = async args => + { + var contentHashes = args.Select(p => p.Hash).ToList(); + var globalGetBulkResult = + await ContentLocationStore.GetBulkAsync( + context, + contentHashes, + context.Token, + urgencyHint, + GetBulkOrigin.Global); + globalGetBulkResult = + globalGetBulkResult.Subtract(localGetBulkResult.Value); + return await FetchFromContentLocationStoreThenPutAsync( + context, + args, + GetBulkOrigin.Global, + urgencyHint, + globalGetBulkResult, + reason); + }; + + // If ColdStorage is ON try to place files from it before use remote locations + if (_coldStorage != null) + { + return Workflows.RunWithFallback( + hashesWithPaths, + initialFunc: async args => { return await _coldStorage.FetchThenPutBulkAsync(context, args, Inner); }, + fallbackFunc: initialFunc, + secondFallbackFunc: fallbackFunc, + thirdFallbackFunc: async args => + { + var coldStorageGetBulkResult = _coldStorage.GetBulkLocations(context, args); + return await FetchFromContentLocationStoreThenPutAsync( + context, + args, + GetBulkOrigin.ColdStorage, + urgencyHint, + coldStorageGetBulkResult, + reason); + }, + isSuccessFunc: result => IsPlaceFileSuccess(result)); + } + + return Workflows.RunWithFallback( + hashesWithPaths, + initialFunc: initialFunc, + fallbackFunc: fallbackFunc, + isSuccessFunc: result => IsPlaceFileSuccess(result)); + } + + private async Task>>> FetchFromContentLocationStoreThenPutAsync( + OperationContext context, + IReadOnlyList hashesWithPaths, + GetBulkOrigin origin, + UrgencyHint urgencyHint, + GetBulkLocationsResult getBulkResult, + CopyReason reason) + { + try + { + // Tracing the hashes here for the entire list, instead of tracing one hash at a time inside TryCopyAndPutAsync method. + + // This returns failure if any item in the batch wasn't copied locally + // TODO: split results and call PlaceFile on successfully copied files (bug 1365340) + if (!getBulkResult.Succeeded || !getBulkResult.ContentHashesInfo.Any()) + { + return hashesWithPaths.Select( + _ => new PlaceFileResult( + getBulkResult, + PlaceFileResult.ResultCode.NotPlacedContentNotFound, + "Metadata records not found in content location store")) + .AsIndexedTasks(); + } + + async Task> copyAndPut(ContentHashWithSizeAndLocations contentHashWithSizeAndLocations, int index) + { + PlaceFileResult result; + try + { + if (!CanCopyContentHash( + context, + contentHashWithSizeAndLocations, + isGlobal: origin == GetBulkOrigin.Global, + out var useInRingMachineLocations, + out var errorMessage)) + { + result = PlaceFileResult.CreateContentNotFound(errorMessage); + } + else + { + var copyResult = await TryCopyAndPutAsync( + context, + contentHashWithSizeAndLocations, + urgencyHint, + reason, + // We just traced all the hashes as a result of GetBulk call, no need to trace each individual hash. + trace: false, + // Using in-ring locations as well if the feature is on. + useInRingMachineLocations: useInRingMachineLocations, + outputPath: hashesWithPaths[index].Path); + + if (!copyResult) + { + // For ColdStorage we should treat all errors as cache misses + result = origin != GetBulkOrigin.ColdStorage + ? new PlaceFileResult(copyResult) + : new PlaceFileResult(copyResult, PlaceFileResult.ResultCode.NotPlacedContentNotFound); + } + else + { + var source = origin == GetBulkOrigin.ColdStorage + ? PlaceFileResult.Source.ColdStorage + : PlaceFileResult.Source.DatacenterCache; + result = PlaceFileResult.CreateSuccess(PlaceFileResult.ResultCode.PlacedWithMove, copyResult.ContentSize, source); + } + } + } + catch (Exception e) + { + // The transform block should not fail with an exception otherwise the block's state will be changed to failed state and the exception + // won't be propagated to the caller. + result = new PlaceFileResult(e); + } + + return result.WithIndex(index); + } + + var copyFilesLocallyActionQueue = new ActionQueue(Settings.ParallelCopyFilesLimit); + var copyFilesLocally = await copyFilesLocallyActionQueue.SelectAsync( + items: getBulkResult.ContentHashesInfo, + body: (item, index) => copyAndPut(item, index)); + + var updateResults = await UpdateContentTrackerWithNewReplicaAsync( + context, + copyFilesLocally.Where(r => r.Item.Succeeded).Select(r => new ContentHashWithSize(hashesWithPaths[r.Index].Hash, r.Item.FileSize)) + .ToList(), + urgencyHint); + + if (!updateResults.Succeeded) + { + return copyFilesLocally.Select(result => new PlaceFileResult(updateResults).WithIndex(result.Index)).AsTasks(); + } + + return copyFilesLocally.AsTasks(); + } + catch (Exception ex) + { + return hashesWithPaths.Select((hash, index) => new PlaceFileResult(ex).WithIndex(index)).AsTasks(); + } + } + + private bool CanCopyContentHash( + Context context, + ContentHashWithSizeAndLocations result, + bool isGlobal, + out bool useInRingMachineLocations, + [NotNullWhen(false)] out string message) + { + useInRingMachineLocations = isGlobal && Settings.UseInRingMachinesForCopies; + if (!isLocationsAvailable(out message)) + { + if (useInRingMachineLocations && GetInRingActiveMachines().Length != 0) + { + string useInRingLocationsMessage = + $", but {nameof(DistributedContentStoreSettings.UseInRingMachinesForCopies)} is true. Trying to copy the content from in-ring machines."; + Tracer.Debug(context, message + useInRingLocationsMessage); + // Still trying copying the content from in-ring machines, even though the location information is lacking. + return true; + } + + // Tracing only for global locations because they come last. + if (isGlobal) + { + Tracer.Warning(context, message); + } + + return false; + } + + return true; + + bool isLocationsAvailable([NotNullWhen(false)] out string errorMessage) + { + errorMessage = null; + // Null represents no replicas were ever registered, where as empty list implies content is missing from all replicas + if (result.Locations == null) + { + errorMessage = $"No replicas registered for hash {result.ContentHash.ToShortString()}"; + return false; + } + + if (!result.Locations.Any()) + { + errorMessage = $"No replicas currently exist in content tracker for hash {result.ContentHash.ToShortString()}"; + return false; + } + + return true; + } + } + + private async Task TryCopyAndPutAsync( + OperationContext operationContext, + ContentHashWithSizeAndLocations hashInfo, + UrgencyHint urgencyHint, + CopyReason reason, + bool trace, + bool useInRingMachineLocations = false, + AbsolutePath outputPath = null) + { + Context context = operationContext; + CancellationToken cts = operationContext.Token; + + if (trace) + { + Tracer.Debug(operationContext, $"Copying {hashInfo.ContentHash.ToShortString()} with {hashInfo.Locations?.Count ?? 0} locations"); + } + + var copyCompression = CopyCompression.None; + var copyCompressionThreshold = Settings.GrpcCopyCompressionSizeThreshold ?? 0; + if (copyCompressionThreshold > 0 && hashInfo.Size > copyCompressionThreshold) + { + copyCompression = Settings.GrpcCopyCompressionAlgorithm; + } + + var copyRequest = new DistributedContentCopier.CopyRequest( + _contentStore, + hashInfo, + reason, + HandleCopyAsync: async args => + { + (CopyFileResult copyFileResult, AbsolutePath tempLocation, _) = args; + + PutResult innerPutResult; + long actualSize = copyFileResult.Size ?? hashInfo.Size; + if (Settings.UseTrustedHash(actualSize) && Inner is ITrustedContentSession trustedInner) + { + // The file has already been hashed, so we can trust the hash of the file. + innerPutResult = await trustedInner.PutTrustedFileAsync( + context, + new ContentHashWithSize(hashInfo.ContentHash, actualSize), + tempLocation, + FileRealizationMode.Move, + cts, + urgencyHint); + } + else + { + // Pass the HashType, not the Hash. This prompts a re-hash of the file, which places it where its actual hash requires. + // If the actual hash differs from the expected hash, then we fail below and move to the next location. + // Also, record the bytes if the file is small enough to be put into the ContentLocationStore. + innerPutResult = await Inner.PutFileAsync( + context, + hashInfo.ContentHash.HashType, + tempLocation, + FileRealizationMode.Move, + cts, + urgencyHint); + } + + return innerPutResult; + }, + copyCompression, + OverrideWorkingFolder: (Inner as ITrustedContentSession)?.TryGetWorkingDirectory(outputPath)); + + if (useInRingMachineLocations) + { + copyRequest = copyRequest with { InRingMachines = GetInRingActiveMachines() }; + } + + var putResult = await DistributedCopier.TryCopyAndPutAsync(operationContext, copyRequest); + + return putResult; + } + + /// + /// Runs a given function in the cancellation context of the outer store. + /// + /// + /// The lifetime of some session-based operation is longer than the session itself. + /// For instance, proactive copies, or put blob can outlive the lifetime of the session. + /// But these operations should not outlive the lifetime of the store, because when the store is closed + /// most likely all the operations will fail with some weird errors like "ObjectDisposedException". + /// + /// This helper method allows running the operations that may outlive the lifetime of the session but should not outlive the lifetime of the store. + /// + protected Task WithStoreCancellationAsync( + Context context, + Func> func, + CancellationToken token = default) + { + return ((StartupShutdownSlimBase)ContentLocationStore).WithOperationContext(context, token, func); + } + + /// + /// Returns a cancellation token that is triggered when the outer store shutdown is started. + /// + protected CancellationToken StoreShutdownStartedCancellationToken => + ((StartupShutdownSlimBase)ContentLocationStore).ShutdownStartedCancellationToken; + + private Task UpdateContentTrackerWithNewReplicaAsync( + OperationContext context, + IReadOnlyList contentHashes, + UrgencyHint urgencyHint) + { + if (contentHashes.Count == 0) + { + return BoolResult.SuccessTask; + } + + // TODO: Pass location store option (seems to only be used to prevent updating TTL when replicating for proactive replication) (bug 1365340) + return ContentLocationStore.RegisterLocalLocationAsync(context, contentHashes, context.Token, urgencyHint); + } + + private Task>>> PinFromMultiLevelContentLocationStore( + OperationContext context, + IReadOnlyList contentHashes, + bool succeedWithOneLocation, + UrgencyHint urgencyHint = UrgencyHint.Nominal) + { + // Pinning a content based on a number of location is inherently dangerous in case of eviction storms in a system. + // The LLS data might be stale because of event processing delay and the global information can be inaccurate because + // the trim events are not sent to the global store. + // We can't do anything when the LLS data is stale, but in some stamps its beneficial not to rely on the global loctions. + if (Settings.PinConfiguration.UseLocalLocationsOnlyOnUnverifiedPin) + { + return PinFromContentLocationStoreOriginAsync( + context, + contentHashes, + GetBulkOrigin.Local, + succeedWithOneLocation: succeedWithOneLocation, + urgencyHint); + } + + return Workflows.RunWithFallback( + contentHashes, + hashes => PinFromContentLocationStoreOriginAsync( + context, + hashes, + GetBulkOrigin.Local, + succeedWithOneLocation: succeedWithOneLocation, + urgencyHint), + hashes => PinFromContentLocationStoreOriginAsync( + context, + hashes, + GetBulkOrigin.Global, + succeedWithOneLocation: succeedWithOneLocation, + urgencyHint), + result => result.Succeeded); + } + + // This method creates pages of hashes, makes one bulk call to the content location store to get content location record sets for all the hashes on the page, + // and fires off processing of the returned content location record sets while proceeding to the next page of hashes in parallel. + private async Task>>> PinFromContentLocationStoreOriginAsync( + OperationContext operationContext, + IReadOnlyList hashes, + GetBulkOrigin origin, + bool succeedWithOneLocation, + UrgencyHint urgency = UrgencyHint.Nominal) + { + CancellationToken cancel = operationContext.Token; + // Create an action block to process all the requested remote pins while limiting the number of simultaneously executed. + var pinnings = new List(hashes.Count); + var pinningAction = ActionBlockSlim.CreateWithAsyncAction( + degreeOfParallelism: Settings.PinConfiguration?.MaxIOOperations ?? 1, + async pinning => await PinRemoteAsync( + operationContext, + pinning, + isLocal: origin == GetBulkOrigin.Local, + updateContentTracker: false, + succeedWithOneLocation: succeedWithOneLocation), + cancellationToken: cancel); + + // Make a bulk call to content location store to get location records for all hashes on the page. + // NOTE: We use GetBulkStackedAsync so that when Global results are retrieved we also include Local results to ensure we get a full view of available content + GetBulkLocationsResult pageLookup = await ContentLocationStoreExtensions.GetBulkStackedAsync( + ContentLocationStore, + operationContext, + hashes, + cancel, + urgency, + origin); + + // If successful, fire off the remote pinning logic for each hash. If not, set all pins to failed. + if (pageLookup.Succeeded) + { + foreach (ContentHashWithSizeAndLocations record in pageLookup.ContentHashesInfo) + { + RemotePinning pinning = new RemotePinning(record); + pinnings.Add(pinning); + await pinningAction.PostAsync(pinning, cancel); + } + } + else + { + foreach (ContentHash hash in hashes) + { + Tracer.Warning( + operationContext, + $"Pin failed for hash {hash.ToShortString()}: directory query failed with error {pageLookup.ErrorMessage}"); + RemotePinning pinning = new RemotePinning(new ContentHashWithSizeAndLocations(hash, -1L)) { Result = new PinResult(pageLookup) }; + pinnings.Add(pinning); + } + } + + Contract.Assert(pinnings.Count == hashes.Count); + + // Wait for all the pinning actions to complete. + pinningAction.Complete(); + + try + { + await pinningAction.Completion; + } + catch (TaskCanceledException) + { + // Cancellation token provided to an action block can be canceled. + // Ignoring the exception in this case. + } + + // Inform the content directory that we copied the files. + // Looking for distributed pin results that were successful by copying the content locally. + var localCopies = pinnings.Select((rp, index) => (result: rp, index)) + .Where(x => x.result.Result is DistributedPinResult dpr && dpr.CopyLocally).ToList(); + + BoolResult updated = await UpdateContentTrackerWithNewReplicaAsync( + operationContext, + localCopies.Select(lc => new ContentHashWithSize(lc.result.Record.ContentHash, lc.result.Record.Size)).ToList(), + UrgencyHint.Nominal); + if (!updated.Succeeded) + { + // We failed to update the tracker. Need to update the results. + string hashesAsString = string.Join(", ", localCopies.Select(lc => lc.result.Record.ContentHash.ToShortString())); + Tracer.Warning( + operationContext, + $"Pin failed for hashes {hashesAsString}: local copy succeeded, but could not inform content directory due to {updated.ErrorMessage}."); + foreach (var (_, index) in localCopies) + { + pinnings[index].Result = new PinResult(updated); + } + } + + // The return type should probably be just Task>, but higher callers require the Indexed wrapper and that the PinResults be encased in Tasks. + return pinnings.Select(x => x.Result ?? createCanceledPutResult()).AsIndexed().AsTasks(); + + static PinResult createCanceledPutResult() => new ErrorResult("The operation was canceled").AsResult(); + } + + // The dataflow framework can process only a single object, and returns no output from that processing. By combining the input and output of each remote pinning into a single object, + // we can nonetheless use the dataflow framework to process pinnings and read the output from the updated objects afterward. + private class RemotePinning + { + public ContentHashWithSizeAndLocations Record { get; } + + private PinResult _result; + + public PinResult Result + { + get => _result; + set + { + value!.ContentSize = Record.Size; + _result = value; + } + } + + public RemotePinning(ContentHashWithSizeAndLocations record) + => Record = record; + } + + // This method processes each remote pinning, setting the output when the operation is completed. + private async Task PinRemoteAsync( + OperationContext context, + RemotePinning pinning, + bool isLocal, + bool updateContentTracker = true, + bool succeedWithOneLocation = false) + { + pinning.Result = await PinRemoteAsync( + context, + pinning.Record, + isLocal, + updateContentTracker, + succeedWithOneLocation: succeedWithOneLocation); + } + + // This method processes a single content location record set for pinning. + private async Task PinRemoteAsync( + OperationContext operationContext, + ContentHashWithSizeAndLocations remote, + bool isLocal, + bool updateContentTracker = true, + bool succeedWithOneLocation = false) + { + IReadOnlyList locations = remote.Locations; + + // If no remote locations are recorded, we definitely can't pin + if (locations == null || locations.Count == 0) + { + if (!isLocal) + { + // Trace only when pin failed based on the data from the global store. + Tracer.Warning(operationContext, $"Pin failed for hash {remote.ContentHash.ToShortString()}: no remote records."); + } + + return DistributedPinResult.ContentNotFound(replicaCount: 0, "No locations found"); + } + + // When we only require the content to exist at least once anywhere, we can ignore pin thresholds + // and return success after finding a single location. + if (succeedWithOneLocation && locations.Count >= 1) + { + return DistributedPinResult.EnoughReplicas(locations.Count, "Global succeeds"); + } + + if (locations.Count >= Settings.PinConfiguration.PinMinUnverifiedCount) + { + SessionCounters[Counters.PinUnverifiedCountSatisfied].Increment(); + + // Tracing extra data if the locations were merged to separate the local locations and the global locations that we added to the final result. + string extraMessage = null; + if (!isLocal) + { + // Extra locations does make sense only for the global case when the entries were merged. + extraMessage = $"ExtraGlobal: {remote.ExtraMergedLocations}"; + } + + var result = DistributedPinResult.EnoughReplicas(locations.Count, extraMessage); + + // Triggering an async copy if the number of replicas are close to a PinMinUnverifiedCount threshold. + int threshold = Settings.PinConfiguration.PinMinUnverifiedCount + + Settings.PinConfiguration.AsyncCopyOnPinThreshold; + if (locations.Count < threshold) + { + Tracer.Info( + operationContext, + $"Starting asynchronous copy of the content for hash {remote.ContentHash.ToShortString()} because the number of locations '{locations.Count}' is less then a threshold of '{threshold}'."); + SessionCounters[Counters.StartCopyForPinWhenUnverifiedCountSatisfied].Increment(); + + // For "synchronous" pins the tracker is updated at once for all the hashes for performance reasons, + // but for asynchronous copy we always need to update the tracker with a new location. + var task = WithStoreCancellationAsync( + operationContext.TracingContext, + opContext => TryCopyAndPutAndUpdateContentTrackerAsync( + opContext, + remote, + updateContentTracker: true, + CopyReason.AsyncCopyOnPin)); + if (Settings.InlineOperationsForTests) + { + (await task).TraceIfFailure(operationContext); + } + else + { + task.FireAndForget(operationContext.TracingContext, traceErrorResult: true, operation: "AsynchronousCopyOnPin"); + } + + // Note: Pin result traces with CpA (copied asynchronously) code is to provide the information that the content is being copied asynchronously, and that replica count is enough but not above async copy threshold. + // This trace result does not represent that of the async copy since that is done FireAndForget. + result = DistributedPinResult.AsynchronousCopy(locations.Count); + } + + return result; + } + + if (isLocal) + { + // Don't copy content locally based on locally cached result. So stop here and return content not found. + // This method will be called again with global locations at which time we will attempt to copy the files locally. + // When allowing global locations to succeed a put, report success. + return DistributedPinResult.ContentNotFound(locations.Count); + } + + // Previous checks were not sufficient, so copy the file locally. + PutResult copy = await TryCopyAndPutAsync(operationContext, remote, UrgencyHint.Nominal, CopyReason.Pin, trace: false); + if (copy) + { + if (!updateContentTracker) + { + return DistributedPinResult.SynchronousCopy(locations.Count); + } + + // Inform the content directory that we have the file. + // We wait for this to complete, rather than doing it fire-and-forget, because another machine in the ring may need the pinned content immediately. + BoolResult updated = await UpdateContentTrackerWithNewReplicaAsync( + operationContext, + new[] { new ContentHashWithSize(remote.ContentHash, copy.ContentSize) }, + UrgencyHint.Nominal); + if (updated.Succeeded) + { + return DistributedPinResult.SynchronousCopy(locations.Count); + } + else + { + // Tracing the error separately. + Tracer.Warning( + operationContext, + $"Pin failed for hash {remote.ContentHash.ToShortString()}: local copy succeeded, but could not inform content directory due to {updated.ErrorMessage}."); + return new DistributedPinResult(locations.Count, updated); + } + } + else + { + // Tracing the error separately. + Tracer.Warning(operationContext, $"Pin failed for hash {remote.ContentHash.ToShortString()}: local copy failed with {copy}."); + return DistributedPinResult.ContentNotFound(locations.Count); + } + } + + private async Task TryCopyAndPutAndUpdateContentTrackerAsync( + OperationContext operationContext, + ContentHashWithSizeAndLocations remote, + bool updateContentTracker, + CopyReason reason) + { + PutResult copy = await TryCopyAndPutAsync(operationContext, remote, UrgencyHint.Nominal, reason, trace: true); + if (copy && updateContentTracker) + { + return await UpdateContentTrackerWithNewReplicaAsync( + operationContext, + new[] { new ContentHashWithSize(remote.ContentHash, copy.ContentSize) }, + UrgencyHint.Nominal); + } + + return copy; + } + + private Task UpdateContentTrackerWithLocalHitsAsync( + OperationContext context, + IReadOnlyList contentHashesWithInfo, + UrgencyHint urgencyHint) + { + if (Disposed) + { + // Nothing to do. + return BoolTask.True; + } + + if (contentHashesWithInfo.Count == 0) + { + // Nothing to do. + return BoolTask.True; + } + + IReadOnlyList hashesToEagerUpdate = + contentHashesWithInfo.Select(x => new ContentHashWithSize(x.Hash, x.Size)).ToList(); + + // Wait for update to complete on remaining hashes to cover case where the record has expired and another machine in the ring requests it immediately after this pin succeeds. + return UpdateContentTrackerWithNewReplicaAsync(context, hashesToEagerUpdate, urgencyHint); + } + + internal async Task> GetLocationsForProactiveCopyAsync( + OperationContext context, + IReadOnlyList hashes) + { + var originalLength = hashes.Count; + if (_buildIdHash.HasValue && !_buildRingMachinesCache.IsUpToDate()) + { + Tracer.Debug( + context, + $"{Tracer.Name}.{nameof(GetLocationsForProactiveCopyAsync)}: getting in-ring machines for BuildId='{_buildId}'."); + // Add build id hash to hashes so build ring machines can be updated + hashes = hashes.AppendItem(_buildIdHash.Value).ToList(); + } + + var result = await MultiLevelUtilities.RunMultiLevelWithMergeAsync( + hashes, + inputs => ResultsExtensions.ThrowIfFailureAsync>( + ContentLocationStore.GetBulkAsync(context, inputs, context.Token, UrgencyHint.Nominal, GetBulkOrigin.Local), + g => g.ContentHashesInfo), + inputs => ResultsExtensions.ThrowIfFailureAsync>( + ContentLocationStore.GetBulkAsync(context, inputs, context.Token, UrgencyHint.Nominal, GetBulkOrigin.Global), + g => g.ContentHashesInfo), + mergeResults: ContentHashWithSizeAndLocations.Merge, + useFirstLevelResult: result => + { + if (result.Locations?.Count >= Settings.ProactiveCopyLocationsThreshold) + { + SessionCounters[Counters.GetLocationsSatisfiedFromLocal].Increment(); + return true; + } + else + { + SessionCounters[Counters.GetLocationsSatisfiedFromRemote].Increment(); + return false; + } + }); + + if (hashes.Count != originalLength) + { + // Update build ring machines with retrieved locations + var buildRingMachines = result.Last().Locations?.AppendItem(LocalCacheRootMachineLocation).ToArray() ?? + CollectionUtilities.EmptyArray(); + _buildRingMachinesCache.Update(buildRingMachines); + Tracer.Debug( + context, + $"{Tracer.Name}.{nameof(GetLocationsForProactiveCopyAsync)}: InRingMachines=[{string.Join(", ", buildRingMachines.Select(m => m.Path))}] BuildId='{_buildId}'"); + return result.Take(originalLength).ToList(); + } + else + { + return result; + } + } + + internal async Task ProactiveCopyIfNeededAsync( + OperationContext context, + ContentHash hash, + bool tryBuildRing, + CopyReason reason) + { + var nagleQueue = _proactiveCopyGetBulkNagleQueue; + if (nagleQueue is null) + { + return new ProactiveCopyResult(new ErrorResult("StartupAsync was not called")); + } + + ContentHashWithSizeAndLocations result = await nagleQueue.EnqueueAsync(hash); + return await ProactiveCopyIfNeededAsync(context, result, tryBuildRing, reason); + } + + internal Task ProactiveCopyIfNeededAsync( + OperationContext context, + ContentHashWithSizeAndLocations info, + bool tryBuildRing, + CopyReason reason) + { + var hash = info.ContentHash; + if (!_pendingProactivePuts.Add(hash) + || info.ContentHash.IsEmptyHash()) // No reason to push an empty hash to another machine. + { + return Task.FromResult(ProactiveCopyResult.CopyNotRequiredResult); + } + + // Don't trace this case since it would add too much log traffic. + var replicatedLocations = (info.Locations ?? CollectionUtilities.EmptyArray()).ToList(); + + if (replicatedLocations.Count >= Settings.ProactiveCopyLocationsThreshold) + { + SessionCounters[Counters.ProactiveCopiesSkipped].Increment(); + return Task.FromResult(ProactiveCopyResult.CopyNotRequiredResult); + } + + // By adding the master to replicatedLocations, it will be excluded from proactive replication + var masterLocation = _contentStore.LocalLocationStore?.MasterElectionMechanism.Master; + if (masterLocation is not null && masterLocation.Value.IsValid) + { + replicatedLocations.Add(masterLocation.Value); + } + + return context.PerformOperationAsync( + Tracer, + operation: async () => + { + try + { + var outsideRingCopyTask = ProactiveCopyOutsideBuildRingWithRetryAsync(context, info, replicatedLocations, reason); + var insideRingCopyTask = ProactiveCopyInsideBuildRingWithRetryAsync( + context, + info, + tryBuildRing, + replicatedLocations, + reason); + await Task.WhenAll(outsideRingCopyTask, insideRingCopyTask); + + var (insideRingRetries, insideRingResult) = await insideRingCopyTask; + var (outsideRingRetries, outsideRingResult) = await outsideRingCopyTask; + + int totalRetries = insideRingRetries + outsideRingRetries; + return new ProactiveCopyResult(insideRingResult, outsideRingResult, totalRetries, info.Entry); + } + finally + { + _pendingProactivePuts.Remove(hash); + } + }, + extraEndMessage: r => $"Hash={info.ContentHash}, Retries={r.TotalRetries}, Reason=[{reason}]"); + } + + private async Task<(int retries, ProactivePushResult outsideRingCopyResult)> ProactiveCopyOutsideBuildRingWithRetryAsync( + OperationContext context, + ContentHashWithSize hash, + IReadOnlyList replicatedLocations, + CopyReason reason) + { + var outsideRingCopyResult = await ProactiveCopyOutsideBuildRingAsync(context, hash, replicatedLocations, reason, retries: 0); + int retries = 0; + while (outsideRingCopyResult.QualifiesForRetry && retries < Settings.ProactiveCopyMaxRetries) + { + SessionCounters[Counters.ProactiveCopyRetries].Increment(); + SessionCounters[Counters.ProactiveCopyOutsideRingRetries].Increment(); + retries++; + outsideRingCopyResult = await ProactiveCopyOutsideBuildRingAsync(context, hash, replicatedLocations, reason, retries); + } + + return (retries, outsideRingCopyResult); + } + + private async Task ProactiveCopyOutsideBuildRingAsync( + OperationContext context, + ContentHashWithSize hash, + IReadOnlyList replicatedLocations, + CopyReason reason, + int retries) + { + // The first attempt is not considered as retry + int attempt = retries + 1; + if ((Settings.ProactiveCopyMode & ProactiveCopyMode.OutsideRing) == 0) + { + return ProactivePushResult.FromPushFileResult(PushFileResult.Disabled(), attempt); + } + + Result getLocationResult = null; + var source = ProactiveCopyLocationSource.Random; + + // Make sure that the machine is not in the build ring and does not already have the content. + var machinesToSkip = replicatedLocations.Concat(BuildRingMachines).ToArray(); + + // Try to select one of the designated machines for this hash. + if (Settings.ProactiveCopyUsePreferredLocations) + { + var designatedLocationsResult = ContentLocationStore.GetDesignatedLocations(hash); + if (designatedLocationsResult.Succeeded) + { + // A machine in the build may be a designated location for the hash, + // but we won't pushing to the same machine twice, because 'replicatedLocations' argument + // has a local machine that we're about to push for inside the ring copy. + // We also want to skip inside ring machines for outsidecopy task + var candidates = Enumerable.Except(designatedLocationsResult.Value, machinesToSkip).ToArray(); + + if (candidates.Length > 0) + { + getLocationResult = candidates[ThreadSafeRandom.Generator.Next(0, candidates.Length)]; + source = ProactiveCopyLocationSource.DesignatedLocation; + SessionCounters[Counters.ProactiveCopy_OutsideRingFromPreferredLocations].Increment(); + } + } + } + + // Try to select one machine at random. + if (getLocationResult?.Succeeded != true) + { + getLocationResult = ContentLocationStore.GetRandomMachineLocation(except: machinesToSkip); + source = ProactiveCopyLocationSource.Random; + } + + if (!getLocationResult.Succeeded) + { + return ProactivePushResult.FromPushFileResult(new PushFileResult(getLocationResult), attempt); + } + + var candidate = getLocationResult.Value; + SessionCounters[Counters.ProactiveCopy_OutsideRingCopies].Increment(); + PushFileResult pushFileResult = await PushContentAsync(context, hash, candidate, isInsideRing: false, reason, source, attempt); + return ProactivePushResult.FromPushFileResult(pushFileResult, attempt); + } + + /// + /// Gets all the active in-ring machines (excluding the current one). + /// + public MachineLocation[] GetInRingActiveMachines() + { + return Enumerable.Where(BuildRingMachines, m => !m.Equals(LocalCacheRootMachineLocation)) + .Where(m => ContentLocationStore.IsMachineActive(m)) + .ToArray(); + } + + private async Task<(int retries, ProactivePushResult insideRingCopyResult)> ProactiveCopyInsideBuildRingWithRetryAsync( + OperationContext context, + ContentHashWithSize hash, + bool tryBuildRing, + IReadOnlyList replicatedLocations, + CopyReason reason) + { + ProactivePushResult insideRingCopyResult = await ProactiveCopyInsideBuildRing( + context, + hash, + tryBuildRing, + replicatedLocations, + reason, + retries: 0); + int retries = 0; + while (insideRingCopyResult.QualifiesForRetry && retries < Settings.ProactiveCopyMaxRetries) + { + SessionCounters[Counters.ProactiveCopyRetries].Increment(); + SessionCounters[Counters.ProactiveCopyInsideRingRetries].Increment(); + retries++; + insideRingCopyResult = await ProactiveCopyInsideBuildRing(context, hash, tryBuildRing, replicatedLocations, reason, retries); + } + + return (retries, insideRingCopyResult); + } + + private async Task ProactiveCopyInsideBuildRing( + OperationContext context, + ContentHashWithSize hash, + bool tryBuildRing, + IReadOnlyList replicatedLocations, + CopyReason reason, + int retries) + { + // The first attempt is not considered as retry + int attempt = retries + 1; + + // Get random machine inside build ring + if (!tryBuildRing || (Settings.ProactiveCopyMode & ProactiveCopyMode.InsideRing) == 0) + { + return ProactivePushResult.FromPushFileResult(PushFileResult.Disabled(), attempt); + } + + if (_buildIdHash == null) + { + return ProactivePushResult.FromStatus(ProactivePushStatus.BuildIdNotSpecified, attempt); + } + + // Having an explicit case to check if the in-ring machine list is empty to separate the case + // when the machine list is not empty but all the candidates are unavailable. + if (BuildRingMachines.Length == 0) + { + return ProactivePushResult.FromStatus(ProactivePushStatus.InRingMachineListIsEmpty, attempt); + } + + var candidates = GetInRingActiveMachines(); + + if (candidates.Length == 0) + { + return ProactivePushResult.FromStatus(ProactivePushStatus.MachineNotFound, attempt); + } + + candidates = candidates.Except(replicatedLocations).ToArray(); + if (candidates.Length == 0) + { + SessionCounters[Counters.ProactiveCopy_InsideRingFullyReplicated].Increment(); + return ProactivePushResult.FromStatus(ProactivePushStatus.MachineAlreadyHasCopy, attempt); + } + + SessionCounters[Counters.ProactiveCopy_InsideRingCopies].Increment(); + var candidate = candidates[ThreadSafeRandom.Generator.Next(0, candidates.Length)]; + PushFileResult pushFileResult = await PushContentAsync( + context, + hash, + candidate, + isInsideRing: true, + reason, + ProactiveCopyLocationSource.Random, + attempt); + return ProactivePushResult.FromPushFileResult(pushFileResult, attempt); + } + + private async Task PushContentAsync( + OperationContext context, + ContentHashWithSize hash, + MachineLocation target, + bool isInsideRing, + CopyReason reason, + ProactiveCopyLocationSource source, + int attempt) + { + // This is here to avoid hanging ProactiveCopyIfNeededAsync on inside/outside ring copies before starting + // the other one. + await Task.Yield(); + + if (Settings.PushProactiveCopies) + { + // It is possible that this method is used during proactive replication + // and the hash was already evicted at the time this method is called. + var streamResult = await Inner.OpenStreamAsync(context, hash, context.Token); + if (!streamResult.Succeeded) + { + return PushFileResult.SkipContentUnavailable(); + } + + using var stream = streamResult.Stream!; + + return await DistributedCopier.PushFileAsync( + context, + hash, + target, + stream, + isInsideRing, + reason, + source, + attempt); + } + else + { + var requestResult = await DistributedCopier.RequestCopyFileAsync(context, hash, target, isInsideRing, attempt); + if (requestResult) + { + return PushFileResult.PushSucceeded(size: null); + } + + return new PushFileResult(requestResult, "Failed requesting a copy"); + } + } + + /// + protected override CounterSet GetCounters() => + base.GetCounters() + .Merge(DistributedCopier.GetCounters()) + .Merge(CounterUtilities.ToCounterSet((CounterCollection)SessionCounters)); + /// protected override Task PutFileCoreAsync( OperationContext operationContext, @@ -56,13 +1765,15 @@ namespace BuildXL.Cache.ContentStore.Distributed.Sessions UrgencyHint urgencyHint, Counter retryCounter) { - return PerformPutFileGatedOperationAsync(operationContext, () => - { - return PutCoreAsync( - operationContext, - urgencyHint, - session => session.PutFileAsync(operationContext, hashType, path, realizationMode, operationContext.Token, urgencyHint)); - }); + return PerformPutFileGatedOperationAsync( + operationContext, + () => + { + return PutCoreAsync( + operationContext, + urgencyHint, + session => session.PutFileAsync(operationContext, hashType, path, realizationMode, operationContext.Token, urgencyHint)); + }); } /// @@ -76,26 +1787,30 @@ namespace BuildXL.Cache.ContentStore.Distributed.Sessions { // We are intentionally not gating PutStream operations because we don't expect a high number of them at // the same time. - return PerformPutFileGatedOperationAsync(operationContext, () => - { - return PutCoreAsync( - operationContext, - urgencyHint, - session => session.PutFileAsync(operationContext, contentHash, path, realizationMode, operationContext.Token, urgencyHint)); - }); + return PerformPutFileGatedOperationAsync( + operationContext, + () => + { + return PutCoreAsync( + operationContext, + urgencyHint, + session => session.PutFileAsync(operationContext, contentHash, path, realizationMode, operationContext.Token, urgencyHint)); + }); } private Task PerformPutFileGatedOperationAsync(OperationContext operationContext, Func> func) { - return PutAndPlaceFileGate.GatedOperationAsync(async (timeWaiting, currentCount) => - { - var gateOccupiedCount = Settings.MaximumConcurrentPutAndPlaceFileOperations - currentCount; + return PutAndPlaceFileGate.GatedOperationAsync( + async (timeWaiting, currentCount) => + { + var gateOccupiedCount = Settings.MaximumConcurrentPutAndPlaceFileOperations - currentCount; - var result = await func(); - result.MetaData = new ResultMetaData(timeWaiting, gateOccupiedCount); + var result = await func(); + result.MetaData = new ResultMetaData(timeWaiting, gateOccupiedCount); - return result; - }, operationContext.Token); + return result; + }, + operationContext.Token); } /// @@ -150,7 +1865,11 @@ namespace BuildXL.Cache.ContentStore.Distributed.Sessions // Since the rest of the operation is done asynchronously, create new context to stop cancelling operation prematurely. var proactiveCopyTask = WithStoreCancellationAsync( context, - operationContext => ProactiveCopyIfNeededAsync(operationContext, result.ContentHash, tryBuildRing: true, CopyReason.ProactiveCopyOnPut) + operationContext => ProactiveCopyIfNeededAsync( + operationContext, + result.ContentHash, + tryBuildRing: true, + CopyReason.ProactiveCopyOnPut) ); if (Settings.InlineOperationsForTests) @@ -166,7 +1885,11 @@ namespace BuildXL.Cache.ContentStore.Distributed.Sessions else { // Tracing task-related errors because normal failures already traced by the operation provider - proactiveCopyTask.TraceIfFailure(context, failureSeverity: Severity.Debug, traceTaskExceptionsOnly: true, operation: "ProactiveCopyIfNeeded"); + proactiveCopyTask.TraceIfFailure( + context, + failureSeverity: Severity.Debug, + traceTaskExceptionsOnly: true, + operation: "ProactiveCopyIfNeeded"); } } @@ -185,7 +1908,7 @@ namespace BuildXL.Cache.ContentStore.Distributed.Sessions // is definitely available in the global store. urgencyHint = UrgencyHint.RegisterEagerly; } - + var updateResult = await ContentLocationStore.RegisterLocalLocationAsync( context, new[] { new ContentHashWithSize(putResult.ContentHash, putResult.ContentSize) }, diff --git a/Public/Src/Cache/ContentStore/Distributed/Sessions/ReadOnlyDistributedContentSession.cs b/Public/Src/Cache/ContentStore/Distributed/Sessions/ReadOnlyDistributedContentSession.cs deleted file mode 100644 index 401fffe49..000000000 --- a/Public/Src/Cache/ContentStore/Distributed/Sessions/ReadOnlyDistributedContentSession.cs +++ /dev/null @@ -1,1562 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -using System; -using System.Collections.Generic; -using System.Diagnostics.CodeAnalysis; -using System.Diagnostics.ContractsLight; -using System.IO; -using System.Linq; -using System.Threading; -using System.Threading.Tasks; -using BuildXL.Cache.ContentStore.Distributed.Stores; -using BuildXL.Cache.ContentStore.Distributed.Utilities; -using BuildXL.Cache.ContentStore.Extensions; -using BuildXL.Cache.ContentStore.Hashing; -using BuildXL.Cache.ContentStore.Interfaces.Distributed; -using BuildXL.Cache.ContentStore.Interfaces.Extensions; -using BuildXL.Cache.ContentStore.Interfaces.FileSystem; -using BuildXL.Cache.ContentStore.Interfaces.Results; -using BuildXL.Cache.ContentStore.Interfaces.Sessions; -using BuildXL.Cache.ContentStore.Interfaces.Stores; -using BuildXL.Cache.ContentStore.Interfaces.Time; -using BuildXL.Cache.ContentStore.Interfaces.Tracing; -using BuildXL.Cache.ContentStore.Interfaces.Utils; -using BuildXL.Cache.ContentStore.Service.Grpc; -using BuildXL.Cache.ContentStore.Sessions; -using BuildXL.Cache.ContentStore.Sessions.Internal; -using BuildXL.Cache.ContentStore.Tracing; -using BuildXL.Cache.ContentStore.Tracing.Internal; -using BuildXL.Cache.ContentStore.UtilitiesCore; -using BuildXL.Cache.ContentStore.Utils; -using BuildXL.Utilities.Collections; -using BuildXL.Utilities.ParallelAlgorithms; -using BuildXL.Utilities.Core.Tasks; -using BuildXL.Utilities.Tracing; -using ContentStore.Grpc; -using PlaceBulkResult = System.Collections.Generic.IEnumerable>>; - -#nullable enable - -namespace BuildXL.Cache.ContentStore.Distributed.Sessions -{ - /// - /// A read only content location based content session with an inner session for storage. - /// - public class ReadOnlyDistributedContentSession : ContentSessionBase, IHibernateContentSession, IConfigurablePin - { - internal enum Counters - { - GetLocationsSatisfiedFromLocal, - GetLocationsSatisfiedFromRemote, - PinUnverifiedCountSatisfied, - StartCopyForPinWhenUnverifiedCountSatisfied, - ProactiveCopiesSkipped, - ProactiveCopy_OutsideRingFromPreferredLocations, - ProactiveCopy_OutsideRingCopies, - ProactiveCopyRetries, - ProactiveCopyInsideRingRetries, - ProactiveCopyOutsideRingRetries, - ProactiveCopy_InsideRingCopies, - ProactiveCopy_InsideRingFullyReplicated, - } - - internal CounterCollection SessionCounters { get; } = new CounterCollection(); - - private string? _buildId = null; - private ContentHash? _buildIdHash = null; - private readonly ExpiringValue _buildRingMachinesCache; - - private MachineLocation[] BuildRingMachines => _buildRingMachinesCache.GetValueOrDefault() ?? Array.Empty(); - - private readonly ConcurrentBigSet _pendingProactivePuts = new ConcurrentBigSet(); - private ResultNagleQueue? _proactiveCopyGetBulkNagleQueue; - - // The method used for remote pins depends on which pin configuration is enabled. - private readonly RemotePinAsync _remotePinner; - - /// - /// The store that persists content locations to a persistent store. - /// - internal readonly IContentLocationStore ContentLocationStore; - - private readonly ColdStorage? _coldStorage; - - /// - /// The machine location for the current cache. - /// - protected readonly MachineLocation LocalCacheRootMachineLocation; - - /// - /// The content session that actually stores content. - /// - public IContentSession Inner { get; } - - /// - protected override Tracer Tracer { get; } = new Tracer(nameof(DistributedContentSession)); - - /// - protected readonly DistributedContentCopier DistributedCopier; - - /// - /// Settings for the session. - /// - protected readonly DistributedContentStoreSettings Settings; - - /// - /// Trace only stops and errors to reduce the Kusto traffic. - /// - protected override bool TraceOperationStarted => false; - - /// - /// Semaphore that limits the maximum number of concurrent put and place operations - /// - protected readonly SemaphoreSlim PutAndPlaceFileGate; - - private readonly DistributedContentStore _contentStore; - - /// - public ReadOnlyDistributedContentSession( - string name, - IContentSession inner, - IContentLocationStore contentLocationStore, - DistributedContentCopier contentCopier, - DistributedContentStore distributedContentStore, - MachineLocation localMachineLocation, - ColdStorage? coldStorage, - DistributedContentStoreSettings? settings = default) - : base(name) - { - Contract.Requires(name != null); - Contract.Requires(inner != null); - Contract.Requires(contentLocationStore != null); - Contract.Requires(contentLocationStore is StartupShutdownSlimBase, "The store must derive from StartupShutdownSlimBase"); - Contract.Requires(localMachineLocation.IsValid); - - Inner = inner; - ContentLocationStore = contentLocationStore; - LocalCacheRootMachineLocation = localMachineLocation; - Settings = settings ?? DistributedContentStoreSettings.DefaultSettings; - _contentStore = distributedContentStore; - _remotePinner = PinFromMultiLevelContentLocationStore; - DistributedCopier = contentCopier; - PutAndPlaceFileGate = new SemaphoreSlim(Settings.MaximumConcurrentPutAndPlaceFileOperations); - - _coldStorage = coldStorage; - - _buildRingMachinesCache = new ExpiringValue( - Settings.ProactiveCopyInRingMachineLocationsExpiryCache, - SystemClock.Instance, - originalValue: Array.Empty()); - } - - /// - protected override async Task StartupCoreAsync(OperationContext context) - { - var canHibernate = Inner is IHibernateContentSession ? "can" : "cannot"; - Tracer.Debug(context, $"Session {Name} {canHibernate} hibernate"); - await Inner.StartupAsync(context).ThrowIfFailure(); - - _proactiveCopyGetBulkNagleQueue = new ResultNagleQueue( - execute: hashes => GetLocationsForProactiveCopyAsync(context.CreateNested(Tracer.Name), hashes), - maxDegreeOfParallelism: 1, - interval: Settings.ProactiveCopyGetBulkInterval, - batchSize: Settings.ProactiveCopyGetBulkBatchSize); - _proactiveCopyGetBulkNagleQueue.Start(); - - TryRegisterMachineWithBuildId(context); - - return BoolResult.Success; - } - - private void TryRegisterMachineWithBuildId(OperationContext context) - { - if (Constants.TryExtractBuildId(Name, out _buildId) && Guid.TryParse(_buildId, out var buildIdGuid)) - { - // Generate a fake hash for the build and register a content entry in the location store to represent - // machines in the build ring - var buildIdContent = buildIdGuid.ToByteArray(); - - var buildIdHash = GetBuildIdHash(buildIdContent); - - var arguments = $"Build={_buildId}, BuildIdHash={buildIdHash.ToShortString()}"; - - context.PerformOperationAsync(Tracer, async () => - { - // Storing the build id in the cache to prevent this data to be removed during reconciliation. - using var stream = new MemoryStream(buildIdContent); - var result = await Inner.PutStreamAsync(context, buildIdHash, stream, CancellationToken.None, UrgencyHint.Nominal); - result.ThrowIfFailure(); - - // Even if 'StoreBuildIdInCache' is true we need to register the content manually, - // because Inner.PutStreamAsync will just add the content to the cache but won't send a registration message. - await ContentLocationStore.RegisterLocalLocationAsync(context, new[] { new ContentHashWithSize(buildIdHash, buildIdContent.Length) }, context.Token, UrgencyHint.Nominal).ThrowIfFailure(); - - // Updating the build id field only if the registration or the put operation succeeded. - _buildIdHash = buildIdHash; - - return BoolResult.Success; - }, - extraStartMessage: arguments, - extraEndMessage: r => arguments).FireAndForget(context); - } - } - - private static ContentHash GetBuildIdHash(byte[] buildId) => buildId.CalculateHash(HashType.MD5); - - /// - protected override async Task ShutdownCoreAsync(OperationContext context) - { - var counterSet = new CounterSet(); - counterSet.Merge(GetCounters(), $"{Tracer.Name}."); - - // Unregister from build machine location set - if (_buildIdHash.HasValue) - { - Guid.TryParse(_buildId, out var buildIdGuid); - var buildIdHash = GetBuildIdHash(buildIdGuid.ToByteArray()); - Tracer.Debug(context, $"Deleting in-ring mapping from cache: Build={_buildId}, BuildIdHash={buildIdHash.ToShortString()}"); - - // DeleteAsync will unregister the content as well. No need for calling 'TrimBulkAsync'. - await _contentStore.DeleteAsync(context, _buildIdHash.Value, new DeleteContentOptions() { DeleteLocalOnly = true }) - .IgnoreErrorsAndReturnCompletion(); - } - - await Inner.ShutdownAsync(context).ThrowIfFailure(); - - _proactiveCopyGetBulkNagleQueue?.Dispose(); - Tracer.TraceStatisticsAtShutdown(context, counterSet, prefix: "DistributedContentSessionStats"); - - return BoolResult.Success; - } - - /// - protected override void DisposeCore() - { - base.DisposeCore(); - - Inner.Dispose(); - } - - /// - protected override async Task PinCoreAsync( - OperationContext operationContext, - ContentHash contentHash, - UrgencyHint urgencyHint, - Counter retryCounter) - { - // Call bulk API - var result = await PinHelperAsync(operationContext, new[] { contentHash }, urgencyHint, PinOperationConfiguration.Default()); - return (await result.First()).Item; - } - - /// - protected override async Task OpenStreamCoreAsync( - OperationContext operationContext, - ContentHash contentHash, - UrgencyHint urgencyHint, - Counter retryCounter) - { - OpenStreamResult streamResult = - await Inner.OpenStreamAsync(operationContext, contentHash, operationContext.Token, urgencyHint); - if (streamResult.Code == OpenStreamResult.ResultCode.Success) - { - return streamResult; - } - - var contentRegistrationUrgencyHint = urgencyHint; - - long? size = null; - GetBulkLocationsResult? localGetBulkResult = null; - - // First try to fetch file based on locally stored locations for the hash - // Then fallback to fetching file based on global locations minus the locally stored locations which were already checked - foreach (var getBulkTask in ContentLocationStore.MultiLevelGetLocations(operationContext, new[] { contentHash }, operationContext.Token, urgencyHint, subtractLocalResults: true)) - { - var getBulkResult = await getBulkTask; - // There is an issue with GetBulkLocationsResult construction from Exception that may loose the information about the origin. - // So we rely on the result order of MultiLevelGetLocations method: the first result is always local and the second one is global. - - GetBulkOrigin origin = localGetBulkResult == null ? GetBulkOrigin.Local : GetBulkOrigin.Global; - if (origin == GetBulkOrigin.Local) - { - localGetBulkResult = getBulkResult; - } - - // Local function: Use content locations for GetBulk to copy file locally - async Task tryCopyContentLocalAsync() - { - if (!getBulkResult || !getBulkResult.ContentHashesInfo.Any()) - { - return new BoolResult($"Metadata records for hash {contentHash.ToShortString()} not found in content location store."); - } - - // Don't reconsider locally stored results that were checked in prior iteration - getBulkResult = getBulkResult.Subtract(localGetBulkResult); - - var hashInfo = getBulkResult.ContentHashesInfo.Single(); - - if (!CanCopyContentHash(operationContext, hashInfo, isGlobal: getBulkResult.Origin == GetBulkOrigin.Global, out var useInRingLocations, out var errorMessage)) - { - return new BoolResult(errorMessage); - } - - // Using in-ring machines if configured - var copyResult = await TryCopyAndPutAsync(operationContext, hashInfo, urgencyHint, CopyReason.OpenStream, trace: false, useInRingLocations); - if (!copyResult) - { - return new BoolResult(copyResult); - } - - size = copyResult.ContentSize; - - // If configured, register the content eagerly on put to make sure the content is discoverable by the other builders, - // even if the current machine used to have the content and evicted it recently. - if (Settings.RegisterEagerlyOnPut && !copyResult.ContentAlreadyExistsInCache) - { - contentRegistrationUrgencyHint = UrgencyHint.RegisterEagerly; - } - - return BoolResult.Success; - } - - var copyLocalResult = await tryCopyContentLocalAsync(); - - // Throw operation canceled to avoid operations below which are not value for canceled case. - operationContext.Token.ThrowIfCancellationRequested(); - - if (copyLocalResult.Succeeded) - { - // Succeeded in copying content locally. No need to try with more content locations - break; - } - else if (origin == GetBulkOrigin.Global) - { - return new OpenStreamResult(copyLocalResult, OpenStreamResult.ResultCode.ContentNotFound); - } - } - - Contract.Assert(size != null, "Size should be set if operation succeeded"); - - var updateResult = await UpdateContentTrackerWithNewReplicaAsync(operationContext, new[] { new ContentHashWithSize(contentHash, size.Value) }, contentRegistrationUrgencyHint); - if (!updateResult.Succeeded) - { - return new OpenStreamResult(updateResult); - } - - return await Inner.OpenStreamAsync(operationContext, contentHash, operationContext.Token, urgencyHint); - } - - /// - Task>>> IConfigurablePin.PinAsync(Context context, IReadOnlyList contentHashes, PinOperationConfiguration pinOperationConfiguration) - { - // The lifetime of this operation should be detached from the lifetime of the session. - // But still tracking the lifetime of the store. - return WithStoreCancellationAsync(context, - opContext => PinHelperAsync(opContext, contentHashes, pinOperationConfiguration.UrgencyHint, pinOperationConfiguration), - pinOperationConfiguration.CancellationToken); - } - - /// - protected override Task>>> PinCoreAsync(OperationContext operationContext, IReadOnlyList contentHashes, UrgencyHint urgencyHint, Counter retryCounter, Counter fileCounter) - { - return PinHelperAsync(operationContext, contentHashes, urgencyHint, PinOperationConfiguration.Default()); - } - - private async Task>>> PinHelperAsync(OperationContext operationContext, IReadOnlyList contentHashes, UrgencyHint urgencyHint, PinOperationConfiguration pinOperationConfiguration) - { - Contract.Requires(contentHashes != null); - - IEnumerable>>? pinResults = null; - - IEnumerable>>? intermediateResult = null; - if (pinOperationConfiguration.ReturnGlobalExistenceFast) - { - Tracer.Debug(operationContext.TracingContext, $"Detected {nameof(PinOperationConfiguration.ReturnGlobalExistenceFast)}"); - - // Check globally for existence, but do not copy locally and do not update content tracker. - pinResults = await Workflows.RunWithFallback( - contentHashes, - async hashes => - { - intermediateResult = await Inner.PinAsync(operationContext, hashes, operationContext.Token, urgencyHint); - return intermediateResult; - }, - hashes => _remotePinner(operationContext, hashes, succeedWithOneLocation: true, urgencyHint), - result => result.Succeeded); - - // Replace operation context with a new cancellation token so it can outlast this call. - // Using a cancellation token from the store avoid the operations lifetime to be greater than the lifetime of the outer store. - operationContext = new OperationContext(operationContext.TracingContext, token: StoreShutdownStartedCancellationToken); - } - - // Default pin action - var pinTask = Workflows.RunWithFallback( - contentHashes, - hashes => intermediateResult == null - ? Inner.PinAsync(operationContext, hashes, operationContext.Token, urgencyHint) - : Task.FromResult(intermediateResult), - hashes => _remotePinner(operationContext, hashes, succeedWithOneLocation: false, urgencyHint), - result => result.Succeeded, - // Exclude the empty hash because it is a special case which is hard coded for place/openstream/pin. - async hits => await UpdateContentTrackerWithLocalHitsAsync( - operationContext, - hits.Where(x => !contentHashes[x.Index].IsEmptyHash()).Select( - x => new ContentHashWithSizeAndLastAccessTime(contentHashes[x.Index], x.Item.ContentSize, x.Item.LastAccessTime)).ToList(), - urgencyHint)); - - // Initiate a proactive copy if just pinned content is under-replicated - if (Settings.ProactiveCopyOnPin && Settings.ProactiveCopyMode != ProactiveCopyMode.Disabled) - { - pinTask = ProactiveCopyOnPinAsync(operationContext, contentHashes, pinTask); - } - - if (pinOperationConfiguration.ReturnGlobalExistenceFast) - { - // Fire off the default pin action, but do not await the result. - // - // Creating a new OperationContext instance without existing 'CancellationToken', - // because the operation we triggered that stored in 'pinTask' can outlive the lifetime of the current instance. - // And we don't want the PerformNonResultOperationAsync to fail because the current instance is shut down (or disposed). - new OperationContext(operationContext.TracingContext).PerformNonResultOperationAsync( - Tracer, - () => pinTask, - extraEndMessage: results => - { - var resultString = string.Join(",", results.Select(async task => - { - // Since all bulk operations are constructed with Task.FromResult, it is safe to just access the result; - Indexed result = await task; - return result != null ? $"{contentHashes[result.Index].ToShortString()}:{result.Item}" : string.Empty; - })); - - return $"ConfigurablePin Count={contentHashes.Count}, Hashes=[{resultString}]"; - }, - traceErrorsOnly: TraceErrorsOnly, - traceOperationStarted: TraceOperationStarted, - traceOperationFinished: true, - isCritical: false).FireAndForget(operationContext); - } - else - { - pinResults = await pinTask; - } - - Contract.Assert(pinResults != null); - return pinResults; - } - - private async Task>>> ProactiveCopyOnPinAsync(OperationContext context, IReadOnlyList contentHashes, Task>>> pinTask) - { - var results = await pinTask; - - // Since the rest of the operation is done asynchronously, using a cancellation context from the outer store. - var proactiveCopyTask = WithStoreCancellationAsync( - context, - async opContext => - { - var proactiveTasks = results.Select(resultTask => proactiveCopyOnSinglePinAsync(opContext, resultTask)).ToList(); - - // Ensure all tasks are completed, after awaiting outer task - await Task.WhenAll(proactiveTasks); - - return proactiveTasks; - }); - - if (Settings.InlineOperationsForTests) - { - return await proactiveCopyTask; - } - else - { - proactiveCopyTask.FireAndForget(context); - return results; - } - - // Proactive copy an individual pin - async Task> proactiveCopyOnSinglePinAsync(OperationContext opContext, Task> resultTask) - { - Indexed indexedPinResult = await resultTask; - var pinResult = indexedPinResult.Item; - - // Local pins and distributed pins which are copied locally allow proactive copy - if (pinResult.Succeeded && (!(pinResult is DistributedPinResult distributedPinResult) || distributedPinResult.CopyLocally)) - { - var proactiveCopyResult = await ProactiveCopyIfNeededAsync(opContext, contentHashes[indexedPinResult.Index], tryBuildRing: true, CopyReason.ProactiveCopyOnPin); - - // Only fail if all copies failed. - if (!proactiveCopyResult.Succeeded) - { - return new PinResult(proactiveCopyResult).WithIndex(indexedPinResult.Index); - } - } - - return indexedPinResult; - } - } - - /// - protected override async Task PlaceFileCoreAsync( - OperationContext operationContext, - ContentHash contentHash, - AbsolutePath path, - FileAccessMode accessMode, - FileReplacementMode replacementMode, - FileRealizationMode realizationMode, - UrgencyHint urgencyHint, - Counter retryCounter) - { - var resultWithData = await PerformPlaceFileGatedOperationAsync( - operationContext, - () => PlaceHelperAsync( - operationContext, - new[] { new ContentHashWithPath(contentHash, path) }, - accessMode, - replacementMode, - realizationMode, - urgencyHint - )); - - var result = await resultWithData.Result.SingleAwaitIndexed(); - result.Metadata = resultWithData.Metadata; - return result; - } - - /// - protected override async Task PlaceFileCoreAsync( - OperationContext operationContext, - IReadOnlyList hashesWithPaths, - FileAccessMode accessMode, - FileReplacementMode replacementMode, - FileRealizationMode realizationMode, - UrgencyHint urgencyHint, - Counter retryCounter) - { - // The fallback is invoked for cache misses only. This preserves existing behavior of - // bubbling up errors with Inner store instead of trying remote. - var resultWithData = await PerformPlaceFileGatedOperationAsync( - operationContext, - () => PlaceHelperAsync( - operationContext, - hashesWithPaths, - accessMode, - replacementMode, - realizationMode, - urgencyHint - )); - - // We are tracing here because we did not want to change the signature for PlaceFileCoreAsync, which is implemented in multiple locations - operationContext.TracingContext.Debug( - $"PlaceFileBulk, Gate.OccupiedCount={resultWithData.Metadata.GateOccupiedCount} Gate.Wait={resultWithData.Metadata.GateWaitTime.TotalMilliseconds}ms Hashes.Count={hashesWithPaths.Count}", - component: nameof(ReadOnlyDistributedContentSession), - operation: "PlaceFileBulk"); - - return resultWithData.Result; - } - - private Task PlaceHelperAsync( - OperationContext operationContext, - IReadOnlyList hashesWithPaths, - FileAccessMode accessMode, - FileReplacementMode replacementMode, - FileRealizationMode realizationMode, - UrgencyHint urgencyHint) - { - return Workflows.RunWithFallback( - hashesWithPaths, - args => Inner.PlaceFileAsync( - operationContext, - args, - accessMode, - replacementMode, - realizationMode, - operationContext.Token, - urgencyHint), - args => fetchFromMultiLevelContentLocationStoreThenPlaceFileAsync(args), - result => IsPlaceFileSuccess(result), - async hits => await UpdateContentTrackerWithLocalHitsAsync( - operationContext, - hits.Select( - x => new ContentHashWithSizeAndLastAccessTime( - hashesWithPaths[x.Index].Hash, - x.Item.FileSize, - x.Item.LastAccessTime)) - .ToList(), - urgencyHint)); - - Task fetchFromMultiLevelContentLocationStoreThenPlaceFileAsync(IReadOnlyList fetchedContentInfo) - { - return MultiLevelUtilities.RunMultiLevelAsync( - fetchedContentInfo, - runFirstLevelAsync: args => FetchFromMultiLevelContentLocationStoreThenPutAsync(operationContext, args, urgencyHint, CopyReason.Place), - runSecondLevelAsync: args => innerPlaceAsync(args), - // NOTE: We just use the first level result if the fetch using content location store fails because the place cannot succeed since the - // content will not have been put into the local CAS - useFirstLevelResult: result => !IsPlaceFileSuccess(result)); - } - - async Task innerPlaceAsync(IReadOnlyList input) - { - // When the content is obtained from remote we still have to place it from the local. - // So in order to return the right source of the file placement - // we have to place locally but change the source for each result. - var results = await Inner.PlaceFileAsync(operationContext, input, accessMode, replacementMode, realizationMode, operationContext.Token, urgencyHint); - var updatedResults = new List>(); - foreach (var resultTask in results) - { - var result = await resultTask; - updatedResults.Add( - result.Item - .WithMaterializationSource(PlaceFileResult.Source.DatacenterCache) - .WithIndex(result.Index)); - } - - return updatedResults.AsTasks(); - } - } - - private Task> PerformPlaceFileGatedOperationAsync(OperationContext operationContext, Func> func) - { - return PutAndPlaceFileGate.GatedOperationAsync(async (timeWaiting, currentCount) => - { - var gateOccupiedCount = Settings.MaximumConcurrentPutAndPlaceFileOperations - currentCount; - - var result = await func(); - - return new ResultWithMetaData( - new ResultMetaData(timeWaiting, gateOccupiedCount), - result); - }, operationContext.Token); - } - - private static bool IsPlaceFileSuccess(PlaceFileResult result) - { - return result.Code != PlaceFileResult.ResultCode.Error && result.Code != PlaceFileResult.ResultCode.NotPlacedContentNotFound; - } - - /// - public IEnumerable EnumeratePinnedContentHashes() - { - return Inner is IHibernateContentSession session - ? session.EnumeratePinnedContentHashes() - : Enumerable.Empty(); - } - - /// - public Task PinBulkAsync(Context context, IEnumerable contentHashes) - { - // TODO: Replace PinBulkAsync in hibernate with PinAsync bulk call (bug 1365340) - return Inner is IHibernateContentSession session - ? session.PinBulkAsync(context, contentHashes) - : Task.FromResult(0); - } - - /// - public Task ShutdownEvictionAsync(Context context) - { - return Inner is IHibernateContentSession session - ? session.ShutdownEvictionAsync(context) - : BoolResult.SuccessTask; - } - - private Task FetchFromMultiLevelContentLocationStoreThenPutAsync( - OperationContext context, - IReadOnlyList hashesWithPaths, - UrgencyHint urgencyHint, - CopyReason reason) - { - // First try to place file by fetching files based on locally stored locations for the hash - // Then fallback to fetching file based on global locations minus the locally stored locations which were already checked - - var localGetBulkResult = new BuildXL.Utilities.AsyncOut(); - - GetIndexedResults initialFunc = async args => - { - var contentHashes = args.Select(p => p.Hash).ToList(); - localGetBulkResult.Value = await ContentLocationStore.GetBulkAsync(context, contentHashes, context.Token, urgencyHint, GetBulkOrigin.Local); - return await FetchFromContentLocationStoreThenPutAsync(context, args, GetBulkOrigin.Local, urgencyHint, localGetBulkResult.Value, reason); - }; - - GetIndexedResults fallbackFunc = async args => - { - var contentHashes = args.Select(p => p.Hash).ToList(); - var globalGetBulkResult = await ContentLocationStore.GetBulkAsync(context, contentHashes, context.Token, urgencyHint, GetBulkOrigin.Global); - globalGetBulkResult = globalGetBulkResult.Subtract(localGetBulkResult.Value); - return await FetchFromContentLocationStoreThenPutAsync(context, args, GetBulkOrigin.Global, urgencyHint, globalGetBulkResult, reason); - }; - - // If ColdStorage is ON try to place files from it before use remote locations - if (_coldStorage != null) - { - return Workflows.RunWithFallback( - hashesWithPaths, - initialFunc: async args => - { - return await _coldStorage.FetchThenPutBulkAsync(context, args, Inner); - }, - fallbackFunc: initialFunc, - secondFallbackFunc: fallbackFunc, - thirdFallbackFunc: async args => - { - var coldStorageGetBulkResult = _coldStorage.GetBulkLocations(context, args); - return await FetchFromContentLocationStoreThenPutAsync(context, args, GetBulkOrigin.ColdStorage, urgencyHint, coldStorageGetBulkResult, reason); - }, - isSuccessFunc: result => IsPlaceFileSuccess(result)); - } - - return Workflows.RunWithFallback( - hashesWithPaths, - initialFunc: initialFunc, - fallbackFunc: fallbackFunc, - isSuccessFunc: result => IsPlaceFileSuccess(result)); - } - - private async Task FetchFromContentLocationStoreThenPutAsync( - OperationContext context, - IReadOnlyList hashesWithPaths, - GetBulkOrigin origin, - UrgencyHint urgencyHint, - GetBulkLocationsResult getBulkResult, - CopyReason reason) - { - try - { - // Tracing the hashes here for the entire list, instead of tracing one hash at a time inside TryCopyAndPutAsync method. - - // This returns failure if any item in the batch wasn't copied locally - // TODO: split results and call PlaceFile on successfully copied files (bug 1365340) - if (!getBulkResult.Succeeded || !getBulkResult.ContentHashesInfo.Any()) - { - return hashesWithPaths.Select( - _ => new PlaceFileResult( - getBulkResult, - PlaceFileResult.ResultCode.NotPlacedContentNotFound, - "Metadata records not found in content location store")) - .AsIndexedTasks(); - } - - async Task> copyAndPut(ContentHashWithSizeAndLocations contentHashWithSizeAndLocations, int index) - { - PlaceFileResult result; - try - { - if (!CanCopyContentHash(context, contentHashWithSizeAndLocations, isGlobal: origin == GetBulkOrigin.Global, out var useInRingMachineLocations, out var errorMessage)) - { - result = PlaceFileResult.CreateContentNotFound(errorMessage); - } - else - { - var copyResult = await TryCopyAndPutAsync( - context, - contentHashWithSizeAndLocations, - urgencyHint, - reason, - // We just traced all the hashes as a result of GetBulk call, no need to trace each individual hash. - trace: false, - // Using in-ring locations as well if the feature is on. - useInRingMachineLocations: useInRingMachineLocations, - outputPath: hashesWithPaths[index].Path); - - if (!copyResult) - { - // For ColdStorage we should treat all errors as cache misses - result = origin != GetBulkOrigin.ColdStorage ? new PlaceFileResult(copyResult) : new PlaceFileResult(copyResult, PlaceFileResult.ResultCode.NotPlacedContentNotFound); - } - else - { - var source = origin == GetBulkOrigin.ColdStorage - ? PlaceFileResult.Source.ColdStorage - : PlaceFileResult.Source.DatacenterCache; - result = PlaceFileResult.CreateSuccess(PlaceFileResult.ResultCode.PlacedWithMove, copyResult.ContentSize, source); - } - } - } - catch (Exception e) - { - // The transform block should not fail with an exception otherwise the block's state will be changed to failed state and the exception - // won't be propagated to the caller. - result = new PlaceFileResult(e); - } - - return result.WithIndex(index); - } - - var copyFilesLocallyActionQueue = new ActionQueue(Settings.ParallelCopyFilesLimit); - var copyFilesLocally = await copyFilesLocallyActionQueue.SelectAsync( - items: getBulkResult.ContentHashesInfo, - body: (item, index) => copyAndPut(item, index)); - - var updateResults = await UpdateContentTrackerWithNewReplicaAsync( - context, - copyFilesLocally.Where(r => r.Item.Succeeded).Select(r => new ContentHashWithSize(hashesWithPaths[r.Index].Hash, r.Item.FileSize)).ToList(), - urgencyHint); - - if (!updateResults.Succeeded) - { - return copyFilesLocally.Select(result => new PlaceFileResult(updateResults).WithIndex(result.Index)).AsTasks(); - } - - return copyFilesLocally.AsTasks(); - } - catch (Exception ex) - { - return hashesWithPaths.Select((hash, index) => new PlaceFileResult(ex).WithIndex(index)).AsTasks(); - } - } - - private bool CanCopyContentHash(Context context, ContentHashWithSizeAndLocations result, bool isGlobal, out bool useInRingMachineLocations, [NotNullWhen(false)]out string? message) - { - useInRingMachineLocations = isGlobal && Settings.UseInRingMachinesForCopies; - if (!isLocationsAvailable(out message)) - { - if (useInRingMachineLocations && GetInRingActiveMachines().Length != 0) - { - string useInRingLocationsMessage = $", but {nameof(Settings.UseInRingMachinesForCopies)} is true. Trying to copy the content from in-ring machines."; - Tracer.Debug(context, message + useInRingLocationsMessage); - // Still trying copying the content from in-ring machines, even though the location information is lacking. - return true; - } - - // Tracing only for global locations because they come last. - if (isGlobal) - { - Tracer.Warning(context, message); - } - - return false; - } - - return true; - - bool isLocationsAvailable([NotNullWhen(false)] out string? errorMessage) - { - errorMessage = null; - // Null represents no replicas were ever registered, where as empty list implies content is missing from all replicas - if (result.Locations == null) - { - errorMessage = $"No replicas registered for hash {result.ContentHash.ToShortString()}"; - return false; - } - - if (!result.Locations.Any()) - { - errorMessage = $"No replicas currently exist in content tracker for hash {result.ContentHash.ToShortString()}"; - return false; - } - - return true; - } - } - - private async Task TryCopyAndPutAsync(OperationContext operationContext, ContentHashWithSizeAndLocations hashInfo, UrgencyHint urgencyHint, CopyReason reason, bool trace, bool useInRingMachineLocations = false, AbsolutePath? outputPath = null) - { - Context context = operationContext; - CancellationToken cts = operationContext.Token; - - if (trace) - { - Tracer.Debug(operationContext, $"Copying {hashInfo.ContentHash.ToShortString()} with {hashInfo.Locations?.Count ?? 0 } locations"); - } - - var copyCompression = CopyCompression.None; - var copyCompressionThreshold = Settings.GrpcCopyCompressionSizeThreshold ?? 0; - if (copyCompressionThreshold > 0 && hashInfo.Size > copyCompressionThreshold) - { - copyCompression = Settings.GrpcCopyCompressionAlgorithm; - } - - var copyRequest = new DistributedContentCopier.CopyRequest( - _contentStore, - hashInfo, - reason, - HandleCopyAsync: async args => - { - (CopyFileResult copyFileResult, AbsolutePath tempLocation, _) = args; - - PutResult innerPutResult; - long actualSize = copyFileResult.Size ?? hashInfo.Size; - if (Settings.UseTrustedHash(actualSize) && Inner is ITrustedContentSession trustedInner) - { - // The file has already been hashed, so we can trust the hash of the file. - innerPutResult = await trustedInner.PutTrustedFileAsync( - context, - new ContentHashWithSize(hashInfo.ContentHash, actualSize), - tempLocation, - FileRealizationMode.Move, - cts, - urgencyHint); - } - else - { - // Pass the HashType, not the Hash. This prompts a re-hash of the file, which places it where its actual hash requires. - // If the actual hash differs from the expected hash, then we fail below and move to the next location. - // Also, record the bytes if the file is small enough to be put into the ContentLocationStore. - innerPutResult = await Inner.PutFileAsync( - context, - hashInfo.ContentHash.HashType, - tempLocation, - FileRealizationMode.Move, - cts, - urgencyHint); - } - - return innerPutResult; - }, - copyCompression, - OverrideWorkingFolder: (Inner as ITrustedContentSession)?.TryGetWorkingDirectory(outputPath)); - - if (useInRingMachineLocations) - { - copyRequest = copyRequest with { InRingMachines = GetInRingActiveMachines() }; - } - - var putResult = await DistributedCopier.TryCopyAndPutAsync(operationContext, copyRequest); - - return putResult; - } - - /// - /// Runs a given function in the cancellation context of the outer store. - /// - /// - /// The lifetime of some session-based operation is longer than the session itself. - /// For instance, proactive copies, or put blob can outlive the lifetime of the session. - /// But these operations should not outlive the lifetime of the store, because when the store is closed - /// most likely all the operations will fail with some weird errors like "ObjectDisposedException". - /// - /// This helper method allows running the operations that may outlive the lifetime of the session but should not outlive the lifetime of the store. - /// - protected Task WithStoreCancellationAsync(Context context, Func> func, CancellationToken token = default) - { - return ((StartupShutdownSlimBase)ContentLocationStore).WithOperationContext(context, token, func); - } - - /// - /// Returns a cancellation token that is triggered when the outer store shutdown is started. - /// - protected CancellationToken StoreShutdownStartedCancellationToken => - ((StartupShutdownSlimBase)ContentLocationStore).ShutdownStartedCancellationToken; - - private Task UpdateContentTrackerWithNewReplicaAsync(OperationContext context, IReadOnlyList contentHashes, UrgencyHint urgencyHint) - { - if (contentHashes.Count == 0) - { - return BoolResult.SuccessTask; - } - - // TODO: Pass location store option (seems to only be used to prevent updating TTL when replicating for proactive replication) (bug 1365340) - return ContentLocationStore.RegisterLocalLocationAsync(context, contentHashes, context.Token, urgencyHint); - } - - private Task>>> PinFromMultiLevelContentLocationStore( - OperationContext context, - IReadOnlyList contentHashes, - bool succeedWithOneLocation, - UrgencyHint urgencyHint = UrgencyHint.Nominal) - { - // Pinning a content based on a number of location is inherently dangerous in case of eviction storms in a system. - // The LLS data might be stale because of event processing delay and the global information can be inaccurate because - // the trim events are not sent to the global store. - // We can't do anything when the LLS data is stale, but in some stamps its beneficial not to rely on the global loctions. - if (Settings.PinConfiguration.UseLocalLocationsOnlyOnUnverifiedPin) - { - return PinFromContentLocationStoreOriginAsync( - context, - contentHashes, - GetBulkOrigin.Local, - succeedWithOneLocation: succeedWithOneLocation, - urgencyHint); - } - - return Workflows.RunWithFallback( - contentHashes, - hashes => PinFromContentLocationStoreOriginAsync(context, hashes, GetBulkOrigin.Local, succeedWithOneLocation: succeedWithOneLocation, urgencyHint), - hashes => PinFromContentLocationStoreOriginAsync(context, hashes, GetBulkOrigin.Global, succeedWithOneLocation: succeedWithOneLocation, urgencyHint), - result => result.Succeeded); - } - - // This method creates pages of hashes, makes one bulk call to the content location store to get content location record sets for all the hashes on the page, - // and fires off processing of the returned content location record sets while proceeding to the next page of hashes in parallel. - private async Task>>> PinFromContentLocationStoreOriginAsync( - OperationContext operationContext, IReadOnlyList hashes, GetBulkOrigin origin, bool succeedWithOneLocation, UrgencyHint urgency = UrgencyHint.Nominal) - { - CancellationToken cancel = operationContext.Token; - // Create an action block to process all the requested remote pins while limiting the number of simultaneously executed. - var pinnings = new List(hashes.Count); - var pinningAction = ActionBlockSlim.CreateWithAsyncAction( - degreeOfParallelism: Settings.PinConfiguration?.MaxIOOperations ?? 1, - async pinning => await PinRemoteAsync( - operationContext, - pinning, - isLocal: origin == GetBulkOrigin.Local, - updateContentTracker: false, - succeedWithOneLocation: succeedWithOneLocation), - cancellationToken: cancel); - - // Make a bulk call to content location store to get location records for all hashes on the page. - // NOTE: We use GetBulkStackedAsync so that when Global results are retrieved we also include Local results to ensure we get a full view of available content - GetBulkLocationsResult pageLookup = await ContentLocationStore.GetBulkStackedAsync(operationContext, hashes, cancel, urgency, origin); - - // If successful, fire off the remote pinning logic for each hash. If not, set all pins to failed. - if (pageLookup.Succeeded) - { - foreach (ContentHashWithSizeAndLocations record in pageLookup.ContentHashesInfo) - { - RemotePinning pinning = new RemotePinning(record); - pinnings.Add(pinning); - await pinningAction.PostAsync(pinning, cancel); - } - } - else - { - foreach (ContentHash hash in hashes) - { - Tracer.Warning(operationContext, $"Pin failed for hash {hash.ToShortString()}: directory query failed with error {pageLookup.ErrorMessage}"); - RemotePinning pinning = new RemotePinning(new ContentHashWithSizeAndLocations(hash, -1L)) - { - Result = new PinResult(pageLookup) - }; - pinnings.Add(pinning); - } - } - - Contract.Assert(pinnings.Count == hashes.Count); - - // Wait for all the pinning actions to complete. - pinningAction.Complete(); - - try - { - await pinningAction.Completion; - } - catch (TaskCanceledException) - { - // Cancellation token provided to an action block can be canceled. - // Ignoring the exception in this case. - } - - // Inform the content directory that we copied the files. - // Looking for distributed pin results that were successful by copying the content locally. - var localCopies = pinnings.Select((rp, index) => (result: rp, index)).Where(x => x.result.Result is DistributedPinResult dpr && dpr.CopyLocally).ToList(); - - BoolResult updated = await UpdateContentTrackerWithNewReplicaAsync(operationContext, localCopies.Select(lc => new ContentHashWithSize(lc.result.Record.ContentHash, lc.result.Record.Size)).ToList(), UrgencyHint.Nominal); - if (!updated.Succeeded) - { - // We failed to update the tracker. Need to update the results. - string hashesAsString = string.Join(", ", localCopies.Select(lc => lc.result.Record.ContentHash.ToShortString())); - Tracer.Warning(operationContext, $"Pin failed for hashes {hashesAsString}: local copy succeeded, but could not inform content directory due to {updated.ErrorMessage}."); - foreach (var (_, index) in localCopies) - { - pinnings[index].Result = new PinResult(updated); - } - } - - // The return type should probably be just Task>, but higher callers require the Indexed wrapper and that the PinResults be encased in Tasks. - return pinnings.Select(x => x.Result ?? createCanceledPutResult()).AsIndexed().AsTasks(); - - static PinResult createCanceledPutResult() => new ErrorResult("The operation was canceled").AsResult(); - } - - // The dataflow framework can process only a single object, and returns no output from that processing. By combining the input and output of each remote pinning into a single object, - // we can nonetheless use the dataflow framework to process pinnings and read the output from the updated objects afterward. - private class RemotePinning - { - public ContentHashWithSizeAndLocations Record { get; } - - private PinResult? _result; - public PinResult? Result - { - get => _result; - set - { - value!.ContentSize = Record.Size; - _result = value; - } - } - - public RemotePinning(ContentHashWithSizeAndLocations record) - => Record = record; - } - - // This method processes each remote pinning, setting the output when the operation is completed. - private async Task PinRemoteAsync( - OperationContext context, - RemotePinning pinning, - bool isLocal, - bool updateContentTracker = true, - bool succeedWithOneLocation = false) - { - pinning.Result = await PinRemoteAsync(context, pinning.Record, isLocal, updateContentTracker, succeedWithOneLocation: succeedWithOneLocation); - } - - // This method processes a single content location record set for pinning. - private async Task PinRemoteAsync( - OperationContext operationContext, - ContentHashWithSizeAndLocations remote, - bool isLocal, - bool updateContentTracker = true, - bool succeedWithOneLocation = false) - { - IReadOnlyList? locations = remote.Locations; - - // If no remote locations are recorded, we definitely can't pin - if (locations == null || locations.Count == 0) - { - if (!isLocal) - { - // Trace only when pin failed based on the data from the global store. - Tracer.Warning(operationContext, $"Pin failed for hash {remote.ContentHash.ToShortString()}: no remote records."); - } - - return DistributedPinResult.ContentNotFound(replicaCount: 0, "No locations found"); - } - - // When we only require the content to exist at least once anywhere, we can ignore pin thresholds - // and return success after finding a single location. - if (succeedWithOneLocation && locations.Count >= 1) - { - return DistributedPinResult.EnoughReplicas(locations.Count, "Global succeeds"); - } - - if (locations.Count >= Settings.PinConfiguration.PinMinUnverifiedCount) - { - SessionCounters[Counters.PinUnverifiedCountSatisfied].Increment(); - - // Tracing extra data if the locations were merged to separate the local locations and the global locations that we added to the final result. - string? extraMessage = null; - if (!isLocal) - { - // Extra locations does make sense only for the global case when the entries were merged. - extraMessage = $"ExtraGlobal: {remote.ExtraMergedLocations}"; - } - - var result = DistributedPinResult.EnoughReplicas(locations.Count, extraMessage); - - // Triggering an async copy if the number of replicas are close to a PinMinUnverifiedCount threshold. - int threshold = Settings.PinConfiguration.PinMinUnverifiedCount + - Settings.PinConfiguration.AsyncCopyOnPinThreshold; - if (locations.Count < threshold) - { - Tracer.Info(operationContext, $"Starting asynchronous copy of the content for hash {remote.ContentHash.ToShortString()} because the number of locations '{locations.Count}' is less then a threshold of '{threshold}'."); - SessionCounters[Counters.StartCopyForPinWhenUnverifiedCountSatisfied].Increment(); - - // For "synchronous" pins the tracker is updated at once for all the hashes for performance reasons, - // but for asynchronous copy we always need to update the tracker with a new location. - var task = WithStoreCancellationAsync( - operationContext.TracingContext, - opContext => TryCopyAndPutAndUpdateContentTrackerAsync(opContext, remote, updateContentTracker: true, CopyReason.AsyncCopyOnPin)); - if (Settings.InlineOperationsForTests) - { - (await task).TraceIfFailure(operationContext); - } - else - { - task.FireAndForget(operationContext.TracingContext, traceErrorResult: true, operation: "AsynchronousCopyOnPin"); - } - - // Note: Pin result traces with CpA (copied asynchronously) code is to provide the information that the content is being copied asynchronously, and that replica count is enough but not above async copy threshold. - // This trace result does not represent that of the async copy since that is done FireAndForget. - result = DistributedPinResult.AsynchronousCopy(locations.Count); - } - - return result; - } - - if (isLocal) - { - // Don't copy content locally based on locally cached result. So stop here and return content not found. - // This method will be called again with global locations at which time we will attempt to copy the files locally. - // When allowing global locations to succeed a put, report success. - return DistributedPinResult.ContentNotFound(locations.Count); - } - - // Previous checks were not sufficient, so copy the file locally. - PutResult copy = await TryCopyAndPutAsync(operationContext, remote, UrgencyHint.Nominal, CopyReason.Pin, trace: false); - if (copy) - { - if (!updateContentTracker) - { - return DistributedPinResult.SynchronousCopy(locations.Count); - } - - // Inform the content directory that we have the file. - // We wait for this to complete, rather than doing it fire-and-forget, because another machine in the ring may need the pinned content immediately. - BoolResult updated = await UpdateContentTrackerWithNewReplicaAsync(operationContext, new[] { new ContentHashWithSize(remote.ContentHash, copy.ContentSize) }, UrgencyHint.Nominal); - if (updated.Succeeded) - { - return DistributedPinResult.SynchronousCopy(locations.Count); - } - else - { - // Tracing the error separately. - Tracer.Warning(operationContext, $"Pin failed for hash {remote.ContentHash.ToShortString()}: local copy succeeded, but could not inform content directory due to {updated.ErrorMessage}."); - return new DistributedPinResult(locations.Count, updated); - } - } - else - { - // Tracing the error separately. - Tracer.Warning(operationContext, $"Pin failed for hash {remote.ContentHash.ToShortString()}: local copy failed with {copy}."); - return DistributedPinResult.ContentNotFound(locations.Count); - } - } - - private async Task TryCopyAndPutAndUpdateContentTrackerAsync( - OperationContext operationContext, - ContentHashWithSizeAndLocations remote, - bool updateContentTracker, - CopyReason reason) - { - PutResult copy = await TryCopyAndPutAsync(operationContext, remote, UrgencyHint.Nominal, reason, trace: true); - if (copy && updateContentTracker) - { - return await UpdateContentTrackerWithNewReplicaAsync(operationContext, new[] { new ContentHashWithSize(remote.ContentHash, copy.ContentSize) }, UrgencyHint.Nominal); - } - - return copy; - } - - private Task UpdateContentTrackerWithLocalHitsAsync(OperationContext context, IReadOnlyList contentHashesWithInfo, UrgencyHint urgencyHint) - { - if (Disposed) - { - // Nothing to do. - return BoolTask.True; - } - - if (contentHashesWithInfo.Count == 0) - { - // Nothing to do. - return BoolTask.True; - } - - IReadOnlyList hashesToEagerUpdate = contentHashesWithInfo.Select(x => new ContentHashWithSize(x.Hash, x.Size)).ToList(); - - // Wait for update to complete on remaining hashes to cover case where the record has expired and another machine in the ring requests it immediately after this pin succeeds. - return UpdateContentTrackerWithNewReplicaAsync(context, hashesToEagerUpdate, urgencyHint); - } - - internal async Task> GetLocationsForProactiveCopyAsync( - OperationContext context, - IReadOnlyList hashes) - { - var originalLength = hashes.Count; - if (_buildIdHash.HasValue && !_buildRingMachinesCache.IsUpToDate()) - { - Tracer.Debug(context, $"{Tracer.Name}.{nameof(GetLocationsForProactiveCopyAsync)}: getting in-ring machines for BuildId='{_buildId}'."); - // Add build id hash to hashes so build ring machines can be updated - hashes = hashes.AppendItem(_buildIdHash.Value).ToList(); - } - - var result = await MultiLevelUtilities.RunMultiLevelWithMergeAsync( - hashes, - inputs => ContentLocationStore.GetBulkAsync(context, inputs, context.Token, UrgencyHint.Nominal, GetBulkOrigin.Local).ThrowIfFailureAsync(g => g.ContentHashesInfo), - inputs => ContentLocationStore.GetBulkAsync(context, inputs, context.Token, UrgencyHint.Nominal, GetBulkOrigin.Global).ThrowIfFailureAsync(g => g.ContentHashesInfo), - mergeResults: ContentHashWithSizeAndLocations.Merge, - useFirstLevelResult: result => - { - if (result.Locations?.Count >= Settings.ProactiveCopyLocationsThreshold) - { - SessionCounters[Counters.GetLocationsSatisfiedFromLocal].Increment(); - return true; - } - else - { - SessionCounters[Counters.GetLocationsSatisfiedFromRemote].Increment(); - return false; - } - }); - - if (hashes.Count != originalLength) - { - // Update build ring machines with retrieved locations - var buildRingMachines = result.Last().Locations?.AppendItem(LocalCacheRootMachineLocation).ToArray() ?? CollectionUtilities.EmptyArray(); - _buildRingMachinesCache.Update(buildRingMachines); - Tracer.Debug(context, $"{Tracer.Name}.{nameof(GetLocationsForProactiveCopyAsync)}: InRingMachines=[{string.Join(", ", buildRingMachines.Select(m => m.Path))}] BuildId='{_buildId}'"); - return result.Take(originalLength).ToList(); - } - else - { - return result; - } - } - - internal async Task ProactiveCopyIfNeededAsync( - OperationContext context, - ContentHash hash, - bool tryBuildRing, - CopyReason reason) - { - var nagleQueue = _proactiveCopyGetBulkNagleQueue; - if (nagleQueue is null) - { - return new ProactiveCopyResult(new ErrorResult("StartupAsync was not called")); - } - - ContentHashWithSizeAndLocations result = await nagleQueue.EnqueueAsync(hash); - return await ProactiveCopyIfNeededAsync(context, result, tryBuildRing, reason); - } - - internal Task ProactiveCopyIfNeededAsync( - OperationContext context, - ContentHashWithSizeAndLocations info, - bool tryBuildRing, - CopyReason reason) - { - var hash = info.ContentHash; - if (!_pendingProactivePuts.Add(hash) - || info.ContentHash.IsEmptyHash()) // No reason to push an empty hash to another machine. - { - return Task.FromResult(ProactiveCopyResult.CopyNotRequiredResult); - } - - // Don't trace this case since it would add too much log traffic. - var replicatedLocations = (info.Locations ?? CollectionUtilities.EmptyArray()).ToList(); - - if (replicatedLocations.Count >= Settings.ProactiveCopyLocationsThreshold) - { - SessionCounters[Counters.ProactiveCopiesSkipped].Increment(); - return Task.FromResult(ProactiveCopyResult.CopyNotRequiredResult); - } - - // By adding the master to replicatedLocations, it will be excluded from proactive replication - var masterLocation = _contentStore.LocalLocationStore?.MasterElectionMechanism.Master; - if (masterLocation is not null && masterLocation.Value.IsValid) - { - replicatedLocations.Add(masterLocation.Value); - } - - return context.PerformOperationAsync( - Tracer, - operation: async () => - { - try - { - var outsideRingCopyTask = ProactiveCopyOutsideBuildRingWithRetryAsync(context, info, replicatedLocations, reason); - var insideRingCopyTask = ProactiveCopyInsideBuildRingWithRetryAsync(context, info, tryBuildRing, replicatedLocations, reason); - await Task.WhenAll(outsideRingCopyTask, insideRingCopyTask); - - var (insideRingRetries, insideRingResult) = await insideRingCopyTask; - var (outsideRingRetries, outsideRingResult) = await outsideRingCopyTask; - - int totalRetries = insideRingRetries + outsideRingRetries; - return new ProactiveCopyResult(insideRingResult, outsideRingResult, totalRetries, info.Entry); - } - finally - { - _pendingProactivePuts.Remove(hash); - } - }, - extraEndMessage: r => $"Hash={info.ContentHash}, Retries={r.TotalRetries}, Reason=[{reason}]"); - } - - private async Task<(int retries, ProactivePushResult outsideRingCopyResult)> ProactiveCopyOutsideBuildRingWithRetryAsync( - OperationContext context, - ContentHashWithSize hash, - IReadOnlyList replicatedLocations, - CopyReason reason) - { - var outsideRingCopyResult = await ProactiveCopyOutsideBuildRingAsync(context, hash, replicatedLocations, reason, retries: 0); - int retries = 0; - while (outsideRingCopyResult.QualifiesForRetry && retries < Settings.ProactiveCopyMaxRetries) - { - SessionCounters[Counters.ProactiveCopyRetries].Increment(); - SessionCounters[Counters.ProactiveCopyOutsideRingRetries].Increment(); - retries++; - outsideRingCopyResult = await ProactiveCopyOutsideBuildRingAsync(context, hash, replicatedLocations, reason, retries); - } - return (retries, outsideRingCopyResult); - } - - private async Task ProactiveCopyOutsideBuildRingAsync( - OperationContext context, - ContentHashWithSize hash, - IReadOnlyList replicatedLocations, - CopyReason reason, - int retries) - { - // The first attempt is not considered as retry - int attempt = retries + 1; - if ((Settings.ProactiveCopyMode & ProactiveCopyMode.OutsideRing) == 0) - { - return ProactivePushResult.FromPushFileResult(PushFileResult.Disabled(), attempt); - } - - Result? getLocationResult = null; - var source = ProactiveCopyLocationSource.Random; - - // Make sure that the machine is not in the build ring and does not already have the content. - var machinesToSkip = replicatedLocations.Concat(BuildRingMachines).ToArray(); - - // Try to select one of the designated machines for this hash. - if (Settings.ProactiveCopyUsePreferredLocations) - { - var designatedLocationsResult = ContentLocationStore.GetDesignatedLocations(hash); - if (designatedLocationsResult.Succeeded) - { - // A machine in the build may be a designated location for the hash, - // but we won't pushing to the same machine twice, because 'replicatedLocations' argument - // has a local machine that we're about to push for inside the ring copy. - // We also want to skip inside ring machines for outsidecopy task - var candidates = designatedLocationsResult.Value - .Except(machinesToSkip).ToArray(); - - if (candidates.Length > 0) - { - getLocationResult = candidates[ThreadSafeRandom.Generator.Next(0, candidates.Length)]; - source = ProactiveCopyLocationSource.DesignatedLocation; - SessionCounters[Counters.ProactiveCopy_OutsideRingFromPreferredLocations].Increment(); - } - } - } - - // Try to select one machine at random. - if (getLocationResult?.Succeeded != true) - { - getLocationResult = ContentLocationStore.GetRandomMachineLocation(except: machinesToSkip); - source = ProactiveCopyLocationSource.Random; - } - - if (!getLocationResult.Succeeded) - { - return ProactivePushResult.FromPushFileResult(new PushFileResult(getLocationResult), attempt); - } - var candidate = getLocationResult.Value; - SessionCounters[Counters.ProactiveCopy_OutsideRingCopies].Increment(); - PushFileResult pushFileResult = await PushContentAsync(context, hash, candidate, isInsideRing: false, reason, source, attempt); - return ProactivePushResult.FromPushFileResult(pushFileResult, attempt); - } - - /// - /// Gets all the active in-ring machines (excluding the current one). - /// - public MachineLocation[] GetInRingActiveMachines() - { - return BuildRingMachines - .Where(m => !m.Equals(LocalCacheRootMachineLocation)) - .Where(m => ContentLocationStore.IsMachineActive(m)) - .ToArray(); - } - - private async Task<(int retries, ProactivePushResult insideRingCopyResult)> ProactiveCopyInsideBuildRingWithRetryAsync( - OperationContext context, - ContentHashWithSize hash, - bool tryBuildRing, - IReadOnlyList replicatedLocations, - CopyReason reason) - { - ProactivePushResult insideRingCopyResult = await ProactiveCopyInsideBuildRing(context, hash, tryBuildRing, replicatedLocations, reason, retries: 0); - int retries = 0; - while (insideRingCopyResult.QualifiesForRetry && retries < Settings.ProactiveCopyMaxRetries) - { - SessionCounters[Counters.ProactiveCopyRetries].Increment(); - SessionCounters[Counters.ProactiveCopyInsideRingRetries].Increment(); - retries++; - insideRingCopyResult = await ProactiveCopyInsideBuildRing(context, hash, tryBuildRing, replicatedLocations, reason, retries); - } - return (retries, insideRingCopyResult); - } - - private async Task ProactiveCopyInsideBuildRing( - OperationContext context, - ContentHashWithSize hash, - bool tryBuildRing, - IReadOnlyList replicatedLocations, - CopyReason reason, - int retries) - { - // The first attempt is not considered as retry - int attempt = retries + 1; - - // Get random machine inside build ring - if (!tryBuildRing || (Settings.ProactiveCopyMode & ProactiveCopyMode.InsideRing) == 0) - { - return ProactivePushResult.FromPushFileResult(PushFileResult.Disabled(), attempt); - } - - if (_buildIdHash == null) - { - return ProactivePushResult.FromStatus(ProactivePushStatus.BuildIdNotSpecified, attempt); - } - - // Having an explicit case to check if the in-ring machine list is empty to separate the case - // when the machine list is not empty but all the candidates are unavailable. - if (BuildRingMachines.Length == 0) - { - return ProactivePushResult.FromStatus(ProactivePushStatus.InRingMachineListIsEmpty, attempt); - } - - var candidates = GetInRingActiveMachines(); - - if (candidates.Length == 0) - { - return ProactivePushResult.FromStatus(ProactivePushStatus.MachineNotFound, attempt); - } - - candidates = candidates.Except(replicatedLocations).ToArray(); - if (candidates.Length == 0) - { - SessionCounters[Counters.ProactiveCopy_InsideRingFullyReplicated].Increment(); - return ProactivePushResult.FromStatus(ProactivePushStatus.MachineAlreadyHasCopy, attempt); - } - - SessionCounters[Counters.ProactiveCopy_InsideRingCopies].Increment(); - var candidate = candidates[ThreadSafeRandom.Generator.Next(0, candidates.Length)]; - PushFileResult pushFileResult = await PushContentAsync(context, hash, candidate, isInsideRing: true, reason, ProactiveCopyLocationSource.Random, attempt); - return ProactivePushResult.FromPushFileResult(pushFileResult, attempt); - } - - private async Task PushContentAsync( - OperationContext context, - ContentHashWithSize hash, - MachineLocation target, - bool isInsideRing, - CopyReason reason, - ProactiveCopyLocationSource source, - int attempt) - { - // This is here to avoid hanging ProactiveCopyIfNeededAsync on inside/outside ring copies before starting - // the other one. - await Task.Yield(); - - if (Settings.PushProactiveCopies) - { - // It is possible that this method is used during proactive replication - // and the hash was already evicted at the time this method is called. - var streamResult = await Inner.OpenStreamAsync(context, hash, context.Token); - if (!streamResult.Succeeded) - { - return PushFileResult.SkipContentUnavailable(); - } - - using var stream = streamResult.Stream!; - - return await DistributedCopier.PushFileAsync( - context, - hash, - target, - stream, - isInsideRing, - reason, - source, - attempt); - } - else - { - var requestResult = await DistributedCopier.RequestCopyFileAsync(context, hash, target, isInsideRing, attempt); - if (requestResult) - { - return PushFileResult.PushSucceeded(size: null); - } - - return new PushFileResult(requestResult, "Failed requesting a copy"); - } - } - - /// - protected override CounterSet GetCounters() => - base.GetCounters() - .Merge(DistributedCopier.GetCounters()) - .Merge(SessionCounters.ToCounterSet()); - } -} diff --git a/Public/Src/Cache/ContentStore/Distributed/Sessions/RemotePinAsync.cs b/Public/Src/Cache/ContentStore/Distributed/Sessions/RemotePinAsync.cs index aa69a72f8..9cb285452 100644 --- a/Public/Src/Cache/ContentStore/Distributed/Sessions/RemotePinAsync.cs +++ b/Public/Src/Cache/ContentStore/Distributed/Sessions/RemotePinAsync.cs @@ -12,7 +12,7 @@ using BuildXL.Cache.ContentStore.Tracing.Internal; namespace BuildXL.Cache.ContentStore.Distributed.Sessions { /// - /// Performs remote pinning for + /// Performs remote pinning for /// public delegate Task>>> RemotePinAsync( OperationContext context, diff --git a/Public/Src/Cache/ContentStore/Distributed/Stores/DistributedContentStore.cs b/Public/Src/Cache/ContentStore/Distributed/Stores/DistributedContentStore.cs index d0a9bc09f..551f01ade 100644 --- a/Public/Src/Cache/ContentStore/Distributed/Stores/DistributedContentStore.cs +++ b/Public/Src/Cache/ContentStore/Distributed/Stores/DistributedContentStore.cs @@ -99,8 +99,8 @@ namespace BuildXL.Cache.ContentStore.Distributed.Stores private readonly DistributedContentCopier _distributedCopier; private readonly DisposableDirectory _copierWorkingDirectory; - private Lazy>>? _proactiveCopySession; - internal Lazy>> ProactiveCopySession => NotNull(_proactiveCopySession, nameof(_proactiveCopySession)); + private Lazy>>? _proactiveCopySession; + internal Lazy>> ProactiveCopySession => NotNull(_proactiveCopySession, nameof(_proactiveCopySession)); private readonly DistributedContentSettings? _distributedContentSettings; @@ -226,7 +226,7 @@ namespace BuildXL.Cache.ContentStore.Distributed.Stores return $"Origin=[{info.Origin}] IsPresentInGcs=[{isPresentInGcs}]"; } - private Task> CreateCopySession(Context context) + private Task> CreateCopySession(Context context) { var sessionId = Guid.NewGuid().ToString(); @@ -235,11 +235,11 @@ namespace BuildXL.Cache.ContentStore.Distributed.Stores async () => { // NOTE: We use ImplicitPin.None so that the OpenStream calls triggered by RequestCopy will only pull the content, NOT pin it in the local store. - var sessionResult = CreateReadOnlySession(operationContext, $"{sessionId}-DefaultCopy", ImplicitPin.None).ThrowIfFailure(); + var sessionResult = CreateSession(operationContext, $"{sessionId}-DefaultCopy", ImplicitPin.None).ThrowIfFailure(); var session = sessionResult.Session!; await session.StartupAsync(context).ThrowIfFailure(); - return Result.Success((ReadOnlyDistributedContentSession)session); + return Result.Success((DistributedContentSession)session); }); } @@ -250,7 +250,7 @@ namespace BuildXL.Cache.ContentStore.Distributed.Stores var startupTask = base.StartupAsync(context); - _proactiveCopySession = new Lazy>>(() => CreateCopySession(context)); + _proactiveCopySession = new Lazy>>(() => CreateCopySession(context)); if (_settings.SetPostInitializationCompletionAfterStartup) { @@ -324,7 +324,7 @@ namespace BuildXL.Cache.ContentStore.Distributed.Stores private Task ProactiveReplicationIterationAsync( OperationContext context, - ReadOnlyDistributedContentSession proactiveCopySession, + DistributedContentSession proactiveCopySession, ILocalContentStore localContentStore, TransitioningContentLocationStore contentLocationStore) { @@ -476,31 +476,6 @@ namespace BuildXL.Cache.ContentStore.Distributed.Stores return BoolResult.Success; } - /// - public CreateSessionResult CreateReadOnlySession(Context context, string name, ImplicitPin implicitPin) - { - return CreateReadOnlySessionCall.Run(_tracer, OperationContext(context), name, () => - { - CreateSessionResult innerSessionResult = InnerContentStore.CreateSession(context, name, implicitPin); - - if (innerSessionResult.Succeeded) - { - var session = new ReadOnlyDistributedContentSession( - name, - innerSessionResult.Session, - ContentLocationStore, - _distributedCopier, - this, - LocalMachineLocation, - ColdStorage, - settings: _settings); - return new CreateSessionResult(session); - } - - return new CreateSessionResult(innerSessionResult, "Could not initialize inner content session with error"); - }); - } - /// public CreateSessionResult CreateSession(Context context, string name, ImplicitPin implicitPin) { @@ -513,7 +488,7 @@ namespace BuildXL.Cache.ContentStore.Distributed.Stores var session = new DistributedContentSession( name, innerSessionResult.Session, - _contentLocationStore, + ContentLocationStore, _distributedCopier, this, LocalMachineLocation, diff --git a/Public/Src/Cache/ContentStore/DistributedTest/ContentLocation/LocalLocationStoreDistributedContentTests.cs b/Public/Src/Cache/ContentStore/DistributedTest/ContentLocation/LocalLocationStoreDistributedContentTests.cs index 1f92cfe5e..07c8e4622 100644 --- a/Public/Src/Cache/ContentStore/DistributedTest/ContentLocation/LocalLocationStoreDistributedContentTests.cs +++ b/Public/Src/Cache/ContentStore/DistributedTest/ContentLocation/LocalLocationStoreDistributedContentTests.cs @@ -3,6 +3,7 @@ using System; using System.Collections.Generic; +using System.Diagnostics; using System.IO; using System.Linq; using System.Threading; diff --git a/Public/Src/Cache/ContentStore/DistributedTest/ContentLocation/ProactiveCopyTests.cs b/Public/Src/Cache/ContentStore/DistributedTest/ContentLocation/ProactiveCopyTests.cs index da7706a65..94a991f0e 100644 --- a/Public/Src/Cache/ContentStore/DistributedTest/ContentLocation/ProactiveCopyTests.cs +++ b/Public/Src/Cache/ContentStore/DistributedTest/ContentLocation/ProactiveCopyTests.cs @@ -20,7 +20,7 @@ using BuildXL.Cache.ContentStore.Service.Grpc; using FluentAssertions; using Xunit; -using static BuildXL.Cache.ContentStore.Distributed.Sessions.ReadOnlyDistributedContentSession.Counters; +using static BuildXL.Cache.ContentStore.Distributed.Sessions.DistributedContentSession.Counters; namespace ContentStoreTest.Distributed.Sessions { diff --git a/Public/Src/Cache/ContentStore/DistributedTest/Sessions/DistributedContentSessionTests.cs b/Public/Src/Cache/ContentStore/DistributedTest/Sessions/DistributedContentSessionTests.cs index d81c9ba00..1537e1ba6 100644 --- a/Public/Src/Cache/ContentStore/DistributedTest/Sessions/DistributedContentSessionTests.cs +++ b/Public/Src/Cache/ContentStore/DistributedTest/Sessions/DistributedContentSessionTests.cs @@ -75,7 +75,7 @@ namespace ContentStoreTest.Distributed.Sessions }, implicitPin); } - protected override Task RunReadOnlyTestAsync(ImplicitPin implicitPin, Func funcAsync) + protected override Task RunReadOnlyTestAsync(ImplicitPin implicitPin, Func funcAsync) { return RunTestAsync(implicitPin, null, (ctx, session) => funcAsync(ctx, session)); } diff --git a/Public/Src/Cache/ContentStore/DistributedTest/Sessions/DistributedContentTests.cs b/Public/Src/Cache/ContentStore/DistributedTest/Sessions/DistributedContentTests.cs index 76755bbec..95804eaf0 100644 --- a/Public/Src/Cache/ContentStore/DistributedTest/Sessions/DistributedContentTests.cs +++ b/Public/Src/Cache/ContentStore/DistributedTest/Sessions/DistributedContentTests.cs @@ -1009,7 +1009,7 @@ namespace ContentStoreTest.Distributed.Sessions } protected async Task OpenStreamReturnsExpectedFile( - IReadOnlyContentSession session, Context context, ContentHash hash, byte[] expected) + IContentSession session, Context context, ContentHash hash, byte[] expected) { OpenStreamResult openResult = await session.OpenStreamAsync(context, hash, Token); _tracer.Debug(context, $"Validating stream for content hash {hash} returned result {openResult.Code} with diagnostics {openResult} with ErrorMessage {openResult.ErrorMessage} diagnostics {openResult.Diagnostics}"); diff --git a/Public/Src/Cache/ContentStore/DistributedTest/Stores/AzureBlobStorageContentSessionTests.cs b/Public/Src/Cache/ContentStore/DistributedTest/Stores/AzureBlobStorageContentSessionTests.cs index 038fc8c88..644179523 100644 --- a/Public/Src/Cache/ContentStore/DistributedTest/Stores/AzureBlobStorageContentSessionTests.cs +++ b/Public/Src/Cache/ContentStore/DistributedTest/Stores/AzureBlobStorageContentSessionTests.cs @@ -52,7 +52,7 @@ namespace BuildXL.Cache.ContentStore.Distributed.Test throw new NotImplementedException(); } - protected override async Task RunReadOnlyTestAsync(ImplicitPin implicitPin, Func funcAsync) + protected override async Task RunReadOnlyTestAsync(ImplicitPin implicitPin, Func funcAsync) { using var directory = new DisposableDirectory(FileSystem); diff --git a/Public/Src/Cache/ContentStore/GrpcTest/TestHangingContentStore.cs b/Public/Src/Cache/ContentStore/GrpcTest/TestHangingContentStore.cs index 180eda036..fe43a0fa9 100644 --- a/Public/Src/Cache/ContentStore/GrpcTest/TestHangingContentStore.cs +++ b/Public/Src/Cache/ContentStore/GrpcTest/TestHangingContentStore.cs @@ -50,12 +50,6 @@ namespace ContentStoreTest.Grpc return Task.FromResult(BoolResult.Success); } - [SuppressMessage("Microsoft.Reliability", "CA2000:Dispose objects before losing scope")] - public CreateSessionResult CreateReadOnlySession(Context context, string name, ImplicitPin implicitPin) - { - return new CreateSessionResult(new TestHangingContentSession(name, _useCancellationToken, _hangHasStartedSemaphore)); - } - [SuppressMessage("Microsoft.Reliability", "CA2000:Dispose objects before losing scope")] public CreateSessionResult CreateSession(Context context, string name, ImplicitPin implicitPin) { diff --git a/Public/Src/Cache/ContentStore/GrpcTest/TestTempFileDeletingContentStore.cs b/Public/Src/Cache/ContentStore/GrpcTest/TestTempFileDeletingContentStore.cs index 388af544f..e92584aaf 100644 --- a/Public/Src/Cache/ContentStore/GrpcTest/TestTempFileDeletingContentStore.cs +++ b/Public/Src/Cache/ContentStore/GrpcTest/TestTempFileDeletingContentStore.cs @@ -48,13 +48,6 @@ namespace ContentStoreTest.Grpc return Task.FromResult(BoolResult.Success); } - [SuppressMessage("Microsoft.Reliability", "CA2000:Dispose objects before losing scope")] - public CreateSessionResult CreateReadOnlySession(Context context, string name, ImplicitPin implicitPin) - { - return new CreateSessionResult(new TestTempFileDeletingContentSession(name, _fileSystem)); - } - - [SuppressMessage("Microsoft.Reliability", "CA2000:Dispose objects before losing scope")] public CreateSessionResult CreateSession(Context context, string name, ImplicitPin implicitPin) { return new CreateSessionResult(new TestTempFileDeletingContentSession(name, _fileSystem)); diff --git a/Public/Src/Cache/ContentStore/Interfaces/Sessions/IContentSession.cs b/Public/Src/Cache/ContentStore/Interfaces/Sessions/IContentSession.cs index 5c69900f3..3d57d137c 100644 --- a/Public/Src/Cache/ContentStore/Interfaces/Sessions/IContentSession.cs +++ b/Public/Src/Cache/ContentStore/Interfaces/Sessions/IContentSession.cs @@ -1,12 +1,14 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. +using System.Collections.Generic; using System.IO; using System.Threading; using System.Threading.Tasks; using BuildXL.Cache.ContentStore.Hashing; using BuildXL.Cache.ContentStore.Interfaces.FileSystem; using BuildXL.Cache.ContentStore.Interfaces.Results; +using BuildXL.Cache.ContentStore.Interfaces.Stores; using BuildXL.Cache.ContentStore.Interfaces.Tracing; // ReSharper disable UnusedParameter.Global @@ -15,8 +17,60 @@ namespace BuildXL.Cache.ContentStore.Interfaces.Sessions /// /// A related set of accesses to a content store. /// - public interface IContentSession : IReadOnlyContentSession + public interface IContentSession : IName, IStartupShutdown, IConfigurablePin { + /// + /// Ensure content does not get deleted. + /// + Task PinAsync( + Context context, + ContentHash contentHash, + CancellationToken cts, + UrgencyHint urgencyHint = UrgencyHint.Nominal); + + /// + /// Open a stream to content. + /// + Task OpenStreamAsync( + Context context, + ContentHash contentHash, + CancellationToken cts, + UrgencyHint urgencyHint = UrgencyHint.Nominal); + + /// + /// Materialize content to the filesystem. + /// + Task PlaceFileAsync( + Context context, + ContentHash contentHash, + AbsolutePath path, + FileAccessMode accessMode, + FileReplacementMode replacementMode, + FileRealizationMode realizationMode, + CancellationToken cts, + UrgencyHint urgencyHint = UrgencyHint.Nominal); + + /// + /// Ensure content does not get deleted in bulk. + /// + Task>>> PinAsync( + Context context, + IReadOnlyList contentHashes, + CancellationToken cts, + UrgencyHint urgencyHint = UrgencyHint.Nominal); + + /// + /// Materialize content to the filesystem in bulk. + /// + Task>>> PlaceFileAsync( + Context context, + IReadOnlyList hashesWithPaths, + FileAccessMode accessMode, + FileReplacementMode replacementMode, + FileRealizationMode realizationMode, + CancellationToken cts, + UrgencyHint urgencyHint = UrgencyHint.Nominal); + /// /// Add content from a file. /// diff --git a/Public/Src/Cache/ContentStore/Interfaces/Sessions/IReadOnlyContentSession.cs b/Public/Src/Cache/ContentStore/Interfaces/Sessions/IReadOnlyContentSession.cs deleted file mode 100644 index 49369a64d..000000000 --- a/Public/Src/Cache/ContentStore/Interfaces/Sessions/IReadOnlyContentSession.cs +++ /dev/null @@ -1,72 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -using System.Collections.Generic; -using System.Threading; -using System.Threading.Tasks; -using BuildXL.Cache.ContentStore.Hashing; -using BuildXL.Cache.ContentStore.Interfaces.FileSystem; -using BuildXL.Cache.ContentStore.Interfaces.Results; -using BuildXL.Cache.ContentStore.Interfaces.Stores; -using BuildXL.Cache.ContentStore.Interfaces.Tracing; - -namespace BuildXL.Cache.ContentStore.Interfaces.Sessions -{ - /// - /// A related set of read accesses to a content store. - /// - public interface IReadOnlyContentSession : IName, IStartupShutdown, IConfigurablePin - { - /// - /// Ensure content does not get deleted. - /// - Task PinAsync( - Context context, - ContentHash contentHash, - CancellationToken cts, - UrgencyHint urgencyHint = UrgencyHint.Nominal); - - /// - /// Open a stream to content. - /// - Task OpenStreamAsync( - Context context, - ContentHash contentHash, - CancellationToken cts, - UrgencyHint urgencyHint = UrgencyHint.Nominal); - - /// - /// Materialize content to the filesystem. - /// - Task PlaceFileAsync( - Context context, - ContentHash contentHash, - AbsolutePath path, - FileAccessMode accessMode, - FileReplacementMode replacementMode, - FileRealizationMode realizationMode, - CancellationToken cts, - UrgencyHint urgencyHint = UrgencyHint.Nominal); - - /// - /// Ensure content does not get deleted in bulk. - /// - Task>>> PinAsync( - Context context, - IReadOnlyList contentHashes, - CancellationToken cts, - UrgencyHint urgencyHint = UrgencyHint.Nominal); - - /// - /// Materialize content to the filesystem in bulk. - /// - Task>>> PlaceFileAsync( - Context context, - IReadOnlyList hashesWithPaths, - FileAccessMode accessMode, - FileReplacementMode replacementMode, - FileRealizationMode realizationMode, - CancellationToken cts, - UrgencyHint urgencyHint = UrgencyHint.Nominal); - } -} diff --git a/Public/Src/Cache/ContentStore/Interfaces/Stores/IContentStore.cs b/Public/Src/Cache/ContentStore/Interfaces/Stores/IContentStore.cs index c71f46213..11fd6a512 100644 --- a/Public/Src/Cache/ContentStore/Interfaces/Stores/IContentStore.cs +++ b/Public/Src/Cache/ContentStore/Interfaces/Stores/IContentStore.cs @@ -26,11 +26,6 @@ namespace BuildXL.Cache.ContentStore.Interfaces.Stores /// public interface IContentStore : IStartupShutdown { - /// - /// Create a new session that can only read. - /// - CreateSessionResult CreateReadOnlySession(Context context, string name, ImplicitPin implicitPin); - /// /// Create a new session that can add content as well as read. /// diff --git a/Public/Src/Cache/ContentStore/InterfacesTest/Sessions/ContentSessionTestsBase.cs b/Public/Src/Cache/ContentStore/InterfacesTest/Sessions/ContentSessionTestsBase.cs index 86ef5e1ed..c26f9698a 100644 --- a/Public/Src/Cache/ContentStore/InterfacesTest/Sessions/ContentSessionTestsBase.cs +++ b/Public/Src/Cache/ContentStore/InterfacesTest/Sessions/ContentSessionTestsBase.cs @@ -63,7 +63,7 @@ namespace BuildXL.Cache.ContentStore.InterfacesTest.Sessions protected abstract IContentStore CreateStore(DisposableDirectory testDirectory, ContentStoreConfiguration configuration); - protected virtual async Task RunReadOnlyTestAsync(ImplicitPin implicitPin, Func funcAsync) + protected virtual async Task RunReadOnlyTestAsync(ImplicitPin implicitPin, Func funcAsync) { var context = new Context(Logger); using (var directory = new DisposableDirectory(FileSystem)) @@ -76,7 +76,7 @@ namespace BuildXL.Cache.ContentStore.InterfacesTest.Sessions { await store.StartupAsync(context).ShouldBeSuccess(); - var createResult = store.CreateReadOnlySession(context, Name, implicitPin).ShouldBeSuccess(); + var createResult = store.CreateSession(context, Name, implicitPin).ShouldBeSuccess(); using (var session = createResult.Session) { try diff --git a/Public/Src/Cache/ContentStore/Library/Sessions/FileSystemContentSession.cs b/Public/Src/Cache/ContentStore/Library/Sessions/FileSystemContentSession.cs index 4bde19218..7e458906c 100644 --- a/Public/Src/Cache/ContentStore/Library/Sessions/FileSystemContentSession.cs +++ b/Public/Src/Cache/ContentStore/Library/Sessions/FileSystemContentSession.cs @@ -2,10 +2,13 @@ // Licensed under the MIT License. using System; +using System.Collections.Generic; using System.IO; +using System.Linq; using System.Threading; using System.Threading.Tasks; using BuildXL.Cache.ContentStore.Hashing; +using BuildXL.Cache.ContentStore.Interfaces.Extensions; using BuildXL.Cache.ContentStore.Interfaces.FileSystem; using BuildXL.Cache.ContentStore.Interfaces.Results; using BuildXL.Cache.ContentStore.Interfaces.Sessions; @@ -13,22 +16,165 @@ using BuildXL.Cache.ContentStore.Interfaces.Stores; using BuildXL.Cache.ContentStore.Interfaces.Tracing; using BuildXL.Cache.ContentStore.Sessions.Internal; using BuildXL.Cache.ContentStore.Stores; +using BuildXL.Cache.ContentStore.Tracing; using BuildXL.Cache.ContentStore.Tracing.Internal; using BuildXL.Utilities.Tracing; namespace BuildXL.Cache.ContentStore.Sessions { - /// - /// An implemented over a - /// - public class FileSystemContentSession : ReadOnlyFileSystemContentSession, ITrustedContentSession + public class FileSystemContentSession : ContentSessionBase, ITrustedContentSession, IHibernateContentSession { /// - /// Initializes a new instance of the class. + /// The internal content store backing the session. /// - public FileSystemContentSession(string name, ImplicitPin implicitPin, FileSystemContentStoreInternal store) - : base(name, store, implicitPin) + protected readonly FileSystemContentStoreInternal Store; + + private readonly PinContext _pinContext; + private readonly ImplicitPin _implicitPin; + + /// + protected override Tracer Tracer { get; } = new Tracer(nameof(FileSystemContentSession)); + + /// + protected override bool TraceErrorsOnly => true; // This type adds nothing in terms of tracing. So configure it to trace errors only. + + /// + /// Initializes a new instance of the class. + /// + public FileSystemContentSession(string name, FileSystemContentStoreInternal store, ImplicitPin implicitPin) + : base(name) { + Store = store; + _pinContext = Store.CreatePinContext(); + _implicitPin = implicitPin; + } + + /// + protected override async Task ShutdownCoreAsync(OperationContext operationContext) + { + await _pinContext.DisposeAsync(); + + var statsResult = await Store.GetStatsAsync(operationContext); + if (statsResult.Succeeded) + { + Tracer.TraceStatisticsAtShutdown(operationContext, statsResult.CounterSet, prefix: "FileSystemContentSessionStats"); + } + + return BoolResult.Success; + } + + /// + protected override void DisposeCore() + { + base.DisposeCore(); + _pinContext.Dispose(); + } + + /// + protected override Task PinCoreAsync( + OperationContext operationContext, + ContentHash contentHash, + UrgencyHint urgencyHint, + Counter retryCounter) + { + return Store.PinAsync(operationContext, contentHash, _pinContext); + } + + /// + protected override Task OpenStreamCoreAsync( + OperationContext operationContext, + ContentHash contentHash, + UrgencyHint urgencyHint, + Counter retryCounter) + { + return Store.OpenStreamAsync(operationContext, contentHash, MakePinRequest(ImplicitPin.Get)); + } + + /// + protected override Task PlaceFileCoreAsync( + OperationContext operationContext, + ContentHash contentHash, + AbsolutePath path, + FileAccessMode accessMode, + FileReplacementMode replacementMode, + FileRealizationMode realizationMode, + UrgencyHint urgencyHint, + Counter retryCounter) + { + return Store.PlaceFileAsync( + operationContext, + contentHash, + path, + accessMode, + replacementMode, + realizationMode, + MakePinRequest(ImplicitPin.Put)); + } + + /// + protected override async Task>>> PinCoreAsync( + OperationContext operationContext, + IReadOnlyList contentHashes, + UrgencyHint urgencyHint, + Counter retryCounter, + Counter fileCounter) + { + return EnumerableExtensions.AsTasks>( + (await Store.PinAsync(operationContext, contentHashes, _pinContext, options: null))); + } + + /// + protected override Task>>> PlaceFileCoreAsync( + OperationContext operationContext, + IReadOnlyList hashesWithPaths, + FileAccessMode accessMode, + FileReplacementMode replacementMode, + FileRealizationMode realizationMode, + UrgencyHint urgencyHint, + Counter retryCounter) + { + return Store.PlaceFileAsync( + operationContext, + hashesWithPaths, + accessMode, + replacementMode, + realizationMode, + MakePinRequest(ImplicitPin.Get)); + } + + /// + public IEnumerable EnumeratePinnedContentHashes() + { + return _pinContext.GetContentHashes(); + } + + /// + async Task IHibernateContentSession.PinBulkAsync(Context context, IEnumerable contentHashes) + { + var contentHashList = contentHashes as List ?? contentHashes.ToList(); + // Passing 'RePinFromHibernation' to use more optimal pinning logic. + var results = Enumerable.ToList>( + (await Store.PinAsync(context, contentHashList, _pinContext, new PinBulkOptions() { RePinFromHibernation = true }))); + + var failed = results.Where(r => !r.Item.Succeeded); + foreach (var result in failed) + { + Tracer.Warning(context, $"Failed to pin contentHash=[{contentHashList[result.Index]}]"); + } + } + + /// + Task IHibernateContentSession.ShutdownEvictionAsync(Context context) + { + return Store.ShutdownEvictionAsync(context); + } + + /// + /// Build a PinRequest based on whether auto-pin configuration matches request. + /// + protected PinRequest? MakePinRequest(ImplicitPin implicitPin) + { + return (implicitPin & _implicitPin) != ImplicitPin.None ? new PinRequest(_pinContext) : (PinRequest?)null; } /// diff --git a/Public/Src/Cache/ContentStore/Library/Sessions/ReadOnlyFileSystemContentSession.cs b/Public/Src/Cache/ContentStore/Library/Sessions/ReadOnlyFileSystemContentSession.cs deleted file mode 100644 index 6d7b99c9a..000000000 --- a/Public/Src/Cache/ContentStore/Library/Sessions/ReadOnlyFileSystemContentSession.cs +++ /dev/null @@ -1,158 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -using System.Collections.Generic; -using System.Linq; -using System.Threading.Tasks; -using BuildXL.Cache.ContentStore.Hashing; -using BuildXL.Cache.ContentStore.Interfaces.Extensions; -using BuildXL.Cache.ContentStore.Interfaces.FileSystem; -using BuildXL.Cache.ContentStore.Interfaces.Logging; -using BuildXL.Cache.ContentStore.Interfaces.Results; -using BuildXL.Cache.ContentStore.Interfaces.Sessions; -using BuildXL.Cache.ContentStore.Interfaces.Stores; -using BuildXL.Cache.ContentStore.Interfaces.Tracing; -using BuildXL.Cache.ContentStore.Stores; -using BuildXL.Cache.ContentStore.Tracing; -using BuildXL.Cache.ContentStore.Tracing.Internal; -using BuildXL.Utilities.Tracing; - -namespace BuildXL.Cache.ContentStore.Sessions -{ - /// - /// An implemented over a - /// - public class ReadOnlyFileSystemContentSession : ContentSessionBase, IHibernateContentSession - { - /// - /// The internal content store backing the session. - /// - protected readonly FileSystemContentStoreInternal Store; - - private readonly PinContext _pinContext; - private readonly ImplicitPin _implicitPin; - - /// - protected override Tracer Tracer { get; } = new Tracer(nameof(FileSystemContentSession)); - - /// - protected override bool TraceErrorsOnly => true; // This type adds nothing in terms of tracing. So configure it to trace errors only. - - /// - /// Initializes a new instance of the class. - /// - public ReadOnlyFileSystemContentSession(string name, FileSystemContentStoreInternal store, ImplicitPin implicitPin) - : base(name) - { - Store = store; - _pinContext = Store.CreatePinContext(); - _implicitPin = implicitPin; - } - - /// - protected override async Task ShutdownCoreAsync(OperationContext operationContext) - { - await _pinContext.DisposeAsync(); - - var statsResult = await Store.GetStatsAsync(operationContext); - if (statsResult.Succeeded) - { - Tracer.TraceStatisticsAtShutdown(operationContext, statsResult.CounterSet, prefix: "FileSystemContentSessionStats"); - } - - return BoolResult.Success; - } - - /// - protected override void DisposeCore() - { - base.DisposeCore(); - _pinContext.Dispose(); - } - - /// - protected override Task PinCoreAsync(OperationContext operationContext, ContentHash contentHash, UrgencyHint urgencyHint, Counter retryCounter) - { - return Store.PinAsync(operationContext, contentHash, _pinContext); - } - - /// - protected override Task OpenStreamCoreAsync( - OperationContext operationContext, ContentHash contentHash, UrgencyHint urgencyHint, Counter retryCounter) - { - return Store.OpenStreamAsync(operationContext, contentHash, MakePinRequest(ImplicitPin.Get)); - } - - /// - protected override Task PlaceFileCoreAsync( - OperationContext operationContext, - ContentHash contentHash, - AbsolutePath path, - FileAccessMode accessMode, - FileReplacementMode replacementMode, - FileRealizationMode realizationMode, - UrgencyHint urgencyHint, - Counter retryCounter) - { - return Store.PlaceFileAsync(operationContext, contentHash, path, accessMode, replacementMode, realizationMode, MakePinRequest(ImplicitPin.Put)); - } - - /// - protected override async Task>>> PinCoreAsync( - OperationContext operationContext, - IReadOnlyList contentHashes, - UrgencyHint urgencyHint, - Counter retryCounter, - Counter fileCounter) - { - return (await Store.PinAsync(operationContext, contentHashes, _pinContext, options: null)).AsTasks(); - } - - /// - protected override Task>>> PlaceFileCoreAsync( - OperationContext operationContext, - IReadOnlyList hashesWithPaths, - FileAccessMode accessMode, - FileReplacementMode replacementMode, - FileRealizationMode realizationMode, - UrgencyHint urgencyHint, - Counter retryCounter) - { - return Store.PlaceFileAsync(operationContext, hashesWithPaths, accessMode, replacementMode, realizationMode, MakePinRequest(ImplicitPin.Get)); - } - - /// - public IEnumerable EnumeratePinnedContentHashes() - { - return _pinContext.GetContentHashes(); - } - - /// - async Task IHibernateContentSession.PinBulkAsync(Context context, IEnumerable contentHashes) - { - var contentHashList = contentHashes as List ?? contentHashes.ToList(); - // Passing 'RePinFromHibernation' to use more optimal pinning logic. - var results = (await Store.PinAsync(context, contentHashList, _pinContext, new PinBulkOptions() { RePinFromHibernation = true })).ToList(); - - var failed = results.Where(r => !r.Item.Succeeded); - foreach (var result in failed) - { - Tracer.Warning(context, $"Failed to pin contentHash=[{contentHashList[result.Index]}]"); - } - } - - /// - Task IHibernateContentSession.ShutdownEvictionAsync(Context context) - { - return Store.ShutdownEvictionAsync(context); - } - - /// - /// Build a PinRequest based on whether auto-pin configuration matches request. - /// - protected PinRequest? MakePinRequest(ImplicitPin implicitPin) - { - return (implicitPin & _implicitPin) != ImplicitPin.None ? new PinRequest(_pinContext) : (PinRequest?)null; - } - } -} diff --git a/Public/Src/Cache/ContentStore/Library/Sessions/ReadOnlyServiceClientContentSession.cs b/Public/Src/Cache/ContentStore/Library/Sessions/ReadOnlyServiceClientContentSession.cs deleted file mode 100644 index 00a2c1fb3..000000000 --- a/Public/Src/Cache/ContentStore/Library/Sessions/ReadOnlyServiceClientContentSession.cs +++ /dev/null @@ -1,274 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -using System; -using System.Collections.Generic; -using System.Diagnostics; -using System.Diagnostics.ContractsLight; -using System.Linq; -using System.Runtime.CompilerServices; -using System.Threading; -using System.Threading.Tasks; -using BuildXL.Cache.ContentStore.FileSystem; -using BuildXL.Cache.ContentStore.Hashing; -using BuildXL.Cache.ContentStore.Interfaces.FileSystem; -using BuildXL.Cache.ContentStore.Interfaces.Logging; -using BuildXL.Cache.ContentStore.Interfaces.Results; -using BuildXL.Cache.ContentStore.Interfaces.Sessions; -using BuildXL.Cache.ContentStore.Interfaces.Stores; -using BuildXL.Cache.ContentStore.Service; -using BuildXL.Cache.ContentStore.Service.Grpc; -using BuildXL.Cache.ContentStore.Stores; -using BuildXL.Cache.ContentStore.Tracing; -using BuildXL.Cache.ContentStore.Tracing.Internal; -using BuildXL.Cache.ContentStore.UtilitiesCore; -using BuildXL.Cache.ContentStore.Utils; -using BuildXL.Utilities.Tracing; - -namespace BuildXL.Cache.ContentStore.Sessions -{ - /// - /// An IReadOnlyContentSession implemented over a ServiceClientContentStore. - /// - public class ReadOnlyServiceClientContentSession : ContentSessionBase - { - /// - /// The filesystem backing the session. - /// - protected readonly IAbsFileSystem FileSystem; - - /// - /// Generator of temporary, seekable streams. - /// - protected readonly TempFileStreamFactory TempFileStreamFactory; - - /// - protected override Tracer Tracer { get; } = new Tracer(nameof(ReadOnlyServiceClientContentSession)); - - /// - /// Request to server retry policy. - /// - protected readonly IRetryPolicy RetryPolicy; - - /// - /// The client backing the session. - /// - protected readonly IRpcClient RpcClient; - - /// - protected readonly ServiceClientContentStoreConfiguration Configuration; - - /// - protected readonly ServiceClientContentSessionTracer SessionTracer; - - /// - protected readonly ILogger Logger; - - /// - protected readonly ImplicitPin ImplicitPin; - - /// - protected override bool TraceOperationStarted => Configuration.TraceOperationStarted; - - /// - /// Initializes a new instance of the class. - /// - public ReadOnlyServiceClientContentSession( - OperationContext context, - string name, - ImplicitPin implicitPin, - ILogger logger, - IAbsFileSystem fileSystem, - ServiceClientContentSessionTracer sessionTracer, - ServiceClientContentStoreConfiguration configuration, - Func? rpcClientFactory = null) - : base(name) - { - Contract.Requires(name != null); - Contract.Requires(logger != null); - Contract.Requires(fileSystem != null); - - ImplicitPin = implicitPin; - SessionTracer = sessionTracer; - Logger = logger; - FileSystem = fileSystem; - Configuration = configuration; - TempFileStreamFactory = new TempFileStreamFactory(FileSystem); - - RpcClient = (rpcClientFactory ?? (() => GetRpcClient(context)))(); - RetryPolicy = configuration.RetryPolicy; - } - - /// - protected virtual IRpcClient GetRpcClient(OperationContext context) - { - var rpcConfiguration = Configuration.RpcConfiguration; - - return new GrpcContentClient(context, SessionTracer, FileSystem, rpcConfiguration, Configuration.Scenario); - } - - /// - protected override async Task StartupCoreAsync(OperationContext operationContext) - { - BoolResult result; - - try - { - result = await RetryPolicy.ExecuteAsync(() => RpcClient.CreateSessionAsync(operationContext, Name, Configuration.CacheName, ImplicitPin), CancellationToken.None); - } - catch (Exception ex) - { - result = new BoolResult(ex); - } - - if (!result) - { - await RetryPolicy.ExecuteAsync(() => RpcClient.ShutdownAsync(operationContext), CancellationToken.None).ThrowIfFailure(); - } - - return result; - } - - /// - protected override async Task ShutdownCoreAsync(OperationContext operationContext) - { - var result = await RetryPolicy.ExecuteAsync(() => RpcClient.ShutdownAsync(operationContext), CancellationToken.None); - - var counterSet = new CounterSet(); - counterSet.Merge(GetCounters(), $"{Tracer.Name}."); - Tracer.TraceStatisticsAtShutdown(operationContext, counterSet, prefix: "ServiceClientContentSessionStats"); - - return result; - } - - /// - protected override void DisposeCore() - { - base.DisposeCore(); - RpcClient.Dispose(); - TempFileStreamFactory.Dispose(); - } - - /// - protected override Task PinCoreAsync(OperationContext operationContext, ContentHash contentHash, UrgencyHint urgencyHint, Counter retryCounter) - { - return PerformRetries( - operationContext, - () => RpcClient.PinAsync(operationContext, contentHash, urgencyHint), - retryCounter: retryCounter); - } - - /// - protected override Task OpenStreamCoreAsync( - OperationContext operationContext, ContentHash contentHash, UrgencyHint urgencyHint, Counter retryCounter) - { - return PerformRetries( - operationContext, - () => RpcClient.OpenStreamAsync(operationContext, contentHash, urgencyHint), - retryCounter: retryCounter); - } - - /// - protected override Task PlaceFileCoreAsync( - OperationContext operationContext, - ContentHash contentHash, - AbsolutePath path, - FileAccessMode accessMode, - FileReplacementMode replacementMode, - FileRealizationMode realizationMode, - UrgencyHint urgencyHint, - Counter retryCounter) - { - if (replacementMode != FileReplacementMode.ReplaceExisting && FileSystem.FileExists(path)) - { - if (replacementMode == FileReplacementMode.SkipIfExists) - { - return Task.FromResult(PlaceFileResult.AlreadyExists); - } - else if (replacementMode == FileReplacementMode.FailIfExists) - { - return Task.FromResult(new PlaceFileResult( - PlaceFileResult.ResultCode.Error, - $"File exists at destination {path} with FailIfExists specified")); - } - } - - return PerformRetries( - operationContext, - () => RpcClient.PlaceFileAsync(operationContext, contentHash, path, accessMode, replacementMode, realizationMode, urgencyHint), - retryCounter: retryCounter); - } - - /// - protected override async Task>>> PinCoreAsync( - OperationContext operationContext, - IReadOnlyList contentHashes, - UrgencyHint urgencyHint, - Counter retryCounter, - Counter fileCounter) - { - var retry = 0; - - try - { - return await RetryPolicy.ExecuteAsync(PinBulkFunc, operationContext.Token); - } - catch (Exception ex) - { - Tracer.Warning(operationContext, $"PinBulk failed with exception {ex}"); - return contentHashes.Select((hash, index) => Task.FromResult(new PinResult(ex).WithIndex(index))); - } - - async Task>>> PinBulkFunc() - { - var sw = Stopwatch.StartNew(); - try - { - fileCounter.Add(contentHashes.Count); - - if (retry > 0) - { - Tracer.Debug(operationContext, $"{Tracer.Name}.PinBulk retry #{retry}"); - retryCounter.Increment(); - } - return await RpcClient.PinAsync(operationContext, contentHashes, urgencyHint); - } - finally - { - Tracer.Debug(operationContext, $"{Tracer.Name}.PinBulk({contentHashes.Count}) stop {sw.Elapsed.TotalMilliseconds}ms. Hashes: [{string.Join(",", contentHashes)}]"); - } - } - } - - /// - protected override Task>>> PlaceFileCoreAsync(OperationContext operationContext, IReadOnlyList hashesWithPaths, FileAccessMode accessMode, FileReplacementMode replacementMode, FileRealizationMode realizationMode, UrgencyHint urgencyHint, Counter retryCounter) - { - throw new NotImplementedException(); - } - - /// - protected Task PerformRetries(OperationContext operationContext, Func> action, Action? onRetry = null, Counter? retryCounter = null, [CallerMemberName] string? operationName = null) - { - Contract.Requires(operationName != null); - var retry = 0; - - return RetryPolicy.ExecuteAsync(Wrapper, operationContext.Token); - - Task Wrapper() - { - if (retry > 0) - { - // Normalize operation name - operationName = operationName.Replace("Async", "").Replace("Core", ""); - Tracer.Debug(operationContext, $"{Tracer.Name}.{operationName} retry #{retry}"); - Tracer.TrackMetric(operationContext, $"{operationName}Retry", 1); - retryCounter?.Increment(); - onRetry?.Invoke(retry); - } - - retry++; - return action(); - } - } - } -} diff --git a/Public/Src/Cache/ContentStore/Library/Sessions/ServiceClientContentSession.cs b/Public/Src/Cache/ContentStore/Library/Sessions/ServiceClientContentSession.cs index a8de99f30..496c43709 100644 --- a/Public/Src/Cache/ContentStore/Library/Sessions/ServiceClientContentSession.cs +++ b/Public/Src/Cache/ContentStore/Library/Sessions/ServiceClientContentSession.cs @@ -2,8 +2,15 @@ // Licensed under the MIT License. using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Diagnostics.ContractsLight; using System.IO; +using System.Linq; +using System.Runtime.CompilerServices; +using System.Threading; using System.Threading.Tasks; +using BuildXL.Cache.ContentStore.FileSystem; using BuildXL.Cache.ContentStore.Hashing; using BuildXL.Cache.ContentStore.Interfaces.FileSystem; using BuildXL.Cache.ContentStore.Interfaces.Logging; @@ -11,21 +18,56 @@ using BuildXL.Cache.ContentStore.Interfaces.Results; using BuildXL.Cache.ContentStore.Interfaces.Sessions; using BuildXL.Cache.ContentStore.Interfaces.Stores; using BuildXL.Cache.ContentStore.Service; +using BuildXL.Cache.ContentStore.Service.Grpc; using BuildXL.Cache.ContentStore.Stores; using BuildXL.Cache.ContentStore.Tracing; using BuildXL.Cache.ContentStore.Tracing.Internal; +using BuildXL.Cache.ContentStore.UtilitiesCore; +using BuildXL.Cache.ContentStore.Utils; using BuildXL.Utilities.Tracing; namespace BuildXL.Cache.ContentStore.Sessions { - /// - /// An IContentSession implemented over a ServiceClientsContentStore. - /// - public class ServiceClientContentSession : ReadOnlyServiceClientContentSession, IContentSession + public class ServiceClientContentSession : ContentSessionBase, IContentSession { + /// + /// The filesystem backing the session. + /// + protected readonly IAbsFileSystem FileSystem; + + /// + /// Generator of temporary, seekable streams. + /// + protected readonly TempFileStreamFactory TempFileStreamFactory; + /// protected override Tracer Tracer { get; } = new Tracer(nameof(ServiceClientContentSession)); + /// + /// Request to server retry policy. + /// + protected readonly IRetryPolicy RetryPolicy; + + /// + /// The client backing the session. + /// + protected readonly IRpcClient RpcClient; + + /// + protected readonly ServiceClientContentStoreConfiguration Configuration; + + /// + protected readonly ServiceClientContentSessionTracer SessionTracer; + + /// + protected readonly ILogger Logger; + + /// + protected readonly ImplicitPin ImplicitPin; + + /// + protected override bool TraceOperationStarted => Configuration.TraceOperationStarted; + /// /// Initializes a new instance of the class. /// @@ -38,13 +80,227 @@ namespace BuildXL.Cache.ContentStore.Sessions ServiceClientContentSessionTracer sessionTracer, ServiceClientContentStoreConfiguration configuration, Func? rpcClientFactory = null) - : base(context, name, implicitPin, logger, fileSystem, sessionTracer, configuration, rpcClientFactory) + : base(name) { + Contract.Requires(name != null); + Contract.Requires(logger != null); + Contract.Requires(fileSystem != null); + + ImplicitPin = implicitPin; + SessionTracer = sessionTracer; + Logger = logger; + FileSystem = fileSystem; + Configuration = configuration; + TempFileStreamFactory = new TempFileStreamFactory(FileSystem); + + RpcClient = (rpcClientFactory ?? (() => GetRpcClient(context)))(); + RetryPolicy = configuration.RetryPolicy; + } + + /// + protected virtual IRpcClient GetRpcClient(OperationContext context) + { + var rpcConfiguration = Configuration.RpcConfiguration; + + return new GrpcContentClient(context, SessionTracer, FileSystem, rpcConfiguration, Configuration.Scenario); + } + + /// + protected override async Task StartupCoreAsync(OperationContext operationContext) + { + BoolResult result; + + try + { + result = await RetryPolicy.ExecuteAsync( + () => RpcClient.CreateSessionAsync(operationContext, Name, Configuration.CacheName, ImplicitPin), + CancellationToken.None); + } + catch (Exception ex) + { + result = new BoolResult(ex); + } + + if (!result) + { + await RetryPolicy.ExecuteAsync(() => RpcClient.ShutdownAsync(operationContext), CancellationToken.None).ThrowIfFailure(); + } + + return result; + } + + /// + protected override async Task ShutdownCoreAsync(OperationContext operationContext) + { + var result = await RetryPolicy.ExecuteAsync(() => RpcClient.ShutdownAsync(operationContext), CancellationToken.None); + + var counterSet = new CounterSet(); + counterSet.Merge(GetCounters(), $"{Tracer.Name}."); + Tracer.TraceStatisticsAtShutdown(operationContext, counterSet, prefix: "ServiceClientContentSessionStats"); + + return result; + } + + /// + protected override void DisposeCore() + { + base.DisposeCore(); + RpcClient.Dispose(); + TempFileStreamFactory.Dispose(); + } + + /// + protected override Task PinCoreAsync( + OperationContext operationContext, + ContentHash contentHash, + UrgencyHint urgencyHint, + Counter retryCounter) + { + return PerformRetries( + operationContext, + () => RpcClient.PinAsync(operationContext, contentHash, urgencyHint), + retryCounter: retryCounter); + } + + /// + protected override Task OpenStreamCoreAsync( + OperationContext operationContext, + ContentHash contentHash, + UrgencyHint urgencyHint, + Counter retryCounter) + { + return PerformRetries( + operationContext, + () => RpcClient.OpenStreamAsync(operationContext, contentHash, urgencyHint), + retryCounter: retryCounter); + } + + /// + protected override Task PlaceFileCoreAsync( + OperationContext operationContext, + ContentHash contentHash, + AbsolutePath path, + FileAccessMode accessMode, + FileReplacementMode replacementMode, + FileRealizationMode realizationMode, + UrgencyHint urgencyHint, + Counter retryCounter) + { + if (replacementMode != FileReplacementMode.ReplaceExisting && FileSystem.FileExists(path)) + { + if (replacementMode == FileReplacementMode.SkipIfExists) + { + return Task.FromResult(PlaceFileResult.AlreadyExists); + } + else if (replacementMode == FileReplacementMode.FailIfExists) + { + return Task.FromResult( + new PlaceFileResult( + PlaceFileResult.ResultCode.Error, + $"File exists at destination {path} with FailIfExists specified")); + } + } + + return PerformRetries( + operationContext, + () => RpcClient.PlaceFileAsync(operationContext, contentHash, path, accessMode, replacementMode, realizationMode, urgencyHint), + retryCounter: retryCounter); + } + + /// + protected override async Task>>> PinCoreAsync( + OperationContext operationContext, + IReadOnlyList contentHashes, + UrgencyHint urgencyHint, + Counter retryCounter, + Counter fileCounter) + { + var retry = 0; + + try + { + return await RetryPolicy.ExecuteAsync(PinBulkFunc, operationContext.Token); + } + catch (Exception ex) + { + Tracer.Warning(operationContext, $"PinBulk failed with exception {ex}"); + return contentHashes.Select((hash, index) => Task.FromResult(new PinResult(ex).WithIndex(index))); + } + + async Task>>> PinBulkFunc() + { + var sw = Stopwatch.StartNew(); + try + { + fileCounter.Add(contentHashes.Count); + + if (retry > 0) + { + Tracer.Debug(operationContext, $"{Tracer.Name}.PinBulk retry #{retry}"); + retryCounter.Increment(); + } + + return await RpcClient.PinAsync(operationContext, contentHashes, urgencyHint); + } + finally + { + Tracer.Debug( + operationContext, + $"{Tracer.Name}.PinBulk({contentHashes.Count}) stop {sw.Elapsed.TotalMilliseconds}ms. Hashes: [{string.Join(",", contentHashes)}]"); + } + } + } + + /// + protected override Task>>> PlaceFileCoreAsync( + OperationContext operationContext, + IReadOnlyList hashesWithPaths, + FileAccessMode accessMode, + FileReplacementMode replacementMode, + FileRealizationMode realizationMode, + UrgencyHint urgencyHint, + Counter retryCounter) + { + throw new NotImplementedException(); + } + + /// + protected Task PerformRetries( + OperationContext operationContext, + Func> action, + Action? onRetry = null, + Counter? retryCounter = null, + [CallerMemberName] string? operationName = null) + { + Contract.Requires(operationName != null); + var retry = 0; + + return RetryPolicy.ExecuteAsync(Wrapper, operationContext.Token); + + Task Wrapper() + { + if (retry > 0) + { + // Normalize operation name + operationName = operationName.Replace("Async", "").Replace("Core", ""); + Tracer.Debug(operationContext, $"{Tracer.Name}.{operationName} retry #{retry}"); + Tracer.TrackMetric(operationContext, $"{operationName}Retry", 1); + retryCounter?.Increment(); + onRetry?.Invoke(retry); + } + + retry++; + return action(); + } } /// protected override Task PutStreamCoreAsync( - OperationContext operationContext, HashType hashType, Stream stream, UrgencyHint urgencyHint, Counter retryCounter) + OperationContext operationContext, + HashType hashType, + Stream stream, + UrgencyHint urgencyHint, + Counter retryCounter) { return PutStreamCoreAsync( operationContext, @@ -55,7 +311,11 @@ namespace BuildXL.Cache.ContentStore.Sessions /// protected override Task PutStreamCoreAsync( - OperationContext operationContext, ContentHash contentHash, Stream stream, UrgencyHint urgencyHint, Counter retryCounter) + OperationContext operationContext, + ContentHash contentHash, + Stream stream, + UrgencyHint urgencyHint, + Counter retryCounter) { return PutStreamCoreAsync( operationContext, @@ -64,7 +324,11 @@ namespace BuildXL.Cache.ContentStore.Sessions args => RpcClient.PutStreamAsync(operationContext, contentHash, args.putStream, args.createDirectory, urgencyHint)); } - private async Task PutStreamCoreAsync(OperationContext operationContext, Stream stream, Counter retryCounter, Func<(Stream putStream, bool createDirectory), Task> putStreamAsync) + private async Task PutStreamCoreAsync( + OperationContext operationContext, + Stream stream, + Counter retryCounter, + Func<(Stream putStream, bool createDirectory), Task> putStreamAsync) { // We need a seekable stream, that can give its length. If the input stream is seekable, we can use it directly. // Otherwise, we need to create a temp file for this purpose. diff --git a/Public/Src/Cache/ContentStore/Library/Stores/FileSystemContentStore.cs b/Public/Src/Cache/ContentStore/Library/Stores/FileSystemContentStore.cs index b6973c010..d562e805e 100644 --- a/Public/Src/Cache/ContentStore/Library/Stores/FileSystemContentStore.cs +++ b/Public/Src/Cache/ContentStore/Library/Stores/FileSystemContentStore.cs @@ -169,22 +169,12 @@ namespace BuildXL.Cache.ContentStore.Stores _directoryLock.Dispose(); } - /// - public virtual CreateSessionResult CreateReadOnlySession(Context context, string name, ImplicitPin implicitPin) - { - return CreateReadOnlySessionCall.Run(_tracer, OperationContext(context), name, () => - { - var session = new ReadOnlyFileSystemContentSession(name, Store, implicitPin); - return new CreateSessionResult(session); - }); - } - /// public virtual CreateSessionResult CreateSession(Context context, string name, ImplicitPin implicitPin) { return CreateSessionCall.Run(_tracer, OperationContext(context), name, () => { - var session = new FileSystemContentSession(name, implicitPin, Store); + var session = new FileSystemContentSession(name, Store, implicitPin); return new CreateSessionResult(session); }); } diff --git a/Public/Src/Cache/ContentStore/Library/Stores/ReadOnlyEmptyContentStore.cs b/Public/Src/Cache/ContentStore/Library/Stores/ReadOnlyEmptyContentStore.cs index f65585631..999aa515f 100644 --- a/Public/Src/Cache/ContentStore/Library/Stores/ReadOnlyEmptyContentStore.cs +++ b/Public/Src/Cache/ContentStore/Library/Stores/ReadOnlyEmptyContentStore.cs @@ -22,24 +22,21 @@ namespace BuildXL.Cache.ContentStore.Stores /// /// ContentStore that is empty and does not accept content. Useful when we want to disable content; specifically, to bypass talking to VSTS/CASaaS when unnecessary. /// - public class ReadOnlyEmptyContentStore : StartupShutdownBase, IContentStore + public class EmptyContentStore : StartupShutdownBase, IContentStore { - private readonly ContentStoreTracer _tracer = new ContentStoreTracer(nameof(ReadOnlyEmptyContentStore)); + private readonly ContentStoreTracer _tracer = new ContentStoreTracer(nameof(EmptyContentStore)); /// protected override Tracer Tracer => _tracer; /// - public CreateSessionResult CreateReadOnlySession(Context context, string name, ImplicitPin implicitPin) => new CreateSessionResult(new ReadOnlyEmptyContentSession(name)); - - /// - public CreateSessionResult CreateSession(Context context, string name, ImplicitPin implicitPin) => new CreateSessionResult(new ReadOnlyEmptyContentSession(name)); + public CreateSessionResult CreateSession(Context context, string name, ImplicitPin implicitPin) => new CreateSessionResult(new EmptyContentSession(name)); /// public Task GetStatsAsync(Context context) => Task.FromResult(new GetStatsResult(_tracer.GetCounters())); /// - Task IContentStore.DeleteAsync(Context context, ContentHash contentHash, DeleteContentOptions? deleteOptions) => Task.FromResult(new DeleteResult(DeleteResult.ResultCode.ContentNotDeleted, $"{nameof(ReadOnlyEmptyContentStore)} cannot contain any content to delete")); + Task IContentStore.DeleteAsync(Context context, ContentHash contentHash, DeleteContentOptions? deleteOptions) => Task.FromResult(new DeleteResult(DeleteResult.ResultCode.ContentNotDeleted, $"{nameof(EmptyContentStore)} cannot contain any content to delete")); /// public void PostInitializationCompleted(Context context, BoolResult result) { } @@ -48,10 +45,10 @@ namespace BuildXL.Cache.ContentStore.Stores /// /// ContentSession is empty and does not accept content. Useful when we want to disable content; specifically, to bypass talking to VSTS/CASaaS when unnecessary. /// - public class ReadOnlyEmptyContentSession : ContentSessionBase + public class EmptyContentSession : ContentSessionBase { /// - public ReadOnlyEmptyContentSession(string name) + public EmptyContentSession(string name) : base(name) { } @@ -59,7 +56,7 @@ namespace BuildXL.Cache.ContentStore.Stores private const string ErrorMessage = "Unsupported operation."; /// - protected override Tracer Tracer { get; } = new Tracer(nameof(ReadOnlyEmptyContentSession)); + protected override Tracer Tracer { get; } = new Tracer(nameof(EmptyContentSession)); /// protected override Task OpenStreamCoreAsync(OperationContext operationContext, ContentHash contentHash, UrgencyHint urgencyHint, Counter retryCounter) diff --git a/Public/Src/Cache/ContentStore/Library/Stores/RocksDbFileSystemContentStore.cs b/Public/Src/Cache/ContentStore/Library/Stores/RocksDbFileSystemContentStore.cs index 3f0148ae3..d5218e1da 100644 --- a/Public/Src/Cache/ContentStore/Library/Stores/RocksDbFileSystemContentStore.cs +++ b/Public/Src/Cache/ContentStore/Library/Stores/RocksDbFileSystemContentStore.cs @@ -121,18 +121,6 @@ namespace BuildXL.Cache.ContentStore.Stores return base.ShutdownCoreAsync(context); } - /// - public CreateSessionResult CreateReadOnlySession(Context context, string name, ImplicitPin implicitPin) - { - return CreateReadOnlySessionCall.Run(_tracer, OperationContext(context), name, () => - { -#pragma warning disable CS8604 // Possible null reference argument. - var session = new RocksDbFileSystemContentSession(name, _lockSet, _fileSystem, _clock, _rootPath, _storePath, _tempDisposableDirectory, _accessor); -#pragma warning restore CS8604 // Possible null reference argument. - return new CreateSessionResult(session); - }); - } - /// public CreateSessionResult CreateSession(Context context, string name, ImplicitPin implicitPin) { diff --git a/Public/Src/Cache/ContentStore/Library/Stores/ServiceClientContentStore.cs b/Public/Src/Cache/ContentStore/Library/Stores/ServiceClientContentStore.cs index e35b3ea6d..7d53341ab 100644 --- a/Public/Src/Cache/ContentStore/Library/Stores/ServiceClientContentStore.cs +++ b/Public/Src/Cache/ContentStore/Library/Stores/ServiceClientContentStore.cs @@ -194,25 +194,6 @@ namespace BuildXL.Cache.ContentStore.Stores _grpcClient?.Dispose(); } - /// - public virtual CreateSessionResult CreateReadOnlySession( - Context context, string name, ImplicitPin implicitPin) - { - return CreateReadOnlySessionCall.Run(ExecutionTracer, OperationContext(context), name, () => - { - var session = new ReadOnlyServiceClientContentSession( - // Its fine to re-create an operation context without cancellation tokens because its only used for tracing purposes. - new OperationContext(context), - name, - implicitPin, - Logger, - FileSystem, - SessionTracer, - Configuration); - return new CreateSessionResult(session); - }); - } - /// public virtual CreateSessionResult CreateSession(Context context, string name, ImplicitPin implicitPin) { diff --git a/Public/Src/Cache/ContentStore/Library/Stores/StreamPathContentStore.cs b/Public/Src/Cache/ContentStore/Library/Stores/StreamPathContentStore.cs index 3b45cea85..601cc5ee5 100644 --- a/Public/Src/Cache/ContentStore/Library/Stores/StreamPathContentStore.cs +++ b/Public/Src/Cache/ContentStore/Library/Stores/StreamPathContentStore.cs @@ -54,35 +54,6 @@ namespace BuildXL.Cache.ContentStore.Stores /// protected override ContentStoreTracer Tracer => _tracer; - /// - public override CreateSessionResult CreateReadOnlySession( - Context context, - string name, - ImplicitPin implicitPin) - { - return CreateReadOnlySessionCall.Run(_tracer, new OperationContext(context), name, () => - { - var sessionForStream = ContentStoreForStream.CreateSession(context, name, implicitPin); - if (!sessionForStream.Succeeded) - { - return new CreateSessionResult(sessionForStream, "creation of stream content session failed"); - } - - var sessionForPath = ContentStoreForPath.CreateSession(context, name, implicitPin); - if (!sessionForPath.Succeeded) - { - return new CreateSessionResult(sessionForPath, "creation of path content session failed"); - } - - var session = new StreamPathContentSession( - name, - sessionForStream.Session, - sessionForPath.Session); - - return new CreateSessionResult(session); - }); - } - /// public override CreateSessionResult CreateSession(Context context, string name, ImplicitPin implicitPin) { diff --git a/Public/Src/Cache/ContentStore/Library/Stores/TwoContentStore.cs b/Public/Src/Cache/ContentStore/Library/Stores/TwoContentStore.cs index 2dd29b7c6..df5a300bd 100644 --- a/Public/Src/Cache/ContentStore/Library/Stores/TwoContentStore.cs +++ b/Public/Src/Cache/ContentStore/Library/Stores/TwoContentStore.cs @@ -167,14 +167,11 @@ namespace BuildXL.Cache.ContentStore.Stores } /// - public abstract CreateSessionResult CreateReadOnlySession( + public abstract CreateSessionResult CreateSession( Context context, string name, ImplicitPin implicitPin); - /// - public abstract CreateSessionResult CreateSession(Context context, string name, ImplicitPin implicitPin); - /// public virtual Task GetStatsAsync(Context context) { diff --git a/Public/Src/Cache/ContentStore/Library/Tracing/ContentStoreTracer.cs b/Public/Src/Cache/ContentStore/Library/Tracing/ContentStoreTracer.cs index 2ec5d8f73..fbf4dde91 100644 --- a/Public/Src/Cache/ContentStore/Library/Tracing/ContentStoreTracer.cs +++ b/Public/Src/Cache/ContentStore/Library/Tracing/ContentStoreTracer.cs @@ -39,22 +39,6 @@ namespace BuildXL.Cache.ContentStore.Tracing } - public void CreateReadOnlySessionStart(Context context, string name) - { - if (context.IsEnabled) - { - Debug(context, $"{Name}.CreateReadOnlySession({name}) start"); - } - } - - public void CreateReadOnlySessionStop(Context context, CreateSessionResult result) - { - if (context.IsEnabled) - { - TracerOperationFinished(context, result, $"{Name}.CreateReadOnlySession() stop {result.DurationMs}ms result=[{result}]"); - } - } - public void CreateSessionStart(Context context, string name) { if (context.IsEnabled) diff --git a/Public/Src/Cache/ContentStore/Library/Tracing/CreateReadOnlySessionCall.cs b/Public/Src/Cache/ContentStore/Library/Tracing/CreateReadOnlySessionCall.cs deleted file mode 100644 index 36bdfacac..000000000 --- a/Public/Src/Cache/ContentStore/Library/Tracing/CreateReadOnlySessionCall.cs +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -using System; -using BuildXL.Cache.ContentStore.Interfaces.Results; -using BuildXL.Cache.ContentStore.Interfaces.Sessions; -using BuildXL.Cache.ContentStore.Tracing.Internal; - -namespace BuildXL.Cache.ContentStore.Tracing -{ - /// - /// Instance of a CreateReadOnlySession operation for tracing purposes. - /// - public sealed class CreateReadOnlySessionCall - : TracedCall>, IDisposable - { - /// - /// Run the call. - /// - public static CreateSessionResult Run( - ContentStoreTracer tracer, OperationContext context, string name, Func> func) - { - using (var call = new CreateReadOnlySessionCall(tracer, context, name)) - { - return call.Run(func); - } - } - - /// - /// Initializes a new instance of the class. - /// - private CreateReadOnlySessionCall(ContentStoreTracer tracer, OperationContext context, string name) - : base(tracer, context) - { - Tracer.CreateReadOnlySessionStart(Context, name); - } - - /// - protected override CreateSessionResult CreateErrorResult(Exception exception) - { - return new CreateSessionResult(exception); - } - - /// - public void Dispose() - { - Tracer.CreateReadOnlySessionStop(Context, Result); - } - } -} diff --git a/Public/Src/Cache/ContentStore/Test/Sessions/ContentPerformanceTests.cs b/Public/Src/Cache/ContentStore/Test/Sessions/ContentPerformanceTests.cs index b928860ad..a34b2ccda 100644 --- a/Public/Src/Cache/ContentStore/Test/Sessions/ContentPerformanceTests.cs +++ b/Public/Src/Cache/ContentStore/Test/Sessions/ContentPerformanceTests.cs @@ -42,7 +42,7 @@ namespace ContentStoreTest.Performance.Sessions private const int SmallFileSize = 1024; private const int LargeFileSize = 5 * 1024 * 1024; private const HashType ContentHashType = HashType.Vso0; - private static readonly Func EmptySetupFuncAsync = session => Task.FromResult(0); + private static readonly Func EmptySetupFuncAsync = session => Task.FromResult(0); private static readonly CancellationToken Token = CancellationToken.None; private readonly Context _context; private readonly int _itemCount; @@ -62,7 +62,7 @@ namespace ContentStoreTest.Performance.Sessions protected abstract IContentStore CreateStore(AbsolutePath rootPath, string cacheName, ContentStoreConfiguration configuration); - protected abstract Task> EnumerateContentHashesAsync(IReadOnlyContentSession session); + protected abstract Task> EnumerateContentHashesAsync(IContentSession session); protected ContentPerformanceTests ( @@ -118,7 +118,7 @@ namespace ContentStoreTest.Performance.Sessions } private async Task PinAsync( - IReadOnlyContentSession session, IReadOnlyCollection hashes, List results) + IContentSession session, IReadOnlyCollection hashes, List results) { var tasks = hashes.Select(contentHash => Task.Run(async () => await session.PinAsync(_context, contentHash, Token))); @@ -164,7 +164,7 @@ namespace ContentStoreTest.Performance.Sessions } private async Task OpenStreamAsync( - IReadOnlyContentSession session, IReadOnlyCollection hashes, List results) + IContentSession session, IReadOnlyCollection hashes, List results) { var tasks = hashes.Select(contentHash => Task.Run(async () => await session.OpenStreamAsync(_context, contentHash, Token))); @@ -223,7 +223,7 @@ namespace ContentStoreTest.Performance.Sessions } private async Task PlaceFileAsync( - IReadOnlyContentSession session, IReadOnlyCollection> args, List results) + IContentSession session, IReadOnlyCollection> args, List results) { var tasks = args.Select(t => Task.Run(async () => await session.PlaceFileAsync( _context, @@ -384,7 +384,7 @@ namespace ContentStoreTest.Performance.Sessions ( string method, Func setupFuncAsync, - Func testFuncAsync + Func testFuncAsync ) { return RunImpl(method, AccessNeeded.ReadOnly, setupFuncAsync, async session => diff --git a/Public/Src/Cache/ContentStore/Test/Sessions/FileSystemContentPerformanceTests.cs b/Public/Src/Cache/ContentStore/Test/Sessions/FileSystemContentPerformanceTests.cs index 9c6724b3a..48fca5f82 100644 --- a/Public/Src/Cache/ContentStore/Test/Sessions/FileSystemContentPerformanceTests.cs +++ b/Public/Src/Cache/ContentStore/Test/Sessions/FileSystemContentPerformanceTests.cs @@ -37,7 +37,7 @@ namespace ContentStoreTest.Performance.Sessions return new TestFileSystemContentStore(FileSystem, SystemClock.Instance, rootPath, configurationModel); } - protected override Task> EnumerateContentHashesAsync(IReadOnlyContentSession session) + protected override Task> EnumerateContentHashesAsync(IContentSession session) { var testSession = (TestFileSystemContentSession)session; return testSession.EnumerateHashes(); diff --git a/Public/Src/Cache/ContentStore/Test/Sessions/InProcessServiceClientContentPerformanceTests.cs b/Public/Src/Cache/ContentStore/Test/Sessions/InProcessServiceClientContentPerformanceTests.cs index 08f1e2506..f678878e0 100644 --- a/Public/Src/Cache/ContentStore/Test/Sessions/InProcessServiceClientContentPerformanceTests.cs +++ b/Public/Src/Cache/ContentStore/Test/Sessions/InProcessServiceClientContentPerformanceTests.cs @@ -56,7 +56,7 @@ namespace ContentStoreTest.Performance.Sessions FileSystem, Logger, cacheName, _scenario, null, serviceConfig); } - protected override Task> EnumerateContentHashesAsync(IReadOnlyContentSession session) + protected override Task> EnumerateContentHashesAsync(IContentSession session) { var testSession = (TestServiceClientContentSession)session; return testSession.EnumerateHashes(); diff --git a/Public/Src/Cache/ContentStore/Test/Sessions/ServiceClientContentPerformanceTests.cs b/Public/Src/Cache/ContentStore/Test/Sessions/ServiceClientContentPerformanceTests.cs index 05fc0b77a..0a265b324 100644 --- a/Public/Src/Cache/ContentStore/Test/Sessions/ServiceClientContentPerformanceTests.cs +++ b/Public/Src/Cache/ContentStore/Test/Sessions/ServiceClientContentPerformanceTests.cs @@ -55,7 +55,7 @@ namespace ContentStoreTest.Performance.Sessions serviceConfiguration); } - protected override Task> EnumerateContentHashesAsync(IReadOnlyContentSession session) + protected override Task> EnumerateContentHashesAsync(IContentSession session) { var testSession = (TestServiceClientContentSession)session; return testSession.EnumerateHashes(); diff --git a/Public/Src/Cache/ContentStore/Test/Sessions/TestFileSystemContentSession.cs b/Public/Src/Cache/ContentStore/Test/Sessions/TestFileSystemContentSession.cs index 83ef1a00b..bfb1f79fc 100644 --- a/Public/Src/Cache/ContentStore/Test/Sessions/TestFileSystemContentSession.cs +++ b/Public/Src/Cache/ContentStore/Test/Sessions/TestFileSystemContentSession.cs @@ -14,7 +14,7 @@ namespace ContentStoreTest.Sessions public class TestFileSystemContentSession : FileSystemContentSession { public TestFileSystemContentSession(string name, ImplicitPin implicitPin, FileSystemContentStoreInternal store) - : base(name, implicitPin, store) + : base(name, store, implicitPin) { } diff --git a/Public/Src/Cache/ContentStore/Test/Stores/TestFailingContentStore.cs b/Public/Src/Cache/ContentStore/Test/Stores/TestFailingContentStore.cs index 51a739e4c..13547b3fd 100644 --- a/Public/Src/Cache/ContentStore/Test/Stores/TestFailingContentStore.cs +++ b/Public/Src/Cache/ContentStore/Test/Stores/TestFailingContentStore.cs @@ -48,12 +48,6 @@ namespace ContentStoreTest.Stores return Task.FromResult(new BoolResult(FailureMessage)); } - // - public CreateSessionResult CreateReadOnlySession(Context context, string name, ImplicitPin implicitPin) - { - return new CreateSessionResult(FailureMessage); - } - // public CreateSessionResult CreateSession(Context context, string name, ImplicitPin implicitPin) { diff --git a/Public/Src/Cache/ContentStore/Test/Stores/TestFileSystemContentStore.cs b/Public/Src/Cache/ContentStore/Test/Stores/TestFileSystemContentStore.cs index 430b2a9b3..304d27ab9 100644 --- a/Public/Src/Cache/ContentStore/Test/Stores/TestFileSystemContentStore.cs +++ b/Public/Src/Cache/ContentStore/Test/Stores/TestFileSystemContentStore.cs @@ -27,12 +27,6 @@ namespace ContentStoreTest.Stores public ContentStoreConfiguration Configuration => Store.Configuration; - public override CreateSessionResult CreateReadOnlySession(Context context, string name, ImplicitPin implicitPin) - { - var session = new TestFileSystemContentSession(name, implicitPin, Store); - return new CreateSessionResult(session); - } - public override CreateSessionResult CreateSession(Context context, string name, ImplicitPin implicitPin) { var session = new TestFileSystemContentSession(name, implicitPin, Store); diff --git a/Public/Src/Cache/ContentStore/Test/Stores/TestInProcessServiceClientContentStore.cs b/Public/Src/Cache/ContentStore/Test/Stores/TestInProcessServiceClientContentStore.cs index 5c86b4dc9..968c9626e 100644 --- a/Public/Src/Cache/ContentStore/Test/Stores/TestInProcessServiceClientContentStore.cs +++ b/Public/Src/Cache/ContentStore/Test/Stores/TestInProcessServiceClientContentStore.cs @@ -163,29 +163,8 @@ namespace ContentStoreTest.Stores return new ServiceClientRpcConfiguration(grpcPort, _heartbeatInterval); } - public override CreateSessionResult CreateReadOnlySession( + public override CreateSessionResult CreateSession( Context context, string name, ImplicitPin implicitPin) - { - return CreateReadOnlySessionCall.Run(ExecutionTracer, OperationContext(context), name, () => - { - var session = new TestServiceClientContentSession( - new OperationContext(context), - name, - implicitPin, - Configuration.RetryPolicy, - _configuration.DataRootPath, - _overrideCacheName ?? Configuration.CacheName, - context.Logger, - FileSystem, - Configuration.Scenario, - this, - SessionTracer, - GetRpcConfig()); - return new CreateSessionResult(session); - }); - } - - public override CreateSessionResult CreateSession(Context context, string name, ImplicitPin implicitPin) { return CreateSessionCall.Run(ExecutionTracer, OperationContext(context), name, () => { diff --git a/Public/Src/Cache/ContentStore/Test/Stores/TestServiceClientContentStore.cs b/Public/Src/Cache/ContentStore/Test/Stores/TestServiceClientContentStore.cs index b32a987d7..d94074102 100644 --- a/Public/Src/Cache/ContentStore/Test/Stores/TestServiceClientContentStore.cs +++ b/Public/Src/Cache/ContentStore/Test/Stores/TestServiceClientContentStore.cs @@ -117,29 +117,6 @@ namespace ContentStoreTest.Stores return new ServiceClientRpcConfiguration(grpcPort, _heartbeatInterval); } - public override CreateSessionResult CreateReadOnlySession( - Context context, string name, ImplicitPin implicitPin) - { - return CreateReadOnlySessionCall.Run(ExecutionTracer, OperationContext(context), name, () => - { - var session = new TestServiceClientContentSession( - new OperationContext(context), - name, - implicitPin, - Configuration.RetryPolicy, - _configuration.DataRootPath, - _overrideCacheName ?? Configuration.CacheName, - context.Logger, - FileSystem, - Configuration.Scenario, - this, - SessionTracer, - GetRpcConfig() - ); - return new CreateSessionResult(session); - }); - } - public override CreateSessionResult CreateSession(Context context, string name, ImplicitPin implicitPin) { return CreateSessionCall.Run(ExecutionTracer, OperationContext(context), name, () => diff --git a/Public/Src/Cache/ContentStore/Vsts/BlobReadOnlyContentSession.cs b/Public/Src/Cache/ContentStore/Vsts/BlobReadOnlyContentSession.cs index c71e7690b..302774df9 100644 --- a/Public/Src/Cache/ContentStore/Vsts/BlobReadOnlyContentSession.cs +++ b/Public/Src/Cache/ContentStore/Vsts/BlobReadOnlyContentSession.cs @@ -42,7 +42,7 @@ using OperationContext = BuildXL.Cache.ContentStore.Tracing.Internal.OperationCo namespace BuildXL.Cache.ContentStore.Vsts { /// - /// IReadOnlyContentSession for BlobBuildXL.ContentStore. + /// IContentSession for BlobBuildXL.ContentStore. /// [SuppressMessage("ReSharper", "MemberCanBePrivate.Global")] public class BlobReadOnlyContentSession : ContentSessionBase, IReadOnlyBackingContentSession diff --git a/Public/Src/Cache/ContentStore/Vsts/DedupReadOnlyContentSession.cs b/Public/Src/Cache/ContentStore/Vsts/DedupReadOnlyContentSession.cs index d965e1155..2f5498e6f 100644 --- a/Public/Src/Cache/ContentStore/Vsts/DedupReadOnlyContentSession.cs +++ b/Public/Src/Cache/ContentStore/Vsts/DedupReadOnlyContentSession.cs @@ -34,7 +34,7 @@ using OperationContext = BuildXL.Cache.ContentStore.Tracing.Internal.OperationCo namespace BuildXL.Cache.ContentStore.Vsts { /// - /// IReadOnlyContentSession for DedupContentStore. + /// IContentSession for DedupContentStore. /// [SuppressMessage("ReSharper", "MemberCanBePrivate.Global")] public class DedupReadOnlyContentSession : ContentSessionBase, IReadOnlyBackingContentSession diff --git a/Public/Src/Cache/ContentStore/Vsts/IBackingContentSession.cs b/Public/Src/Cache/ContentStore/Vsts/IBackingContentSession.cs index 93f92eeb1..b771f05de 100644 --- a/Public/Src/Cache/ContentStore/Vsts/IBackingContentSession.cs +++ b/Public/Src/Cache/ContentStore/Vsts/IBackingContentSession.cs @@ -12,7 +12,7 @@ using BuildXL.Cache.ContentStore.Tracing.Internal; namespace BuildXL.Cache.ContentStore.Vsts { /// - public interface IReadOnlyBackingContentSession : IReadOnlyContentSession + public interface IReadOnlyBackingContentSession : IContentSession { /// /// Bulk operations for pins with a specific TTL diff --git a/Public/Src/Cache/DistributedCache.Host/Service/Internal/MultiLevelContentSession.cs b/Public/Src/Cache/DistributedCache.Host/Service/Internal/MultiLevelContentSession.cs index e045c5349..894edad23 100644 --- a/Public/Src/Cache/DistributedCache.Host/Service/Internal/MultiLevelContentSession.cs +++ b/Public/Src/Cache/DistributedCache.Host/Service/Internal/MultiLevelContentSession.cs @@ -2,23 +2,31 @@ // Licensed under the MIT License. using System; +using System.Collections.Generic; +using System.Diagnostics.ContractsLight; using System.IO; +using System.Linq; using System.Threading.Tasks; using BuildXL.Cache.ContentStore.Hashing; using BuildXL.Cache.ContentStore.Interfaces.FileSystem; using BuildXL.Cache.ContentStore.Interfaces.Results; using BuildXL.Cache.ContentStore.Interfaces.Sessions; +using BuildXL.Cache.ContentStore.Interfaces.Tracing; +using BuildXL.Cache.ContentStore.Sessions; +using BuildXL.Cache.ContentStore.Tracing; using BuildXL.Cache.ContentStore.Tracing.Internal; using BuildXL.Utilities.Tracing; namespace BuildXL.Cache.Host.Service.Internal { - /// - /// Session which aggregates a local and backing content store. The backing content store is - /// used to populate local content store in cases of local misses. - /// - public class MultiLevelContentSession : MultiLevelReadOnlyContentSession, IContentSession + public class MultiLevelContentSession : ContentSessionBase, IContentSession, IHibernateContentSession { + protected readonly IContentSession LocalSession; + protected readonly IContentSession BackingSession; + + /// + protected override Tracer Tracer { get; } = new Tracer(nameof(MultiLevelContentStore)); + /// /// Initializes a new instance of the class. /// @@ -26,32 +34,202 @@ namespace BuildXL.Cache.Host.Service.Internal string name, IContentSession localSession, IContentSession backingSession) - : base(name, localSession, backingSession, isLocalWritable: true) + : base(name) { + Contract.Requires(name != null); + Contract.Requires(localSession != null); + Contract.Requires(backingSession != null); + + LocalSession = localSession; + BackingSession = backingSession; } /// - protected override Task PutFileCoreAsync(OperationContext operationContext, ContentHash contentHash, AbsolutePath path, FileRealizationMode realizationMode, UrgencyHint urgencyHint, Counter retryCounter) + protected override async Task StartupCoreAsync(OperationContext context) { - return MultiLevelWriteAsync(session => session.PutFileAsync( + return (await LocalSession.StartupAsync(context) & await BackingSession.StartupAsync(context)); + } + + /// + protected override async Task ShutdownCoreAsync(OperationContext context) + { + return (await LocalSession.ShutdownAsync(context) & await BackingSession.ShutdownAsync(context)); + } + + private async Task MultiLevelReadAsync( + OperationContext context, + ContentHash hash, + Func> runAsync) + where TResult : ResultBase + { + var result = await runAsync(LocalSession); + if (!result.Succeeded) + { + var ensureLocalResult = await EnsureLocalAsync(context, hash); + if (ensureLocalResult.Succeeded) + { + return await runAsync(LocalSession); + } + } + + return result; + } + + private Task EnsureLocalAsync(OperationContext context, ContentHash hash) + { + return context.PerformOperationAsync( + Tracer, + async () => + { + var streamResult = await BackingSession.OpenStreamAsync(context, hash, context.Token).ThrowIfFailure(); + + return await LocalSession.PutStreamAsync(context, hash, streamResult.Stream, context.Token); + }); + } + + /// + public IEnumerable EnumeratePinnedContentHashes() + { + return LocalSession is IHibernateContentSession session + ? session.EnumeratePinnedContentHashes() + : Enumerable.Empty(); + } + + /// + public Task PinBulkAsync(Context context, IEnumerable contentHashes) + { + return LocalSession is IHibernateContentSession session + ? session.PinBulkAsync(context, contentHashes) + : BoolResult.SuccessTask; + } + + /// + public Task ShutdownEvictionAsync(Context context) + { + return LocalSession is IHibernateContentSession session + ? session.ShutdownEvictionAsync(context) + : BoolResult.SuccessTask; + } + + /// + protected override Task OpenStreamCoreAsync( + OperationContext operationContext, + ContentHash contentHash, + UrgencyHint urgencyHint, + Counter retryCounter) + { + return MultiLevelReadAsync( operationContext, contentHash, - path, - CoerceRealizationMode(realizationMode, session), - operationContext.Token, - urgencyHint)); + session => session.OpenStreamAsync(operationContext, contentHash, operationContext.Token, urgencyHint)); } /// - protected override Task PutFileCoreAsync(OperationContext operationContext, HashType hashType, AbsolutePath path, FileRealizationMode realizationMode, UrgencyHint urgencyHint, Counter retryCounter) + protected override Task PinCoreAsync( + OperationContext operationContext, + ContentHash contentHash, + UrgencyHint urgencyHint, + Counter retryCounter) { - return MultiLevelWriteAsync(session => session.PutFileAsync( + // TODO: decide if we want to pin on the backing session as well. The issue here is as follows: on the use- + // case we need this for (permanent distributed CASaaS running + local CAS on a different drive with drop + // as client against distributed), it doesn't really matters what pin does. For the general scenario, it + // depends. + return LocalSession.PinAsync(operationContext, contentHash, operationContext.Token, urgencyHint); + } + + /// + protected override Task>>> PinCoreAsync( + OperationContext operationContext, + IReadOnlyList contentHashes, + UrgencyHint urgencyHint, + Counter retryCounter, + Counter fileCounter) + { + // TODO: decide if we want to pin on the backing session as well. The issue here is as follows: on the use- + // case we need this for (permanent distributed CASaaS running + local CAS on a different drive with drop + // as client against distributed), it doesn't really matters what pin does. For the general scenario, it + // depends. + return LocalSession.PinAsync(operationContext, contentHashes, operationContext.Token, urgencyHint); + } + + /// + protected override Task PlaceFileCoreAsync( + OperationContext operationContext, + ContentHash contentHash, + AbsolutePath path, + FileAccessMode accessMode, + FileReplacementMode replacementMode, + FileRealizationMode realizationMode, + UrgencyHint urgencyHint, + Counter retryCounter) + { + return MultiLevelReadAsync( operationContext, - hashType, - path, - CoerceRealizationMode(realizationMode, session), - operationContext.Token, - urgencyHint)); + contentHash, + session => session.PlaceFileAsync( + operationContext, + contentHash, + path, + accessMode, + replacementMode, + realizationMode, + operationContext.Token, + urgencyHint)); + } + + /// + protected override Task>>> PlaceFileCoreAsync( + OperationContext operationContext, + IReadOnlyList hashesWithPaths, + FileAccessMode accessMode, + FileReplacementMode replacementMode, + FileRealizationMode realizationMode, + UrgencyHint urgencyHint, + Counter retryCounter) + { + // NOTE: Most of the IContentSession implementations throw NotImplementedException, most notably + // the ReadOnlyServiceClientContentSession which is used to communicate with this session. Given that, + // it is safe for this method to not be implemented here as well. + throw new NotImplementedException(); + } + + /// + protected override Task PutFileCoreAsync( + OperationContext operationContext, + ContentHash contentHash, + AbsolutePath path, + FileRealizationMode realizationMode, + UrgencyHint urgencyHint, + Counter retryCounter) + { + return MultiLevelWriteAsync( + session => session.PutFileAsync( + operationContext, + contentHash, + path, + CoerceRealizationMode(realizationMode, session), + operationContext.Token, + urgencyHint)); + } + + /// + protected override Task PutFileCoreAsync( + OperationContext operationContext, + HashType hashType, + AbsolutePath path, + FileRealizationMode realizationMode, + UrgencyHint urgencyHint, + Counter retryCounter) + { + return MultiLevelWriteAsync( + session => session.PutFileAsync( + operationContext, + hashType, + path, + CoerceRealizationMode(realizationMode, session), + operationContext.Token, + urgencyHint)); } private FileRealizationMode CoerceRealizationMode(FileRealizationMode mode, IContentSession session) @@ -66,13 +244,24 @@ namespace BuildXL.Cache.Host.Service.Internal } /// - protected override Task PutStreamCoreAsync(OperationContext operationContext, ContentHash contentHash, Stream stream, UrgencyHint urgencyHint, Counter retryCounter) + protected override Task PutStreamCoreAsync( + OperationContext operationContext, + ContentHash contentHash, + Stream stream, + UrgencyHint urgencyHint, + Counter retryCounter) { - return MultiLevelWriteAsync(session => session.PutStreamAsync(operationContext, contentHash, stream, operationContext.Token, urgencyHint)); + return MultiLevelWriteAsync( + session => session.PutStreamAsync(operationContext, contentHash, stream, operationContext.Token, urgencyHint)); } /// - protected override Task PutStreamCoreAsync(OperationContext operationContext, HashType hashType, Stream stream, UrgencyHint urgencyHint, Counter retryCounter) + protected override Task PutStreamCoreAsync( + OperationContext operationContext, + HashType hashType, + Stream stream, + UrgencyHint urgencyHint, + Counter retryCounter) { return MultiLevelWriteAsync(session => session.PutStreamAsync(operationContext, hashType, stream, operationContext.Token, urgencyHint)); } diff --git a/Public/Src/Cache/DistributedCache.Host/Service/Internal/MultiLevelContentStore.cs b/Public/Src/Cache/DistributedCache.Host/Service/Internal/MultiLevelContentStore.cs index d8a4623c2..6ca90e041 100644 --- a/Public/Src/Cache/DistributedCache.Host/Service/Internal/MultiLevelContentStore.cs +++ b/Public/Src/Cache/DistributedCache.Host/Service/Internal/MultiLevelContentStore.cs @@ -58,18 +58,6 @@ namespace BuildXL.Cache.Host.Service.Internal _backingContentStore.Dispose(); } - /// - public CreateSessionResult CreateReadOnlySession(Context context, string name, ImplicitPin implicitPin) - { - return CreateReadOnlySessionCall.Run((ContentStoreTracer)Tracer, new OperationContext(context), name, () => - { - var localSession = _localContentStore.CreateSession(context, name, implicitPin).ThrowIfFailure(); - var backingSession = _backingContentStore.CreateReadOnlySession(context, name, implicitPin).ThrowIfFailure(); - - return new CreateSessionResult(new MultiLevelReadOnlyContentSession(name, localSession.Session, backingSession.Session, isLocalWritable: true)); - }); - } - /// public CreateSessionResult CreateSession(Context context, string name, ImplicitPin implicitPin) { diff --git a/Public/Src/Cache/DistributedCache.Host/Service/Internal/MultiLevelReadOnlyContentSession.cs b/Public/Src/Cache/DistributedCache.Host/Service/Internal/MultiLevelReadOnlyContentSession.cs deleted file mode 100644 index 9aeb2e862..000000000 --- a/Public/Src/Cache/DistributedCache.Host/Service/Internal/MultiLevelReadOnlyContentSession.cs +++ /dev/null @@ -1,168 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -using System; -using System.Collections.Generic; -using System.Diagnostics.ContractsLight; -using System.Linq; -using System.Threading.Tasks; -using BuildXL.Cache.ContentStore.Hashing; -using BuildXL.Cache.ContentStore.Interfaces.FileSystem; -using BuildXL.Cache.ContentStore.Interfaces.Results; -using BuildXL.Cache.ContentStore.Interfaces.Sessions; -using BuildXL.Cache.ContentStore.Interfaces.Tracing; -using BuildXL.Cache.ContentStore.Sessions; -using BuildXL.Cache.ContentStore.Tracing; -using BuildXL.Cache.ContentStore.Tracing.Internal; -using BuildXL.Utilities.Tracing; - -namespace BuildXL.Cache.Host.Service.Internal -{ - using PlaceBulkResult = IEnumerable>>; - - /// - /// Session which aggregates a local and backing content store. If local content store is marked writable, the - /// backing content store is used to populate local content store in cases of local misses. - /// - public class MultiLevelReadOnlyContentSession : ContentSessionBase, IHibernateContentSession - where TSession : IReadOnlyContentSession - { - protected readonly TSession LocalSession; - protected readonly TSession BackingSession; - - /// - /// Gets a local session in which content can be put if writes are allowed - /// - protected readonly IContentSession WritableLocalSession; - - /// - protected override Tracer Tracer { get; } = new Tracer(nameof(MultiLevelContentStore)); - - /// - /// Initializes a new instance of the class. - /// - public MultiLevelReadOnlyContentSession( - string name, - TSession localSession, - TSession backingSession, - bool isLocalWritable) - : base(name) - { - Contract.Requires(name != null); - Contract.Requires(localSession != null); - Contract.Requires(backingSession != null); - - LocalSession = localSession; - BackingSession = backingSession; - WritableLocalSession = isLocalWritable ? (IContentSession)localSession : null; - } - - /// - protected override async Task StartupCoreAsync(OperationContext context) - { - return (await LocalSession.StartupAsync(context) & await BackingSession.StartupAsync(context)); - } - - /// - protected override async Task ShutdownCoreAsync(OperationContext context) - { - return (await LocalSession.ShutdownAsync(context) & await BackingSession.ShutdownAsync(context)); - } - - private async Task MultiLevelReadAsync( - OperationContext context, - ContentHash hash, - Func> runAsync) - where TResult : ResultBase - { - var result = await runAsync(LocalSession); - if (!result.Succeeded && WritableLocalSession != null) - { - var ensureLocalResult = await EnsureLocalAsync(context, hash); - if (ensureLocalResult.Succeeded) - { - return await runAsync(LocalSession); - } - } - - return result; - } - - private Task EnsureLocalAsync(OperationContext context, ContentHash hash) - { - return context.PerformOperationAsync( - Tracer, - async () => - { - var streamResult = await BackingSession.OpenStreamAsync(context, hash, context.Token).ThrowIfFailure(); - - return await WritableLocalSession.PutStreamAsync(context, hash, streamResult.Stream, context.Token); - }); - } - - /// - public IEnumerable EnumeratePinnedContentHashes() - { - return LocalSession is IHibernateContentSession session - ? session.EnumeratePinnedContentHashes() - : Enumerable.Empty(); - } - - /// - public Task PinBulkAsync(Context context, IEnumerable contentHashes) - { - return LocalSession is IHibernateContentSession session - ? session.PinBulkAsync(context, contentHashes) - : BoolResult.SuccessTask; - } - - /// - public Task ShutdownEvictionAsync(Context context) - { - return LocalSession is IHibernateContentSession session - ? session.ShutdownEvictionAsync(context) - : BoolResult.SuccessTask; - } - - /// - protected override Task OpenStreamCoreAsync(OperationContext operationContext, ContentHash contentHash, UrgencyHint urgencyHint, Counter retryCounter) - { - return MultiLevelReadAsync(operationContext, contentHash, session => session.OpenStreamAsync(operationContext, contentHash, operationContext.Token, urgencyHint)); - } - - /// - protected override Task PinCoreAsync(OperationContext operationContext, ContentHash contentHash, UrgencyHint urgencyHint, Counter retryCounter) - { - // TODO: decide if we want to pin on the backing session as well. The issue here is as follows: on the use- - // case we need this for (permanent distributed CASaaS running + local CAS on a different drive with drop - // as client against distributed), it doesn't really matters what pin does. For the general scenario, it - // depends. - return LocalSession.PinAsync(operationContext, contentHash, operationContext.Token, urgencyHint); - } - - /// - protected override Task>>> PinCoreAsync(OperationContext operationContext, IReadOnlyList contentHashes, UrgencyHint urgencyHint, Counter retryCounter, Counter fileCounter) - { - // TODO: decide if we want to pin on the backing session as well. The issue here is as follows: on the use- - // case we need this for (permanent distributed CASaaS running + local CAS on a different drive with drop - // as client against distributed), it doesn't really matters what pin does. For the general scenario, it - // depends. - return LocalSession.PinAsync(operationContext, contentHashes, operationContext.Token, urgencyHint); - } - - /// - protected override Task PlaceFileCoreAsync(OperationContext operationContext, ContentHash contentHash, AbsolutePath path, FileAccessMode accessMode, FileReplacementMode replacementMode, FileRealizationMode realizationMode, UrgencyHint urgencyHint, Counter retryCounter) - { - return MultiLevelReadAsync(operationContext, contentHash, session => session.PlaceFileAsync(operationContext, contentHash, path, accessMode, replacementMode, realizationMode, operationContext.Token, urgencyHint)); - } - - /// - protected override Task PlaceFileCoreAsync(OperationContext operationContext, IReadOnlyList hashesWithPaths, FileAccessMode accessMode, FileReplacementMode replacementMode, FileRealizationMode realizationMode, UrgencyHint urgencyHint, Counter retryCounter) - { - // NOTE: Most of the IContentSession implementations throw NotImplementedException, most notably - // the ReadOnlyServiceClientContentSession which is used to communicate with this session. Given that, - // it is safe for this method to not be implemented here as well. - throw new NotImplementedException(); - } - } -} diff --git a/Public/Src/Cache/DistributedCache.Host/Service/Internal/MultiplexedContentSession.cs b/Public/Src/Cache/DistributedCache.Host/Service/Internal/MultiplexedContentSession.cs index e47f467fb..25f9e039f 100644 --- a/Public/Src/Cache/DistributedCache.Host/Service/Internal/MultiplexedContentSession.cs +++ b/Public/Src/Cache/DistributedCache.Host/Service/Internal/MultiplexedContentSession.cs @@ -3,26 +3,416 @@ using System; using System.Collections.Generic; +using System.Diagnostics.ContractsLight; using System.IO; +using System.Linq; +using System.Runtime.CompilerServices; using System.Threading; using System.Threading.Tasks; +using BuildXL.Cache.ContentStore.Distributed.Utilities; using BuildXL.Cache.ContentStore.Hashing; using BuildXL.Cache.ContentStore.Interfaces.FileSystem; using BuildXL.Cache.ContentStore.Interfaces.Results; using BuildXL.Cache.ContentStore.Interfaces.Sessions; using BuildXL.Cache.ContentStore.Interfaces.Tracing; using BuildXL.Cache.ContentStore.Sessions.Internal; +using BuildXL.Cache.ContentStore.Tracing; +using BuildXL.Cache.ContentStore.Tracing.Internal; +using BuildXL.Cache.ContentStore.Utils; namespace BuildXL.Cache.Host.Service.Internal { - public class MultiplexedContentSession : MultiplexedReadOnlyContentSession, IContentSession, ITrustedContentSession + public class MultiplexedContentSession : StartupShutdownBase, IContentSession, ITrustedContentSession, IHibernateContentSession { + /// + public readonly IContentSession PreferredContentSession; + + /// + public readonly IDictionary SessionsByCacheRoot; + + protected readonly MultiplexedContentStore Store; + + /// + protected override Tracer Tracer { get; } = new Tracer(nameof(MultiplexedContentSession)); + + /// + public string Name { get; } + + /// + protected override async Task StartupCoreAsync(OperationContext context) + { + var finalResult = BoolResult.Success; + + var sessions = Enumerable.ToArray(SessionsByCacheRoot.Values); + for (var i = 0; i < sessions.Length; i++) + { + var canHibernate = sessions[i] is IHibernateContentSession ? "can" : "cannot"; + Tracer.Debug(context, $"Session {sessions[i].Name} {canHibernate} hibernate"); + var startupResult = await sessions[i].StartupAsync(context).ConfigureAwait(false); + + if (!startupResult.Succeeded) + { + finalResult = startupResult; + for (var j = 0; j < i; j++) + { + var shutdownResult = await sessions[j].ShutdownAsync(context).ConfigureAwait(false); + if (!shutdownResult.Succeeded) + { + finalResult = new BoolResult(finalResult, shutdownResult.ErrorMessage); + } + } + } + } + + return finalResult; + } + + /// + protected override async Task ShutdownCoreAsync(OperationContext context) + { + var finalResult = BoolResult.Success; + + foreach (var session in SessionsByCacheRoot.Values) + { + var result = await session.ShutdownAsync(context).ConfigureAwait(false); + if (!result.Succeeded) + { + finalResult = new BoolResult(finalResult, result.ErrorMessage); + } + } + + return finalResult; + } + + /// + protected override void DisposeCore() + { + foreach (var session in SessionsByCacheRoot.Values) + { + session.Dispose(); + } + } + + /// + public Task PinAsync( + Context context, + ContentHash contentHash, + CancellationToken cts, + UrgencyHint urgencyHint = UrgencyHint.Nominal) + { + return PerformAggregateSessionOperationAsync( + context, + session => session.PinAsync(context, contentHash, cts, urgencyHint), + (r1, r2) => r1.Succeeded ? r1 : r2, + shouldBreak: r => r.Succeeded); + } + + /// + public Task OpenStreamAsync( + Context context, + ContentHash contentHash, + CancellationToken cts, + UrgencyHint urgencyHint = UrgencyHint.Nominal) + { + return PerformAggregateSessionOperationAsync( + context, + session => session.OpenStreamAsync(context, contentHash, cts, urgencyHint), + (r1, r2) => r1.Succeeded ? r1 : r2, + shouldBreak: r => r.Succeeded); + } + + /// + public Task PlaceFileAsync( + Context context, + ContentHash contentHash, + AbsolutePath path, + FileAccessMode accessMode, + FileReplacementMode replacementMode, + FileRealizationMode realizationMode, + CancellationToken cts, + UrgencyHint urgencyHint = UrgencyHint.Nominal) + { + IContentSession hardlinkSession = null; + if (realizationMode == FileRealizationMode.HardLink) + { + var drive = path.GetPathRoot(); + if (SessionsByCacheRoot.TryGetValue(drive, out var session) && session is IContentSession writeableSession) + { + hardlinkSession = writeableSession; + } + else + { + return Task.FromResult(new PlaceFileResult("Requested hardlink but there is no session on the same drive as destination path.")); + } + } + + return PerformAggregateSessionOperationAsync( + context, + executeAsync: placeFileCore, + (r1, r2) => r1.Succeeded ? r1 : r2, + shouldBreak: r => r.Succeeded, + pathHint: path); + + async Task placeFileCore(IContentSession session) + { + // If we exclusively want a hardlink, we should make sure that we can copy from other drives to satisfy the request. + if (realizationMode != FileRealizationMode.HardLink || session == hardlinkSession) + { + return await session.PlaceFileAsync(context, contentHash, path, accessMode, replacementMode, realizationMode, cts, urgencyHint); + } + + // See if session has the content. + var streamResult = await session.OpenStreamAsync(context, contentHash, cts, urgencyHint).ThrowIfFailure(); + + // Put it into correct store + var putResult = await hardlinkSession.PutStreamAsync(context, contentHash, streamResult.Stream, cts, urgencyHint).ThrowIfFailure(); + + // Try the hardlink on the correct drive. + return await hardlinkSession.PlaceFileAsync( + context, + contentHash, + path, + accessMode, + replacementMode, + realizationMode, + cts, + urgencyHint); + } + } + + /// + public Task>>> PinAsync( + Context context, + IReadOnlyList contentHashes, + CancellationToken cts, + UrgencyHint urgencyHint = UrgencyHint.Nominal) + { + return MultiLevelUtilities.RunManyLevelAsync( + GetSessionsInOrder().ToArray(), + contentHashes, + (session, hashes) => session.PinAsync(context, hashes, cts, urgencyHint), + p => p.Succeeded); + } + + public Task>>> PinAsync( + Context context, + IReadOnlyList contentHashes, + PinOperationConfiguration config) + { + return MultiLevelUtilities.RunManyLevelAsync( + GetSessionsInOrder().ToArray(), + contentHashes, + (session, hashes) => session.PinAsync(context, hashes, config), + p => p.Succeeded); + } + + protected TCache GetCache(AbsolutePath path = null) + { + if (path != null) + { + var drive = path.GetPathRoot(); + if (SessionsByCacheRoot.TryGetValue(drive, out var contentSession)) + { + return (TCache)contentSession; + } + } + + return (TCache)PreferredContentSession; + } + + public Task>>> PlaceFileAsync( + Context context, + IReadOnlyList hashesWithPaths, + FileAccessMode accessMode, + FileReplacementMode replacementMode, + FileRealizationMode realizationMode, + CancellationToken cts, + UrgencyHint urgencyHint = UrgencyHint.Nominal) + { + // NOTE: this goes around the FileSystemContentStore's bulk place, rendering it useless. The reason + // we do it this way is so that the hardlink logic stays consistent. This means that multiple place + // operations from the same request may run at a higher rate than estipulated by the store. + IEnumerable>> materializations = hashesWithPaths.Select( + async (hashWithPath, index) => + { + var result = await PlaceFileAsync( + context, + hashWithPath.Hash, + hashWithPath.Path, + accessMode, + replacementMode, + realizationMode, + cts, + urgencyHint); + return new Indexed(result, index); + }); + + return Task.FromResult(materializations.ToList().AsEnumerable()); + } + + /// + public IEnumerable EnumeratePinnedContentHashes() + { + return PerformAggregateSessionOperationCoreAsync>>( + session => + { + var hashes = session.EnumeratePinnedContentHashes(); + return Task.FromResult(Result.Success(hashes)); + }, + (r1, r2) => Result.Success(r1.Value.Concat(r2.Value)), + shouldBreak: r => false).GetAwaiter().GetResult().Value; + } + + /// + public Task PinBulkAsync(Context context, IEnumerable contentHashes) + { + return PerformAggregateSessionOperationAsync( + context, + async session => + { + await session.PinBulkAsync(context, contentHashes); + return BoolResult.Success; + }, + (r1, r2) => r1 & r2, + shouldBreak: r => false); + } + + /// + public Task ShutdownEvictionAsync(Context context) + { + return PerformAggregateSessionOperationAsync( + context, + session => session.ShutdownEvictionAsync(context), + (r1, r2) => r1 & r2, + shouldBreak: r => false); + } + + private IEnumerable GetSessionsInOrder(AbsolutePath path = null) + { + var drive = path != null ? path.GetPathRoot() : Store.PreferredCacheDrive; + + if (!SessionsByCacheRoot.ContainsKey(drive)) + { + drive = Store.PreferredCacheDrive; + } + + if (SessionsByCacheRoot[drive] is TSession session) + { + yield return session; + } + + foreach (var kvp in SessionsByCacheRoot) + { + if (StringComparer.OrdinalIgnoreCase.Equals((string)kvp.Key, drive)) + { + // Already yielded the preferred cache + continue; + } + + if (kvp.Value is TSession otherSession) + { + yield return otherSession; + } + } + } + + private async Task PerformSessionOperationAsync(Func> executeAsync) + where TResult : ResultBase + { + TResult result = null; + + foreach (var session in GetSessionsInOrder()) + { + result = await executeAsync(session); + + if (result.Succeeded) + { + return result; + } + } + + return result ?? new ErrorResult( + $"Could not find a content session which implements {typeof(TSession).Name} in {nameof(MultiplexedContentSession)}.") + .AsResult(); + } + + private Task PerformAggregateSessionOperationAsync( + Context context, + Func> executeAsync, + Func aggregate, + Func shouldBreak, + AbsolutePath pathHint = null, + [CallerMemberName] string caller = null) + where TResult : ResultBase + { + var operationContext = context is null ? new OperationContext() : new OperationContext(context); + return operationContext.PerformOperationAsync( + Tracer, + () => PerformAggregateSessionOperationCoreAsync(executeAsync, aggregate, shouldBreak, pathHint), + traceOperationStarted: false, + traceOperationFinished: false, + caller: caller); + } + + private async Task PerformAggregateSessionOperationCoreAsync( + Func> executeAsync, + Func aggregate, + Func shouldBreak, + AbsolutePath pathHint = null) + where TResult : ResultBase + { + TResult result = null; + + // Go through all the sessions + foreach (var session in GetSessionsInOrder(pathHint)) + { + var priorResult = result; + + try + { + result = await executeAsync(session); + } + catch (Exception e) + { + result = new ErrorResult(e).AsResult(); + } + + // Aggregate with previous result + if (priorResult != null) + { + result = aggregate(priorResult, result); + } + + // If result is sufficient, stop trying other stores and return result + if (shouldBreak(result)) + { + return result; + } + } + + Contract.Assert( + result != null, + $"Could not find a content session which implements {typeof(TSession).Name} in {nameof(MultiplexedContentSession)}."); + return result; + } + /// /// Initializes a new instance of the class. /// - public MultiplexedContentSession(Dictionary cacheSessionsByRoot, string name, MultiplexedContentStore store) - : base(cacheSessionsByRoot, name, store) + public MultiplexedContentSession(Dictionary cacheSessionsByRoot, string name, MultiplexedContentStore store) { + Contract.Requires(name != null); + Contract.Requires(cacheSessionsByRoot != null); + Contract.Requires(cacheSessionsByRoot.Count > 0); + + Name = name; + SessionsByCacheRoot = cacheSessionsByRoot; + Store = store; + + if (!SessionsByCacheRoot.TryGetValue(store.PreferredCacheDrive, out PreferredContentSession)) + { + throw new ArgumentException(nameof(store.PreferredCacheDrive)); + } } /// @@ -76,7 +466,13 @@ namespace BuildXL.Cache.Host.Service.Internal } /// - public Task PutTrustedFileAsync(Context context, ContentHashWithSize contentHashWithSize, AbsolutePath path, FileRealizationMode realizationMode, CancellationToken cts, UrgencyHint urgencyHint) + public Task PutTrustedFileAsync( + Context context, + ContentHashWithSize contentHashWithSize, + AbsolutePath path, + FileRealizationMode realizationMode, + CancellationToken cts, + UrgencyHint urgencyHint) { var session = GetCache(path); return session.PutTrustedFileAsync(context, contentHashWithSize, path, realizationMode, cts, urgencyHint); diff --git a/Public/Src/Cache/DistributedCache.Host/Service/Internal/MultiplexedContentStore.cs b/Public/Src/Cache/DistributedCache.Host/Service/Internal/MultiplexedContentStore.cs index 0ff77a9a0..4a131d4f3 100644 --- a/Public/Src/Cache/DistributedCache.Host/Service/Internal/MultiplexedContentStore.cs +++ b/Public/Src/Cache/DistributedCache.Host/Service/Internal/MultiplexedContentStore.cs @@ -106,38 +106,12 @@ namespace BuildXL.Cache.Host.Service.Internal } } - /// - public CreateSessionResult CreateReadOnlySession(Context context, string name, ImplicitPin implicitPin) - { - return CreateReadOnlySessionCall.Run(StoreTracer, new OperationContext(context), name, () => - { - var sessions = new Dictionary(StringComparer.OrdinalIgnoreCase); - foreach (KeyValuePair entry in DrivesWithContentStore) - { - var result = entry.Value.CreateReadOnlySession(context, name, implicitPin); - if (!result.Succeeded) - { - foreach (var session in sessions.Values) - { - session.Dispose(); - } - - return new CreateSessionResult(result); - } - sessions.Add(entry.Key, result.Session); - } - - var multiCacheSession = new MultiplexedReadOnlyContentSession(sessions, name, this); - return new CreateSessionResult(multiCacheSession); - }); - } - /// public CreateSessionResult CreateSession(Context context, string name, ImplicitPin implicitPin) { return CreateSessionCall.Run(StoreTracer, new OperationContext(context), name, () => { - var sessions = new Dictionary(StringComparer.OrdinalIgnoreCase); + var sessions = new Dictionary(StringComparer.OrdinalIgnoreCase); foreach (KeyValuePair entry in DrivesWithContentStore) { var result = entry.Value.CreateSession(context, name, implicitPin); diff --git a/Public/Src/Cache/DistributedCache.Host/Service/Internal/MultiplexedReadOnlyContentSession.cs b/Public/Src/Cache/DistributedCache.Host/Service/Internal/MultiplexedReadOnlyContentSession.cs deleted file mode 100644 index 34d1ef6f7..000000000 --- a/Public/Src/Cache/DistributedCache.Host/Service/Internal/MultiplexedReadOnlyContentSession.cs +++ /dev/null @@ -1,397 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -using System; -using System.Collections.Generic; -using System.Diagnostics.ContractsLight; -using System.Linq; -using System.Runtime.CompilerServices; -using System.Threading; -using System.Threading.Tasks; -using BuildXL.Cache.ContentStore.Distributed.Utilities; -using BuildXL.Cache.ContentStore.Hashing; -using BuildXL.Cache.ContentStore.Interfaces.FileSystem; -using BuildXL.Cache.ContentStore.Interfaces.Results; -using BuildXL.Cache.ContentStore.Interfaces.Sessions; -using BuildXL.Cache.ContentStore.Interfaces.Tracing; -using BuildXL.Cache.ContentStore.Tracing; -using BuildXL.Cache.ContentStore.Tracing.Internal; -using BuildXL.Cache.ContentStore.Utils; - -namespace BuildXL.Cache.Host.Service.Internal -{ - public class MultiplexedReadOnlyContentSession : StartupShutdownBase, IReadOnlyContentSession, IHibernateContentSession - { - /// - public readonly IReadOnlyContentSession PreferredContentSession; - - /// - public readonly IDictionary SessionsByCacheRoot; - protected readonly MultiplexedContentStore Store; - - /// - protected override Tracer Tracer { get; } = new Tracer(nameof(MultiplexedContentSession)); - - /// - public string Name { get; } - - /// - /// Initializes a new instance of the class. - /// - public MultiplexedReadOnlyContentSession( - Dictionary sessionsByCacheRoot, - string name, - MultiplexedContentStore store) - { - Contract.Requires(name != null); - Contract.Requires(sessionsByCacheRoot != null); - Contract.Requires(sessionsByCacheRoot.Count > 0); - - Name = name; - SessionsByCacheRoot = sessionsByCacheRoot; - Store = store; - - if (!SessionsByCacheRoot.TryGetValue(store.PreferredCacheDrive, out PreferredContentSession)) - { - throw new ArgumentException(nameof(store.PreferredCacheDrive)); - } - } - - /// - protected override async Task StartupCoreAsync(OperationContext context) - { - var finalResult = BoolResult.Success; - - var sessions = SessionsByCacheRoot.Values.ToArray(); - for (var i = 0; i < sessions.Length; i++) - { - var canHibernate = sessions[i] is IHibernateContentSession ? "can" : "cannot"; - Tracer.Debug(context, $"Session {sessions[i].Name} {canHibernate} hibernate"); - var startupResult = await sessions[i].StartupAsync(context).ConfigureAwait(false); - - if (!startupResult.Succeeded) - { - finalResult = startupResult; - for (var j = 0; j < i; j++) - { - var shutdownResult = await sessions[j].ShutdownAsync(context).ConfigureAwait(false); - if (!shutdownResult.Succeeded) - { - finalResult = new BoolResult(finalResult, shutdownResult.ErrorMessage); - } - } - } - } - - return finalResult; - } - - /// - protected override async Task ShutdownCoreAsync(OperationContext context) - { - var finalResult = BoolResult.Success; - - foreach (var session in SessionsByCacheRoot.Values) - { - var result = await session.ShutdownAsync(context).ConfigureAwait(false); - if (!result.Succeeded) - { - finalResult = new BoolResult(finalResult, result.ErrorMessage); - } - } - - return finalResult; - } - - /// - protected override void DisposeCore() - { - foreach (var session in SessionsByCacheRoot.Values) - { - session.Dispose(); - } - } - - /// - public Task PinAsync( - Context context, - ContentHash contentHash, - CancellationToken cts, - UrgencyHint urgencyHint = UrgencyHint.Nominal) - { - return PerformAggregateSessionOperationAsync( - context, - session => session.PinAsync(context, contentHash, cts, urgencyHint), - (r1, r2) => r1.Succeeded ? r1 : r2, - shouldBreak: r => r.Succeeded); - } - - /// - public Task OpenStreamAsync( - Context context, - ContentHash contentHash, - CancellationToken cts, - UrgencyHint urgencyHint = UrgencyHint.Nominal) - { - return PerformAggregateSessionOperationAsync( - context, - session => session.OpenStreamAsync(context, contentHash, cts, urgencyHint), - (r1, r2) => r1.Succeeded ? r1 : r2, - shouldBreak: r => r.Succeeded); - } - - /// - public Task PlaceFileAsync( - Context context, - ContentHash contentHash, - AbsolutePath path, - FileAccessMode accessMode, - FileReplacementMode replacementMode, - FileRealizationMode realizationMode, - CancellationToken cts, - UrgencyHint urgencyHint = UrgencyHint.Nominal) - { - IContentSession hardlinkSession = null; - if (realizationMode == FileRealizationMode.HardLink) - { - var drive = path.GetPathRoot(); - if (SessionsByCacheRoot.TryGetValue(drive, out var session) && session is IContentSession writeableSession) - { - hardlinkSession = writeableSession; - } - else - { - return Task.FromResult(new PlaceFileResult("Requested hardlink but there is no session on the same drive as destination path.")); - } - } - - return PerformAggregateSessionOperationAsync( - context, - executeAsync: placeFileCore, - (r1, r2) => r1.Succeeded ? r1 : r2, - shouldBreak: r => r.Succeeded, - pathHint: path); - - async Task placeFileCore(IReadOnlyContentSession session) - { - // If we exclusively want a hardlink, we should make sure that we can copy from other drives to satisfy the request. - if (realizationMode != FileRealizationMode.HardLink || session == hardlinkSession) - { - return await session.PlaceFileAsync(context, contentHash, path, accessMode, replacementMode, realizationMode, cts, urgencyHint); - } - - // See if session has the content. - var streamResult = await session.OpenStreamAsync(context, contentHash, cts, urgencyHint).ThrowIfFailure(); - - // Put it into correct store - var putResult = await hardlinkSession.PutStreamAsync(context, contentHash, streamResult.Stream, cts, urgencyHint).ThrowIfFailure(); - - // Try the hardlink on the correct drive. - return await hardlinkSession.PlaceFileAsync(context, contentHash, path, accessMode, replacementMode, realizationMode, cts, urgencyHint); - } - } - - /// - public Task>>> PinAsync( - Context context, - IReadOnlyList contentHashes, - CancellationToken cts, - UrgencyHint urgencyHint = UrgencyHint.Nominal) - { - return MultiLevelUtilities.RunManyLevelAsync( - GetSessionsInOrder().ToArray(), - contentHashes, - (session, hashes) => session.PinAsync(context, hashes, cts, urgencyHint), - p => p.Succeeded); - } - - public Task>>> PinAsync( - Context context, - IReadOnlyList contentHashes, - PinOperationConfiguration config) - { - return MultiLevelUtilities.RunManyLevelAsync( - GetSessionsInOrder().ToArray(), - contentHashes, - (session, hashes) => session.PinAsync(context, hashes, config), - p => p.Succeeded); - } - - protected TCache GetCache(AbsolutePath path = null) - { - if (path != null) - { - var drive = path.GetPathRoot(); - if (SessionsByCacheRoot.TryGetValue(drive, out var contentSession)) - { - return (TCache)contentSession; - } - } - - return (TCache)PreferredContentSession; - } - - public Task>>> PlaceFileAsync( - Context context, - IReadOnlyList hashesWithPaths, - FileAccessMode accessMode, - FileReplacementMode replacementMode, - FileRealizationMode realizationMode, - CancellationToken cts, - UrgencyHint urgencyHint = UrgencyHint.Nominal) - { - // NOTE: this goes around the FileSystemContentStore's bulk place, rendering it useless. The reason - // we do it this way is so that the hardlink logic stays consistent. This means that multiple place - // operations from the same request may run at a higher rate than estipulated by the store. - IEnumerable>> materializations = hashesWithPaths.Select(async (hashWithPath, index) => - { - var result = await PlaceFileAsync(context, hashWithPath.Hash, hashWithPath.Path, accessMode, replacementMode, realizationMode, cts, urgencyHint); - return new Indexed(result, index); - }); - - return Task.FromResult(materializations.ToList().AsEnumerable()); - } - - /// - public IEnumerable EnumeratePinnedContentHashes() - { - return PerformAggregateSessionOperationCoreAsync>>( - session => - { - var hashes = session.EnumeratePinnedContentHashes(); - return Task.FromResult(Result.Success(hashes)); - }, - (r1, r2) => Result.Success(r1.Value.Concat(r2.Value)), - shouldBreak: r => false).GetAwaiter().GetResult().Value; - } - - /// - public Task PinBulkAsync(Context context, IEnumerable contentHashes) - { - return PerformAggregateSessionOperationAsync( - context, - async session => - { - await session.PinBulkAsync(context, contentHashes); - return BoolResult.Success; - }, - (r1, r2) => r1 & r2, - shouldBreak: r => false); - } - - /// - public Task ShutdownEvictionAsync(Context context) - { - return PerformAggregateSessionOperationAsync( - context, - session => session.ShutdownEvictionAsync(context), - (r1, r2) => r1 & r2, - shouldBreak: r => false); - } - - private IEnumerable GetSessionsInOrder(AbsolutePath path = null) - { - var drive = path != null ? path.GetPathRoot() : Store.PreferredCacheDrive; - - if (!SessionsByCacheRoot.ContainsKey(drive)) - { - drive = Store.PreferredCacheDrive; - } - - if (SessionsByCacheRoot[drive] is TSession session) - { - yield return session; - } - - foreach (var kvp in SessionsByCacheRoot) - { - if (StringComparer.OrdinalIgnoreCase.Equals(kvp.Key, drive)) - { - // Already yielded the preferred cache - continue; - } - - if (kvp.Value is TSession otherSession) - { - yield return otherSession; - } - } - } - - private async Task PerformSessionOperationAsync(Func> executeAsync) - where TResult : ResultBase - { - TResult result = null; - - foreach (var session in GetSessionsInOrder()) - { - result = await executeAsync(session); - - if (result.Succeeded) - { - return result; - } - } - - return result ?? new ErrorResult($"Could not find a content session which implements {typeof(TSession).Name} in {nameof(MultiplexedContentSession)}.").AsResult(); - } - - private Task PerformAggregateSessionOperationAsync( - Context context, - Func> executeAsync, - Func aggregate, - Func shouldBreak, - AbsolutePath pathHint = null, - [CallerMemberName] string caller = null) - where TResult : ResultBase - { - var operationContext = context is null ? new OperationContext() : new OperationContext(context); - return operationContext.PerformOperationAsync( - Tracer, - () => PerformAggregateSessionOperationCoreAsync(executeAsync, aggregate, shouldBreak, pathHint), - traceOperationStarted: false, - traceOperationFinished: false, - caller: caller); - } - - private async Task PerformAggregateSessionOperationCoreAsync( - Func> executeAsync, - Func aggregate, - Func shouldBreak, - AbsolutePath pathHint = null) - where TResult : ResultBase - { - TResult result = null; - - // Go through all the sessions - foreach (var session in GetSessionsInOrder(pathHint)) - { - var priorResult = result; - - try - { - result = await executeAsync(session); - } - catch (Exception e) - { - result = new ErrorResult(e).AsResult(); - } - - // Aggregate with previous result - if (priorResult != null) - { - result = aggregate(priorResult, result); - } - - // If result is sufficient, stop trying other stores and return result - if (shouldBreak(result)) - { - return result; - } - } - - Contract.Assert(result != null, $"Could not find a content session which implements {typeof(TSession).Name} in {nameof(MultiplexedContentSession)}."); - return result; - } - } -} diff --git a/Public/Src/Cache/MemoizationStore/Distributed/Sessions/PublishingCacheSession.cs b/Public/Src/Cache/MemoizationStore/Distributed/Sessions/PublishingCacheSession.cs index 9e4773e22..20be54795 100644 --- a/Public/Src/Cache/MemoizationStore/Distributed/Sessions/PublishingCacheSession.cs +++ b/Public/Src/Cache/MemoizationStore/Distributed/Sessions/PublishingCacheSession.cs @@ -25,7 +25,7 @@ using BuildXL.Utilities.Core.Tasks; namespace BuildXL.Cache.MemoizationStore.Distributed.Sessions { - internal class PublishingCacheSession : StartupShutdownBase, ICacheSession, IReadOnlyCacheSessionWithLevelSelectors, IHibernateCacheSession, IConfigurablePin, IAsyncShutdown + internal class PublishingCacheSession : StartupShutdownBase, ICacheSession, ICacheSessionWithLevelSelectors, IHibernateCacheSession, IConfigurablePin, IAsyncShutdown { public string Name { get; } protected override Tracer Tracer { get; } = new Tracer(nameof(PublishingCacheSession)); @@ -228,7 +228,7 @@ namespace BuildXL.Cache.MemoizationStore.Distributed.Sessions /// public Task>>> PinAsync(Context context, IReadOnlyList contentHashes, CancellationToken cts, UrgencyHint urgencyHint = UrgencyHint.Nominal) { - return ((IReadOnlyContentSession)_local).PinAsync(context, contentHashes, cts, urgencyHint); + return ((IContentSession)_local).PinAsync(context, contentHashes, cts, urgencyHint); } /// @@ -331,17 +331,17 @@ namespace BuildXL.Cache.MemoizationStore.Distributed.Sessions #endregion - #region IReadOnlyCacheSessionWithLevelSelectors + #region ICacheSessionWithLevelSelectors /// public Task> GetLevelSelectorsAsync(Context context, Fingerprint weakFingerprint, CancellationToken cts, int level) { - if (_local is IReadOnlyCacheSessionWithLevelSelectors withSelectors) + if (_local is ICacheSessionWithLevelSelectors withSelectors) { return withSelectors.GetLevelSelectorsAsync(context, weakFingerprint, cts, level); } - return Task.FromResult(new Result($"{nameof(_local)} does not implement {nameof(IReadOnlyCacheSessionWithLevelSelectors)}.")); + return Task.FromResult(new Result($"{nameof(_local)} does not implement {nameof(ICacheSessionWithLevelSelectors)}.")); } #endregion diff --git a/Public/Src/Cache/MemoizationStore/Distributed/Stores/PublishingCache.cs b/Public/Src/Cache/MemoizationStore/Distributed/Stores/PublishingCache.cs index 0872cf60f..aceded3e7 100644 --- a/Public/Src/Cache/MemoizationStore/Distributed/Stores/PublishingCache.cs +++ b/Public/Src/Cache/MemoizationStore/Distributed/Stores/PublishingCache.cs @@ -87,12 +87,6 @@ namespace BuildXL.Cache.MemoizationStore.Distributed.Stores _local.Dispose(); } - /// - public virtual CreateSessionResult CreateReadOnlySession(Context context, string name, ImplicitPin implicitPin) - { - return ((ICache)_local).CreateReadOnlySession(context, name, implicitPin); - } - /// public virtual CreateSessionResult CreateSession(Context context, string name, ImplicitPin implicitPin) { @@ -149,12 +143,6 @@ namespace BuildXL.Cache.MemoizationStore.Distributed.Stores return ((ICache)_local).GetStatsAsync(context); } - /// - CreateSessionResult IContentStore.CreateReadOnlySession(Context context, string name, ImplicitPin implicitPin) - { - return ((IContentStore)_local).CreateReadOnlySession(context, name, implicitPin); - } - /// CreateSessionResult IContentStore.CreateSession(Context context, string name, ImplicitPin implicitPin) { diff --git a/Public/Src/Cache/MemoizationStore/Distributed/Stores/TwoLevelCache.cs b/Public/Src/Cache/MemoizationStore/Distributed/Stores/TwoLevelCache.cs index 333165b64..ade77b8c0 100644 --- a/Public/Src/Cache/MemoizationStore/Distributed/Stores/TwoLevelCache.cs +++ b/Public/Src/Cache/MemoizationStore/Distributed/Stores/TwoLevelCache.cs @@ -3,7 +3,6 @@ using System; using System.Linq; using System.Threading.Tasks; -using BuildXL.Cache.ContentStore.Hashing; using BuildXL.Cache.ContentStore.Interfaces.Results; using BuildXL.Cache.ContentStore.Interfaces.Stores; using BuildXL.Cache.ContentStore.Interfaces.Tracing; @@ -11,7 +10,6 @@ using BuildXL.Cache.ContentStore.Tracing; using BuildXL.Cache.MemoizationStore.Interfaces.Caches; using BuildXL.Cache.MemoizationStore.Interfaces.Sessions; using BuildXL.Cache.MemoizationStore.Tracing; -using CreateReadOnlySessionCall = BuildXL.Cache.MemoizationStore.Tracing.CreateReadOnlySessionCall; using CreateSessionCall = BuildXL.Cache.MemoizationStore.Tracing.CreateSessionCall; namespace BuildXL.Cache.MemoizationStore.Distributed.Stores @@ -148,12 +146,12 @@ namespace BuildXL.Cache.MemoizationStore.Distributed.Stores public bool ShutdownStarted { get; private set; } /// - public CreateSessionResult CreateReadOnlySession( + public CreateSessionResult CreateSession( Context context, string name, ImplicitPin implicitPin) { - return CreateReadOnlySessionCall.Run( + return CreateSessionCall.Run( _tracer, context, name, @@ -162,34 +160,24 @@ namespace BuildXL.Cache.MemoizationStore.Distributed.Stores CreateSessionResult localSession = _localCache.CreateSession(context, name, implicitPin); if (!localSession.Succeeded) { - return new CreateSessionResult(localSession); + return new CreateSessionResult(localSession); } CreateSessionResult remoteSession = _remoteCache.CreateSession(context, name, implicitPin); if (!remoteSession.Succeeded) { - return new CreateSessionResult(remoteSession); + return new CreateSessionResult(remoteSession); } - IReadOnlyCacheSession cacheSession = new TwoLevelCacheSession( + ICacheSession cacheSession = new TwoLevelCacheSession( name, localSession.Session, remoteSession.Session, _config); - return new CreateSessionResult(cacheSession); + return new CreateSessionResult(cacheSession); }); } - /// - public CreateSessionResult CreateSession(Context context, string name, ImplicitPin implicitPin) - { - return CreateSessionCall.Run( - _tracer, - context, - name, - () => CreateSessionAsync(context, name, implicitPin).GetAwaiter().GetResult()); - } - private async Task> CreateSessionAsync(Context context, string name, ImplicitPin implicitPin) { (CreateSessionResult localResult, CreateSessionResult remoteResult) = await MultiLevel(cache => cache.CreateSession(context, name, implicitPin)); diff --git a/Public/Src/Cache/MemoizationStore/DistributedTest/LocalCacheServerTests.cs b/Public/Src/Cache/MemoizationStore/DistributedTest/LocalCacheServerTests.cs index 102bdf969..010f1ee2f 100644 --- a/Public/Src/Cache/MemoizationStore/DistributedTest/LocalCacheServerTests.cs +++ b/Public/Src/Cache/MemoizationStore/DistributedTest/LocalCacheServerTests.cs @@ -203,9 +203,6 @@ namespace BuildXL.Cache.MemoizationStore.Distributed.Test protected override Tracer Tracer { get; } = new Tracer(nameof(PublishingCacheToContentStore)); public CreateSessionResult CreatePublishingSession(Context context, string name, ImplicitPin implicitPin, PublishingCacheConfiguration publishingConfig, string pat) => _inner.CreatePublishingSession(context, name, implicitPin, publishingConfig, pat); - public CreateSessionResult CreateReadOnlySession(Context context, string name, ImplicitPin implicitPin) - => new CreateSessionResult(_inner.CreateReadOnlySession(context, name, implicitPin).ShouldBeSuccess().Session); - public CreateSessionResult CreateSession(Context context, string name, ImplicitPin implicitPin) => new CreateSessionResult(_inner.CreateSession(context, name, implicitPin).ShouldBeSuccess().Session); @@ -219,7 +216,6 @@ namespace BuildXL.Cache.MemoizationStore.Distributed.Test public IAsyncEnumerable> EnumerateStrongFingerprints(Context context) => _inner.EnumerateStrongFingerprints(context); public Task GetStatsAsync(Context context) => _inner.GetStatsAsync(context); public void PostInitializationCompleted(Context context, BoolResult result) { } - CreateSessionResult ICache.CreateReadOnlySession(Context context, string name, ImplicitPin implicitPin) => _inner.CreateReadOnlySession(context, name, implicitPin); CreateSessionResult ICache.CreateSession(Context context, string name, ImplicitPin implicitPin) => _inner.CreateSession(context, name, implicitPin); } diff --git a/Public/Src/Cache/MemoizationStore/DistributedTest/PublishingCacheTests.cs b/Public/Src/Cache/MemoizationStore/DistributedTest/PublishingCacheTests.cs index c39c55101..7da9dab5a 100644 --- a/Public/Src/Cache/MemoizationStore/DistributedTest/PublishingCacheTests.cs +++ b/Public/Src/Cache/MemoizationStore/DistributedTest/PublishingCacheTests.cs @@ -145,7 +145,7 @@ namespace BuildXL.Cache.MemoizationStore.Test.Sessions } /// - public override CreateSessionResult CreateReadOnlySession(Context context, string name, ImplicitPin implicitPin) + public override CreateSessionResult CreateSession(Context context, string name, ImplicitPin implicitPin) { var sessionResult = CreatePublishingSession( context, @@ -156,15 +156,11 @@ namespace BuildXL.Cache.MemoizationStore.Test.Sessions if (!sessionResult.Succeeded) { - return new CreateSessionResult(sessionResult); + return new CreateSessionResult(sessionResult); } - return new CreateSessionResult(sessionResult.Session); + return new CreateSessionResult(sessionResult.Session); } - - /// - public override CreateSessionResult CreateSession(Context context, string name, ImplicitPin implicitPin) - => base.CreatePublishingSession(context, name, implicitPin, _configFactory(), pat: _pat); } internal class BlockingPublishingStore : StartupShutdownSlimBase, IPublishingStore @@ -226,9 +222,6 @@ namespace BuildXL.Cache.MemoizationStore.Test.Sessions protected override Tracer Tracer { get; } = new Tracer(nameof(CacheToContentStore)); - public CreateSessionResult CreateReadOnlySession(Context context, string name, ImplicitPin implicitPin) - => new CreateSessionResult(_inner.CreateReadOnlySession(context, name, implicitPin).ShouldBeSuccess().Session); - public CreateSessionResult CreateSession(Context context, string name, ImplicitPin implicitPin) => new CreateSessionResult(_inner.CreateSession(context, name, implicitPin).ShouldBeSuccess().Session); @@ -239,7 +232,6 @@ namespace BuildXL.Cache.MemoizationStore.Test.Sessions public IAsyncEnumerable> EnumerateStrongFingerprints(Context context) => _inner.EnumerateStrongFingerprints(context); public Task GetStatsAsync(Context context) => _inner.GetStatsAsync(context); public void PostInitializationCompleted(Context context, BoolResult result) { } - CreateSessionResult ICache.CreateReadOnlySession(Context context, string name, ImplicitPin implicitPin) => _inner.CreateReadOnlySession(context, name, implicitPin); CreateSessionResult ICache.CreateSession(Context context, string name, ImplicitPin implicitPin) => _inner.CreateSession(context, name, implicitPin); } } diff --git a/Public/Src/Cache/MemoizationStore/Interfaces/Caches/ICache.cs b/Public/Src/Cache/MemoizationStore/Interfaces/Caches/ICache.cs index 9eda5569d..f86c1e64a 100644 --- a/Public/Src/Cache/MemoizationStore/Interfaces/Caches/ICache.cs +++ b/Public/Src/Cache/MemoizationStore/Interfaces/Caches/ICache.cs @@ -25,11 +25,6 @@ namespace BuildXL.Cache.MemoizationStore.Interfaces.Caches /// Guid Id { get; } - /// - /// Create a new session that can only read. - /// - CreateSessionResult CreateReadOnlySession(Context context, string name, ImplicitPin implicitPin); - /// /// Create a new session that can change the cache. /// diff --git a/Public/Src/Cache/MemoizationStore/Interfaces/Sessions/ICacheSession.cs b/Public/Src/Cache/MemoizationStore/Interfaces/Sessions/ICacheSession.cs index 6c4eca743..45fe1ce30 100644 --- a/Public/Src/Cache/MemoizationStore/Interfaces/Sessions/ICacheSession.cs +++ b/Public/Src/Cache/MemoizationStore/Interfaces/Sessions/ICacheSession.cs @@ -5,10 +5,13 @@ using BuildXL.Cache.ContentStore.Interfaces.Sessions; namespace BuildXL.Cache.MemoizationStore.Interfaces.Sessions { - /// - /// Writable cache session. - /// - public interface ICacheSession : IReadOnlyCacheSession, IMemoizationSession, IContentSession + /// + public interface ICacheSession : IMemoizationSession, IContentSession, IConfigurablePin + { + } + + /// + public interface ICacheSessionWithLevelSelectors : ICacheSession, IMemoizationSessionWithLevelSelectors { } } diff --git a/Public/Src/Cache/MemoizationStore/Interfaces/Sessions/IMemoizationSession.cs b/Public/Src/Cache/MemoizationStore/Interfaces/Sessions/IMemoizationSession.cs index 31dc8c3d3..190f27384 100644 --- a/Public/Src/Cache/MemoizationStore/Interfaces/Sessions/IMemoizationSession.cs +++ b/Public/Src/Cache/MemoizationStore/Interfaces/Sessions/IMemoizationSession.cs @@ -6,6 +6,7 @@ using System.Threading; using System.Threading.Tasks; using BuildXL.Cache.ContentStore.Interfaces.Results; using BuildXL.Cache.ContentStore.Interfaces.Sessions; +using BuildXL.Cache.ContentStore.Interfaces.Stores; using BuildXL.Cache.ContentStore.Interfaces.Tracing; using BuildXL.Cache.MemoizationStore.Interfaces.Results; @@ -14,8 +15,27 @@ namespace BuildXL.Cache.MemoizationStore.Interfaces.Sessions /// /// A related set of accesses to a cache. /// - public interface IMemoizationSession : IReadOnlyMemoizationSession + public interface IMemoizationSession : IName, IStartupShutdown { + /// + /// Gets known selectors for a given weak fingerprint. + /// + System.Collections.Generic.IAsyncEnumerable GetSelectors( + Context context, + Fingerprint weakFingerprint, + CancellationToken cts, + UrgencyHint urgencyHint = UrgencyHint.Nominal + ); + + /// + /// Load a ContentHashList. + /// + Task GetContentHashListAsync( + Context context, + StrongFingerprint strongFingerprint, + CancellationToken cts, + UrgencyHint urgencyHint = UrgencyHint.Nominal); + /// /// Store a ContentHashList /// diff --git a/Public/Src/Cache/MemoizationStore/Interfaces/Sessions/IReadOnlyCacheSession.cs b/Public/Src/Cache/MemoizationStore/Interfaces/Sessions/IReadOnlyCacheSession.cs deleted file mode 100644 index 85fa2681f..000000000 --- a/Public/Src/Cache/MemoizationStore/Interfaces/Sessions/IReadOnlyCacheSession.cs +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -using BuildXL.Cache.ContentStore.Interfaces.Sessions; - -namespace BuildXL.Cache.MemoizationStore.Interfaces.Sessions -{ - /// - /// Read-only cache session. - /// - public interface IReadOnlyCacheSession : IReadOnlyMemoizationSession, IReadOnlyContentSession, IConfigurablePin - { - } - - /// - /// Read-only cache session. - /// - public interface IReadOnlyCacheSessionWithLevelSelectors : IReadOnlyCacheSession, IReadOnlyMemoizationSessionWithLevelSelectors - { - } -} diff --git a/Public/Src/Cache/MemoizationStore/Interfaces/Sessions/IReadOnlyMemoizationSession.cs b/Public/Src/Cache/MemoizationStore/Interfaces/Sessions/IReadOnlyMemoizationSession.cs index e3c0750f3..de27849c6 100644 --- a/Public/Src/Cache/MemoizationStore/Interfaces/Sessions/IReadOnlyMemoizationSession.cs +++ b/Public/Src/Cache/MemoizationStore/Interfaces/Sessions/IReadOnlyMemoizationSession.cs @@ -11,35 +11,10 @@ using BuildXL.Cache.MemoizationStore.Interfaces.Results; namespace BuildXL.Cache.MemoizationStore.Interfaces.Sessions { - /// - /// A related set of read accesses to a cache. - /// - public interface IReadOnlyMemoizationSession : IName, IStartupShutdown - { - /// - /// Gets known selectors for a given weak fingerprint. - /// - System.Collections.Generic.IAsyncEnumerable GetSelectors( - Context context, - Fingerprint weakFingerprint, - CancellationToken cts, - UrgencyHint urgencyHint = UrgencyHint.Nominal - ); - - /// - /// Load a ContentHashList. - /// - Task GetContentHashListAsync( - Context context, - StrongFingerprint strongFingerprint, - CancellationToken cts, - UrgencyHint urgencyHint = UrgencyHint.Nominal); - } - /// /// A related set of read accesses to a cache with support for multi-level GetSelectors. /// - public interface IReadOnlyMemoizationSessionWithLevelSelectors : IReadOnlyMemoizationSession, ILevelSelectorsProvider + public interface IMemoizationSessionWithLevelSelectors : IMemoizationSession, ILevelSelectorsProvider { } @@ -52,7 +27,7 @@ namespace BuildXL.Cache.MemoizationStore.Interfaces.Sessions /// Gets known selectors for a given weak fingerprint for a given "level". /// /// - /// Unlike , this method is RPC friendly. + /// Unlike , this method is RPC friendly. /// Task> GetLevelSelectorsAsync( Context context, diff --git a/Public/Src/Cache/MemoizationStore/Interfaces/Sessions/ReadOnlyMemoizationSessionExtensions.cs b/Public/Src/Cache/MemoizationStore/Interfaces/Sessions/ReadOnlyMemoizationSessionExtensions.cs index 801293562..4aab7a803 100644 --- a/Public/Src/Cache/MemoizationStore/Interfaces/Sessions/ReadOnlyMemoizationSessionExtensions.cs +++ b/Public/Src/Cache/MemoizationStore/Interfaces/Sessions/ReadOnlyMemoizationSessionExtensions.cs @@ -15,7 +15,7 @@ using BuildXL.Cache.MemoizationStore.Interfaces.Results; namespace BuildXL.Cache.MemoizationStore.Interfaces.Sessions { /// - /// Set of extension methods for interface. + /// Set of extension methods for interface. /// public static class ReadOnlyMemoizationSessionExtensions { diff --git a/Public/Src/Cache/MemoizationStore/Interfaces/Stores/IMemoizationStore.cs b/Public/Src/Cache/MemoizationStore/Interfaces/Stores/IMemoizationStore.cs index 075005ca3..689cb71be 100644 --- a/Public/Src/Cache/MemoizationStore/Interfaces/Stores/IMemoizationStore.cs +++ b/Public/Src/Cache/MemoizationStore/Interfaces/Stores/IMemoizationStore.cs @@ -16,11 +16,6 @@ namespace BuildXL.Cache.MemoizationStore.Interfaces.Stores /// public interface IMemoizationStore : IStartupShutdown { - /// - /// Create a new session that can only read. - /// - CreateSessionResult CreateReadOnlySession(Context context, string name); - /// /// Create a new session that can add as well as read. /// diff --git a/Public/Src/Cache/MemoizationStore/InterfacesTest/Results/CreateSessionResultTests.cs b/Public/Src/Cache/MemoizationStore/InterfacesTest/Results/CreateSessionResultTests.cs index ee447fdfc..9952d1d42 100644 --- a/Public/Src/Cache/MemoizationStore/InterfacesTest/Results/CreateSessionResultTests.cs +++ b/Public/Src/Cache/MemoizationStore/InterfacesTest/Results/CreateSessionResultTests.cs @@ -9,28 +9,28 @@ using Xunit; namespace BuildXL.Cache.MemoizationStore.InterfacesTest.Results { - public class CreateSessionResultTests : ResultTests> + public class CreateSessionResultTests : ResultTests> { - protected override CreateSessionResult CreateFrom(Exception exception) + protected override CreateSessionResult CreateFrom(Exception exception) { - return new CreateSessionResult(exception); + return new CreateSessionResult(exception); } - protected override CreateSessionResult CreateFrom(string errorMessage) + protected override CreateSessionResult CreateFrom(string errorMessage) { - return new CreateSessionResult(errorMessage); + return new CreateSessionResult(errorMessage); } - protected override CreateSessionResult CreateFrom(string errorMessage, string diagnostics) + protected override CreateSessionResult CreateFrom(string errorMessage, string diagnostics) { - return new CreateSessionResult(errorMessage, diagnostics); + return new CreateSessionResult(errorMessage, diagnostics); } [Fact] public void ConstructFromResultBase() { var other = new BoolResult("error"); - Assert.False(new CreateSessionResult(other, "message").Succeeded); + Assert.False(new CreateSessionResult(other, "message").Succeeded); } [Fact] @@ -56,14 +56,14 @@ namespace BuildXL.Cache.MemoizationStore.InterfacesTest.Results { using (var session = new ThrowingCacheSession()) { - Assert.True(new CreateSessionResult(session).Succeeded); + Assert.True(new CreateSessionResult(session).Succeeded); } } [Fact] public void CodePropertyError() { - Assert.False(new CreateSessionResult("error").Succeeded); + Assert.False(new CreateSessionResult("error").Succeeded); } [Fact] @@ -71,7 +71,7 @@ namespace BuildXL.Cache.MemoizationStore.InterfacesTest.Results { using (var session = new ThrowingCacheSession()) { - Assert.Equal(session, new CreateSessionResult(session).Session); + Assert.Equal(session, new CreateSessionResult(session).Session); } } @@ -80,8 +80,8 @@ namespace BuildXL.Cache.MemoizationStore.InterfacesTest.Results { using (var session = new ThrowingCacheSession()) { - var v1 = new CreateSessionResult(session); - var v2 = new CreateSessionResult(session) as object; + var v1 = new CreateSessionResult(session); + var v2 = new CreateSessionResult(session) as object; Assert.True(v1.Equals(v2)); } } @@ -91,7 +91,7 @@ namespace BuildXL.Cache.MemoizationStore.InterfacesTest.Results { using (var session = new ThrowingCacheSession()) { - var v1 = new CreateSessionResult(session); + var v1 = new CreateSessionResult(session); var v2 = new object(); Assert.False(v1.Equals(v2)); } @@ -102,8 +102,8 @@ namespace BuildXL.Cache.MemoizationStore.InterfacesTest.Results { using (var session = new ThrowingCacheSession()) { - var v1 = new CreateSessionResult(session); - var v2 = new CreateSessionResult(session); + var v1 = new CreateSessionResult(session); + var v2 = new CreateSessionResult(session); Assert.True(v1.Equals(v2)); } } @@ -111,8 +111,8 @@ namespace BuildXL.Cache.MemoizationStore.InterfacesTest.Results [Fact] public void EqualsTrueForInvalidSessions() { - var v1 = new CreateSessionResult("error1"); - var v2 = new CreateSessionResult("error1"); + var v1 = new CreateSessionResult("error1"); + var v2 = new CreateSessionResult("error1"); Assert.True(v1.Equals(v2)); } @@ -122,8 +122,8 @@ namespace BuildXL.Cache.MemoizationStore.InterfacesTest.Results using (var session1 = new ThrowingCacheSession()) using (var session2 = new ThrowingCacheSession()) { - var v1 = new CreateSessionResult(session1); - var v2 = new CreateSessionResult(session2); + var v1 = new CreateSessionResult(session1); + var v2 = new CreateSessionResult(session2); Assert.True(v1.Equals(v2)); } } @@ -133,8 +133,8 @@ namespace BuildXL.Cache.MemoizationStore.InterfacesTest.Results { using (var session1 = new ThrowingCacheSession()) { - var v1 = new CreateSessionResult(session1); - var v2 = new CreateSessionResult("error"); + var v1 = new CreateSessionResult(session1); + var v2 = new CreateSessionResult("error"); Assert.False(v1.Equals(v2)); } } @@ -145,8 +145,8 @@ namespace BuildXL.Cache.MemoizationStore.InterfacesTest.Results using (var session1 = new ThrowingCacheSession("session1")) using (var session2 = new ThrowingCacheSession("session2")) { - var v1 = new CreateSessionResult(session1); - var v2 = new CreateSessionResult(session2); + var v1 = new CreateSessionResult(session1); + var v2 = new CreateSessionResult(session2); Assert.False(v1.Equals(v2)); } } @@ -157,8 +157,8 @@ namespace BuildXL.Cache.MemoizationStore.InterfacesTest.Results using (var session1 = new ThrowingCacheSession("session1")) using (var session2 = new ThrowingCacheSession("session1")) { - var v1 = new CreateSessionResult(session1); - var v2 = new CreateSessionResult(session2); + var v1 = new CreateSessionResult(session1); + var v2 = new CreateSessionResult(session2); Assert.Equal(v1.GetHashCode(), v2.GetHashCode()); } } @@ -169,8 +169,8 @@ namespace BuildXL.Cache.MemoizationStore.InterfacesTest.Results using (var session1 = new ThrowingCacheSession("session1")) using (var session2 = new ThrowingCacheSession("session2")) { - var v1 = new CreateSessionResult(session1); - var v2 = new CreateSessionResult(session2); + var v1 = new CreateSessionResult(session1); + var v2 = new CreateSessionResult(session2); Assert.NotEqual(v1.GetHashCode(), v2.GetHashCode()); } } @@ -179,7 +179,7 @@ namespace BuildXL.Cache.MemoizationStore.InterfacesTest.Results public void ToStringWithError() { Assert.Contains( - "something", new CreateSessionResult("something").ToString(), StringComparison.OrdinalIgnoreCase); + "something", new CreateSessionResult("something").ToString(), StringComparison.OrdinalIgnoreCase); } [Fact] @@ -187,7 +187,7 @@ namespace BuildXL.Cache.MemoizationStore.InterfacesTest.Results { using (var session = new ThrowingCacheSession()) { - var result = new CreateSessionResult(session); + var result = new CreateSessionResult(session); Assert.Contains("Success", result.ToString(), StringComparison.OrdinalIgnoreCase); } } diff --git a/Public/Src/Cache/MemoizationStore/InterfacesTest/Sessions/CacheTests.cs b/Public/Src/Cache/MemoizationStore/InterfacesTest/Sessions/CacheTests.cs index 3d7341633..ad1648a93 100644 --- a/Public/Src/Cache/MemoizationStore/InterfacesTest/Sessions/CacheTests.cs +++ b/Public/Src/Cache/MemoizationStore/InterfacesTest/Sessions/CacheTests.cs @@ -182,22 +182,6 @@ namespace BuildXL.Cache.MemoizationStore.InterfacesTest.Sessions return RunTestAsync(context, (ICache cache) => Task.FromResult(0)); } - [Fact] - public Task CreateReadOnlySession() - { - var context = new Context(Logger); - return RunTestAsync(context, async cache => - { - var result = cache.CreateReadOnlySession(context, Name, ImplicitPin.None).ShouldBeSuccess(); - - using (IReadOnlyCacheSession session = result.Session) - { - await session.StartupAsync(context).ShouldBeSuccess(); - await session.ShutdownAsync(context).ShouldBeSuccess(); - } - }); - } - [Fact] public Task CreateSession() { @@ -1119,19 +1103,19 @@ namespace BuildXL.Cache.MemoizationStore.InterfacesTest.Sessions return strongFingerprints; } - private Task RunReadOnlySessionTestAsync(Context context, Func funcAsync) + private Task RunReadOnlySessionTestAsync(Context context, Func funcAsync) { return RunReadOnlySessionTestAsync(context, funcAsync, CreateCache); } protected virtual Task RunReadOnlySessionTestAsync( - Context context, Func funcAsync, Func createCacheFunc) + Context context, Func funcAsync, Func createCacheFunc) { return RunTestAsync( context, async cache => { - var createSessionResult = cache.CreateReadOnlySession(context, Name, ImplicitPinPolicy).ShouldBeSuccess(); + var createSessionResult = cache.CreateSession(context, Name, ImplicitPinPolicy).ShouldBeSuccess(); using (var session = createSessionResult.Session) { try diff --git a/Public/Src/Cache/MemoizationStore/InterfacesTest/Sessions/MemoizationSessionTestBase.cs b/Public/Src/Cache/MemoizationStore/InterfacesTest/Sessions/MemoizationSessionTestBase.cs index 73a338a3f..6f7855750 100644 --- a/Public/Src/Cache/MemoizationStore/InterfacesTest/Sessions/MemoizationSessionTestBase.cs +++ b/Public/Src/Cache/MemoizationStore/InterfacesTest/Sessions/MemoizationSessionTestBase.cs @@ -424,11 +424,11 @@ namespace BuildXL.Cache.MemoizationStore.InterfacesTest.Sessions } } - private Task RunReadOnlyTestAsync(Context context, Func funcAsync) + private Task RunReadOnlyTestAsync(Context context, Func funcAsync) { return RunTestAsync(context, async store => { - var createResult = store.CreateReadOnlySession(context, Name); + var createResult = store.CreateSession(context, Name); createResult.ShouldBeSuccess(); using (var session = createResult.Session) { diff --git a/Public/Src/Cache/MemoizationStore/Library/Service/GrpcCacheServer.cs b/Public/Src/Cache/MemoizationStore/Library/Service/GrpcCacheServer.cs index 247dd8950..26fc2faa3 100644 --- a/Public/Src/Cache/MemoizationStore/Library/Service/GrpcCacheServer.cs +++ b/Public/Src/Cache/MemoizationStore/Library/Service/GrpcCacheServer.cs @@ -126,7 +126,7 @@ namespace BuildXL.Cache.MemoizationStore.Sessions.Grpc async c => { Fingerprint fingerprint = request.WeakFingerprint.DeserializeFingerprintFromGrpc(); - if (c.Session is IReadOnlyMemoizationSessionWithLevelSelectors withSelectors) + if (c.Session is IMemoizationSessionWithLevelSelectors withSelectors) { var result = await withSelectors.GetLevelSelectorsAsync(c.Context, fingerprint, c.Context.Token, request.Level).ThrowIfFailure(); var selectors = result.Value!.Selectors.Select(s => s.ToGrpc()); diff --git a/Public/Src/Cache/MemoizationStore/Library/Service/ServiceClientCache.cs b/Public/Src/Cache/MemoizationStore/Library/Service/ServiceClientCache.cs index b95106819..1d184c781 100644 --- a/Public/Src/Cache/MemoizationStore/Library/Service/ServiceClientCache.cs +++ b/Public/Src/Cache/MemoizationStore/Library/Service/ServiceClientCache.cs @@ -43,19 +43,6 @@ namespace BuildXL.Cache.MemoizationStore.Service { } - /// - public new CreateSessionResult CreateReadOnlySession(Context context, string name, ImplicitPin implicitPin) - { - var operationContext = OperationContext(context); - return operationContext.PerformOperation( - Tracer, - () => - { - var session = new ServiceClientCacheSession(new OperationContext(context), name, implicitPin, Logger, FileSystem, SessionTracer, Configuration); - return new CreateSessionResult(session); - }); - } - /// public new virtual CreateSessionResult CreateSession(Context context, string name, ImplicitPin implicitPin) { @@ -76,13 +63,7 @@ namespace BuildXL.Cache.MemoizationStore.Service } /// - public CreateSessionResult CreateReadOnlySession(Context context, string name) - { - return CreateReadOnlySession(context, name, ImplicitPin.None).Map(session => (IReadOnlyMemoizationSession)session); - } - - /// - public CreateSessionResult CreateSession(Context context, string name) + CreateSessionResult IMemoizationStore.CreateSession(Context context, string name) { return CreateSession(context, name, ImplicitPin.None).Map(session => (IMemoizationSession)session); } diff --git a/Public/Src/Cache/MemoizationStore/Library/Service/ServiceClientCacheSession.cs b/Public/Src/Cache/MemoizationStore/Library/Service/ServiceClientCacheSession.cs index f8da54858..387e6f936 100644 --- a/Public/Src/Cache/MemoizationStore/Library/Service/ServiceClientCacheSession.cs +++ b/Public/Src/Cache/MemoizationStore/Library/Service/ServiceClientCacheSession.cs @@ -30,7 +30,7 @@ using BuildXL.Utilities.Tracing; namespace BuildXL.Cache.MemoizationStore.Service { /// - public class ServiceClientCacheSession : ServiceClientContentSession, ICacheSession, IReadOnlyMemoizationSessionWithLevelSelectors + public class ServiceClientCacheSession : ServiceClientContentSession, ICacheSession, IMemoizationSessionWithLevelSelectors { private CounterCollection _memoizationCounters { get; } = new CounterCollection(); diff --git a/Public/Src/Cache/MemoizationStore/Library/Sessions/CacheAsContentStore.cs b/Public/Src/Cache/MemoizationStore/Library/Sessions/CacheAsContentStore.cs index 410718498..6e032701b 100644 --- a/Public/Src/Cache/MemoizationStore/Library/Sessions/CacheAsContentStore.cs +++ b/Public/Src/Cache/MemoizationStore/Library/Sessions/CacheAsContentStore.cs @@ -60,12 +60,6 @@ namespace BuildXL.Cache.MemoizationStore.Sessions return _cache.GetStatsAsync(context); } - /// - public CreateSessionResult CreateReadOnlySession(Context context, string name, ImplicitPin implicitPin) - { - return _cache.CreateReadOnlySession(context, name, implicitPin).Map(s => s); - } - /// public CreateSessionResult CreateSession(Context context, string name, ImplicitPin implicitPin) { diff --git a/Public/Src/Cache/MemoizationStore/Library/Sessions/CacheAsMemoizationStore.cs b/Public/Src/Cache/MemoizationStore/Library/Sessions/CacheAsMemoizationStore.cs index 0794283e4..308885f5a 100644 --- a/Public/Src/Cache/MemoizationStore/Library/Sessions/CacheAsMemoizationStore.cs +++ b/Public/Src/Cache/MemoizationStore/Library/Sessions/CacheAsMemoizationStore.cs @@ -56,12 +56,6 @@ namespace BuildXL.Cache.MemoizationStore.Sessions return await base.ShutdownCoreAsync(context); } - /// - public CreateSessionResult CreateReadOnlySession(Context context, string name) - { - return _cache.CreateReadOnlySession(context, name, ImplicitPin.None).Map(s => s); - } - /// public CreateSessionResult CreateSession(Context context, string name) { diff --git a/Public/Src/Cache/MemoizationStore/Library/Sessions/DatabaseMemoizationSession.cs b/Public/Src/Cache/MemoizationStore/Library/Sessions/DatabaseMemoizationSession.cs index ac5ff1bff..0e389117d 100644 --- a/Public/Src/Cache/MemoizationStore/Library/Sessions/DatabaseMemoizationSession.cs +++ b/Public/Src/Cache/MemoizationStore/Library/Sessions/DatabaseMemoizationSession.cs @@ -2,22 +2,59 @@ // Licensed under the MIT License. using System.Collections.Generic; +using System.Diagnostics.ContractsLight; using System.Threading; using System.Threading.Tasks; using BuildXL.Cache.ContentStore.Interfaces.Results; using BuildXL.Cache.ContentStore.Interfaces.Sessions; using BuildXL.Cache.ContentStore.Interfaces.Tracing; +using BuildXL.Cache.ContentStore.Tracing; +using BuildXL.Cache.ContentStore.Utils; using BuildXL.Cache.MemoizationStore.Interfaces.Results; using BuildXL.Cache.MemoizationStore.Interfaces.Sessions; using BuildXL.Cache.MemoizationStore.Stores; namespace BuildXL.Cache.MemoizationStore.Sessions { - /// - /// An IMemoizationSession implemented using a database - /// - public class DatabaseMemoizationSession : ReadOnlyDatabaseMemoizationSession, IMemoizationSession + /// + public class DatabaseMemoizationSession : StartupShutdownBase, IMemoizationSession, IMemoizationSessionWithLevelSelectors { + /// + public string Name { get; } + + /// + protected override Tracer Tracer { get; } + + /// + protected readonly DatabaseMemoizationStore MemoizationStore; + + /// + public Task GetContentHashListAsync( + Context context, + StrongFingerprint strongFingerprint, + CancellationToken cts, + UrgencyHint urgencyHint = UrgencyHint.Nominal) + { + bool preferShared = urgencyHint == UrgencyHint.PreferShared; + return MemoizationStore.GetContentHashListAsync(context, strongFingerprint, cts, preferShared); + } + + /// + public Task> GetLevelSelectorsAsync(Context context, Fingerprint weakFingerprint, CancellationToken cts, int level) + { + return MemoizationStore.GetLevelSelectorsAsync(context, weakFingerprint, cts, level); + } + + /// + public System.Collections.Generic.IAsyncEnumerable GetSelectors( + Context context, + Fingerprint weakFingerprint, + CancellationToken cts, + UrgencyHint urgencyHint = UrgencyHint.Nominal) + { + return this.GetSelectorsAsAsyncEnumerable(context, weakFingerprint, cts, urgencyHint); + } + private readonly IContentSession _contentSession; /// @@ -30,8 +67,13 @@ namespace BuildXL.Cache.MemoizationStore.Sessions /// overwritten because we're unable to check whether or not content is missing. /// public DatabaseMemoizationSession(string name, DatabaseMemoizationStore memoizationStore, IContentSession contentSession = null) - : base(name, memoizationStore) { + Contract.Requires(name != null); + Contract.Requires(memoizationStore != null); + + Tracer = new Tracer(nameof(DatabaseMemoizationSession)); + Name = name; + MemoizationStore = memoizationStore; _contentSession = contentSession; } @@ -44,7 +86,11 @@ namespace BuildXL.Cache.MemoizationStore.Sessions UrgencyHint urgencyHint) { return MemoizationStore.AddOrGetContentHashListAsync( - context, strongFingerprint, contentHashListWithDeterminism, _contentSession, cts); + context, + strongFingerprint, + contentHashListWithDeterminism, + _contentSession, + cts); } /// diff --git a/Public/Src/Cache/MemoizationStore/Library/Sessions/MemoryMemoizationSession.cs b/Public/Src/Cache/MemoizationStore/Library/Sessions/MemoryMemoizationSession.cs index 4e8f0b3ee..05466d1f8 100644 --- a/Public/Src/Cache/MemoizationStore/Library/Sessions/MemoryMemoizationSession.cs +++ b/Public/Src/Cache/MemoizationStore/Library/Sessions/MemoryMemoizationSession.cs @@ -2,21 +2,22 @@ // Licensed under the MIT License. using System.Collections.Generic; +using System.Diagnostics.ContractsLight; using System.Threading; using System.Threading.Tasks; using BuildXL.Cache.ContentStore.Interfaces.Results; using BuildXL.Cache.ContentStore.Interfaces.Sessions; using BuildXL.Cache.ContentStore.Interfaces.Tracing; +using BuildXL.Cache.ContentStore.Tracing; +using BuildXL.Cache.ContentStore.Utils; using BuildXL.Cache.MemoizationStore.Interfaces.Results; using BuildXL.Cache.MemoizationStore.Interfaces.Sessions; using BuildXL.Cache.MemoizationStore.Stores; namespace BuildXL.Cache.MemoizationStore.Sessions { - /// - /// An IMemoizationSession implemented in Memory - /// - public class MemoryMemoizationSession : ReadOnlyMemoryMemoizationSession, IMemoizationSession + /// + public class MemoryMemoizationSession : StartupShutdownBase, IMemoizationSession, IMemoizationSessionWithLevelSelectors { private readonly IContentSession _contentSession; @@ -30,8 +31,13 @@ namespace BuildXL.Cache.MemoizationStore.Sessions /// overwritten because we're unable to check whether or not content is missing. /// public MemoryMemoizationSession(string name, MemoryMemoizationStore memoizationStore, IContentSession contentSession = null) - : base(name, memoizationStore) { + Contract.Requires(name != null); + Contract.Requires(memoizationStore != null); + + Name = name; + Tracer = new Tracer(nameof(MemoryMemoizationSession)); + MemoizationStore = memoizationStore; _contentSession = contentSession; } @@ -44,14 +50,56 @@ namespace BuildXL.Cache.MemoizationStore.Sessions UrgencyHint urgencyHint) { return MemoizationStore.AddOrGetContentHashListAsync( - context, strongFingerprint, contentHashListWithDeterminism, _contentSession, cts); + context, + strongFingerprint, + contentHashListWithDeterminism, + _contentSession, + cts); } /// public Task IncorporateStrongFingerprintsAsync( - Context context, IEnumerable> strongFingerprints, CancellationToken cts, UrgencyHint urgencyHint) + Context context, + IEnumerable> strongFingerprints, + CancellationToken cts, + UrgencyHint urgencyHint) { return Task.FromResult(BoolResult.Success); } + + /// + protected readonly MemoryMemoizationStore MemoizationStore; + + /// + protected override Tracer Tracer { get; } + + /// + public string Name { get; } + + /// + public System.Collections.Generic.IAsyncEnumerable GetSelectors( + Context context, + Fingerprint weakFingerprint, + CancellationToken cts, + UrgencyHint urgencyHint = UrgencyHint.Nominal) + { + return this.GetSelectorsAsAsyncEnumerable(context, weakFingerprint, cts, urgencyHint); + } + + /// + public Task> GetLevelSelectorsAsync(Context context, Fingerprint weakFingerprint, CancellationToken cts, int level) + { + return Task.FromResult(LevelSelectors.Single(MemoizationStore.GetSelectorsCore(context, weakFingerprint, cts))); + } + + /// + public Task GetContentHashListAsync( + Context context, + StrongFingerprint strongFingerprint, + CancellationToken cts, + UrgencyHint urgencyHint) + { + return MemoizationStore.GetContentHashListAsync(context, strongFingerprint, cts); + } } } diff --git a/Public/Src/Cache/MemoizationStore/Library/Sessions/OneLevelCacheBase.cs b/Public/Src/Cache/MemoizationStore/Library/Sessions/OneLevelCacheBase.cs index 789dfa07d..0d7353b38 100644 --- a/Public/Src/Cache/MemoizationStore/Library/Sessions/OneLevelCacheBase.cs +++ b/Public/Src/Cache/MemoizationStore/Library/Sessions/OneLevelCacheBase.cs @@ -185,35 +185,6 @@ namespace BuildXL.Cache.MemoizationStore.Sessions } } - /// - public CreateSessionResult CreateReadOnlySession(Context context, string name, ImplicitPin implicitPin) - { - Contract.Requires(ContentStore != null); - Contract.Requires(MemoizationStore != null); - - return Tracing.CreateReadOnlySessionCall.Run(CacheTracer, context, name, () => - { - var createContentResult = ContentStore.CreateReadOnlySession(context, name, implicitPin); - if (!createContentResult.Succeeded) - { - return new CreateSessionResult(createContentResult, "Content session creation failed"); - } - - var contentReadOnlySession = createContentResult.Session; - - var createMemoizationResult = MemoizationStore.CreateReadOnlySession(context, name); - if (!createMemoizationResult.Succeeded) - { - return new CreateSessionResult(createMemoizationResult, "Memoization session creation failed"); - } - - var memoizationReadOnlySession = createMemoizationResult.Session; - - var session = new ReadOnlyOneLevelCacheSession(this, name, implicitPin, memoizationReadOnlySession, contentReadOnlySession); - return new CreateSessionResult(session); - }); - } - /// public CreateSessionResult CreateSession(Context context, string name, ImplicitPin implicitPin) { @@ -241,7 +212,7 @@ namespace BuildXL.Cache.MemoizationStore.Sessions var memoizationSession = createMemoizationResult.Session; - var session = new OneLevelCacheSession(this, name, implicitPin, memoizationSession, contentSession); + var session = new OneLevelCacheSession(this, name, implicitPin, memoizationSession!, contentSession!); return new CreateSessionResult(session); }); } @@ -339,11 +310,6 @@ namespace BuildXL.Cache.MemoizationStore.Sessions return new BoolResult($"{ContentStore} does not implement {nameof(ICopyRequestHandler)} in {nameof(OneLevelCache)}."); } - CreateSessionResult IContentStore.CreateReadOnlySession(Context context, string name, ImplicitPin implicitPin) - { - return CreateReadOnlySession(context, name, implicitPin).Map(session => (IReadOnlyContentSession)session); - } - CreateSessionResult IContentStore.CreateSession(Context context, string name, ImplicitPin implicitPin) { return CreateSession(context, name, implicitPin).Map(session => (IContentSession)session); diff --git a/Public/Src/Cache/MemoizationStore/Library/Sessions/OneLevelCacheSession.cs b/Public/Src/Cache/MemoizationStore/Library/Sessions/OneLevelCacheSession.cs index 052a3da32..b2e2ac771 100644 --- a/Public/Src/Cache/MemoizationStore/Library/Sessions/OneLevelCacheSession.cs +++ b/Public/Src/Cache/MemoizationStore/Library/Sessions/OneLevelCacheSession.cs @@ -3,25 +3,405 @@ using System; using System.Collections.Generic; +using System.Diagnostics.ContractsLight; using System.IO; +using System.Linq; +using System.Text; using System.Threading; using System.Threading.Tasks; using BuildXL.Cache.ContentStore.Hashing; +using BuildXL.Cache.ContentStore.Interfaces.Extensions; using BuildXL.Cache.ContentStore.Interfaces.FileSystem; using BuildXL.Cache.ContentStore.Interfaces.Results; using BuildXL.Cache.ContentStore.Interfaces.Sessions; using BuildXL.Cache.ContentStore.Interfaces.Stores; using BuildXL.Cache.ContentStore.Interfaces.Tracing; +using BuildXL.Cache.ContentStore.Interfaces.Utils; using BuildXL.Cache.MemoizationStore.Interfaces.Results; using BuildXL.Cache.MemoizationStore.Sessions; +#nullable enable + namespace BuildXL.Cache.MemoizationStore.Interfaces.Sessions { - /// - /// An ICacheSession implemented with one level of content and memoization. - /// - public class OneLevelCacheSession : ReadOnlyOneLevelCacheSession, ICacheSession + /// + public class OneLevelCacheSession : ICacheSessionWithLevelSelectors, IHibernateCacheSession { + /// + /// Auto-pinning behavior configuration. + /// + protected readonly ImplicitPin ImplicitPin; + + private IContentSession? _contentReadOnlySession; + + private IMemoizationSession? _memoizationReadOnlySession; + + /// + /// The content session backing the session. + /// + protected IContentSession ContentReadOnlySession + { + get + { + if (_disposed) + { + throw new InvalidOperationException("Can't obtain an inner session because the instance was already being disposed."); + } + + return _contentReadOnlySession!; + } + } + + /// + /// The memoization store backing the session. + /// + protected IMemoizationSession MemoizationReadOnlySession + { + get + { + if (_disposed) + { + throw new InvalidOperationException("Can't obtain an inner session because the instance was already being disposed."); + } + + return _memoizationReadOnlySession!; + } + } + + private bool _disposed; + + /// + protected OneLevelCacheBase? Parent; + + /// + /// Initializes a new instance of the class. + /// + public OneLevelCacheSession( + OneLevelCacheBase? parent, + string name, + ImplicitPin implicitPin, + IMemoizationSession memoizationSession, + IContentSession contentSession) + { + Contract.Requires(name != null); + Contract.Requires(memoizationSession != null); + Contract.Requires(contentSession != null); + + Parent = parent; + Name = name; + ImplicitPin = implicitPin; + _memoizationReadOnlySession = memoizationSession; + _contentReadOnlySession = contentSession; + } + + /// + public string Name { get; } + + /// + public bool StartupStarted { get; private set; } + + /// + public bool StartupCompleted { get; private set; } + + /// + public bool ShutdownStarted { get; private set; } + + /// + public bool ShutdownCompleted { get; private set; } + + /// + public virtual async Task StartupAsync(Context context) + { + StartupStarted = true; + + var startupContentResult = await ContentReadOnlySession.StartupAsync(context).ConfigureAwait(false); + if (!startupContentResult.Succeeded) + { + StartupCompleted = true; + return new BoolResult(startupContentResult, "Content session startup failed"); + } + + var startupMemoizationResult = await MemoizationReadOnlySession.StartupAsync(context).ConfigureAwait(false); + if (!startupMemoizationResult.Succeeded) + { + var sb = new StringBuilder(); + var shutdownContentResult = await ContentReadOnlySession.ShutdownAsync(context).ConfigureAwait(false); + if (!shutdownContentResult.Succeeded) + { + sb.Append($"Content session shutdown failed, error=[{shutdownContentResult}]"); + } + + sb.Append(sb.Length > 0 ? ", " : string.Empty); + sb.Append($"Memoization session startup failed, error=[{startupMemoizationResult}]"); + StartupCompleted = true; + return new BoolResult(sb.ToString()); + } + + StartupCompleted = true; + return BoolResult.Success; + } + + /// + public virtual async Task ShutdownAsync(Context context) + { + ShutdownStarted = true; + var shutdownMemoizationResult = MemoizationReadOnlySession != null + ? await MemoizationReadOnlySession.ShutdownAsync(context).ConfigureAwait(false) + : BoolResult.Success; + var shutdownContentResult = ContentReadOnlySession != null + ? await ContentReadOnlySession.ShutdownAsync(context).ConfigureAwait(false) + : BoolResult.Success; + + BoolResult result; + if (shutdownMemoizationResult.Succeeded && shutdownContentResult.Succeeded) + { + result = BoolResult.Success; + } + else + { + var sb = new StringBuilder(); + if (!shutdownMemoizationResult.Succeeded) + { + sb.Append($"Memoization session shutdown failed, error=[{shutdownMemoizationResult}]"); + } + + if (!shutdownContentResult.Succeeded) + { + sb.Append($"Content session shutdown failed, error=[{shutdownContentResult}]"); + } + + result = new BoolResult(sb.ToString()); + } + + ShutdownCompleted = true; + return result; + } + + /// + public void Dispose() + { + if (_disposed) + { + return; + } + + Dispose(true); + GC.SuppressFinalize(this); + + _disposed = true; + } + + /// + /// Dispose pattern. + /// + private void Dispose(bool disposing) + { + if (disposing) + { + _memoizationReadOnlySession?.Dispose(); + _memoizationReadOnlySession = null; + + _contentReadOnlySession?.Dispose(); + _contentReadOnlySession = null; + } + } + + /// + public IAsyncEnumerable GetSelectors( + Context context, + Fingerprint weakFingerprint, + CancellationToken cts, + UrgencyHint urgencyHint = UrgencyHint.Nominal) + { + return ReadOnlyMemoizationSessionExtensions.GetSelectorsAsAsyncEnumerable(this, context, weakFingerprint, cts, urgencyHint); + } + + /// + public async Task> GetLevelSelectorsAsync( + Context context, + Fingerprint weakFingerprint, + CancellationToken cts, + int level) + { + if (MemoizationReadOnlySession is IMemoizationSessionWithLevelSelectors withLevelSelectors) + { + var result = await withLevelSelectors.GetLevelSelectorsAsync(context, weakFingerprint, cts, level); + + if (result.Succeeded && Parent is not null) + { + foreach (var selector in result.Value.Selectors) + { + Parent.AddOrExtendPin(context, selector.ContentHash); + } + } + + return result; + } + + throw new NotSupportedException( + $"ReadOnlyMemoization session {MemoizationReadOnlySession.GetType().Name} does not support GetLevelSelectors functionality."); + } + + /// + public async Task GetContentHashListAsync( + Context context, + StrongFingerprint strongFingerprint, + CancellationToken cts, + UrgencyHint urgencyHint) + { + var result = await MemoizationReadOnlySession.GetContentHashListAsync(context, strongFingerprint, cts, urgencyHint); + if (result.Succeeded && Parent is not null && result.ContentHashListWithDeterminism.ContentHashList is not null) + { + Parent.AddOrExtendPin(context, strongFingerprint.Selector.ContentHash); + + var contentHashList = result.ContentHashListWithDeterminism.ContentHashList.Hashes; + foreach (var contentHash in contentHashList) + { + Parent.AddOrExtendPin(context, contentHash); + } + } + + return result; + } + + /// + public Task PinAsync(Context context, ContentHash contentHash, CancellationToken cts, UrgencyHint urgencyHint) + { + if (Parent is not null && Parent.CanElidePin(context, contentHash)) + { + return PinResult.SuccessTask; + } + + return ContentReadOnlySession.PinAsync(context, contentHash, cts, urgencyHint); + } + + /// + public Task>>> PinAsync( + Context context, + IReadOnlyList contentHashes, + PinOperationConfiguration configuration) + { + return Workflows.RunWithFallback( + contentHashes, + initialFunc: (contentHashes) => + { + return Task.FromResult( + contentHashes.Select( + contentHash => + { + if (Parent is not null && Parent.CanElidePin(context, contentHash)) + { + return PinResult.Success; + } + + return PinResult.ContentNotFound; + }).AsIndexedTasks()); + }, + fallbackFunc: (contentHashes) => { return ContentReadOnlySession.PinAsync(context, contentHashes, configuration); }, + isSuccessFunc: r => r.Succeeded); + } + + /// + public Task OpenStreamAsync( + Context context, + ContentHash contentHash, + CancellationToken cts, + UrgencyHint urgencyHint) + { + return ContentReadOnlySession.OpenStreamAsync(context, contentHash, cts, urgencyHint); + } + + /// + public Task PlaceFileAsync( + Context context, + ContentHash contentHash, + AbsolutePath path, + FileAccessMode accessMode, + FileReplacementMode replacementMode, + FileRealizationMode realizationMode, + CancellationToken cts, + UrgencyHint urgencyHint + ) + { + return ContentReadOnlySession.PlaceFileAsync(context, contentHash, path, accessMode, replacementMode, realizationMode, cts, urgencyHint); + } + + /// + public Task>>> PinAsync( + Context context, + IReadOnlyList contentHashes, + CancellationToken cts, + UrgencyHint urgencyHint = UrgencyHint.Nominal) + { + return Workflows.RunWithFallback( + contentHashes, + initialFunc: (contentHashes) => + { + return Task.FromResult( + contentHashes.Select( + contentHash => + { + if (Parent is not null && Parent.CanElidePin(context, contentHash)) + { + return PinResult.Success; + } + + return PinResult.ContentNotFound; + }).AsIndexedTasks()); + }, + fallbackFunc: (contentHashes) => { return ContentReadOnlySession.PinAsync(context, contentHashes, cts, urgencyHint); }, + isSuccessFunc: r => r.Succeeded); + } + + /// + public Task>>> PlaceFileAsync( + Context context, + IReadOnlyList hashesWithPaths, + FileAccessMode accessMode, + FileReplacementMode replacementMode, + FileRealizationMode realizationMode, + CancellationToken cts, + UrgencyHint urgencyHint = UrgencyHint.Nominal) + { + return ContentReadOnlySession.PlaceFileAsync(context, hashesWithPaths, accessMode, replacementMode, realizationMode, cts, urgencyHint); + } + + /// + public IEnumerable EnumeratePinnedContentHashes() + { + return ContentReadOnlySession is IHibernateContentSession session + ? session.EnumeratePinnedContentHashes() + : Enumerable.Empty(); + } + + /// + public Task PinBulkAsync(Context context, IEnumerable contentHashes) + { + return ContentReadOnlySession is IHibernateContentSession session + ? session.PinBulkAsync(context, contentHashes) + : Task.FromResult(0); + } + + /// + public Task ShutdownEvictionAsync(Context context) + { + return ContentReadOnlySession is IHibernateContentSession session + ? session.ShutdownEvictionAsync(context) + : BoolResult.SuccessTask; + } + + /// + public IList GetPendingPublishingOperations() + => MemoizationReadOnlySession is IHibernateCacheSession session + ? session.GetPendingPublishingOperations() + : new List(); + + /// + public Task SchedulePublishingOperationsAsync(Context context, IEnumerable pendingOperations) + => MemoizationReadOnlySession is IHibernateCacheSession session + ? session.SchedulePublishingOperationsAsync(context, pendingOperations) + : Task.FromResult(0); + /// /// Gets the writable content session. /// @@ -32,19 +412,6 @@ namespace BuildXL.Cache.MemoizationStore.Interfaces.Sessions /// public IMemoizationSession MemoizationSession => (IMemoizationSession)MemoizationReadOnlySession; - /// - /// Initializes a new instance of the class. - /// - public OneLevelCacheSession( - OneLevelCacheBase parent, - string name, - ImplicitPin implicitPin, - IMemoizationSession memoizationSession, - IContentSession contentSession) - : base(parent, name, implicitPin, memoizationSession, contentSession) - { - } - /// public async Task AddOrGetContentHashListAsync( Context context, @@ -54,7 +421,11 @@ namespace BuildXL.Cache.MemoizationStore.Interfaces.Sessions UrgencyHint urgencyHint) { var result = await MemoizationSession.AddOrGetContentHashListAsync( - context, strongFingerprint, contentHashListWithDeterminism, cts, urgencyHint); + context, + strongFingerprint, + contentHashListWithDeterminism, + cts, + urgencyHint); if (result.Succeeded && Parent is not null && result.ContentHashListWithDeterminism.ContentHashList is not null) { @@ -93,15 +464,14 @@ namespace BuildXL.Cache.MemoizationStore.Interfaces.Sessions } /// - public Task PutFileAsync - ( + public Task PutFileAsync( Context context, ContentHash contentHash, AbsolutePath path, FileRealizationMode realizationMode, CancellationToken cts, UrgencyHint urgencyHint - ) + ) { return ContentSession.PutFileAsync(context, contentHash, path, realizationMode, cts, urgencyHint); } diff --git a/Public/Src/Cache/MemoizationStore/Library/Sessions/ReadOnlyDatabaseMemoizationSession.cs b/Public/Src/Cache/MemoizationStore/Library/Sessions/ReadOnlyDatabaseMemoizationSession.cs deleted file mode 100644 index 343e1b8d3..000000000 --- a/Public/Src/Cache/MemoizationStore/Library/Sessions/ReadOnlyDatabaseMemoizationSession.cs +++ /dev/null @@ -1,62 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -using System.Diagnostics.ContractsLight; -using System.Threading; -using System.Threading.Tasks; -using BuildXL.Cache.ContentStore.Interfaces.Results; -using BuildXL.Cache.ContentStore.Interfaces.Sessions; -using BuildXL.Cache.ContentStore.Interfaces.Tracing; -using BuildXL.Cache.ContentStore.Tracing; -using BuildXL.Cache.ContentStore.Utils; -using BuildXL.Cache.MemoizationStore.Interfaces.Results; -using BuildXL.Cache.MemoizationStore.Interfaces.Sessions; -using BuildXL.Cache.MemoizationStore.Stores; - -namespace BuildXL.Cache.MemoizationStore.Sessions -{ - /// - /// An IReadOnlyMemoizationSession implemented in RocksDb - /// - public class ReadOnlyDatabaseMemoizationSession : StartupShutdownBase, IReadOnlyMemoizationSessionWithLevelSelectors - { - /// - public string Name { get; } - - /// - protected override Tracer Tracer { get; } - - /// - protected readonly DatabaseMemoizationStore MemoizationStore; - - /// - public ReadOnlyDatabaseMemoizationSession(string name, DatabaseMemoizationStore memoizationStore) - { - Contract.Requires(name != null); - Contract.Requires(memoizationStore != null); - - Tracer = new Tracer(nameof(ReadOnlyDatabaseMemoizationSession)); - Name = name; - MemoizationStore = memoizationStore; - } - - /// - public Task GetContentHashListAsync(Context context, StrongFingerprint strongFingerprint, CancellationToken cts, UrgencyHint urgencyHint = UrgencyHint.Nominal) - { - bool preferShared = urgencyHint == UrgencyHint.PreferShared; - return MemoizationStore.GetContentHashListAsync(context, strongFingerprint, cts, preferShared); - } - - /// - public Task> GetLevelSelectorsAsync(Context context, Fingerprint weakFingerprint, CancellationToken cts, int level) - { - return MemoizationStore.GetLevelSelectorsAsync(context, weakFingerprint, cts, level); - } - - /// - public System.Collections.Generic.IAsyncEnumerable GetSelectors(Context context, Fingerprint weakFingerprint, CancellationToken cts, UrgencyHint urgencyHint = UrgencyHint.Nominal) - { - return this.GetSelectorsAsAsyncEnumerable(context, weakFingerprint, cts, urgencyHint); - } - } -} diff --git a/Public/Src/Cache/MemoizationStore/Library/Sessions/ReadOnlyMemoryMemoizationSession.cs b/Public/Src/Cache/MemoizationStore/Library/Sessions/ReadOnlyMemoryMemoizationSession.cs deleted file mode 100644 index 07fee3a8a..000000000 --- a/Public/Src/Cache/MemoizationStore/Library/Sessions/ReadOnlyMemoryMemoizationSession.cs +++ /dev/null @@ -1,62 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -using System.Diagnostics.ContractsLight; -using System.Threading; -using System.Threading.Tasks; -using BuildXL.Cache.ContentStore.Interfaces.Results; -using BuildXL.Cache.ContentStore.Interfaces.Sessions; -using BuildXL.Cache.ContentStore.Interfaces.Tracing; -using BuildXL.Cache.ContentStore.Tracing; -using BuildXL.Cache.ContentStore.Utils; -using BuildXL.Cache.MemoizationStore.Interfaces.Results; -using BuildXL.Cache.MemoizationStore.Interfaces.Sessions; -using BuildXL.Cache.MemoizationStore.Stores; - -namespace BuildXL.Cache.MemoizationStore.Sessions -{ - /// - /// An implemented in memory - /// - public class ReadOnlyMemoryMemoizationSession : StartupShutdownBase, IReadOnlyMemoizationSessionWithLevelSelectors - { - /// - protected readonly MemoryMemoizationStore MemoizationStore; - - /// - protected override Tracer Tracer { get; } - - /// - public ReadOnlyMemoryMemoizationSession(string name, MemoryMemoizationStore memoizationStore) - { - Contract.Requires(name != null); - Contract.Requires(memoizationStore != null); - - Name = name; - Tracer = new Tracer(nameof(ReadOnlyMemoryMemoizationSession)); - MemoizationStore = memoizationStore; - } - - /// - public string Name { get; } - - /// - public System.Collections.Generic.IAsyncEnumerable GetSelectors(Context context, Fingerprint weakFingerprint, CancellationToken cts, UrgencyHint urgencyHint = UrgencyHint.Nominal) - { - return this.GetSelectorsAsAsyncEnumerable(context, weakFingerprint, cts, urgencyHint); - } - - /// - public Task> GetLevelSelectorsAsync(Context context, Fingerprint weakFingerprint, CancellationToken cts, int level) - { - return Task.FromResult(LevelSelectors.Single(MemoizationStore.GetSelectorsCore(context, weakFingerprint, cts))); - } - - /// - public Task GetContentHashListAsync( - Context context, StrongFingerprint strongFingerprint, CancellationToken cts, UrgencyHint urgencyHint) - { - return MemoizationStore.GetContentHashListAsync(context, strongFingerprint, cts); - } - } -} diff --git a/Public/Src/Cache/MemoizationStore/Library/Sessions/ReadOnlyOneLevelCacheSession.cs b/Public/Src/Cache/MemoizationStore/Library/Sessions/ReadOnlyOneLevelCacheSession.cs deleted file mode 100644 index 4dbad29d7..000000000 --- a/Public/Src/Cache/MemoizationStore/Library/Sessions/ReadOnlyOneLevelCacheSession.cs +++ /dev/null @@ -1,384 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -using System; -using System.Collections.Generic; -using System.Diagnostics.ContractsLight; -using System.Linq; -using System.Text; -using System.Threading; -using System.Threading.Tasks; -using BuildXL.Cache.ContentStore.Hashing; -using BuildXL.Cache.ContentStore.Interfaces.Extensions; -using BuildXL.Cache.ContentStore.Interfaces.FileSystem; -using BuildXL.Cache.ContentStore.Interfaces.Results; -using BuildXL.Cache.ContentStore.Interfaces.Sessions; -using BuildXL.Cache.ContentStore.Interfaces.Stores; -using BuildXL.Cache.ContentStore.Interfaces.Tracing; -using BuildXL.Cache.ContentStore.Interfaces.Utils; -using BuildXL.Cache.MemoizationStore.Interfaces.Results; -using BuildXL.Cache.MemoizationStore.Sessions; - -#nullable enable - -namespace BuildXL.Cache.MemoizationStore.Interfaces.Sessions -{ - /// - /// An IReadOnlyCacheSession implemented with one level of content and memoization. - /// - public class ReadOnlyOneLevelCacheSession : IReadOnlyCacheSessionWithLevelSelectors, IHibernateCacheSession, IConfigurablePin - { - /// - /// Auto-pinning behavior configuration. - /// - protected readonly ImplicitPin ImplicitPin; - - private IReadOnlyContentSession? _contentReadOnlySession; - - private IReadOnlyMemoizationSession? _memoizationReadOnlySession; - - /// - /// The content session backing the session. - /// - protected IReadOnlyContentSession ContentReadOnlySession - { - get - { - if (_disposed) - { - throw new InvalidOperationException("Can't obtain an inner session because the instance was already being disposed."); - } - - return _contentReadOnlySession!; - } - } - - /// - /// The memoization store backing the session. - /// - protected IReadOnlyMemoizationSession MemoizationReadOnlySession - { - get - { - if (_disposed) - { - throw new InvalidOperationException("Can't obtain an inner session because the instance was already being disposed."); - } - - return _memoizationReadOnlySession!; - } - } - - private bool _disposed; - - /// - protected OneLevelCacheBase? Parent; - - /// - /// Initializes a new instance of the class. - /// - public ReadOnlyOneLevelCacheSession( - OneLevelCacheBase? parent, - string name, - ImplicitPin implicitPin, - IReadOnlyMemoizationSession memoizationSession, - IReadOnlyContentSession contentSession) - { - Contract.Requires(name != null); - Contract.Requires(memoizationSession != null); - Contract.Requires(contentSession != null); - - Parent = parent; - Name = name; - ImplicitPin = implicitPin; - _memoizationReadOnlySession = memoizationSession; - _contentReadOnlySession = contentSession; - } - - /// - public string Name { get; } - - /// - public bool StartupStarted { get; private set; } - - /// - public bool StartupCompleted { get; private set; } - - /// - public bool ShutdownStarted { get; private set; } - - /// - public bool ShutdownCompleted { get; private set; } - - /// - public virtual async Task StartupAsync(Context context) - { - StartupStarted = true; - - var startupContentResult = await ContentReadOnlySession.StartupAsync(context).ConfigureAwait(false); - if (!startupContentResult.Succeeded) - { - StartupCompleted = true; - return new BoolResult(startupContentResult, "Content session startup failed"); - } - - var startupMemoizationResult = await MemoizationReadOnlySession.StartupAsync(context).ConfigureAwait(false); - if (!startupMemoizationResult.Succeeded) - { - var sb = new StringBuilder(); - var shutdownContentResult = await ContentReadOnlySession.ShutdownAsync(context).ConfigureAwait(false); - if (!shutdownContentResult.Succeeded) - { - sb.Append($"Content session shutdown failed, error=[{shutdownContentResult}]"); - } - - sb.Append(sb.Length > 0 ? ", " : string.Empty); - sb.Append($"Memoization session startup failed, error=[{startupMemoizationResult}]"); - StartupCompleted = true; - return new BoolResult(sb.ToString()); - } - - StartupCompleted = true; - return BoolResult.Success; - } - - /// - public virtual async Task ShutdownAsync(Context context) - { - ShutdownStarted = true; - var shutdownMemoizationResult = MemoizationReadOnlySession != null - ? await MemoizationReadOnlySession.ShutdownAsync(context).ConfigureAwait(false) - : BoolResult.Success; - var shutdownContentResult = ContentReadOnlySession != null - ? await ContentReadOnlySession.ShutdownAsync(context).ConfigureAwait(false) - : BoolResult.Success; - - BoolResult result; - if (shutdownMemoizationResult.Succeeded && shutdownContentResult.Succeeded) - { - result = BoolResult.Success; - } - else - { - var sb = new StringBuilder(); - if (!shutdownMemoizationResult.Succeeded) - { - sb.Append($"Memoization session shutdown failed, error=[{shutdownMemoizationResult}]"); - } - - if (!shutdownContentResult.Succeeded) - { - sb.Append($"Content session shutdown failed, error=[{shutdownContentResult}]"); - } - - result = new BoolResult(sb.ToString()); - } - - ShutdownCompleted = true; - return result; - } - - /// - public void Dispose() - { - if (_disposed) - { - return; - } - - Dispose(true); - GC.SuppressFinalize(this); - - _disposed = true; - } - - /// - /// Dispose pattern. - /// - private void Dispose(bool disposing) - { - if (disposing) - { - _memoizationReadOnlySession?.Dispose(); - _memoizationReadOnlySession = null; - - _contentReadOnlySession?.Dispose(); - _contentReadOnlySession = null; - } - } - - /// - public System.Collections.Generic.IAsyncEnumerable GetSelectors(Context context, Fingerprint weakFingerprint, CancellationToken cts, UrgencyHint urgencyHint = UrgencyHint.Nominal) - { - return this.GetSelectorsAsAsyncEnumerable(context, weakFingerprint, cts, urgencyHint); - } - - /// - public async Task> GetLevelSelectorsAsync(Context context, Fingerprint weakFingerprint, CancellationToken cts, int level) - { - if (MemoizationReadOnlySession is IReadOnlyMemoizationSessionWithLevelSelectors withLevelSelectors) - { - var result = await withLevelSelectors.GetLevelSelectorsAsync(context, weakFingerprint, cts, level); - - if (result.Succeeded && Parent is not null) - { - foreach (var selector in result.Value.Selectors) - { - Parent.AddOrExtendPin(context, selector.ContentHash); - } - } - - return result; - } - - throw new NotSupportedException($"ReadOnlyMemoization session {MemoizationReadOnlySession.GetType().Name} does not support GetLevelSelectors functionality."); - } - - /// - public async Task GetContentHashListAsync(Context context, - StrongFingerprint strongFingerprint, - CancellationToken cts, - UrgencyHint urgencyHint) - { - var result = await MemoizationReadOnlySession.GetContentHashListAsync(context, strongFingerprint, cts, urgencyHint); - if (result.Succeeded && Parent is not null && result.ContentHashListWithDeterminism.ContentHashList is not null) - { - Parent.AddOrExtendPin(context, strongFingerprint.Selector.ContentHash); - - var contentHashList = result.ContentHashListWithDeterminism.ContentHashList.Hashes; - foreach (var contentHash in contentHashList) - { - Parent.AddOrExtendPin(context, contentHash); - } - } - - return result; - } - - /// - public Task PinAsync(Context context, ContentHash contentHash, CancellationToken cts, UrgencyHint urgencyHint) - { - if (Parent is not null && Parent.CanElidePin(context, contentHash)) - { - return PinResult.SuccessTask; - } - - return ContentReadOnlySession.PinAsync(context, contentHash, cts, urgencyHint); - } - - /// - public Task>>> PinAsync(Context context, IReadOnlyList contentHashes, PinOperationConfiguration configuration) - { - return Workflows.RunWithFallback( - contentHashes, - initialFunc: (contentHashes) => - { - return Task.FromResult(contentHashes.Select(contentHash => - { - if (Parent is not null && Parent.CanElidePin(context, contentHash)) - { - return PinResult.Success; - } - - return PinResult.ContentNotFound; - }).AsIndexedTasks()); - }, - fallbackFunc: (contentHashes) => { - return ContentReadOnlySession.PinAsync(context, contentHashes, configuration); - }, - isSuccessFunc: r => r.Succeeded); - } - - /// - public Task OpenStreamAsync( - Context context, ContentHash contentHash, CancellationToken cts, UrgencyHint urgencyHint) - { - return ContentReadOnlySession.OpenStreamAsync(context, contentHash, cts, urgencyHint); - } - - /// - public Task PlaceFileAsync - ( - Context context, - ContentHash contentHash, - AbsolutePath path, - FileAccessMode accessMode, - FileReplacementMode replacementMode, - FileRealizationMode realizationMode, - CancellationToken cts, - UrgencyHint urgencyHint - ) - { - return ContentReadOnlySession.PlaceFileAsync(context, contentHash, path, accessMode, replacementMode, realizationMode, cts, urgencyHint); - } - - /// - public Task>>> PinAsync( - Context context, - IReadOnlyList contentHashes, - CancellationToken cts, - UrgencyHint urgencyHint = UrgencyHint.Nominal) - { - return Workflows.RunWithFallback( - contentHashes, - initialFunc: (contentHashes) => - { - return Task.FromResult(contentHashes.Select(contentHash => - { - if (Parent is not null && Parent.CanElidePin(context, contentHash)) - { - return PinResult.Success; - } - - return PinResult.ContentNotFound; - }).AsIndexedTasks()); - }, - fallbackFunc: (contentHashes) => { - return ContentReadOnlySession.PinAsync(context, contentHashes, cts, urgencyHint); - }, - isSuccessFunc: r => r.Succeeded); - } - - /// - public Task>>> PlaceFileAsync(Context context, IReadOnlyList hashesWithPaths, FileAccessMode accessMode, FileReplacementMode replacementMode, FileRealizationMode realizationMode, CancellationToken cts, UrgencyHint urgencyHint = UrgencyHint.Nominal) - { - return ContentReadOnlySession.PlaceFileAsync(context, hashesWithPaths, accessMode, replacementMode, realizationMode, cts, urgencyHint); - } - - /// - public IEnumerable EnumeratePinnedContentHashes() - { - return ContentReadOnlySession is IHibernateContentSession session - ? session.EnumeratePinnedContentHashes() - : Enumerable.Empty(); - } - - /// - public Task PinBulkAsync(Context context, IEnumerable contentHashes) - { - return ContentReadOnlySession is IHibernateContentSession session - ? session.PinBulkAsync(context, contentHashes) - : Task.FromResult(0); - } - - /// - public Task ShutdownEvictionAsync(Context context) - { - return ContentReadOnlySession is IHibernateContentSession session - ? session.ShutdownEvictionAsync(context) - : BoolResult.SuccessTask; - } - - /// - public IList GetPendingPublishingOperations() - => MemoizationReadOnlySession is IHibernateCacheSession session - ? session.GetPendingPublishingOperations() - : new List(); - - /// - public Task SchedulePublishingOperationsAsync(Context context, IEnumerable pendingOperations) - => MemoizationReadOnlySession is IHibernateCacheSession session - ? session.SchedulePublishingOperationsAsync(context, pendingOperations) - : Task.FromResult(0); - } -} diff --git a/Public/Src/Cache/MemoizationStore/Library/Stores/DatabaseMemoizationStore.cs b/Public/Src/Cache/MemoizationStore/Library/Stores/DatabaseMemoizationStore.cs index 380da0f09..d9c9ea59f 100644 --- a/Public/Src/Cache/MemoizationStore/Library/Stores/DatabaseMemoizationStore.cs +++ b/Public/Src/Cache/MemoizationStore/Library/Stores/DatabaseMemoizationStore.cs @@ -70,13 +70,6 @@ namespace BuildXL.Cache.MemoizationStore.Stores Database = database; } - /// - public CreateSessionResult CreateReadOnlySession(Context context, string name) - { - var session = new ReadOnlyDatabaseMemoizationSession(name, this); - return new CreateSessionResult(session); - } - /// public CreateSessionResult CreateSession(Context context, string name) { diff --git a/Public/Src/Cache/MemoizationStore/Library/Stores/MemoryMemoizationStore.cs b/Public/Src/Cache/MemoizationStore/Library/Stores/MemoryMemoizationStore.cs index a61aedeaa..af85ea415 100644 --- a/Public/Src/Cache/MemoizationStore/Library/Stores/MemoryMemoizationStore.cs +++ b/Public/Src/Cache/MemoizationStore/Library/Stores/MemoryMemoizationStore.cs @@ -60,13 +60,6 @@ namespace BuildXL.Cache.MemoizationStore.Stores return Task.FromResult(BoolResult.Success); } - /// - public CreateSessionResult CreateReadOnlySession(Context context, string name) - { - var session = new ReadOnlyMemoryMemoizationSession(name, this); - return new CreateSessionResult(session); - } - /// public CreateSessionResult CreateSession(Context context, string name) { diff --git a/Public/Src/Cache/MemoizationStore/Library/Tracing/CacheTracer.cs b/Public/Src/Cache/MemoizationStore/Library/Tracing/CacheTracer.cs index 3204fe7b5..f5620587d 100644 --- a/Public/Src/Cache/MemoizationStore/Library/Tracing/CacheTracer.cs +++ b/Public/Src/Cache/MemoizationStore/Library/Tracing/CacheTracer.cs @@ -17,7 +17,7 @@ namespace BuildXL.Cache.MemoizationStore.Tracing { private readonly CallCounter _statsCounter = new CallCounter("CacheTracer.Stats"); private readonly CallCounter _sessionCounter = new CallCounter("CacheTracer.CreateSession"); - private readonly CallCounter _readSessionCounter = new CallCounter("CacheTracer.CreateReadOnlySession"); + private readonly CallCounter _readSessionCounter = new CallCounter("CacheTracer.CreateSession"); public CacheTracer(string name) : base(name) @@ -35,24 +35,6 @@ namespace BuildXL.Cache.MemoizationStore.Tracing base.GetStatsStop(context, result); } - public void CreateReadOnlySessionStart(Context context, string name) - { - _readSessionCounter.Started(); - if (context.IsEnabled) - { - Debug(context, $"{Name}.CreateReadOnlySession({name}) start"); - } - } - - public void CreateReadOnlySessionStop(Context context, CreateSessionResult result) - { - _readSessionCounter.Completed(result.Duration.Ticks); - if (context.IsEnabled) - { - Debug(context, $"{Name}.CreateReadOnlySession() stop {result.DurationMs}ms result=[{result}]"); - } - } - public void CreateSessionStart(Context context, string name) { _sessionCounter.Started(); diff --git a/Public/Src/Cache/MemoizationStore/Library/Tracing/CreateReadOnlySessionCall.cs b/Public/Src/Cache/MemoizationStore/Library/Tracing/CreateReadOnlySessionCall.cs deleted file mode 100644 index 125de1672..000000000 --- a/Public/Src/Cache/MemoizationStore/Library/Tracing/CreateReadOnlySessionCall.cs +++ /dev/null @@ -1,51 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -using System; -using BuildXL.Cache.ContentStore.Interfaces.Results; -using BuildXL.Cache.ContentStore.Interfaces.Tracing; -using BuildXL.Cache.ContentStore.Tracing; -using BuildXL.Cache.MemoizationStore.Interfaces.Sessions; - -namespace BuildXL.Cache.MemoizationStore.Tracing -{ - /// - /// Instance of a CreateReadOnlySession operation for tracing purposes. - /// - public sealed class CreateReadOnlySessionCall - : TracedCall>, IDisposable - { - /// - /// Run the call. - /// - public static CreateSessionResult Run( - CacheTracer tracer, Context context, string name, Func> func) - { - using (var call = new CreateReadOnlySessionCall(tracer, context, name)) - { - return call.Run(func); - } - } - - /// - /// Initializes a new instance of the class. - /// - private CreateReadOnlySessionCall(CacheTracer tracer, Context context, string name) - : base(tracer, context) - { - Tracer.CreateReadOnlySessionStart(context, name); - } - - /// - protected override CreateSessionResult CreateErrorResult(Exception exception) - { - return new CreateSessionResult(exception); - } - - /// - public void Dispose() - { - Tracer.CreateReadOnlySessionStop(Context, Result); - } - } -} diff --git a/Public/Src/Cache/MemoizationStore/Test/Sessions/OneLevelCacheTests.Mocks.cs b/Public/Src/Cache/MemoizationStore/Test/Sessions/OneLevelCacheTests.Mocks.cs index 9239c2e93..f527f25ae 100644 --- a/Public/Src/Cache/MemoizationStore/Test/Sessions/OneLevelCacheTests.Mocks.cs +++ b/Public/Src/Cache/MemoizationStore/Test/Sessions/OneLevelCacheTests.Mocks.cs @@ -37,11 +37,6 @@ namespace BuildXL.Cache.MemoizationStore.Test.Sessions _contentSession = testContentSession; } - public CreateSessionResult CreateReadOnlySession(Context context, string name, ImplicitPin implicitPin) - { - return new CreateSessionResult(_contentSession); - } - public CreateSessionResult CreateSession(Context context, string name, ImplicitPin implicitPin) { return new CreateSessionResult(_contentSession); @@ -184,11 +179,6 @@ namespace BuildXL.Cache.MemoizationStore.Test.Sessions _memoizationSession = iMemoizationSession; } - public CreateSessionResult CreateReadOnlySession(Context context, string name) - { - return new CreateSessionResult(_memoizationSession); - } - public CreateSessionResult CreateSession(Context context, string name) { return new CreateSessionResult(_memoizationSession); @@ -224,7 +214,7 @@ namespace BuildXL.Cache.MemoizationStore.Test.Sessions } } - internal class TestMemoizationSession : IMemoizationSession, IReadOnlyMemoizationSessionWithLevelSelectors + internal class TestMemoizationSession : IMemoizationSession, IMemoizationSessionWithLevelSelectors { public HashSet GetSelectorsParams = new HashSet(); public HashSet GetContentHashListAsyncParams = new HashSet(); diff --git a/Public/Src/Cache/MemoizationStore/Test/Sessions/OneLevelCacheTests.cs b/Public/Src/Cache/MemoizationStore/Test/Sessions/OneLevelCacheTests.cs index 3d6feb996..35f3750d7 100644 --- a/Public/Src/Cache/MemoizationStore/Test/Sessions/OneLevelCacheTests.cs +++ b/Public/Src/Cache/MemoizationStore/Test/Sessions/OneLevelCacheTests.cs @@ -250,7 +250,7 @@ namespace BuildXL.Cache.MemoizationStore.Test.Sessions testDirectory => new OneLevelCache(() => _mockContentStore, () => _mockMemoizationStore, CacheDeterminism.NewCacheGuid())); } - private Task RunMockReadOnlySessionTestAsync(Context context, Func funcAsync) + private Task RunMockReadOnlySessionTestAsync(Context context, Func funcAsync) { return RunReadOnlySessionTestAsync( context, diff --git a/Public/Src/Cache/MemoizationStore/Test/Sessions/TestInProcessServiceClientCache.cs b/Public/Src/Cache/MemoizationStore/Test/Sessions/TestInProcessServiceClientCache.cs index e2ec372ee..084687497 100644 --- a/Public/Src/Cache/MemoizationStore/Test/Sessions/TestInProcessServiceClientCache.cs +++ b/Public/Src/Cache/MemoizationStore/Test/Sessions/TestInProcessServiceClientCache.cs @@ -82,9 +82,6 @@ namespace BuildXL.Cache.MemoizationStore.Test.Sessions /// public Guid Id => _client.Id; - /// - public CreateSessionResult CreateReadOnlySession(Context context, string name, ImplicitPin implicitPin) => _client.CreateReadOnlySession(context, name, implicitPin); - /// public CreateSessionResult CreateSession(Context context, string name, ImplicitPin implicitPin) => _client.CreateSession(context, name, implicitPin); diff --git a/Public/Src/Cache/MemoizationStore/Test/Synchronization/OneLevelCacheFixture.cs b/Public/Src/Cache/MemoizationStore/Test/Synchronization/OneLevelCacheFixture.cs index 0de528ab8..fa5b78801 100644 --- a/Public/Src/Cache/MemoizationStore/Test/Synchronization/OneLevelCacheFixture.cs +++ b/Public/Src/Cache/MemoizationStore/Test/Synchronization/OneLevelCacheFixture.cs @@ -159,8 +159,6 @@ namespace BuildXL.Cache.MemoizationStore.Test.Synchronization /// public class MockContentStore : StartupShutdownMock, IContentStore { - public CreateSessionResult CreateReadOnlySession(Context context, string name, ImplicitPin implicitPin) => null; - public CreateSessionResult CreateSession(Context context, string name, ImplicitPin implicitPin) => null; public Task GetStatsAsync(Context context) => Task.FromResult(new GetStatsResult(new CounterSet())); @@ -177,8 +175,6 @@ namespace BuildXL.Cache.MemoizationStore.Test.Synchronization /// public class MockMemoizationStore : StartupShutdownMock, IMemoizationStore { - public CreateSessionResult CreateReadOnlySession(Context context, string name) => null; - public CreateSessionResult CreateSession(Context context, string name) => null; public CreateSessionResult CreateSession(Context context, string name, IContentSession contentSession) => null; diff --git a/Public/Src/Cache/MemoizationStore/Vsts/BuildCacheCache.cs b/Public/Src/Cache/MemoizationStore/Vsts/BuildCacheCache.cs index 048f1bfff..d2514740e 100644 --- a/Public/Src/Cache/MemoizationStore/Vsts/BuildCacheCache.cs +++ b/Public/Src/Cache/MemoizationStore/Vsts/BuildCacheCache.cs @@ -320,61 +320,6 @@ namespace BuildXL.Cache.MemoizationStore.Vsts /// public bool StartupStarted { get; private set; } - /// - [SuppressMessage("Microsoft.Reliability", "CA2000:Dispose objects before losing scope")] - public CreateSessionResult CreateReadOnlySession(Context context, string name, ImplicitPin implicitPin) - { - return Tracing.CreateReadOnlySessionCall.Run(_tracer, context, name, () => - { - var backingContentSessionResult = _backingContentStore.CreateSession(context, name); - if (!backingContentSessionResult.Succeeded) - { - return new CreateSessionResult(backingContentSessionResult); - } - - IContentSession writeThroughContentSession = null; - if (_writeThroughContentStore != null) - { - var writeThroughContentSessionResult = _writeThroughContentStore.CreateSession(context, name, implicitPin); - if (!writeThroughContentSessionResult.Succeeded) - { - return new CreateSessionResult(writeThroughContentSessionResult); - } - - writeThroughContentSession = writeThroughContentSessionResult.Session; - } - - return new CreateSessionResult( - new BuildCacheReadOnlySession( - _fileSystem, - name, - implicitPin, - _cacheNamespace, - Id, - _contentHashListAdapterFactory.Create(backingContentSessionResult.Session, _includeDownloadUris), - backingContentSessionResult.Session, - _maxFingerprintSelectorsToFetch, - _minimumTimeToKeepContentHashLists, - _rangeOfTimeToKeepContentHashLists, - _fingerprintIncorporationEnabled, - _maxDegreeOfParallelismForIncorporateRequests, - _maxFingerprintsPerIncorporateRequest, - writeThroughContentSession, - _sealUnbackedContentHashLists, - _overrideUnixFileAccessMode, - _tracer, - _enableEagerFingerprintIncorporation, - _inlineFingerprintIncorporationExpiry, - _eagerFingerprintIncorporationNagleInterval, - _eagerFingerprintIncorporationNagleBatchSize, - _manuallyExtendContentLifetime, - _forceUpdateOnAddContentHashList) - { - RequiredContentKeepUntil = _backingContentStoreConfiguration.RequiredContentKeepUntil - }); - }); - } - /// [SuppressMessage("Microsoft.Reliability", "CA2000:Dispose objects before losing scope")] public CreateSessionResult CreateSession(Context context, string name, ImplicitPin implicitPin) @@ -479,16 +424,6 @@ namespace BuildXL.Cache.MemoizationStore.Vsts return AsyncEnumerable.Empty>(); } - /// - public CreateSessionResult CreateReadOnlySession(Context context, string name) - { - var result = CreateReadOnlySession(context, name, ImplicitPin.None); - - return result.Succeeded - ? new CreateSessionResult(result.Session) - : new CreateSessionResult(result); - } - /// public CreateSessionResult CreateSession(Context context, string name) { diff --git a/Public/Src/Cache/MemoizationStore/Vsts/BuildCacheReadOnlySession.cs b/Public/Src/Cache/MemoizationStore/Vsts/BuildCacheReadOnlySession.cs deleted file mode 100644 index b68bcd4aa..000000000 --- a/Public/Src/Cache/MemoizationStore/Vsts/BuildCacheReadOnlySession.cs +++ /dev/null @@ -1,1124 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -using System; -using System.Collections.Generic; -using System.Diagnostics; -using System.Diagnostics.CodeAnalysis; -using System.Diagnostics.ContractsLight; -using System.IO; -using System.Linq; -using System.Runtime.CompilerServices; -using System.Text; -using System.Threading; -using System.Threading.Tasks; -using BuildXL.Cache.ContentStore.Extensions; -using BuildXL.Cache.ContentStore.Hashing; -using BuildXL.Cache.ContentStore.Interfaces.Extensions; -using BuildXL.Cache.ContentStore.Interfaces.FileSystem; -using BuildXL.Cache.ContentStore.Interfaces.Results; -using BuildXL.Cache.ContentStore.Interfaces.Sessions; -using BuildXL.Cache.ContentStore.Interfaces.Stores; -using BuildXL.Cache.ContentStore.Interfaces.Tracing; -using BuildXL.Cache.ContentStore.Interfaces.Utils; -using BuildXL.Cache.ContentStore.Synchronization; -using BuildXL.Cache.ContentStore.Tracing; -using BuildXL.Cache.ContentStore.Tracing.Internal; -using BuildXL.Cache.ContentStore.UtilitiesCore.Internal; -using BuildXL.Cache.ContentStore.Utils; -using BuildXL.Cache.ContentStore.Vsts; -using BuildXL.Cache.MemoizationStore.Interfaces.Results; -using BuildXL.Cache.MemoizationStore.Interfaces.Sessions; -using BuildXL.Cache.MemoizationStore.Vsts.Adapters; -using BuildXL.Cache.MemoizationStore.VstsInterfaces; -using BuildXL.Utilities.ParallelAlgorithms; -using Microsoft.VisualStudio.Services.BlobStore.Common; -using BlobIdentifier = BuildXL.Cache.ContentStore.Hashing.BlobIdentifier; - -namespace BuildXL.Cache.MemoizationStore.Vsts -{ - /// - /// IReadOnlyCacheSession for BuildCacheCache. - /// - public class BuildCacheReadOnlySession : StartupShutdownSlimBase, IReadOnlyCacheSessionWithLevelSelectors - { - /// - protected override Tracer Tracer => CacheTracer; - - /// - protected BuildCacheCacheTracer CacheTracer; - - private const int MaxSealingErrorsToPrintOnShutdown = 10; - - /// - /// Public name for monitoring use. - /// - public const string Component = "BuildCacheSession"; - - /// - /// The only HashType recognizable by the server. - /// - protected const HashType RequiredHashType = HashType.Vso0; - - /// - /// Size for stream buffers to temp files. - /// - protected const int StreamBufferSize = 16384; - - /// - /// Policy determining whether or not content should be automatically pinned on adds or gets. - /// - protected readonly ImplicitPin ImplicitPin; - - /// - /// Backing BuildCache http client - /// - internal readonly IContentHashListAdapter ContentHashListAdapter; - - /// - /// Backing content session - /// - protected readonly IBackingContentSession BackingContentSession; - - /// - /// Optional write-through session to allow writing-behind to BlobStore - /// - protected readonly IContentSession WriteThroughContentSession; - - /// - /// The namespace of the build cache service being communicated with. - /// - protected readonly string CacheNamespace; - - /// - /// The id of the build cache service being communicated with. - /// - protected readonly Guid CacheId; - - private readonly bool _enableEagerFingerprintIncorporation; - private readonly TimeSpan _inlineFingerprintIncorporationExpiry; - private readonly TimeSpan _eagerFingerprintIncorporationInterval; - private readonly int _eagerFingerprintIncorporationBatchSize; - - /// - internal TimeSpan? RequiredContentKeepUntil { get; init; } - - /// - /// Background tracker for handling the upload/sealing of unbacked metadata. - /// - // ReSharper disable once InconsistentNaming - protected BackgroundTaskTracker _taskTracker; - - /// - /// Keep track of all the fingerprints that need to be incorporated later - /// - internal readonly FingerprintTracker FingerprintTracker; - - private readonly int _maxFingerprintSelectorsToFetch; - private readonly DisposableDirectory _tempDirectory; - private readonly bool _sealUnbackedContentHashLists; - private readonly List _sealingErrorsToPrintOnShutdown; - private readonly bool _fingerprintIncorporationEnabled; - private readonly int _maxDegreeOfParallelismForIncorporateRequests; - private readonly int _maxFingerprintsPerIncorporateRequest; - private int _sealingErrorCount; - private readonly bool _overrideUnixFileAccessMode; - - private Context _eagerFingerprintIncorporationTracingContext; // must be set at StartupAsync - private readonly NagleQueue _eagerFingerprintIncorporationNagleQueue; - - /// - protected readonly bool ManuallyExtendContentLifetime; - - /// - protected readonly bool ForceUpdateOnAddContentHashList; - - /// - /// Initializes a new instance of the class. - /// - /// A interface to read/write files. - /// Session name. - /// Policy determining whether or not content should be automatically pinned on adds or gets. - /// the namespace of the cache in VSTS - /// the guid of the cache in VSTS - /// Backing BuildCache http client. - /// Backing content session. - /// Maximum number of selectors to enumerate. - /// Minimum time-to-live for created or referenced ContentHashLists. - /// Range of time beyond the minimum for the time-to-live of created or referenced ContentHashLists. - /// Feature flag to enable fingerprints incorporation - /// Throttle the number of fingerprints chunks sent in parallel - /// Max fingerprints allowed per chunk - /// Optional write-through session to allow writing-behind to BlobStore - /// If true, the client will attempt to seal any unbacked ContentHashLists that it sees. - /// If true, overrides default Unix file access modes when placing files. - /// A tracer for logging and perf counters. - /// - /// - /// - /// - /// Whether to manually extend content lifetime when doing incorporate calls - /// Whether to force an update and ignore existing CHLs when adding. - public BuildCacheReadOnlySession( - IAbsFileSystem fileSystem, - string name, - ImplicitPin implicitPin, - string cacheNamespace, - Guid cacheId, - IContentHashListAdapter contentHashListAdapter, - IBackingContentSession backingContentSession, - int maxFingerprintSelectorsToFetch, - TimeSpan minimumTimeToKeepContentHashLists, - TimeSpan rangeOfTimeToKeepContentHashLists, - bool fingerprintIncorporationEnabled, - int maxDegreeOfParallelismForIncorporateRequests, - int maxFingerprintsPerIncorporateRequest, - IContentSession writeThroughContentSession, - bool sealUnbackedContentHashLists, - bool overrideUnixFileAccessMode, - BuildCacheCacheTracer tracer, - bool enableEagerFingerprintIncorporation, - TimeSpan inlineFingerprintIncorporationExpiry, - TimeSpan eagerFingerprintIncorporationInterval, - int eagerFingerprintIncorporationBatchSize, - bool manuallyExtendContentLifetime, - bool forceUpdateOnAddContentHashList) - { - Contract.Requires(name != null); - Contract.Requires(contentHashListAdapter != null); - Contract.Requires(backingContentSession != null); - Contract.Requires(!backingContentSession.StartupStarted); - - Name = name; - ImplicitPin = implicitPin; - ContentHashListAdapter = contentHashListAdapter; - BackingContentSession = backingContentSession; - _maxFingerprintSelectorsToFetch = maxFingerprintSelectorsToFetch; - CacheNamespace = cacheNamespace; - CacheId = cacheId; - WriteThroughContentSession = writeThroughContentSession; - _sealUnbackedContentHashLists = sealUnbackedContentHashLists; - CacheTracer = tracer; - _enableEagerFingerprintIncorporation = enableEagerFingerprintIncorporation; - _inlineFingerprintIncorporationExpiry = inlineFingerprintIncorporationExpiry; - _eagerFingerprintIncorporationInterval = eagerFingerprintIncorporationInterval; - _eagerFingerprintIncorporationBatchSize = eagerFingerprintIncorporationBatchSize; - - _tempDirectory = new DisposableDirectory(fileSystem); - _sealingErrorsToPrintOnShutdown = new List(); - _fingerprintIncorporationEnabled = fingerprintIncorporationEnabled; - _maxDegreeOfParallelismForIncorporateRequests = maxDegreeOfParallelismForIncorporateRequests; - _maxFingerprintsPerIncorporateRequest = maxFingerprintsPerIncorporateRequest; - _overrideUnixFileAccessMode = overrideUnixFileAccessMode; - - ManuallyExtendContentLifetime = manuallyExtendContentLifetime; - - FingerprintTracker = new FingerprintTracker(DateTime.UtcNow + minimumTimeToKeepContentHashLists, rangeOfTimeToKeepContentHashLists); - - ForceUpdateOnAddContentHashList = forceUpdateOnAddContentHashList; - - if (enableEagerFingerprintIncorporation) - { - _eagerFingerprintIncorporationNagleQueue = NagleQueue.Create(IncorporateBatchAsync, maxDegreeOfParallelismForIncorporateRequests, eagerFingerprintIncorporationInterval, eagerFingerprintIncorporationBatchSize); - } - } - - /// - public void Dispose() - { - _taskTracker?.Dispose(); - BackingContentSession?.Dispose(); - WriteThroughContentSession?.Dispose(); - _tempDirectory.Dispose(); - } - - /// - public string Name { get; } - - /// - protected override async Task StartupCoreAsync(OperationContext context) - { - _eagerFingerprintIncorporationTracingContext = context; - - LogIncorporateOptions(context); - - var backingContentSessionTask = Task.Run(async () => await BackingContentSession.StartupAsync(context).ConfigureAwait(false)); - var writeThroughContentSessionResult = WriteThroughContentSession != null - ? await WriteThroughContentSession.StartupAsync(context).ConfigureAwait(false) - : BoolResult.Success; - var backingContentSessionResult = await backingContentSessionTask.ConfigureAwait(false); - if (backingContentSessionResult.Succeeded && writeThroughContentSessionResult.Succeeded) - { - _taskTracker = new BackgroundTaskTracker(Component, context.CreateNested(Component)); - return BoolResult.Success; - } - - var sb = new StringBuilder(); - - if (backingContentSessionResult.Succeeded) - { - var r = await BackingContentSession.ShutdownAsync(context).ConfigureAwait(false); - if (!r.Succeeded) - { - sb.Append($"Backing content session shutdown failed, error=[{r}]"); - } - } - else - { - sb.Append($"Backing content session startup failed, error=[{backingContentSessionResult}]"); - } - - if (writeThroughContentSessionResult.Succeeded) - { - var r = WriteThroughContentSession != null - ? await WriteThroughContentSession.ShutdownAsync(context).ConfigureAwait(false) - : BoolResult.Success; - if (!r.Succeeded) - { - sb.Append(sb.Length > 0 ? ", " : string.Empty); - sb.Append($"Write-through content session shutdown failed, error=[{r}]"); - } - } - else - { - sb.Append(sb.Length > 0 ? ", " : string.Empty); - sb.Append($"Write-through content session startup failed, error=[{writeThroughContentSessionResult}]"); - } - - return new BoolResult(sb.ToString()); - } - - private void LogIncorporateOptions(Context context) - { - CacheTracer.Debug(context, $"BuildCacheReadOnlySession incorporation options: FingerprintIncorporationEnabled={_fingerprintIncorporationEnabled}, EnableEagerFingerprintIncorporation={_enableEagerFingerprintIncorporation} " + - $"InlineFingerprintIncorporationExpiry={_inlineFingerprintIncorporationExpiry}, " + - $"EagerFingerprintIncorporationInterval={_eagerFingerprintIncorporationInterval}, EagerFingerprintIncorporationBatchSize={_eagerFingerprintIncorporationBatchSize}."); - } - - /// - protected override async Task ShutdownCoreAsync(OperationContext context) - { - _eagerFingerprintIncorporationNagleQueue?.Dispose(); - - CacheTracer.Debug(context, "IncorporateOnShutdown start"); - CacheTracer.Debug(context, $"Incorporate fingerprints feature enabled:[{_fingerprintIncorporationEnabled}]"); - CacheTracer.Debug(context, $"Total fingerprints to be incorporated:[{FingerprintTracker.Count}]"); - CacheTracer.Debug(context, $"Max fingerprints per incorporate request(=chunk size):[{_maxFingerprintsPerIncorporateRequest}]"); - CacheTracer.Debug(context, $"Max incorporate requests allowed in parallel:[{_maxDegreeOfParallelismForIncorporateRequests}]"); - if (_fingerprintIncorporationEnabled) - { - // Incorporating all of the fingerprints for a build, in one request, to a single endpoint causes pain. Incorporation involves - // extending the lifetime of all fingerprints *and* content/s mapped to each fingerprint. Processing a large request payload - // results in, potentially, fanning out a massive number of "lifetime extend" requests to itemstore and blobstore, which can - // bring down the endpoint. Break this down into chunks so that multiple, load-balanced endpoints can share the burden. - List fingerprintsToBump = FingerprintTracker.StaleFingerprints.ToList(); - CacheTracer.Debug(context, $"Total fingerprints to be sent in incorporation requests to the service: {fingerprintsToBump.Count}"); - - List> chunks = fingerprintsToBump.Select( - strongFingerprint => new StrongFingerprintAndExpiration(strongFingerprint, FingerprintTracker.GenerateNewExpiration()) - ).GetPages(_maxFingerprintsPerIncorporateRequest).ToList(); - CacheTracer.Debug(context, $"Total fingerprint incorporation requests to be issued(=number of fingerprint chunks):[{chunks.Count}]"); - - var incorporateBlock = ActionBlockSlim.Create>( - degreeOfParallelism: _maxDegreeOfParallelismForIncorporateRequests, - processItemAction: async chunk => - { - var pinResult = await PinContentManuallyAsync(new OperationContext(context, CancellationToken.None), chunk); - if (!pinResult) - { - return; - } - - await ContentHashListAdapter.IncorporateStrongFingerprints( - context, - CacheNamespace, - new IncorporateStrongFingerprintsRequest(chunk.AsReadOnly()) - ).ConfigureAwait(false); - }); - - foreach (var chunk in chunks) - { - await incorporateBlock.PostAsync(chunk); - } - - incorporateBlock.Complete(); - await incorporateBlock.Completion.ConfigureAwait(false); // TODO: Gracefully handle exceptions so that the rest of shutdown can happen (bug 1365340) - CacheTracer.Debug(context, "IncorporateOnShutdown stop"); - } - - if (_taskTracker != null) - { - await _taskTracker.Synchronize().ConfigureAwait(false); - await _taskTracker.ShutdownAsync(context).ConfigureAwait(false); - } - - var backingContentSessionTask = Task.Run(async () => await BackingContentSession.ShutdownAsync(context).ConfigureAwait(false)); - var writeThroughContentSessionResult = WriteThroughContentSession != null - ? await WriteThroughContentSession.ShutdownAsync(context).ConfigureAwait(false) - : BoolResult.Success; - var backingContentSessionResult = await backingContentSessionTask.ConfigureAwait(false); - - BoolResult result; - if (backingContentSessionResult.Succeeded && writeThroughContentSessionResult.Succeeded) - { - if (_sealingErrorsToPrintOnShutdown.Any()) - { - var sb = new StringBuilder(); - sb.AppendLine("Error(s) during background sealing:"); - foreach (var sealingError in _sealingErrorsToPrintOnShutdown) - { - sb.AppendLine($"[{sealingError}]"); - } - - if (_sealingErrorCount > MaxSealingErrorsToPrintOnShutdown) - { - sb.AppendLine($"See log for the other {MaxSealingErrorsToPrintOnShutdown - _sealingErrorCount} error(s)."); - } - - result = new BoolResult(sb.ToString()); - } - else - { - result = BoolResult.Success; - } - } - else - { - var sb = new StringBuilder(); - if (!backingContentSessionResult.Succeeded) - { - sb.Append($"Backing content session shutdown failed, error=[{backingContentSessionResult}]"); - } - - if (!writeThroughContentSessionResult.Succeeded) - { - sb.Append(sb.Length > 0 ? ", " : string.Empty); - sb.Append($"Write-through content session shutdown failed, error=[{writeThroughContentSessionResult}]"); - } - - result = new BoolResult(sb.ToString()); - } - - return result; - } - - private async Task IncorporateBatchAsync(List fingerprints) - { - // Tracking shutdown to avoid using nagle queue after the shutdown. - using var shutdownContext = TrackShutdown(_eagerFingerprintIncorporationTracingContext); - - var context = shutdownContext.Context; - BoolResult result = await context.CreateOperation( - CacheTracer, - async () => - { - CacheTracer.Debug(context, $"IncorporateBatch: Total fingerprints to be incorporated {fingerprints.Count}, ChunkSize={_maxFingerprintsPerIncorporateRequest}, DegreeOfParallelism={_maxDegreeOfParallelismForIncorporateRequests}."); - - // Incorporating all of the fingerprints for a build, in one request, to a single endpoint causes pain. Incorporation involves - // extending the lifetime of all fingerprints *and* content/s mapped to each fingerprint. Processing a large request payload - // results in, potentially, fanning out a massive number of "lifetime extend" requests to itemstore and blobstore, which can - // bring down the endpoint. Break this down into chunks so that multiple, load-balanced endpoints can share the burden. - - var fingerprintsWithExpiration = - fingerprints - .Select(strongFingerprint => new StrongFingerprintAndExpiration(strongFingerprint, FingerprintTracker.GenerateNewExpiration())) - .ToList().AsReadOnly(); - - var pinResult = await PinContentManuallyAsync(context, fingerprintsWithExpiration); - if (!pinResult) - { - return pinResult; - } - - await ContentHashListAdapter.IncorporateStrongFingerprints( - context, - CacheNamespace, - new IncorporateStrongFingerprintsRequest(fingerprintsWithExpiration) - ).ConfigureAwait(false); - - return BoolResult.Success; - }).RunAsync(); - - // Ignoring the failure, because it was already traced if needed. - result.IgnoreFailure(); - } - - private async Task IncorporateFingerprintAsync(OperationContext context, StrongFingerprint fingerprint) - { - BoolResult result = await context.CreateOperation( - CacheTracer, - async () => - { - var fingerprintWithExpiration = new StrongFingerprintAndExpiration(fingerprint, FingerprintTracker.GenerateNewExpiration()); - - var pinResult = await PinContentManuallyAsync(context, new StrongFingerprintAndExpiration[] { fingerprintWithExpiration }); - if (!pinResult) - { - return pinResult; - } - - await ContentHashListAdapter.IncorporateStrongFingerprints( - context, - CacheNamespace, - new IncorporateStrongFingerprintsRequest(new []{ fingerprintWithExpiration}) - ).ConfigureAwait(false); - - return BoolResult.Success; - }).RunAsync(); - - // Ignoring the failure, because it was already traced if needed. - result.IgnoreFailure(); - } - - private async Task PinContentManuallyAsync(OperationContext context, IEnumerable fingerprints) - { - // Pin the content manually as a workaround for the fact that BuildCache doesn't know how to talk to the backing content store due to it being - // dedup or in a non-default domain and we want the content to be there even if we have to tell BuildCache that the ContentHashList is unbacked. - if (ManuallyExtendContentLifetime) - { - // TODO: optimize and run in parallel if needed. - foreach (var fingerprint in fingerprints) - { - // TODO: Get the content hash list in a more efficient manner which does not require us to talk to BuildCache. - var hashListResult = await ContentHashListAdapter.GetContentHashListAsync(context, CacheNamespace, fingerprint.StrongFingerprint); - if (!hashListResult.Succeeded || hashListResult.Value?.ContentHashListWithDeterminism.ContentHashList == null) - { - return new BoolResult(hashListResult, "Failed to get content hash list when attempting to extend its conetnts' lifetimes."); - } - - var expirationDate = new DateTime(Math.Max(hashListResult.Value.GetRawExpirationTimeUtc()?.Ticks ?? 0, fingerprint.ExpirationDateUtc.Ticks), DateTimeKind.Utc); - - var pinResults = await Task.WhenAll(await BackingContentSession.PinAsync( - context, - hashListResult.Value.ContentHashListWithDeterminism.ContentHashList.Hashes, - expirationDate)); - - if (pinResults.Any(r => !r.Succeeded)) - { - return new BoolResult($"Failed to pin all pieces of content for fingerprint=[{fingerprint.StrongFingerprint}]"); - } - } - } - - return BoolResult.Success; - } - - /// - public System.Collections.Generic.IAsyncEnumerable GetSelectors(Context context, Fingerprint weakFingerprint, CancellationToken cts, UrgencyHint urgencyHint = UrgencyHint.Nominal) - { - return this.GetSelectorsAsAsyncEnumerable(context, weakFingerprint, cts, urgencyHint); - } - - /// - public async Task> GetLevelSelectorsAsync(Context context, Fingerprint weakFingerprint, CancellationToken cts, int level) - { - var result = await GetSelectorsAsync(new OperationContext(context, cts), weakFingerprint); - return LevelSelectors.Single(result); - } - - private async Task> GetSelectorsAsync(OperationContext context, Fingerprint weakFingerprint) - { - CacheTracer.MemoizationStoreTracer.GetSelectorsStart(context, weakFingerprint); - Stopwatch sw = Stopwatch.StartNew(); - try - { - var responseResult = await ContentHashListAdapter.GetSelectorsAsync( - context, - CacheNamespace, - weakFingerprint, - _maxFingerprintSelectorsToFetch).ConfigureAwait(false); - - if (!responseResult) - { - return Result.FromError(responseResult); - } - - if (responseResult.Value == null) - { - return Result.Success(CollectionUtilities.EmptyArray()); - } - - foreach (var selectorAndPossible in responseResult.Value) - { - var selector = selectorAndPossible.Selector; - if (selectorAndPossible.ContentHashList != null) - { - // Store pre-fetched data in-memory - var strongFingerprint = new StrongFingerprint(weakFingerprint, selector); - var unpackResult = UnpackContentHashListWithDeterminismAfterGet( - selectorAndPossible.ContentHashList, - CacheId); - if (unpackResult && unpackResult.ContentHashListWithDeterminism.Determinism.IsDeterministic) - { - CacheTracer.RecordPrefetchedContentHashList(); - ContentHashListWithDeterminismCache.Instance.AddValue( - CacheNamespace, - strongFingerprint, - unpackResult.ContentHashListWithDeterminism); - - BackingContentSession.ExpiryCache.AddExpiry( - selector.ContentHash, - unpackResult.ContentHashListWithDeterminism.Determinism.ExpirationUtc); - } - } - } - - CacheTracer.MemoizationStoreTracer.GetSelectorsCount(context, weakFingerprint, responseResult.Value.Count()); - return responseResult.Value.Select(responseData => responseData.Selector).ToArray(); - } - catch (Exception e) - { - return Result.FromException(e); - } - finally - { - CacheTracer.MemoizationStoreTracer.GetSelectorsStop(context, sw.Elapsed, weakFingerprint); - } - } - - /// - /// Stores the DownloadUris in-memory to reduce the calls to BlobStore. - /// - protected void StorePrefetchedDownloadUris(IDictionary blobDownloadUris) - { - if (blobDownloadUris == null) - { - return; - } - - foreach (var blobDownloadUri in blobDownloadUris) - { - BackingContentSession.UriCache.AddDownloadUri( - BlobIdentifier.Deserialize(blobDownloadUri.Key).ToContentHash(), - new PreauthenticatedUri(blobDownloadUri.Value, EdgeType.Unknown)); // EdgeType value shouldn't matter because we don't use it. - } - } - - /// - public Task GetContentHashListAsync( - Context context, - StrongFingerprint strongFingerprint, - CancellationToken cts, - UrgencyHint urgencyHint) - { - var operationContext = new OperationContext(context, cts); - return operationContext.PerformOperationAsync( - CacheTracer, - async () => - { - // Check for pre-fetched data - ContentHashListWithDeterminism contentHashListWithDeterminism; - - if (ContentHashListWithDeterminismCache.Instance.TryGetValue( - CacheNamespace, strongFingerprint, out contentHashListWithDeterminism)) - { - CacheTracer.RecordUseOfPrefetchedContentHashList(); - await TrackFingerprintAsync( - operationContext, - strongFingerprint, - contentHashListWithDeterminism.Determinism.ExpirationUtc, - contentHashListWithDeterminism.ContentHashList).ConfigureAwait(false); - return new GetContentHashListResult(contentHashListWithDeterminism); - } - - // No pre-fetched data. Need to query the server. - Result responseObject = - await ContentHashListAdapter.GetContentHashListAsync(operationContext, CacheNamespace, strongFingerprint).ConfigureAwait(false); - - if (!responseObject.Succeeded) - { - return new GetContentHashListResult(responseObject); - } - - ContentHashListWithCacheMetadata response = responseObject.Value; - if (response.ContentHashListWithDeterminism.ContentHashList == null) - { - // Miss - return new GetContentHashListResult(new ContentHashListWithDeterminism(null, CacheDeterminism.None)); - } - - GetContentHashListResult unpackResult = UnpackContentHashListWithDeterminismAfterGet(response, CacheId); - if (!unpackResult.Succeeded) - { - return unpackResult; - } - - SealIfNecessaryAfterGet(operationContext, strongFingerprint, response); - - await TrackFingerprintAsync(operationContext, strongFingerprint, response.GetRawExpirationTimeUtc(), unpackResult.ContentHashListWithDeterminism.ContentHashList); - return new GetContentHashListResult(unpackResult.ContentHashListWithDeterminism); - }, - traceOperationStarted: true, - extraStartMessage: $"StrongFingerprint=({strongFingerprint})", - extraEndMessage: result => $"StrongFingerprint=({strongFingerprint})"); - } - - /// - protected async Task TrackFingerprintAsync(OperationContext context, StrongFingerprint strongFingerprint, DateTime? expirationUtc, ContentHashList hashes) - { - context.Token.ThrowIfCancellationRequested(); - if (expirationUtc != null) - { - BackingContentSession.ExpiryCache.AddExpiry(strongFingerprint.Selector.ContentHash, expirationUtc.Value); - - if (hashes != null) - { - foreach (var hash in hashes.Hashes) - { - BackingContentSession.ExpiryCache.AddExpiry(hash, expirationUtc.Value); - } - } - } - - // Currently we have 3 ways for fingerprint incorporation: - // 1. Inline incorporation: If eager fingerprint incorporation enabled and - // the entry will expire in _inlineFingerprintIncorporationExpiry time. - // 2. Eager bulk incorporation: if eager fingerprint incorporation enabled and - // the entry's expiry is not available or it won't expire in _inlineFingerprintIncorporationExpiry time. - // 3. Session shutdown incorporation: if eager fingerprint incorporation is disabled and the normal fingerprint incorporation is enabled. - if (_enableEagerFingerprintIncorporation) - { - if (expirationUtc != null && (expirationUtc.Value - DateTime.UtcNow < _inlineFingerprintIncorporationExpiry)) - { - CacheTracer.Debug(context, $"Incorporating fingerprint inline: StrongFingerprint=[{strongFingerprint}], ExpirationUtc=[{expirationUtc}]."); - - await IncorporateFingerprintAsync(context, strongFingerprint); - } - else - { - // We either don't have an expiration time or the time to expiry is greater then _inlineFingerprintIncorporationExpiry - Contract.Assert(_eagerFingerprintIncorporationNagleQueue != null); - _eagerFingerprintIncorporationNagleQueue.Enqueue(strongFingerprint); - } - } - else - { - FingerprintTracker.Track(strongFingerprint, expirationUtc); - } - } - - /// - public Task PinAsync( - Context context, ContentHash contentHash, CancellationToken cts, UrgencyHint urgencyHint) - { - return PinCall.RunAsync(CacheTracer.ContentSessionTracer, new OperationContext(context), contentHash, async () => - { - var bulkResults = await PinAsync(context, new[] { contentHash }, cts, urgencyHint); - return await bulkResults.SingleAwaitIndexed(); - }); - } - - /// - public Task OpenStreamAsync( - Context context, ContentHash contentHash, CancellationToken cts, UrgencyHint urgencyHint) - { - return OpenStreamCall.RunAsync(CacheTracer.ContentSessionTracer, new OperationContext(context), contentHash, async () => - { - if (WriteThroughContentSession != null) - { - var result = - await WriteThroughContentSession.OpenStreamAsync(context, contentHash, cts, urgencyHint).ConfigureAwait(false); - if (result.Succeeded || result.Code != OpenStreamResult.ResultCode.ContentNotFound) - { - return result; - } - } - - return await BackingContentSession.OpenStreamAsync(context, contentHash, cts, urgencyHint).ConfigureAwait(false); - }); - } - - /// - public Task PlaceFileAsync( - Context context, - ContentHash contentHash, - AbsolutePath path, - FileAccessMode accessMode, - FileReplacementMode replacementMode, - FileRealizationMode realizationMode, - CancellationToken cts, - UrgencyHint urgencyHint) - { - return PlaceFileCall.RunAsync(CacheTracer.ContentSessionTracer, new OperationContext(context), contentHash, path, accessMode, replacementMode, realizationMode, async () => - { - if (WriteThroughContentSession != null) - { - var writeThroughResult = await WriteThroughContentSession.PlaceFileAsync(context, contentHash, path, accessMode, replacementMode, realizationMode, cts, urgencyHint).ConfigureAwait(false); - if (writeThroughResult.Succeeded || writeThroughResult.Code != PlaceFileResult.ResultCode.NotPlacedContentNotFound) - { - UnixHelpers.OverrideFileAccessMode(_overrideUnixFileAccessMode, path.Path); - return writeThroughResult; - } - } - - var backingResult = await BackingContentSession.PlaceFileAsync(context, contentHash, path, accessMode, replacementMode, realizationMode, cts, urgencyHint); - UnixHelpers.OverrideFileAccessMode(_overrideUnixFileAccessMode, path.Path); - return backingResult; - }); - } - - /// - public Task>>> PinAsync(Context context, IReadOnlyList contentHashes, CancellationToken cts, UrgencyHint urgencyHint = UrgencyHint.Nominal) - { - return PinHelperAsync(contentHashes, (session, hashes) => session.PinAsync(context, hashes, cts, urgencyHint)); - } - - /// - public Task>>> PinAsync(Context context, IReadOnlyList contentHashes, PinOperationConfiguration config) - { - return PinHelperAsync(contentHashes, (session, hashes) => session.PinAsync(context, hashes, config)); - } - - private Task>>> PinHelperAsync( - IReadOnlyList contentHashes, - Func, Task>>>> pinAsync) - { - var requiredExpiry = DateTime.UtcNow + RequiredContentKeepUntil; - return Workflows.RunWithFallback( - contentHashes, - hashes => - { - return Task.FromResult(hashes.Select(hash => CheckExpiryCache(hash, requiredExpiry)).ToList().AsIndexedTasks()); - }, - hashes => - { - if (WriteThroughContentSession == null) - { - return pinAsync(BackingContentSession, hashes); - } - - return Workflows.RunWithFallback( - hashes, - hashes => pinAsync(WriteThroughContentSession, hashes), - hashes => pinAsync(BackingContentSession, hashes), - result => result.Succeeded); - }, - result => result.Succeeded); - } - - private PinResult CheckExpiryCache(ContentHash hash, DateTime? requiredExpiry) - { - if (requiredExpiry != null && BackingContentSession.ExpiryCache.TryGetExpiry(hash, out var expiry) && (expiry >= requiredExpiry)) - { - return PinResult.Success; - } - else - { - return PinResult.ContentNotFound; - } - } - - /// - public Task>>> PlaceFileAsync(Context context, IReadOnlyList hashesWithPaths, FileAccessMode accessMode, FileReplacementMode replacementMode, FileRealizationMode realizationMode, CancellationToken cts, UrgencyHint urgencyHint = UrgencyHint.Nominal) - { - throw new NotImplementedException(); - } - - /// - protected async Task AddOrGetContentHashListAsync( - OperationContext context, - StrongFingerprint strongFingerprint, - ContentHashListWithDeterminism contentHashListWithDeterminism, - ContentAvailabilityGuarantee guarantee) - { - using var shutdownContext = TrackShutdown(context); - - context = shutdownContext.Context; - - try - { - DateTime expirationUtc = FingerprintTracker.GenerateNewExpiration(); - var valueToAdd = new ContentHashListWithCacheMetadata( - contentHashListWithDeterminism, expirationUtc, guarantee); - - CacheTracer.Debug( - context, - $"Adding contentHashList=[{valueToAdd.ContentHashListWithDeterminism.ContentHashList}] determinism=[{valueToAdd.ContentHashListWithDeterminism.Determinism}] to VSTS with contentAvailabilityGuarantee=[{valueToAdd.ContentGuarantee}], expirationUtc=[{expirationUtc}], forceUpdate=[{ForceUpdateOnAddContentHashList}]"); - - var contentHashListResponseObject = - await ContentHashListAdapter.AddContentHashListAsync( - context, - CacheNamespace, - strongFingerprint, - valueToAdd, - forceUpdate: ForceUpdateOnAddContentHashList).ConfigureAwait(false); - - if (!contentHashListResponseObject.Succeeded) - { - return new AddOrGetContentHashListResult(contentHashListResponseObject); - } - - var contentHashListResponse = contentHashListResponseObject.Value; - var inconsistencyErrorMessage = CheckForResponseInconsistency(contentHashListResponse); - if (inconsistencyErrorMessage != null) - { - return new AddOrGetContentHashListResult(inconsistencyErrorMessage); - } - - ContentHashList contentHashListToReturn = UnpackContentHashListAfterAdd( - contentHashListWithDeterminism.ContentHashList, contentHashListResponse); - - CacheDeterminism determinismToReturn = UnpackDeterminism(contentHashListResponse, CacheId); - if (guarantee == ContentAvailabilityGuarantee.AllContentBackedByCache && !determinismToReturn.IsDeterministic) - { - return new AddOrGetContentHashListResult( - "Inconsistent BuildCache service response. Unbacked values should never override backed values."); - } - - await TrackFingerprintAsync(context, strongFingerprint, contentHashListResponse.GetRawExpirationTimeUtc(), contentHashListToReturn).ConfigureAwait(false); - return new AddOrGetContentHashListResult(new ContentHashListWithDeterminism(contentHashListToReturn, determinismToReturn)); - } - catch (Exception e) - { - return new AddOrGetContentHashListResult(e); - } - } - - private static GetContentHashListResult UnpackContentHashListWithDeterminismAfterGet( - ContentHashListWithCacheMetadata cacheMetadata, Guid cacheId) - { - var inconsistencyErrorMessage = CheckForResponseInconsistency(cacheMetadata); - if (inconsistencyErrorMessage != null) - { - return new GetContentHashListResult(inconsistencyErrorMessage); - } - - if (cacheMetadata?.ContentHashListWithDeterminism.ContentHashList == null) - { - // Miss - return new GetContentHashListResult(new ContentHashListWithDeterminism(null, CacheDeterminism.None)); - } - - ContentHashList contentHashList = cacheMetadata.ContentHashListWithDeterminism.ContentHashList; - CacheDeterminism determinism = UnpackDeterminism(cacheMetadata, cacheId); - - return new GetContentHashListResult(new ContentHashListWithDeterminism(contentHashList, determinism)); - } - - /// - /// Checks for inconsistencies in the metadata returned by the service, returning an appropriate error message (null if none). - /// - protected static string CheckForResponseInconsistency(ContentHashListWithCacheMetadata cacheMetadata) - { - if (cacheMetadata != null) - { - if (cacheMetadata.GetEffectiveExpirationTimeUtc() == null && - cacheMetadata.ContentGuarantee != ContentAvailabilityGuarantee.NoContentBackedByCache) - { - return - "Inconsistent BuildCache service response. Null ContentHashListExpirationUtc should be iff ContentAvailabilityGuarantee.NoContentBackedByCache."; - } - } - - return null; - } - - /// - /// Determine the ContentHashList to return based on the added value and the service response. - /// - protected static ContentHashList UnpackContentHashListAfterAdd( - ContentHashList addedContentHashList, ContentHashListWithCacheMetadata cacheMetadata) - { - Contract.Assert(cacheMetadata != null); - - if (cacheMetadata.ContentHashListWithDeterminism.ContentHashList != null && - !addedContentHashList.Equals( - cacheMetadata.ContentHashListWithDeterminism.ContentHashList)) - { - // The service returned a ContentHashList different from the one we tried to add, so we'll return that. - return cacheMetadata.ContentHashListWithDeterminism.ContentHashList; - } - - // The added value was accepted, so we return null. - return null; - } - - /// - /// Determine the Determinism to return. - /// - protected static CacheDeterminism UnpackDeterminism(ContentHashListWithCacheMetadata cacheMetadata, Guid cacheId) - { - Contract.Assert(cacheMetadata != null); - - if (cacheMetadata.ContentHashListWithDeterminism.Determinism.Guid == CacheDeterminism.Tool.Guid) - { - // Value is Tool-deterministic - return CacheDeterminism.Tool; - } - - var expirationUtc = cacheMetadata.GetEffectiveExpirationTimeUtc(); - return expirationUtc == null - ? CacheDeterminism.None // Value is unbacked in VSTS - : CacheDeterminism.ViaCache(cacheId, expirationUtc.Value); // Value is backed in VSTS - } - - private void SealIfNecessaryAfterGet(OperationContext context, StrongFingerprint strongFingerprint, ContentHashListWithCacheMetadata cacheMetadata) - { - if (WriteThroughContentSession == null) - { - return; - } - - if (cacheMetadata != null && cacheMetadata.GetEffectiveExpirationTimeUtc() == null) - { - // Value is unbacked in VSTS - SealInTheBackground(context, strongFingerprint, cacheMetadata.ContentHashListWithDeterminism); - } - } - - /// - /// Queue a seal operation in the background. Attempts to upload all content to VSTS before updating the metadata as backed. - /// Lost races will be ignored and any failures will be logged and reported on shutdown. - /// - protected void SealInTheBackground( - OperationContext context, StrongFingerprint strongFingerprint, ContentHashListWithDeterminism contentHashListWithDeterminism) - { - if (_sealUnbackedContentHashLists) - { - _taskTracker.Add(Task.Run(() => SealAsync(context, strongFingerprint, contentHashListWithDeterminism))); - } - } - - private async Task SealAsync( - OperationContext context, - StrongFingerprint strongFingerprint, - ContentHashListWithDeterminism contentHashListWithDeterminism) - { - Contract.Assert(WriteThroughContentSession != null); - Contract.Assert(contentHashListWithDeterminism.ContentHashList != null); - try - { - var uploadResult = await UploadAllContentAsync( - context, - strongFingerprint.Selector.ContentHash, - contentHashListWithDeterminism.ContentHashList.Hashes, - CancellationToken.None, - UrgencyHint.Low).ConfigureAwait(false); - if (uploadResult.Code == PinResult.ResultCode.ContentNotFound) - { - CacheTracer.Debug(context, "Background seal unable to find all content during upload."); - return; - } - - if (!uploadResult.Succeeded) - { - ReportSealingError(context, $"Background seal failed during upload: error=[{uploadResult}]."); - return; - } - - var sealResult = await AddOrGetContentHashListAsync( - context, strongFingerprint, contentHashListWithDeterminism, ContentAvailabilityGuarantee.AllContentBackedByCache).ConfigureAwait(false); - if (sealResult.Succeeded) - { - CacheTracer.Debug( - context, - sealResult.ContentHashListWithDeterminism.ContentHashList == null ? $"Successfully sealed value for strongFingerprint [{strongFingerprint}]." : $"Lost the race in sealing value for strongFingerprint [{strongFingerprint}]."); - } - else - { - ReportSealingError(context, $"Background seal failed during sealing: {sealResult}"); - } - } - catch (Exception e) - { - ReportSealingError(context, $"Background seal threw exception: {e}."); - } - } - - private void ReportSealingError(Context context, string errorMessage, [CallerMemberName] string operation = null) - { - Interlocked.Increment(ref _sealingErrorCount); - CacheTracer.Error(context, errorMessage, operation); - if (_sealingErrorCount < MaxSealingErrorsToPrintOnShutdown) - { - _sealingErrorsToPrintOnShutdown.Add(errorMessage); - } - } - - /// - /// Attempt to ensure that all content is in VSTS, uploading any misses from the WriteThroughContentSession. - /// - private async Task UploadAllContentAsync( - Context context, ContentHash selectorHash, IEnumerable hashes, CancellationToken cts, UrgencyHint urgencyHint) - { - List contentToUpload = new List(); - - var pinResult = await BackingContentSession.PinAsync(context, selectorHash, cts, urgencyHint).ConfigureAwait(false); - if (pinResult.Code == PinResult.ResultCode.ContentNotFound) - { - contentToUpload.Add(selectorHash); - } - else if (!pinResult.Succeeded) - { - return pinResult; - } - - foreach (var contentHash in hashes) - { - pinResult = await BackingContentSession.PinAsync(context, contentHash, cts, urgencyHint).ConfigureAwait(false); - if (pinResult.Code == PinResult.ResultCode.ContentNotFound) - { - contentToUpload.Add(contentHash); - } - else if (!pinResult.Succeeded) - { - return pinResult; - } - } - - foreach (var contentHash in contentToUpload) - { - // TODO: Upload the content efficiently (in parallel and with caching of success) (bug 1365340) - AbsolutePath tempFile = null; - try - { - tempFile = _tempDirectory.CreateRandomFileName(); - var placeResult = await WriteThroughContentSession.PlaceFileAsync( - context, - contentHash, - tempFile, - FileAccessMode.Write, - FileReplacementMode.FailIfExists, - FileRealizationMode.Any, - CancellationToken.None).ConfigureAwait(false); - if (placeResult.Code == PlaceFileResult.ResultCode.NotPlacedContentNotFound) - { - return PinResult.ContentNotFound; - } - else if (!placeResult.Succeeded) - { - return new PinResult(placeResult); - } - - var putResult = await BackingContentSession.PutFileAsync( - context, contentHash, tempFile, FileRealizationMode.Any, CancellationToken.None).ConfigureAwait(false); - if (!putResult.Succeeded) - { - return new PinResult(putResult); - } - } - finally - { - if (tempFile != null) - { - try - { - File.Delete(tempFile.Path); - } - catch (Exception e) - { - CacheTracer.Warning(context, $"Error deleting temporary file at {tempFile.Path}: {e}"); - } - } - } - } - - return PinResult.Success; - } - - private struct IncorporationOptions - { - - } - } -} diff --git a/Public/Src/Cache/MemoizationStore/Vsts/BuildCacheSession.cs b/Public/Src/Cache/MemoizationStore/Vsts/BuildCacheSession.cs index 513a25aad..d5dd22b8d 100644 --- a/Public/Src/Cache/MemoizationStore/Vsts/BuildCacheSession.cs +++ b/Public/Src/Cache/MemoizationStore/Vsts/BuildCacheSession.cs @@ -3,32 +3,1145 @@ using System; using System.Collections.Generic; +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using System.Diagnostics.ContractsLight; using System.IO; +using System.Linq; +using System.Runtime.CompilerServices; +using System.Text; using System.Threading; using System.Threading.Tasks; +using BuildXL.Cache.ContentStore.Extensions; using BuildXL.Cache.ContentStore.Hashing; +using BuildXL.Cache.ContentStore.Interfaces.Extensions; using BuildXL.Cache.ContentStore.Interfaces.FileSystem; using BuildXL.Cache.ContentStore.Interfaces.Results; using BuildXL.Cache.ContentStore.Interfaces.Sessions; using BuildXL.Cache.ContentStore.Interfaces.Stores; using BuildXL.Cache.ContentStore.Interfaces.Tracing; using BuildXL.Cache.ContentStore.Interfaces.Utils; +using BuildXL.Cache.ContentStore.Synchronization; using BuildXL.Cache.ContentStore.Tracing; using BuildXL.Cache.ContentStore.Tracing.Internal; +using BuildXL.Cache.ContentStore.UtilitiesCore.Internal; +using BuildXL.Cache.ContentStore.Utils; using BuildXL.Cache.ContentStore.Vsts; using BuildXL.Cache.MemoizationStore.Interfaces.Results; using BuildXL.Cache.MemoizationStore.Interfaces.Sessions; using BuildXL.Cache.MemoizationStore.Tracing; using BuildXL.Cache.MemoizationStore.Vsts.Adapters; using BuildXL.Cache.MemoizationStore.VstsInterfaces; +using BuildXL.Utilities.ParallelAlgorithms; +using Microsoft.VisualStudio.Services.BlobStore.Common; +using BlobIdentifier = BuildXL.Cache.ContentStore.Hashing.BlobIdentifier; namespace BuildXL.Cache.MemoizationStore.Vsts { - /// - /// ICacheSession for BuildCacheCache. - /// - public class BuildCacheSession : BuildCacheReadOnlySession, ICacheSession + /// + public class BuildCacheSession : StartupShutdownSlimBase, ICacheSessionWithLevelSelectors, ICacheSession { + /// + protected override Tracer Tracer => CacheTracer; + + /// + protected BuildCacheCacheTracer CacheTracer; + + private const int MaxSealingErrorsToPrintOnShutdown = 10; + + /// + /// Public name for monitoring use. + /// + public const string Component = "BuildCacheSession"; + + /// + /// The only HashType recognizable by the server. + /// + protected const HashType RequiredHashType = HashType.Vso0; + + /// + /// Size for stream buffers to temp files. + /// + protected const int StreamBufferSize = 16384; + + /// + /// Policy determining whether or not content should be automatically pinned on adds or gets. + /// + protected readonly ImplicitPin ImplicitPin; + + /// + /// Backing BuildCache http client + /// + internal readonly IContentHashListAdapter ContentHashListAdapter; + + /// + /// Backing content session + /// + protected readonly IBackingContentSession BackingContentSession; + + /// + /// Optional write-through session to allow writing-behind to BlobStore + /// + protected readonly IContentSession WriteThroughContentSession; + + /// + /// The namespace of the build cache service being communicated with. + /// + protected readonly string CacheNamespace; + + /// + /// The id of the build cache service being communicated with. + /// + protected readonly Guid CacheId; + + private readonly bool _enableEagerFingerprintIncorporation; + private readonly TimeSpan _inlineFingerprintIncorporationExpiry; + private readonly TimeSpan _eagerFingerprintIncorporationInterval; + private readonly int _eagerFingerprintIncorporationBatchSize; + + /// + internal TimeSpan? RequiredContentKeepUntil { get; init; } + + /// + /// Background tracker for handling the upload/sealing of unbacked metadata. + /// + // ReSharper disable once InconsistentNaming + protected BackgroundTaskTracker _taskTracker; + + /// + /// Keep track of all the fingerprints that need to be incorporated later + /// + internal readonly FingerprintTracker FingerprintTracker; + + private readonly int _maxFingerprintSelectorsToFetch; + private readonly DisposableDirectory _tempDirectory; + private readonly bool _sealUnbackedContentHashLists; + private readonly List _sealingErrorsToPrintOnShutdown; + private readonly bool _fingerprintIncorporationEnabled; + private readonly int _maxDegreeOfParallelismForIncorporateRequests; + private readonly int _maxFingerprintsPerIncorporateRequest; + private int _sealingErrorCount; + private readonly bool _overrideUnixFileAccessMode; + + private Context _eagerFingerprintIncorporationTracingContext; // must be set at StartupAsync + private readonly NagleQueue _eagerFingerprintIncorporationNagleQueue; + + /// + protected readonly bool ManuallyExtendContentLifetime; + + /// + protected readonly bool ForceUpdateOnAddContentHashList; + + /// + public void Dispose() + { + _taskTracker?.Dispose(); + BackingContentSession?.Dispose(); + WriteThroughContentSession?.Dispose(); + _tempDirectory.Dispose(); + } + + /// + public string Name { get; } + + /// + protected override async Task StartupCoreAsync(OperationContext context) + { + _eagerFingerprintIncorporationTracingContext = context; + + LogIncorporateOptions(context); + + var backingContentSessionTask = Task.Run(async () => await BackingContentSession.StartupAsync(context).ConfigureAwait(false)); + var writeThroughContentSessionResult = WriteThroughContentSession != null + ? await WriteThroughContentSession.StartupAsync(context).ConfigureAwait(false) + : BoolResult.Success; + var backingContentSessionResult = await backingContentSessionTask.ConfigureAwait(false); + if (backingContentSessionResult.Succeeded && writeThroughContentSessionResult.Succeeded) + { + _taskTracker = new BackgroundTaskTracker(Component, context.CreateNested(Component)); + return BoolResult.Success; + } + + var sb = new StringBuilder(); + + if (backingContentSessionResult.Succeeded) + { + var r = await BackingContentSession.ShutdownAsync(context).ConfigureAwait(false); + if (!r.Succeeded) + { + sb.Append($"Backing content session shutdown failed, error=[{r}]"); + } + } + else + { + sb.Append($"Backing content session startup failed, error=[{backingContentSessionResult}]"); + } + + if (writeThroughContentSessionResult.Succeeded) + { + var r = WriteThroughContentSession != null + ? await WriteThroughContentSession.ShutdownAsync(context).ConfigureAwait(false) + : BoolResult.Success; + if (!r.Succeeded) + { + sb.Append(sb.Length > 0 ? ", " : string.Empty); + sb.Append($"Write-through content session shutdown failed, error=[{r}]"); + } + } + else + { + sb.Append(sb.Length > 0 ? ", " : string.Empty); + sb.Append($"Write-through content session startup failed, error=[{writeThroughContentSessionResult}]"); + } + + return new BoolResult(sb.ToString()); + } + + private void LogIncorporateOptions(Context context) + { + CacheTracer.Debug( + context, + $"BuildCacheSession incorporation options: FingerprintIncorporationEnabled={_fingerprintIncorporationEnabled}, EnableEagerFingerprintIncorporation={_enableEagerFingerprintIncorporation} " + + $"InlineFingerprintIncorporationExpiry={_inlineFingerprintIncorporationExpiry}, " + + $"EagerFingerprintIncorporationInterval={_eagerFingerprintIncorporationInterval}, EagerFingerprintIncorporationBatchSize={_eagerFingerprintIncorporationBatchSize}."); + } + + /// + protected override async Task ShutdownCoreAsync(OperationContext context) + { + _eagerFingerprintIncorporationNagleQueue?.Dispose(); + + CacheTracer.Debug(context, "IncorporateOnShutdown start"); + CacheTracer.Debug(context, $"Incorporate fingerprints feature enabled:[{_fingerprintIncorporationEnabled}]"); + CacheTracer.Debug(context, $"Total fingerprints to be incorporated:[{FingerprintTracker.Count}]"); + CacheTracer.Debug(context, $"Max fingerprints per incorporate request(=chunk size):[{_maxFingerprintsPerIncorporateRequest}]"); + CacheTracer.Debug(context, $"Max incorporate requests allowed in parallel:[{_maxDegreeOfParallelismForIncorporateRequests}]"); + if (_fingerprintIncorporationEnabled) + { + // Incorporating all of the fingerprints for a build, in one request, to a single endpoint causes pain. Incorporation involves + // extending the lifetime of all fingerprints *and* content/s mapped to each fingerprint. Processing a large request payload + // results in, potentially, fanning out a massive number of "lifetime extend" requests to itemstore and blobstore, which can + // bring down the endpoint. Break this down into chunks so that multiple, load-balanced endpoints can share the burden. + List fingerprintsToBump = Enumerable.ToList(FingerprintTracker.StaleFingerprints); + CacheTracer.Debug(context, $"Total fingerprints to be sent in incorporation requests to the service: {fingerprintsToBump.Count}"); + + List> chunks = fingerprintsToBump.Select( + strongFingerprint => new StrongFingerprintAndExpiration(strongFingerprint, FingerprintTracker.GenerateNewExpiration()) + ).GetPages(_maxFingerprintsPerIncorporateRequest).ToList(); + CacheTracer.Debug(context, $"Total fingerprint incorporation requests to be issued(=number of fingerprint chunks):[{chunks.Count}]"); + + var incorporateBlock = ActionBlockSlim.Create>( + degreeOfParallelism: _maxDegreeOfParallelismForIncorporateRequests, + processItemAction: async chunk => + { + var pinResult = await PinContentManuallyAsync( + new OperationContext(context, CancellationToken.None), + chunk); + if (!pinResult) + { + return; + } + + await ContentHashListAdapter.IncorporateStrongFingerprints( + context, + CacheNamespace, + new IncorporateStrongFingerprintsRequest(chunk.AsReadOnly()) + ).ConfigureAwait(false); + }); + + foreach (var chunk in chunks) + { + await incorporateBlock.PostAsync(chunk); + } + + incorporateBlock.Complete(); + await incorporateBlock.Completion + .ConfigureAwait(false); // TODO: Gracefully handle exceptions so that the rest of shutdown can happen (bug 1365340) + CacheTracer.Debug(context, "IncorporateOnShutdown stop"); + } + + if (_taskTracker != null) + { + await _taskTracker.Synchronize().ConfigureAwait(false); + await _taskTracker.ShutdownAsync(context).ConfigureAwait(false); + } + + var backingContentSessionTask = + Task.Run(async () => await BackingContentSession.ShutdownAsync(context).ConfigureAwait(false)); + var writeThroughContentSessionResult = WriteThroughContentSession != null + ? await WriteThroughContentSession.ShutdownAsync(context).ConfigureAwait(false) + : BoolResult.Success; + var backingContentSessionResult = await backingContentSessionTask.ConfigureAwait(false); + + BoolResult result; + if (backingContentSessionResult.Succeeded && writeThroughContentSessionResult.Succeeded) + { + if (Enumerable.Any(_sealingErrorsToPrintOnShutdown)) + { + var sb = new StringBuilder(); + sb.AppendLine("Error(s) during background sealing:"); + foreach (var sealingError in _sealingErrorsToPrintOnShutdown) + { + sb.AppendLine($"[{sealingError}]"); + } + + if (_sealingErrorCount > MaxSealingErrorsToPrintOnShutdown) + { + sb.AppendLine($"See log for the other {MaxSealingErrorsToPrintOnShutdown - _sealingErrorCount} error(s)."); + } + + result = new BoolResult(sb.ToString()); + } + else + { + result = BoolResult.Success; + } + } + else + { + var sb = new StringBuilder(); + if (!backingContentSessionResult.Succeeded) + { + sb.Append($"Backing content session shutdown failed, error=[{backingContentSessionResult}]"); + } + + if (!writeThroughContentSessionResult.Succeeded) + { + sb.Append(sb.Length > 0 ? ", " : string.Empty); + sb.Append($"Write-through content session shutdown failed, error=[{writeThroughContentSessionResult}]"); + } + + result = new BoolResult(sb.ToString()); + } + + return result; + } + + private async Task IncorporateBatchAsync(List fingerprints) + { + // Tracking shutdown to avoid using nagle queue after the shutdown. + using var shutdownContext = TrackShutdown(_eagerFingerprintIncorporationTracingContext); + + var context = shutdownContext.Context; + BoolResult result = await context.CreateOperation( + CacheTracer, + async () => + { + CacheTracer.Debug( + context, + $"IncorporateBatch: Total fingerprints to be incorporated {fingerprints.Count}, ChunkSize={_maxFingerprintsPerIncorporateRequest}, DegreeOfParallelism={_maxDegreeOfParallelismForIncorporateRequests}."); + + // Incorporating all of the fingerprints for a build, in one request, to a single endpoint causes pain. Incorporation involves + // extending the lifetime of all fingerprints *and* content/s mapped to each fingerprint. Processing a large request payload + // results in, potentially, fanning out a massive number of "lifetime extend" requests to itemstore and blobstore, which can + // bring down the endpoint. Break this down into chunks so that multiple, load-balanced endpoints can share the burden. + + var fingerprintsWithExpiration = + fingerprints + .Select( + strongFingerprint => new StrongFingerprintAndExpiration( + strongFingerprint, + FingerprintTracker.GenerateNewExpiration())) + .ToList().AsReadOnly(); + + var pinResult = await PinContentManuallyAsync(context, fingerprintsWithExpiration); + if (!pinResult) + { + return pinResult; + } + + await ContentHashListAdapter.IncorporateStrongFingerprints( + context, + CacheNamespace, + new IncorporateStrongFingerprintsRequest(fingerprintsWithExpiration) + ).ConfigureAwait(false); + + return BoolResult.Success; + }).RunAsync(); + + // Ignoring the failure, because it was already traced if needed. + result.IgnoreFailure(); + } + + private async Task IncorporateFingerprintAsync(OperationContext context, StrongFingerprint fingerprint) + { + BoolResult result = await context.CreateOperation( + CacheTracer, + async () => + { + var fingerprintWithExpiration = new StrongFingerprintAndExpiration(fingerprint, FingerprintTracker.GenerateNewExpiration()); + + var pinResult = await PinContentManuallyAsync(context, new StrongFingerprintAndExpiration[] { fingerprintWithExpiration }); + if (!pinResult) + { + return pinResult; + } + + await ContentHashListAdapter.IncorporateStrongFingerprints( + context, + CacheNamespace, + new IncorporateStrongFingerprintsRequest(new[] { fingerprintWithExpiration }) + ).ConfigureAwait(false); + + return BoolResult.Success; + }).RunAsync(); + + // Ignoring the failure, because it was already traced if needed. + result.IgnoreFailure(); + } + + private async Task PinContentManuallyAsync(OperationContext context, IEnumerable fingerprints) + { + // Pin the content manually as a workaround for the fact that BuildCache doesn't know how to talk to the backing content store due to it being + // dedup or in a non-default domain and we want the content to be there even if we have to tell BuildCache that the ContentHashList is unbacked. + if (ManuallyExtendContentLifetime) + { + // TODO: optimize and run in parallel if needed. + foreach (var fingerprint in fingerprints) + { + // TODO: Get the content hash list in a more efficient manner which does not require us to talk to BuildCache. + var hashListResult = await ContentHashListAdapter.GetContentHashListAsync(context, CacheNamespace, fingerprint.StrongFingerprint); + if (!hashListResult.Succeeded || hashListResult.Value?.ContentHashListWithDeterminism.ContentHashList == null) + { + return new BoolResult(hashListResult, "Failed to get content hash list when attempting to extend its conetnts' lifetimes."); + } + + var expirationDate = new DateTime( + Math.Max(hashListResult.Value.GetRawExpirationTimeUtc()?.Ticks ?? 0, fingerprint.ExpirationDateUtc.Ticks), + DateTimeKind.Utc); + + var pinResults = await Task.WhenAll( + await BackingContentSession.PinAsync( + context, + hashListResult.Value.ContentHashListWithDeterminism.ContentHashList.Hashes, + expirationDate)); + + if (Enumerable.Any(pinResults, r => !r.Succeeded)) + { + return new BoolResult($"Failed to pin all pieces of content for fingerprint=[{fingerprint.StrongFingerprint}]"); + } + } + } + + return BoolResult.Success; + } + + /// + public System.Collections.Generic.IAsyncEnumerable GetSelectors( + Context context, + Fingerprint weakFingerprint, + CancellationToken cts, + UrgencyHint urgencyHint = UrgencyHint.Nominal) + { + return this.GetSelectorsAsAsyncEnumerable(context, weakFingerprint, cts, urgencyHint); + } + + /// + public async Task> GetLevelSelectorsAsync( + Context context, + Fingerprint weakFingerprint, + CancellationToken cts, + int level) + { + var result = await GetSelectorsAsync(new OperationContext(context, cts), weakFingerprint); + return LevelSelectors.Single(result); + } + + private async Task> GetSelectorsAsync(OperationContext context, Fingerprint weakFingerprint) + { + CacheTracer.MemoizationStoreTracer.GetSelectorsStart(context, weakFingerprint); + Stopwatch sw = Stopwatch.StartNew(); + try + { + var responseResult = await ContentHashListAdapter.GetSelectorsAsync( + context, + CacheNamespace, + weakFingerprint, + _maxFingerprintSelectorsToFetch).ConfigureAwait(false); + + if (!responseResult) + { + return Result.FromError(responseResult); + } + + if (responseResult.Value == null) + { + return Result.Success(CollectionUtilities.EmptyArray()); + } + + foreach (var selectorAndPossible in responseResult.Value) + { + var selector = selectorAndPossible.Selector; + if (selectorAndPossible.ContentHashList != null) + { + // Store pre-fetched data in-memory + var strongFingerprint = new StrongFingerprint(weakFingerprint, selector); + var unpackResult = UnpackContentHashListWithDeterminismAfterGet( + selectorAndPossible.ContentHashList, + CacheId); + if (unpackResult && unpackResult.ContentHashListWithDeterminism.Determinism.IsDeterministic) + { + CacheTracer.RecordPrefetchedContentHashList(); + ContentHashListWithDeterminismCache.Instance.AddValue( + CacheNamespace, + strongFingerprint, + unpackResult.ContentHashListWithDeterminism); + + BackingContentSession.ExpiryCache.AddExpiry( + selector.ContentHash, + unpackResult.ContentHashListWithDeterminism.Determinism.ExpirationUtc); + } + } + } + + CacheTracer.MemoizationStoreTracer.GetSelectorsCount( + context, + weakFingerprint, + Enumerable.Count(responseResult.Value)); + return Enumerable.Select( + responseResult.Value, + responseData => responseData.Selector).ToArray(); + } + catch (Exception e) + { + return Result.FromException(e); + } + finally + { + CacheTracer.MemoizationStoreTracer.GetSelectorsStop(context, sw.Elapsed, weakFingerprint); + } + } + + /// + /// Stores the DownloadUris in-memory to reduce the calls to BlobStore. + /// + protected void StorePrefetchedDownloadUris(IDictionary blobDownloadUris) + { + if (blobDownloadUris == null) + { + return; + } + + foreach (var blobDownloadUri in blobDownloadUris) + { + BackingContentSession.UriCache.AddDownloadUri( + BlobIdentifier.Deserialize(blobDownloadUri.Key).ToContentHash(), + new PreauthenticatedUri(blobDownloadUri.Value, EdgeType.Unknown)); // EdgeType value shouldn't matter because we don't use it. + } + } + + /// + public Task GetContentHashListAsync( + Context context, + StrongFingerprint strongFingerprint, + CancellationToken cts, + UrgencyHint urgencyHint) + { + var operationContext = new OperationContext(context, cts); + return operationContext.PerformOperationAsync( + CacheTracer, + async () => + { + // Check for pre-fetched data + ContentHashListWithDeterminism contentHashListWithDeterminism; + + if (ContentHashListWithDeterminismCache.Instance.TryGetValue( + CacheNamespace, + strongFingerprint, + out contentHashListWithDeterminism)) + { + CacheTracer.RecordUseOfPrefetchedContentHashList(); + await TrackFingerprintAsync( + operationContext, + strongFingerprint, + contentHashListWithDeterminism.Determinism.ExpirationUtc, + contentHashListWithDeterminism.ContentHashList).ConfigureAwait(false); + return new GetContentHashListResult(contentHashListWithDeterminism); + } + + // No pre-fetched data. Need to query the server. + Result responseObject = + await ContentHashListAdapter.GetContentHashListAsync(operationContext, CacheNamespace, strongFingerprint) + .ConfigureAwait(false); + + if (!responseObject.Succeeded) + { + return new GetContentHashListResult(responseObject); + } + + ContentHashListWithCacheMetadata response = responseObject.Value; + if (response.ContentHashListWithDeterminism.ContentHashList == null) + { + // Miss + return new GetContentHashListResult(new ContentHashListWithDeterminism(null, CacheDeterminism.None)); + } + + GetContentHashListResult unpackResult = UnpackContentHashListWithDeterminismAfterGet(response, CacheId); + if (!unpackResult.Succeeded) + { + return unpackResult; + } + + SealIfNecessaryAfterGet(operationContext, strongFingerprint, response); + + await TrackFingerprintAsync( + operationContext, + strongFingerprint, + response.GetRawExpirationTimeUtc(), + unpackResult.ContentHashListWithDeterminism.ContentHashList); + return new GetContentHashListResult(unpackResult.ContentHashListWithDeterminism); + }, + traceOperationStarted: true, + extraStartMessage: $"StrongFingerprint=({strongFingerprint})", + extraEndMessage: result => $"StrongFingerprint=({strongFingerprint})"); + } + + /// + protected async Task TrackFingerprintAsync( + OperationContext context, + StrongFingerprint strongFingerprint, + DateTime? expirationUtc, + ContentHashList hashes) + { + context.Token.ThrowIfCancellationRequested(); + if (expirationUtc != null) + { + BackingContentSession.ExpiryCache.AddExpiry(strongFingerprint.Selector.ContentHash, expirationUtc.Value); + + if (hashes != null) + { + foreach (var hash in hashes.Hashes) + { + BackingContentSession.ExpiryCache.AddExpiry(hash, expirationUtc.Value); + } + } + } + + // Currently we have 3 ways for fingerprint incorporation: + // 1. Inline incorporation: If eager fingerprint incorporation enabled and + // the entry will expire in _inlineFingerprintIncorporationExpiry time. + // 2. Eager bulk incorporation: if eager fingerprint incorporation enabled and + // the entry's expiry is not available or it won't expire in _inlineFingerprintIncorporationExpiry time. + // 3. Session shutdown incorporation: if eager fingerprint incorporation is disabled and the normal fingerprint incorporation is enabled. + if (_enableEagerFingerprintIncorporation) + { + if (expirationUtc != null && (expirationUtc.Value - DateTime.UtcNow < _inlineFingerprintIncorporationExpiry)) + { + CacheTracer.Debug( + context, + $"Incorporating fingerprint inline: StrongFingerprint=[{strongFingerprint}], ExpirationUtc=[{expirationUtc}]."); + + await IncorporateFingerprintAsync(context, strongFingerprint); + } + else + { + // We either don't have an expiration time or the time to expiry is greater then _inlineFingerprintIncorporationExpiry + Contract.Assert(_eagerFingerprintIncorporationNagleQueue != null); + _eagerFingerprintIncorporationNagleQueue.Enqueue(strongFingerprint); + } + } + else + { + FingerprintTracker.Track(strongFingerprint, expirationUtc); + } + } + + /// + public Task PinAsync( + Context context, + ContentHash contentHash, + CancellationToken cts, + UrgencyHint urgencyHint) + { + return PinCall.RunAsync( + CacheTracer.ContentSessionTracer, + new OperationContext(context), + contentHash, + async () => + { + var bulkResults = await PinAsync(context, new[] { contentHash }, cts, urgencyHint); + return await bulkResults.SingleAwaitIndexed(); + }); + } + + /// + public Task OpenStreamAsync( + Context context, + ContentHash contentHash, + CancellationToken cts, + UrgencyHint urgencyHint) + { + return OpenStreamCall.RunAsync( + CacheTracer.ContentSessionTracer, + new OperationContext(context), + contentHash, + async () => + { + if (WriteThroughContentSession != null) + { + var result = + await WriteThroughContentSession.OpenStreamAsync(context, contentHash, cts, urgencyHint).ConfigureAwait(false); + if (result.Succeeded || result.Code != OpenStreamResult.ResultCode.ContentNotFound) + { + return result; + } + } + + return await BackingContentSession.OpenStreamAsync(context, contentHash, cts, urgencyHint).ConfigureAwait(false); + }); + } + + /// + public Task PlaceFileAsync( + Context context, + ContentHash contentHash, + AbsolutePath path, + FileAccessMode accessMode, + FileReplacementMode replacementMode, + FileRealizationMode realizationMode, + CancellationToken cts, + UrgencyHint urgencyHint) + { + return PlaceFileCall.RunAsync( + CacheTracer.ContentSessionTracer, + new OperationContext(context), + contentHash, + path, + accessMode, + replacementMode, + realizationMode, + async () => + { + if (WriteThroughContentSession != null) + { + var writeThroughResult = await WriteThroughContentSession.PlaceFileAsync( + context, + contentHash, + path, + accessMode, + replacementMode, + realizationMode, + cts, + urgencyHint).ConfigureAwait(false); + if (writeThroughResult.Succeeded || writeThroughResult.Code != PlaceFileResult.ResultCode.NotPlacedContentNotFound) + { + UnixHelpers.OverrideFileAccessMode(_overrideUnixFileAccessMode, path.Path); + return writeThroughResult; + } + } + + var backingResult = await BackingContentSession.PlaceFileAsync( + context, + contentHash, + path, + accessMode, + replacementMode, + realizationMode, + cts, + urgencyHint); + UnixHelpers.OverrideFileAccessMode(_overrideUnixFileAccessMode, path.Path); + return backingResult; + }); + } + + /// + public Task>>> PinAsync( + Context context, + IReadOnlyList contentHashes, + CancellationToken cts, + UrgencyHint urgencyHint = UrgencyHint.Nominal) + { + return PinHelperAsync(contentHashes, (session, hashes) => session.PinAsync(context, hashes, cts, urgencyHint)); + } + + /// + public Task>>> PinAsync( + Context context, + IReadOnlyList contentHashes, + PinOperationConfiguration config) + { + return PinHelperAsync(contentHashes, (session, hashes) => session.PinAsync(context, hashes, config)); + } + + private Task>>> PinHelperAsync( + IReadOnlyList contentHashes, + Func, Task>>>> pinAsync) + { + var requiredExpiry = DateTime.UtcNow + RequiredContentKeepUntil; + return Workflows.RunWithFallback( + contentHashes, + hashes => { return Task.FromResult(hashes.Select(hash => CheckExpiryCache(hash, requiredExpiry)).ToList().AsIndexedTasks()); }, + hashes => + { + if (WriteThroughContentSession == null) + { + return pinAsync(BackingContentSession, hashes); + } + + return Workflows.RunWithFallback( + hashes, + hashes => pinAsync(WriteThroughContentSession, hashes), + hashes => pinAsync(BackingContentSession, hashes), + result => result.Succeeded); + }, + result => result.Succeeded); + } + + private PinResult CheckExpiryCache(ContentHash hash, DateTime? requiredExpiry) + { + if (requiredExpiry != null && BackingContentSession.ExpiryCache.TryGetExpiry(hash, out var expiry) && (expiry >= requiredExpiry)) + { + return PinResult.Success; + } + else + { + return PinResult.ContentNotFound; + } + } + + /// + public Task>>> PlaceFileAsync( + Context context, + IReadOnlyList hashesWithPaths, + FileAccessMode accessMode, + FileReplacementMode replacementMode, + FileRealizationMode realizationMode, + CancellationToken cts, + UrgencyHint urgencyHint = UrgencyHint.Nominal) + { + throw new NotImplementedException(); + } + + /// + protected async Task AddOrGetContentHashListAsync( + OperationContext context, + StrongFingerprint strongFingerprint, + ContentHashListWithDeterminism contentHashListWithDeterminism, + ContentAvailabilityGuarantee guarantee) + { + using var shutdownContext = TrackShutdown(context); + + context = shutdownContext.Context; + + try + { + DateTime expirationUtc = FingerprintTracker.GenerateNewExpiration(); + var valueToAdd = new ContentHashListWithCacheMetadata( + contentHashListWithDeterminism, + expirationUtc, + guarantee); + + CacheTracer.Debug( + context, + $"Adding contentHashList=[{valueToAdd.ContentHashListWithDeterminism.ContentHashList}] determinism=[{valueToAdd.ContentHashListWithDeterminism.Determinism}] to VSTS with contentAvailabilityGuarantee=[{valueToAdd.ContentGuarantee}], expirationUtc=[{expirationUtc}], forceUpdate=[{ForceUpdateOnAddContentHashList}]"); + + var contentHashListResponseObject = + await ContentHashListAdapter.AddContentHashListAsync( + context, + CacheNamespace, + strongFingerprint, + valueToAdd, + forceUpdate: ForceUpdateOnAddContentHashList).ConfigureAwait(false); + + if (!contentHashListResponseObject.Succeeded) + { + return new AddOrGetContentHashListResult(contentHashListResponseObject); + } + + var contentHashListResponse = contentHashListResponseObject.Value; + var inconsistencyErrorMessage = CheckForResponseInconsistency(contentHashListResponse); + if (inconsistencyErrorMessage != null) + { + return new AddOrGetContentHashListResult(inconsistencyErrorMessage); + } + + ContentHashList contentHashListToReturn = UnpackContentHashListAfterAdd( + contentHashListWithDeterminism.ContentHashList, + contentHashListResponse); + + CacheDeterminism determinismToReturn = UnpackDeterminism(contentHashListResponse, CacheId); + if (guarantee == ContentAvailabilityGuarantee.AllContentBackedByCache && !determinismToReturn.IsDeterministic) + { + return new AddOrGetContentHashListResult( + "Inconsistent BuildCache service response. Unbacked values should never override backed values."); + } + + await TrackFingerprintAsync(context, strongFingerprint, contentHashListResponse.GetRawExpirationTimeUtc(), contentHashListToReturn) + .ConfigureAwait(false); + return new AddOrGetContentHashListResult(new ContentHashListWithDeterminism(contentHashListToReturn, determinismToReturn)); + } + catch (Exception e) + { + return new AddOrGetContentHashListResult(e); + } + } + + private static GetContentHashListResult UnpackContentHashListWithDeterminismAfterGet( + ContentHashListWithCacheMetadata cacheMetadata, + Guid cacheId) + { + var inconsistencyErrorMessage = CheckForResponseInconsistency(cacheMetadata); + if (inconsistencyErrorMessage != null) + { + return new GetContentHashListResult(inconsistencyErrorMessage); + } + + if (cacheMetadata?.ContentHashListWithDeterminism.ContentHashList == null) + { + // Miss + return new GetContentHashListResult(new ContentHashListWithDeterminism(null, CacheDeterminism.None)); + } + + ContentHashList contentHashList = cacheMetadata.ContentHashListWithDeterminism.ContentHashList; + CacheDeterminism determinism = UnpackDeterminism(cacheMetadata, cacheId); + + return new GetContentHashListResult(new ContentHashListWithDeterminism(contentHashList, determinism)); + } + + /// + /// Checks for inconsistencies in the metadata returned by the service, returning an appropriate error message (null if none). + /// + protected static string CheckForResponseInconsistency(ContentHashListWithCacheMetadata cacheMetadata) + { + if (cacheMetadata != null) + { + if (cacheMetadata.GetEffectiveExpirationTimeUtc() == null && + cacheMetadata.ContentGuarantee != ContentAvailabilityGuarantee.NoContentBackedByCache) + { + return + "Inconsistent BuildCache service response. Null ContentHashListExpirationUtc should be iff ContentAvailabilityGuarantee.NoContentBackedByCache."; + } + } + + return null; + } + + /// + /// Determine the ContentHashList to return based on the added value and the service response. + /// + protected static ContentHashList UnpackContentHashListAfterAdd( + ContentHashList addedContentHashList, + ContentHashListWithCacheMetadata cacheMetadata) + { + Contract.Assert(cacheMetadata != null); + + if (cacheMetadata.ContentHashListWithDeterminism.ContentHashList != null && + !addedContentHashList.Equals( + cacheMetadata.ContentHashListWithDeterminism.ContentHashList)) + { + // The service returned a ContentHashList different from the one we tried to add, so we'll return that. + return cacheMetadata.ContentHashListWithDeterminism.ContentHashList; + } + + // The added value was accepted, so we return null. + return null; + } + + /// + /// Determine the Determinism to return. + /// + protected static CacheDeterminism UnpackDeterminism(ContentHashListWithCacheMetadata cacheMetadata, Guid cacheId) + { + Contract.Assert(cacheMetadata != null); + + if (cacheMetadata.ContentHashListWithDeterminism.Determinism.Guid == CacheDeterminism.Tool.Guid) + { + // Value is Tool-deterministic + return CacheDeterminism.Tool; + } + + var expirationUtc = cacheMetadata.GetEffectiveExpirationTimeUtc(); + return expirationUtc == null + ? CacheDeterminism.None // Value is unbacked in VSTS + : CacheDeterminism.ViaCache(cacheId, expirationUtc.Value); // Value is backed in VSTS + } + + private void SealIfNecessaryAfterGet( + OperationContext context, + StrongFingerprint strongFingerprint, + ContentHashListWithCacheMetadata cacheMetadata) + { + if (WriteThroughContentSession == null) + { + return; + } + + if (cacheMetadata != null && cacheMetadata.GetEffectiveExpirationTimeUtc() == null) + { + // Value is unbacked in VSTS + SealInTheBackground(context, strongFingerprint, cacheMetadata.ContentHashListWithDeterminism); + } + } + + /// + /// Queue a seal operation in the background. Attempts to upload all content to VSTS before updating the metadata as backed. + /// Lost races will be ignored and any failures will be logged and reported on shutdown. + /// + protected void SealInTheBackground( + OperationContext context, + StrongFingerprint strongFingerprint, + ContentHashListWithDeterminism contentHashListWithDeterminism) + { + if (_sealUnbackedContentHashLists) + { + _taskTracker.Add(Task.Run(() => SealAsync(context, strongFingerprint, contentHashListWithDeterminism))); + } + } + + private async Task SealAsync( + OperationContext context, + StrongFingerprint strongFingerprint, + ContentHashListWithDeterminism contentHashListWithDeterminism) + { + Contract.Assert(WriteThroughContentSession != null); + Contract.Assert(contentHashListWithDeterminism.ContentHashList != null); + try + { + var uploadResult = await UploadAllContentAsync( + context, + strongFingerprint.Selector.ContentHash, + contentHashListWithDeterminism.ContentHashList.Hashes, + CancellationToken.None, + UrgencyHint.Low).ConfigureAwait(false); + if (uploadResult.Code == PinResult.ResultCode.ContentNotFound) + { + CacheTracer.Debug(context, "Background seal unable to find all content during upload."); + return; + } + + if (!uploadResult.Succeeded) + { + ReportSealingError(context, $"Background seal failed during upload: error=[{uploadResult}]."); + return; + } + + var sealResult = await AddOrGetContentHashListAsync( + context, + strongFingerprint, + contentHashListWithDeterminism, + ContentAvailabilityGuarantee.AllContentBackedByCache).ConfigureAwait(false); + if (sealResult.Succeeded) + { + CacheTracer.Debug( + context, + sealResult.ContentHashListWithDeterminism.ContentHashList == null + ? $"Successfully sealed value for strongFingerprint [{strongFingerprint}]." + : $"Lost the race in sealing value for strongFingerprint [{strongFingerprint}]."); + } + else + { + ReportSealingError(context, $"Background seal failed during sealing: {sealResult}"); + } + } + catch (Exception e) + { + ReportSealingError(context, $"Background seal threw exception: {e}."); + } + } + + private void ReportSealingError(Context context, string errorMessage, [CallerMemberName] string operation = null) + { + Interlocked.Increment(ref _sealingErrorCount); + CacheTracer.Error(context, errorMessage, operation); + if (_sealingErrorCount < MaxSealingErrorsToPrintOnShutdown) + { + _sealingErrorsToPrintOnShutdown.Add(errorMessage); + } + } + + /// + /// Attempt to ensure that all content is in VSTS, uploading any misses from the WriteThroughContentSession. + /// + private async Task UploadAllContentAsync( + Context context, + ContentHash selectorHash, + IEnumerable hashes, + CancellationToken cts, + UrgencyHint urgencyHint) + { + List contentToUpload = new List(); + + var pinResult = await BackingContentSession.PinAsync(context, selectorHash, cts, urgencyHint).ConfigureAwait(false); + if (pinResult.Code == PinResult.ResultCode.ContentNotFound) + { + contentToUpload.Add(selectorHash); + } + else if (!pinResult.Succeeded) + { + return pinResult; + } + + foreach (var contentHash in hashes) + { + pinResult = await BackingContentSession.PinAsync(context, contentHash, cts, urgencyHint).ConfigureAwait(false); + if (pinResult.Code == PinResult.ResultCode.ContentNotFound) + { + contentToUpload.Add(contentHash); + } + else if (!pinResult.Succeeded) + { + return pinResult; + } + } + + foreach (var contentHash in contentToUpload) + { + // TODO: Upload the content efficiently (in parallel and with caching of success) (bug 1365340) + AbsolutePath tempFile = null; + try + { + tempFile = _tempDirectory.CreateRandomFileName(); + var placeResult = await WriteThroughContentSession.PlaceFileAsync( + context, + contentHash, + tempFile, + FileAccessMode.Write, + FileReplacementMode.FailIfExists, + FileRealizationMode.Any, + CancellationToken.None).ConfigureAwait(false); + if (placeResult.Code == PlaceFileResult.ResultCode.NotPlacedContentNotFound) + { + return PinResult.ContentNotFound; + } + else if (!placeResult.Succeeded) + { + return new PinResult(placeResult); + } + + var putResult = await BackingContentSession.PutFileAsync( + context, + contentHash, + tempFile, + FileRealizationMode.Any, + CancellationToken.None).ConfigureAwait(false); + if (!putResult.Succeeded) + { + return new PinResult(putResult); + } + } + finally + { + if (tempFile != null) + { + try + { + File.Delete(tempFile.Path); + } + catch (Exception e) + { + CacheTracer.Warning(context, $"Error deleting temporary file at {tempFile.Path}: {e}"); + } + } + } + } + + return PinResult.Success; + } + + private struct IncorporationOptions + { + } + /// /// Initializes a new instance of the class. /// @@ -79,31 +1192,48 @@ namespace BuildXL.Cache.MemoizationStore.Vsts int eagerFingerprintIncorporationBatchSize, bool manuallyExtendContentLifetime, bool forceUpdateOnAddContentHashList) - : base( - fileSystem, - name, - implicitPin, - cacheNamespace, - cacheId, - contentHashListAdapter, - backingContentSession, - maxFingerprintSelectorsToFetch, - minimumTimeToKeepContentHashLists, - rangeOfTimeToKeepContentHashLists, - fingerprintIncorporationEnabled, - maxDegreeOfParallelismForIncorporateRequests, - maxFingerprintsPerIncorporateRequest, - writeThroughContentSession, - sealUnbackedContentHashLists, - overrideUnixFileAccessMode, - tracer, - enableEagerFingerprintIncorporation, - inlineFingerprintIncorporationExpiry, - eagerFingerprintIncorporationInterval, - eagerFingerprintIncorporationBatchSize, - manuallyExtendContentLifetime, - forceUpdateOnAddContentHashList) { + Contract.Requires(name != null); + Contract.Requires(contentHashListAdapter != null); + Contract.Requires(backingContentSession != null); + Contract.Requires(!backingContentSession.StartupStarted); + + Name = name; + ImplicitPin = implicitPin; + ContentHashListAdapter = contentHashListAdapter; + BackingContentSession = backingContentSession; + _maxFingerprintSelectorsToFetch = maxFingerprintSelectorsToFetch; + CacheNamespace = cacheNamespace; + CacheId = cacheId; + WriteThroughContentSession = writeThroughContentSession; + _sealUnbackedContentHashLists = sealUnbackedContentHashLists; + CacheTracer = tracer; + _enableEagerFingerprintIncorporation = enableEagerFingerprintIncorporation; + _inlineFingerprintIncorporationExpiry = inlineFingerprintIncorporationExpiry; + _eagerFingerprintIncorporationInterval = eagerFingerprintIncorporationInterval; + _eagerFingerprintIncorporationBatchSize = eagerFingerprintIncorporationBatchSize; + + _tempDirectory = new DisposableDirectory(fileSystem); + _sealingErrorsToPrintOnShutdown = new List(); + _fingerprintIncorporationEnabled = fingerprintIncorporationEnabled; + _maxDegreeOfParallelismForIncorporateRequests = maxDegreeOfParallelismForIncorporateRequests; + _maxFingerprintsPerIncorporateRequest = maxFingerprintsPerIncorporateRequest; + _overrideUnixFileAccessMode = overrideUnixFileAccessMode; + + ManuallyExtendContentLifetime = manuallyExtendContentLifetime; + + FingerprintTracker = new FingerprintTracker(DateTime.UtcNow + minimumTimeToKeepContentHashLists, rangeOfTimeToKeepContentHashLists); + + ForceUpdateOnAddContentHashList = forceUpdateOnAddContentHashList; + + if (enableEagerFingerprintIncorporation) + { + _eagerFingerprintIncorporationNagleQueue = NagleQueue.Create( + IncorporateBatchAsync, + maxDegreeOfParallelismForIncorporateRequests, + eagerFingerprintIncorporationInterval, + eagerFingerprintIncorporationBatchSize); + } } /// @@ -136,7 +1266,9 @@ namespace BuildXL.Cache.MemoizationStore.Vsts // Ensure that the content exists somewhere before trying to add if (!await EnsureContentIsAvailableAsync( - operationContext, contentHashListWithDeterminism.ContentHashList.Hashes, urgencyHint).ConfigureAwait(false)) + operationContext, + contentHashListWithDeterminism.ContentHashList.Hashes, + urgencyHint).ConfigureAwait(false)) { return new AddOrGetContentHashListResult( "Referenced content must exist in the cache before a new content hash list is added."); @@ -153,8 +1285,8 @@ namespace BuildXL.Cache.MemoizationStore.Vsts for (int addAttempts = 0; addAttempts < addLimit; addAttempts++) { var debugString = $"Adding contentHashList=[{valueToAdd.ContentHashListWithDeterminism.ContentHashList}] " + - $"determinism=[{valueToAdd.ContentHashListWithDeterminism.Determinism}] to VSTS with " + - $"contentAvailabilityGuarantee=[{valueToAdd.ContentGuarantee}], expirationUtc=[{expirationUtc}], forceUpdate=[{ForceUpdateOnAddContentHashList}]"; + $"determinism=[{valueToAdd.ContentHashListWithDeterminism.Determinism}] to VSTS with " + + $"contentAvailabilityGuarantee=[{valueToAdd.ContentGuarantee}], expirationUtc=[{expirationUtc}], forceUpdate=[{ForceUpdateOnAddContentHashList}]"; CacheTracer.Debug(operationContext, debugString); Result responseObject = await ContentHashListAdapter.AddContentHashListAsync( @@ -191,13 +1323,16 @@ namespace BuildXL.Cache.MemoizationStore.Vsts { SealIfNecessaryAfterUnbackedAddOrGet(operationContext, strongFingerprint, contentHashListWithDeterminism, response); - await TrackFingerprintAsync(operationContext, strongFingerprint, rawExpiration, hashes: contentHashListToReturn).ConfigureAwait(false); + await TrackFingerprintAsync(operationContext, strongFingerprint, rawExpiration, hashes: contentHashListToReturn) + .ConfigureAwait(false); return new AddOrGetContentHashListResult( new ContentHashListWithDeterminism(contentHashListToReturn, determinismToReturn)); } var hashOfExistingContentHashList = response.HashOfExistingContentHashList; - CacheTracer.Debug(operationContext, $"Attempting to replace unbacked value with hash {hashOfExistingContentHashList.ToHex()}"); + CacheTracer.Debug( + operationContext, + $"Attempting to replace unbacked value with hash {hashOfExistingContentHashList.ToHex()}"); valueToAdd = new ContentHashListWithCacheMetadata( contentHashListWithDeterminism, expirationUtc, @@ -211,11 +1346,13 @@ namespace BuildXL.Cache.MemoizationStore.Vsts $"Lost the AddOrUpdate race {addLimit} times against unbacked values. Returning as though the add succeeded for now."); await TrackFingerprintAsync(operationContext, strongFingerprint, rawExpiration, hashes: null).ConfigureAwait(false); return new AddOrGetContentHashListResult(new ContentHashListWithDeterminism(null, CacheDeterminism.None)); - }, traceOperationStarted: true, - extraStartMessage: $"StrongFingerprint=({strongFingerprint}), ForceUpdate=({ForceUpdateOnAddContentHashList}) {contentHashListWithDeterminism.ToTraceString()}", - extraEndMessage: _ => $"StrongFingerprint=({strongFingerprint}), ForceUpdate=({ForceUpdateOnAddContentHashList}) {contentHashListWithDeterminism.ToTraceString()}"); + extraStartMessage: + $"StrongFingerprint=({strongFingerprint}), ForceUpdate=({ForceUpdateOnAddContentHashList}) {contentHashListWithDeterminism.ToTraceString()}", + extraEndMessage: + _ => + $"StrongFingerprint=({strongFingerprint}), ForceUpdate=({ForceUpdateOnAddContentHashList}) {contentHashListWithDeterminism.ToTraceString()}"); } private async Task CheckNeedToUpdateExistingValueAsync( @@ -254,7 +1391,8 @@ namespace BuildXL.Cache.MemoizationStore.Vsts { // Our Add won the race, so seal the added value var valueToSeal = new ContentHashListWithDeterminism( - addedValue.ContentHashList, cacheMetadata.ContentHashListWithDeterminism.Determinism); + addedValue.ContentHashList, + cacheMetadata.ContentHashListWithDeterminism.Determinism); SealInTheBackground( context, strongFingerprint, @@ -270,23 +1408,27 @@ namespace BuildXL.Cache.MemoizationStore.Vsts CancellationToken cts, UrgencyHint urgencyHint) { - return IncorporateStrongFingerprintsCall.RunAsync(CacheTracer.MemoizationStoreTracer, context, async () => - { - // BuildCache remembers fingerprints for all content that has been fetched or added through it's APIs. - // However, the build may end up satisfying some fingerprints without using BuildCache. The build or - // other cache instances in the cache topology can track these "other" fingerprints and ask that the - // fingerprints be included in the cache session using the incorporate API. BuildCache will extend the - // expiration of the fingerprints and mapped content, as if they had just been published. - foreach (var strongFingerprintTask in strongFingerprints) + return IncorporateStrongFingerprintsCall.RunAsync( + CacheTracer.MemoizationStoreTracer, + context, + async () => { - var strongFingerprint = await strongFingerprintTask.ConfigureAwait(false); + // BuildCache remembers fingerprints for all content that has been fetched or added through it's APIs. + // However, the build may end up satisfying some fingerprints without using BuildCache. The build or + // other cache instances in the cache topology can track these "other" fingerprints and ask that the + // fingerprints be included in the cache session using the incorporate API. BuildCache will extend the + // expiration of the fingerprints and mapped content, as if they had just been published. + foreach (var strongFingerprintTask in strongFingerprints) + { + var strongFingerprint = await strongFingerprintTask.ConfigureAwait(false); - // The Incorporate API currently does allow passing the expiration, so we can't pass it here. - await TrackFingerprintAsync(new OperationContext(context, cts), strongFingerprint, expirationUtc: null, hashes: null).ConfigureAwait(false); - } + // The Incorporate API currently does allow passing the expiration, so we can't pass it here. + await TrackFingerprintAsync(new OperationContext(context, cts), strongFingerprint, expirationUtc: null, hashes: null) + .ConfigureAwait(false); + } - return BoolResult.Success; - }); + return BoolResult.Success; + }); } /// @@ -298,10 +1440,16 @@ namespace BuildXL.Cache.MemoizationStore.Vsts CancellationToken cts, UrgencyHint urgencyHint) { - return CacheTracer.ContentSessionTracer.PutFileAsync(new OperationContext(context), path, realizationMode, hashType, trustedHash: false, () => + return CacheTracer.ContentSessionTracer.PutFileAsync( + new OperationContext(context), + path, + realizationMode, + hashType, + trustedHash: false, + () => WriteThroughContentSession != null - ? WriteThroughContentSession.PutFileAsync(context, hashType, path, realizationMode, cts, urgencyHint) - : BackingContentSession.PutFileAsync(context, hashType, path, realizationMode, cts, urgencyHint)); + ? WriteThroughContentSession.PutFileAsync(context, hashType, path, realizationMode, cts, urgencyHint) + : BackingContentSession.PutFileAsync(context, hashType, path, realizationMode, cts, urgencyHint)); } /// @@ -313,10 +1461,16 @@ namespace BuildXL.Cache.MemoizationStore.Vsts CancellationToken cts, UrgencyHint urgencyHint) { - return CacheTracer.ContentSessionTracer.PutFileAsync(new OperationContext(context, cts), path, realizationMode, contentHash, trustedHash: false, () => + return CacheTracer.ContentSessionTracer.PutFileAsync( + new OperationContext(context, cts), + path, + realizationMode, + contentHash, + trustedHash: false, + () => WriteThroughContentSession != null - ? WriteThroughContentSession.PutFileAsync(context, contentHash, path, realizationMode, cts, urgencyHint) - : BackingContentSession.PutFileAsync(context, contentHash, path, realizationMode, cts, urgencyHint)); + ? WriteThroughContentSession.PutFileAsync(context, contentHash, path, realizationMode, cts, urgencyHint) + : BackingContentSession.PutFileAsync(context, contentHash, path, realizationMode, cts, urgencyHint)); } /// @@ -327,10 +1481,13 @@ namespace BuildXL.Cache.MemoizationStore.Vsts CancellationToken cts, UrgencyHint urgencyHint) { - return CacheTracer.ContentSessionTracer.PutStreamAsync(new OperationContext(context, cts), hashType, () => - WriteThroughContentSession != null - ? WriteThroughContentSession.PutStreamAsync(context, hashType, stream, cts, urgencyHint) - : BackingContentSession.PutStreamAsync(context, hashType, stream, cts, urgencyHint)); + return CacheTracer.ContentSessionTracer.PutStreamAsync( + new OperationContext(context, cts), + hashType, + () => + WriteThroughContentSession != null + ? WriteThroughContentSession.PutStreamAsync(context, hashType, stream, cts, urgencyHint) + : BackingContentSession.PutStreamAsync(context, hashType, stream, cts, urgencyHint)); } /// @@ -341,14 +1498,19 @@ namespace BuildXL.Cache.MemoizationStore.Vsts CancellationToken cts, UrgencyHint urgencyHint) { - return CacheTracer.ContentSessionTracer.PutStreamAsync(new OperationContext(context, cts), contentHash, () => - WriteThroughContentSession != null - ? WriteThroughContentSession.PutStreamAsync(context, contentHash, stream, cts, urgencyHint) - : BackingContentSession.PutStreamAsync(context, contentHash, stream, cts, urgencyHint)); + return CacheTracer.ContentSessionTracer.PutStreamAsync( + new OperationContext(context, cts), + contentHash, + () => + WriteThroughContentSession != null + ? WriteThroughContentSession.PutStreamAsync(context, contentHash, stream, cts, urgencyHint) + : BackingContentSession.PutStreamAsync(context, contentHash, stream, cts, urgencyHint)); } private async Task EnsureContentIsAvailableAsync( - OperationContext context, IReadOnlyList contentHashes, UrgencyHint urgencyHint) + OperationContext context, + IReadOnlyList contentHashes, + UrgencyHint urgencyHint) { bool missingContent = false; diff --git a/Public/Src/Cache/VerticalStore/MemoizationStoreAdapter/MemoizationStoreAdapterCache.cs b/Public/Src/Cache/VerticalStore/MemoizationStoreAdapter/MemoizationStoreAdapterCache.cs index 74992fcdd..0dd21042b 100644 --- a/Public/Src/Cache/VerticalStore/MemoizationStoreAdapter/MemoizationStoreAdapterCache.cs +++ b/Public/Src/Cache/VerticalStore/MemoizationStoreAdapter/MemoizationStoreAdapterCache.cs @@ -212,7 +212,7 @@ namespace BuildXL.Cache.MemoizationStoreAdapter Contract.Requires(!IsShutdown); var context = new Context(m_logger); - var createSessionResult = m_cache.CreateReadOnlySession( + var createSessionResult = m_cache.CreateSession( context, $"{CacheId}-Anonymous", m_implicitPin); diff --git a/Public/Src/Cache/VerticalStore/MemoizationStoreAdapter/MemoizationStoreAdapterCacheReadOnlySession.cs b/Public/Src/Cache/VerticalStore/MemoizationStoreAdapter/MemoizationStoreAdapterCacheReadOnlySession.cs index 804c001c7..6f26e7c12 100644 --- a/Public/Src/Cache/VerticalStore/MemoizationStoreAdapter/MemoizationStoreAdapterCacheReadOnlySession.cs +++ b/Public/Src/Cache/VerticalStore/MemoizationStoreAdapter/MemoizationStoreAdapterCacheReadOnlySession.cs @@ -32,7 +32,7 @@ namespace BuildXL.Cache.MemoizationStoreAdapter /// /// Backing ReadOnlyCacheSession for content and fingerprint calls. /// - protected readonly IReadOnlyCacheSession ReadOnlyCacheSession; + protected readonly MemoizationStore.Interfaces.Sessions.ICacheSession ReadOnlyCacheSession; /// /// The set of strong fingerprints touched by this cache session. @@ -67,7 +67,7 @@ namespace BuildXL.Cache.MemoizationStoreAdapter /// Telemetry ID for the session. /// When true, replace existing file when placing file. public MemoizationStoreAdapterCacheReadOnlySession( - IReadOnlyCacheSession readOnlyCacheSession, + MemoizationStore.Interfaces.Sessions.ICacheSession readOnlyCacheSession, BuildXL.Cache.MemoizationStore.Interfaces.Caches.ICache cache, CacheId cacheId, ILogger logger,