microsoft
/
BuildXL
зеркало из https://github.com/microsoft/BuildXL.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

Merged PR 522563: New eviction logic

This commit is contained in:
Sergey Tepliakov 2019-12-23 22:10:51 +00:00
Родитель b9716d7690
Коммит 2b93a86b35
32 изменённых файлов: 753 добавлений и 548 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

13
Public/Src/Cache/ContentStore/Distributed/NuCache/ContentLocationDatabase.cs
Просмотреть файл

@ -26,10 +26,21 @@ using AbsolutePath = BuildXL.Cache.ContentStore.Interfaces.FileSystem.AbsolutePa
namespace BuildXL.Cache.ContentStore.Distributed.NuCache
{
/// <summary>
/// Interface for <see cref="ContentLocationDatabase"/>.
/// </summary>
public interface IContentLocationDatabase
{
/// <summary>
/// Tries to locate an entry for a given hash.
/// </summary>
bool TryGetEntry(OperationContext context, ShortHash hash, out ContentLocationEntry entry);
}
/// <summary>
/// Base class that implements the core logic of <see cref="ContentLocationDatabase"/> interface.
/// </summary>
public abstract class ContentLocationDatabase : StartupShutdownSlimBase
public abstract class ContentLocationDatabase : StartupShutdownSlimBase, IContentLocationDatabase
{
private readonly ObjectPool<StreamBinaryWriter> _writerPool = new ObjectPool<StreamBinaryWriter>(() => new StreamBinaryWriter(), w => w.ResetPosition());
private readonly ObjectPool<StreamBinaryReader> _readerPool = new ObjectPool<StreamBinaryReader>(() => new StreamBinaryReader(), r => { });

201
Public/Src/Cache/ContentStore/Distributed/NuCache/EffectiveLastAccessTimeProvider.cs Normal file
Просмотреть файл

@ -0,0 +1,201 @@
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
using System;
using System.Collections.Generic;
using System.Diagnostics.ContractsLight;
using BuildXL.Cache.ContentStore.Hashing;
using BuildXL.Cache.ContentStore.Interfaces.Results;
using BuildXL.Cache.ContentStore.Interfaces.Stores;
using BuildXL.Cache.ContentStore.Interfaces.Time;
using BuildXL.Cache.ContentStore.Stores;
using BuildXL.Cache.ContentStore.Tracing.Internal;
using BuildXL.Utilities;
#nullable enable
namespace BuildXL.Cache.ContentStore.Distributed.NuCache
{
/// <summary>
/// Interfaces that used by <see cref="EffectiveLastAccessTimeProvider"/> in order to resolve content locations.
/// </summary>
public interface IContentResolver
{
/// <summary>
/// Tries to obtain <see cref="ContentInfo"/> from a store and <see cref="ContentLocationEntry"/> from a content location database.
/// </summary>
(ContentInfo info, ContentLocationEntry entry) GetContentInfo(OperationContext context, ContentHash hash);
}
/// <summary>
/// A helper class responsible for computing content's effective age.
/// </summary>
public sealed class EffectiveLastAccessTimeProvider
{
private readonly LocalLocationStoreConfiguration _configuration;
private readonly IContentResolver _contentResolver;
private readonly IClock _clock;
/// <nodoc />
public EffectiveLastAccessTimeProvider(
LocalLocationStoreConfiguration configuration,
IClock clock,
IContentResolver contentResolver)
{
_clock = clock;
_configuration = configuration;
_contentResolver = contentResolver;
}
/// <summary>
/// Returns effective last access time for all the <paramref name="contentHashes"/>.
/// </summary>
/// <remarks>
/// Effective last access time is computed based on entries last access time considering content's size and replica count.
/// This method is used in distributed eviction.
/// </remarks>
public Result<IReadOnlyList<ContentEvictionInfo>> GetEffectiveLastAccessTimes(
OperationContext context,
MachineId localMachineId,
IReadOnlyList<ContentHashWithLastAccessTime> contentHashes)
{
Contract.RequiresNotNull(contentHashes);
Contract.Requires(contentHashes.Count > 0);
// This is required because the code inside could throw.
var effectiveLastAccessTimes = new List<ContentEvictionInfo>();
double logInverseMachineRisk = -Math.Log(_configuration.MachineRisk);
foreach (var contentHash in contentHashes)
{
DateTime lastAccessTime = contentHash.LastAccessTime;
int replicaCount = 1;
bool isImportantReplica = false;
var (contentInfo, entry) = _contentResolver.GetContentInfo(context, contentHash.Hash);
// Getting a size from content directory information first.
long size = contentInfo.Size;
if (entry != null)
{
// Use the latest last access time between LLS and local last access time
DateTime distributedLastAccessTime = entry.LastAccessTimeUtc.ToDateTime();
lastAccessTime = distributedLastAccessTime > lastAccessTime ? distributedLastAccessTime : lastAccessTime;
isImportantReplica = IsImportantReplica(contentHash.Hash, entry, localMachineId, _configuration.DesiredReplicaRetention);
replicaCount = entry.Locations.Count;
if (size == 0)
{
size = entry.ContentSize;
}
}
var age = _clock.UtcNow - lastAccessTime;
var effectiveAge = GetEffectiveLastAccessTime(_configuration, age, replicaCount, size, isImportantReplica, logInverseMachineRisk);
var info = new ContentEvictionInfo(contentHash.Hash, age, effectiveAge, replicaCount, size, isImportantReplica);
effectiveLastAccessTimes.Add(info);
}
return Result.Success<IReadOnlyList<ContentEvictionInfo>>(effectiveLastAccessTimes);
}
/// <summary>
/// Returns true if a given hash considered to be an important replica for the given machine.
/// </summary>
public static bool IsImportantReplica(ContentHash hash, ContentLocationEntry entry, MachineId localMachineId, long desiredReplicaCount)
{
var locationsCount = entry.Locations.Count;
if (locationsCount <= desiredReplicaCount)
{
return true;
}
if (desiredReplicaCount == 0)
{
return false;
}
// Making sure that probabilistically, some locations are considered important for the current machine.
long replicaHash = unchecked((uint)HashCodeHelper.Combine(hash[0] | hash[1] << 8, localMachineId.Index + 1));
var offset = replicaHash % locationsCount;
var importantRangeStart = offset;
var importantRangeStop = (offset + desiredReplicaCount) % locationsCount;
// Getting an index of a current location in the location list
int currentMachineLocationIndex = entry.Locations.GetMachineIdIndex(localMachineId);
if (currentMachineLocationIndex == -1)
{
// This is used for testing only. The machine Id should be part of the machines.
// But in tests it is useful to control the behavior of this method and in some cases to guarantee that some replica won't be important.
return false;
}
// For instance, for locations [1, 20]
// the important range can be [5, 7]
// or [19, 1]
if (importantRangeStart < importantRangeStop)
{
// This is the first case: the start index is less then the stop,
// so the desired location should be within the range
return currentMachineLocationIndex >= importantRangeStart && currentMachineLocationIndex <= importantRangeStop;
}
// Important range is broken because the start is greater then the end.
// like [19, 1], so the location is important if it's index is >= start or <= the stop.
// For instance, for 10 locations the start is 9 and the end is 2
return currentMachineLocationIndex >= importantRangeStart || currentMachineLocationIndex <= importantRangeStop;
}
/// <summary>
/// Gets effective last access time based on the age and importance.
/// </summary>
public static TimeSpan GetEffectiveLastAccessTime(LocalLocationStoreConfiguration configuration, TimeSpan age, int replicaCount, long size, bool isImportantReplica, double logInverseMachineRisk)
{
if (configuration.UseTieredDistributedEviction)
{
var ageBucketIndex = FindAgeBucketIndex(configuration, age);
if (isImportantReplica)
{
ageBucketIndex = Math.Max(0, ageBucketIndex - configuration.ImportantReplicaBucketOffset);
}
return configuration.AgeBuckets[ageBucketIndex];
}
else
{
// Incorporate both replica count and size into an evictability metric.
// It's better to eliminate big content (more bytes freed per eviction) and it's better to eliminate content with more replicas (less chance
// of all replicas being inaccessible).
// A simple model with exponential decay of likelihood-to-use and a fixed probability of each replica being inaccessible shows that the metric
// evictability = age + (time decay parameter) * (-log(risk of content unavailability) * (number of replicas) + log(size of content))
// minimizes the increase in the probability of (content wanted && all replicas inaccessible) / per bytes freed.
// Since this metric is just the age plus a computed quantity, it can be interpreted as an "effective age".
TimeSpan totalReplicaPenalty = TimeSpan.FromMinutes(configuration.ContentLifetime.TotalMinutes * (Math.Max(1, replicaCount) * logInverseMachineRisk + Math.Log(Math.Max(1, size))));
return age + totalReplicaPenalty;
}
}
private static int FindAgeBucketIndex(LocalLocationStoreConfiguration configuration, TimeSpan age)
{
Contract.Requires(configuration.AgeBuckets.Count > 0);
for (int i = 0; i < configuration.AgeBuckets.Count; i++)
{
var bucket = configuration.AgeBuckets[i];
if (age < bucket)
{
return i;
}
}
return configuration.AgeBuckets.Count - 1;
}
}
}

100
Public/Src/Cache/ContentStore/Distributed/NuCache/LocalLocationStore.cs
Просмотреть файл

@ -29,8 +29,6 @@ using BuildXL.Native.IO;
using BuildXL.Utilities.Collections;
using BuildXL.Utilities.Tracing;
using DateTimeUtilities = BuildXL.Cache.ContentStore.Utils.DateTimeUtilities;
using static BuildXL.Cache.ContentStore.Utils.DateTimeUtilities;
using BuildXL.Utilities.Tasks;
using BuildXL.Utilities;
using BuildXL.Cache.ContentStore.Interfaces.Synchronization.Internal;
using System.Runtime.CompilerServices;
@ -1172,7 +1170,7 @@ namespace BuildXL.Cache.ContentStore.Distributed.NuCache
/// <summary>
/// Computes content hashes with effective last access time sorted in LRU manner.
/// </summary>
public IEnumerable<ContentHashWithLastAccessTimeAndReplicaCount> GetHashesInEvictionOrder(
public IEnumerable<ContentEvictionInfo> GetHashesInEvictionOrder(
Context context,
IReadOnlyList<ContentHashWithLastAccessTimeAndReplicaCount> contentHashesWithInfo,
bool reverse)
@ -1197,13 +1195,13 @@ namespace BuildXL.Cache.ContentStore.Distributed.NuCache
// Ideally, we want to remove content we know won't be used again for quite a while. We don't have that
// information, so we use an evictability metric. Here we obtain and sort by that evictability metric.
// Assume that EffectiveLastAccessTime will always have a value.
var comparer = Comparer<ContentHashWithLastAccessTimeAndReplicaCount>.Create((c1, c2) => (reverse ? -1 : 1) * c1.EffectiveLastAccessTime.Value.CompareTo(c2.EffectiveLastAccessTime.Value));
var comparer = ContentEvictionInfo.AgeBucketingPrecedenceComparer.Instance;
Func<List<ContentHashWithLastAccessTimeAndReplicaCount>, IEnumerable<ContentHashWithLastAccessTimeAndReplicaCount>> intoEffectiveLastAccessTimes =
page => GetEffectiveLastAccessTimes(
operationContext,
page.SelectList(v => new ContentHashWithLastAccessTime(v.ContentHash, v.LastAccessTime))).ThrowIfFailure();
IEnumerable<ContentEvictionInfo> getContentEvictionInfos(List<ContentHashWithLastAccessTimeAndReplicaCount> page) =>
GetEffectiveLastAccessTimes(
operationContext,
page.SelectList(v => new ContentHashWithLastAccessTime(v.ContentHash, v.LastAccessTime)))
.ThrowIfFailure();
// We make sure that we select a set of the newer content, to ensure that we at least look at newer
// content to see if it should be evicted first due to having a high number of replicas. We do this by
@ -1214,29 +1212,28 @@ namespace BuildXL.Cache.ContentStore.Distributed.NuCache
// not the evictability metric.
var oldestContentSortedByEvictability = contentHashesWithInfo
.Take(contentHashesWithInfo.Count / 2)
.ApproximateSort(comparer, intoEffectiveLastAccessTimes, _configuration.EvictionPoolSize, _configuration.EvictionWindowSize, _configuration.EvictionRemovalFraction, _configuration.EvictionDiscardFraction);
.ApproximateSort(comparer, getContentEvictionInfos, _configuration.EvictionPoolSize, _configuration.EvictionWindowSize, _configuration.EvictionRemovalFraction, _configuration.EvictionDiscardFraction);
var newestContentSortedByEvictability = contentHashesWithInfo
.SkipOptimized(contentHashesWithInfo.Count / 2)
.ApproximateSort(comparer, intoEffectiveLastAccessTimes, _configuration.EvictionPoolSize, _configuration.EvictionWindowSize, _configuration.EvictionRemovalFraction, _configuration.EvictionDiscardFraction);
.ApproximateSort(comparer, getContentEvictionInfos, _configuration.EvictionPoolSize, _configuration.EvictionWindowSize, _configuration.EvictionRemovalFraction, _configuration.EvictionDiscardFraction);
return NuCacheCollectionUtilities.MergeOrdered(oldestContentSortedByEvictability, newestContentSortedByEvictability, comparer)
.Where((candidate, index) => IsPassEvictionAge(context, candidate, _configuration.EvictionMinAge, index, ref evictionCount));
}
private bool IsPassEvictionAge(Context context, ContentHashWithLastAccessTimeAndReplicaCount candidate, TimeSpan evictionMinAge, int index, ref int evictionCount)
private bool IsPassEvictionAge(Context context, ContentEvictionInfo candidate, TimeSpan evictionMinAge, int index, ref int evictionCount)
{
if (candidate.Age(_clock) >= evictionMinAge)
if (candidate.EffectiveAge >= evictionMinAge)
{
evictionCount++;
return true;
}
context.Debug($"Previous successful eviction attempts = {evictionCount}, Total eviction attempts previously = {index}, minimum eviction age = {evictionMinAge.ToString()}, pool size = {_configuration.EvictionPoolSize}." +
$" Candidate replica count = {candidate.ReplicaCount}, effective age = {candidate.EffectiveAge(_clock)}, age = {candidate.Age(_clock)}.");
$" Candidate replica count = {candidate.ReplicaCount}, effective age = {candidate.EffectiveAge}, age = {candidate.Age}.");
return false;
}
/// <summary>
/// Returns effective last access time for all the <paramref name="contentHashes"/>.
/// </summary>
@ -1244,7 +1241,7 @@ namespace BuildXL.Cache.ContentStore.Distributed.NuCache
/// Effective last access time is computed based on entries last access time considering content's size and replica count.
/// This method is used in distributed eviction.
/// </remarks>
public Result<IReadOnlyList<ContentHashWithLastAccessTimeAndReplicaCount>> GetEffectiveLastAccessTimes(
public Result<IReadOnlyList<ContentEvictionInfo>> GetEffectiveLastAccessTimes(
OperationContext context,
IReadOnlyList<ContentHashWithLastAccessTime> contentHashes)
{
@ -1254,56 +1251,15 @@ namespace BuildXL.Cache.ContentStore.Distributed.NuCache
var postInitializationResult = EnsureInitializedAsync().GetAwaiter().GetResult();
if (!postInitializationResult)
{
return new Result<IReadOnlyList<ContentHashWithLastAccessTimeAndReplicaCount>>(postInitializationResult);
return new Result<IReadOnlyList<ContentEvictionInfo>>(postInitializationResult);
}
// This is required because the code inside could throw.
var effectiveLastAccessTimeProvider = new EffectiveLastAccessTimeProvider(_configuration, _clock, new ContentResolver(this));
return context.PerformOperation(
Tracer,
() =>
{
var effectiveLastAccessTimes = new List<ContentHashWithLastAccessTimeAndReplicaCount>();
double logInverseMachineRisk = -Math.Log(_configuration.MachineRisk);
foreach (var contentHash in contentHashes)
{
DateTime lastAccessTime = contentHash.LastAccessTime;
int replicaCount = 1;
DateTime? effectiveLastAccessTime = null;
if (TryGetContentLocations(context, contentHash.Hash, out var entry))
{
// Use the latest last access time between LLS and local last access time
DateTime distributedLastAccessTime = entry.LastAccessTimeUtc.ToDateTime();
lastAccessTime = distributedLastAccessTime > lastAccessTime ? distributedLastAccessTime : lastAccessTime;
// TODO[LLS]: Maybe some machines should be primary replicas for the content and not prioritize deletion (bug 1365340)
// just because there are many replicas
replicaCount = entry.Locations.Count;
// Incorporate both replica count and size into an evictability metric.
// It's better to eliminate big content (more bytes freed per eviction) and it's better to eliminate content with more replicas (less chance
// of all replicas being inaccessible).
// A simple model with exponential decay of likelihood-to-use and a fixed probability of each replica being inaccessible shows that the metric
// evictability = age + (time decay parameter) * (-log(risk of content unavailability) * (number of replicas) + log(size of content))
// minimizes the increase in the probability of (content wanted && all replicas inaccessible) / per bytes freed.
// Since this metric is just the age plus a computed quantity, it can be intrepreted as an "effective age".
TimeSpan totalReplicaPenalty = TimeSpan.FromMinutes(_configuration.ContentLifetime.TotalMinutes * (Math.Max(1, replicaCount) * logInverseMachineRisk + Math.Log(Math.Max(1, entry.ContentSize))));
effectiveLastAccessTime = lastAccessTime - totalReplicaPenalty;
Counters[ContentLocationStoreCounters.EffectiveLastAccessTimeLookupHit].Increment();
}
else
{
Counters[ContentLocationStoreCounters.EffectiveLastAccessTimeLookupMiss].Increment();
}
effectiveLastAccessTimes.Add(new ContentHashWithLastAccessTimeAndReplicaCount(contentHash.Hash, lastAccessTime, replicaCount, effectiveLastAccessTime: effectiveLastAccessTime ?? lastAccessTime));
}
return Result.Success<IReadOnlyList<ContentHashWithLastAccessTimeAndReplicaCount>>(effectiveLastAccessTimes);
}, Counters[ContentLocationStoreCounters.GetEffectiveLastAccessTimes],
traceOperationStarted: false);
Tracer,
() => effectiveLastAccessTimeProvider.GetEffectiveLastAccessTimes(context, LocalMachineId, contentHashes),
Counters[ContentLocationStoreCounters.GetEffectiveLastAccessTimes],
traceOperationStarted: false);
}
/// <summary>
@ -1607,5 +1563,23 @@ namespace BuildXL.Cache.ContentStore.Distributed.NuCache
}
}
}
private sealed class ContentResolver : IContentResolver
{
private readonly LocalLocationStore _localLocationStore;
public ContentResolver(LocalLocationStore localLocationStore) => _localLocationStore = localLocationStore;
/// <inheritdoc />
public (ContentInfo info, ContentLocationEntry entry) GetContentInfo(OperationContext context, ContentHash hash)
{
ContentInfo info = default;
_localLocationStore._localContentStore?.TryGetContentInfo(hash, out info);
_localLocationStore.TryGetContentLocations(context, hash, out var entry);
return (info, entry);
}
}
}
}

32
Public/Src/Cache/ContentStore/Distributed/NuCache/LocalLocationStoreConfiguration.cs
Просмотреть файл

@ -113,6 +113,38 @@ namespace BuildXL.Cache.ContentStore.Distributed
/// </summary>
public MachineReputationTrackerConfiguration ReputationTrackerConfiguration { get; set; } = new MachineReputationTrackerConfiguration();
/// <summary>
/// Specifies whether tiered eviction comparison should be used when ordering content for eviction
/// </summary>
public bool UseTieredDistributedEviction { get; set; }
/// <summary>
/// The number of buckets to offset by for important replicas. This effectively makes important replicas look younger.
/// </summary>
public int ImportantReplicaBucketOffset { get; set; } = 2;
/// <summary>
/// Age buckets for use with tiered eviction
/// </summary>
public IReadOnlyList<TimeSpan> AgeBuckets { get; set; } =
new TimeSpan[]
{
// Roughly 30 min + 3h^i
TimeSpan.FromMinutes(30),
TimeSpan.FromHours(2),
TimeSpan.FromHours(4),
TimeSpan.FromHours(10),
TimeSpan.FromDays(1),
TimeSpan.FromDays(3),
TimeSpan.FromDays(10),
TimeSpan.FromDays(30),
};
/// <summary>
/// Controls the desired number of replicas to retain.
/// </summary>
public int DesiredReplicaRetention { get; set; } = 3;
/// <summary>
/// Estimated decay time for content re-use.
/// </summary>

17
Public/Src/Cache/ContentStore/Distributed/NuCache/MachineIdSets/ArrayMachineIdSet.cs
Просмотреть файл

@ -39,5 +39,22 @@ namespace BuildXL.Cache.ContentStore.Distributed.NuCache
yield return id;
}
}
/// <inheritdoc />
public override int GetMachineIdIndex(MachineId currentMachineId)
{
int index = 0;
foreach (var machineId in _machineIds)
{
if (new MachineId(machineId) == currentMachineId)
{
return index;
}
index++;
}
return -1;
}
}
}

23
Public/Src/Cache/ContentStore/Distributed/NuCache/MachineIdSets/BitMachineIdSet.cs
Просмотреть файл

@ -149,6 +149,29 @@ namespace BuildXL.Cache.ContentStore.Distributed.NuCache
}
}
/// <inheritdoc />
public override int GetMachineIdIndex(MachineId currentMachineId)
{
for (int i = Offset; i < Data.Length; i++)
{
byte redisChar = Data[i];
int position = 0;
while (redisChar != 0)
{
if ((redisChar & MaxCharBitMask) != 0)
{
return i + position;
}
redisChar <<= 1;
position++;
}
}
return -1;
}
/// <inheritdoc />
public override string ToString()
{

17
Public/Src/Cache/ContentStore/Distributed/NuCache/MachineIdSets/HashSetMachineIdSet.cs
Просмотреть файл

@ -44,5 +44,22 @@ namespace BuildXL.Cache.ContentStore.Distributed.NuCache
yield return machineId;
}
}
/// <inheritdoc />
public override int GetMachineIdIndex(MachineId currentMachineId)
{
int index = 0;
foreach (var machineId in _machineIds)
{
if (new MachineId(machineId) == currentMachineId)
{
return index;
}
index++;
}
return -1;
}
}
}

10
Public/Src/Cache/ContentStore/Distributed/NuCache/MachineIdSets/MachineIdSet.cs
Просмотреть файл

@ -65,6 +65,16 @@ namespace BuildXL.Cache.ContentStore.Distributed.NuCache
/// </summary>
public abstract IEnumerable<MachineId> EnumerateMachineIds();
/// <summary>
/// Returns a position of the <paramref name="currentMachineId"/> in the current machine id list.
/// </summary>
/// <returns>-1 if the given id is not part of the machine id list.</returns>
/// <remarks>
/// This method can be implemented on top of <see cref="EnumerateMachineIds"/> but it is a separate method
/// because different subtypes can implement this operation with no extra allocations.
/// </remarks>
public abstract int GetMachineIdIndex(MachineId currentMachineId);
/// <nodoc />
public void Serialize(BuildXLWriter writer)
{

7
Public/Src/Cache/ContentStore/Distributed/NuCache/NuCacheCollectionUtilities.cs
Просмотреть файл

@ -237,17 +237,18 @@ namespace BuildXL.Cache.ContentStore.Distributed.NuCache
/// If <paramref name="poolSize"/> = <paramref name="pageSize"/>, the algorithm fills up the entire pool and
/// evicts a fraction of it in every iteration.
/// </remarks>
public static IEnumerable<T> ApproximateSort<T>(this IEnumerable<T> original, Comparer<T> comparer, Func<List<T>, IEnumerable<T>> query, int poolSize, int pageSize, float removalFraction, float discardFraction = 0)
public static IEnumerable<TResult> ApproximateSort<T, TResult>(this IEnumerable<T> original, IComparer<TResult> comparer, Func<List<T>, IEnumerable<TResult>> query, int poolSize, int pageSize, float removalFraction, float discardFraction = 0)
{
Contract.RequiresNotNull(original);
Contract.Requires(poolSize > 0);
Contract.Requires(pageSize > 0 && pageSize <= poolSize);
Contract.Requires(removalFraction > 0 && removalFraction <= 1);
Contract.Requires(discardFraction >= 0 && discardFraction <= 1);
// The pool holds up to `poolSize` candidates sorted by the comparer
var pool = new MinMaxHeap<T>(poolSize, comparer);
var pool = new MinMaxHeap<TResult>(poolSize, comparer);
var source = original.GetEnumerator();
using var source = original.GetEnumerator();
var sourceHasItems = true;
while (true)

2
Public/Src/Cache/ContentStore/Distributed/NuCache/TransitioningContentLocationStore.cs
Просмотреть файл

@ -281,7 +281,7 @@ namespace BuildXL.Cache.ContentStore.Distributed.NuCache
}
/// <inheritdoc />
public IEnumerable<ContentHashWithLastAccessTimeAndReplicaCount> GetHashesInEvictionOrder(Context context, IReadOnlyList<ContentHashWithLastAccessTimeAndReplicaCount> contentHashesWithInfo)
public IEnumerable<ContentEvictionInfo> GetHashesInEvictionOrder(Context context, IReadOnlyList<ContentHashWithLastAccessTimeAndReplicaCount> contentHashesWithInfo)
{
Contract.Assert(
_configuration.HasReadOrWriteMode(ContentLocationMode.LocalLocationStore),

2
Public/Src/Cache/ContentStore/Distributed/Stores/DistributedContentStore.cs
Просмотреть файл

@ -503,7 +503,7 @@ namespace BuildXL.Cache.ContentStore.Distributed.Stores
}
/// <nodoc />
public IEnumerable<ContentHashWithLastAccessTimeAndReplicaCount> GetHashesInEvictionOrder(Context context, IReadOnlyList<ContentHashWithLastAccessTimeAndReplicaCount> contentHashesWithInfo)
public IEnumerable<ContentEvictionInfo> GetHashesInEvictionOrder(Context context, IReadOnlyList<ContentHashWithLastAccessTimeAndReplicaCount> contentHashesWithInfo)
{
// Ensure startup was called then wait for it to complete successfully (or error)
// This logic is important to avoid runtime errors when, for instance, QuotaKeeper tries

4
Public/Src/Cache/ContentStore/DistributedTest/ContentLocation/LocalLocationStoreDistributedContentTests.cs
Просмотреть файл

@ -1840,9 +1840,9 @@ namespace ContentStoreTest.Distributed.Sessions
foreach (var item in master.GetHashesInEvictionOrder(context, lruContent))
{
tracer.Debug($"{item}");
tracer.Debug($"LTO: {item.LastAccessTime.Ticks - lastTime}, LOTO: {item.LastAccessTime.Ticks - lastTime}, IsDupe: {!hashes.Add(item.ContentHash)}");
tracer.Debug($"LTO: {item.EffectiveAge.Ticks - lastTime}, LOTO: {item.EffectiveAge.Ticks - lastTime}, IsDupe: {!hashes.Add(item.ContentHash)}");
lastTime = item.LastAccessTime.Ticks;
lastTime = item.Age.Ticks;
}
await Task.Yield();

43
Public/Src/Cache/ContentStore/DistributedTest/Stores/ContentEvictionInfoTests.cs Normal file
Просмотреть файл

@ -0,0 +1,43 @@
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
using System;
using System.Linq;
using BuildXL.Cache.ContentStore.Hashing;
using BuildXL.Cache.ContentStore.Interfaces.Stores;
using Xunit;
namespace BuildXL.Cache.ContentStore.Distributed.Test.Stores
{
public class ContentEvictionInfoTests
{
[Fact]
public void TestContentEvictionInfoComparer()
{
var inputs = new []
{
new ContentEvictionInfo(ContentHash.Random(), TimeSpan.FromHours(1), TimeSpan.FromHours(2), replicaCount: 1, size: 1, isImportantReplica: false),
new ContentEvictionInfo(ContentHash.Random(), TimeSpan.FromHours(1), TimeSpan.FromHours(2), replicaCount: 1, size: 2, isImportantReplica: true),
new ContentEvictionInfo(ContentHash.Random(), TimeSpan.FromHours(1), TimeSpan.FromHours(2), replicaCount: 1, size: 1, isImportantReplica: true),
new ContentEvictionInfo(ContentHash.Random(), TimeSpan.FromHours(2), TimeSpan.FromHours(3), replicaCount: 1, size: 1, isImportantReplica: true),
new ContentEvictionInfo(ContentHash.Random(), TimeSpan.FromHours(2), TimeSpan.FromHours(3), replicaCount: 1, size: 1, isImportantReplica: false),
};
var list = inputs.ToList();
list.Sort(ContentEvictionInfo.AgeBucketingPrecedenceComparer.Instance);
var expected = new[]
{
inputs[4], // EffAge=3, Importance=false
inputs[3], // EffAge=3, Importance=true
inputs[0], // EffAge=2, Importance=false
inputs[1], // EffAge=2, Importance=true, Cost=2
inputs[2], // EffAge=2, Importance=true, Cost=1
};
Assert.Equal(expected, list.ToArray());
}
}
}

199
Public/Src/Cache/ContentStore/DistributedTest/Stores/EffectiveLastAccessTimeProviderTests.cs Normal file
Просмотреть файл

@ -0,0 +1,199 @@
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using BuildXL.Cache.ContentStore.Distributed.NuCache;
using BuildXL.Cache.ContentStore.Hashing;
using BuildXL.Cache.ContentStore.Interfaces.Logging;
using BuildXL.Cache.ContentStore.Interfaces.Stores;
using BuildXL.Cache.ContentStore.Interfaces.Tracing;
using BuildXL.Cache.ContentStore.InterfacesTest.Results;
using BuildXL.Cache.ContentStore.InterfacesTest.Time;
using BuildXL.Cache.ContentStore.Logging;
using BuildXL.Cache.ContentStore.Stores;
using BuildXL.Cache.ContentStore.Tracing.Internal;
using BuildXL.Utilities;
using ContentStoreTest.Test;
using FluentAssertions;
using Xunit;
using Xunit.Abstractions;
namespace BuildXL.Cache.ContentStore.Distributed.Test.Stores
{
public class EffectiveLastAccessTimeProviderTests : TestBase
{
private static LocalLocationStoreConfiguration Configuration { get; } = new LocalLocationStoreConfiguration() {UseTieredDistributedEviction = true, DesiredReplicaRetention = 3};
[Fact]
public void RareContentShouldBeImportant()
{
var hash = ContentHash.Random();
var clock = new MemoryClock();
var entry = ContentLocationEntry.Create(
locations: new ArrayMachineIdSet(new ushort[1]),
contentSize: 42,
lastAccessTimeUtc: clock.UtcNow,
creationTimeUtc: clock.UtcNow);
bool isImportant = EffectiveLastAccessTimeProvider.IsImportantReplica(hash, entry, new MachineId(1), Configuration.DesiredReplicaRetention);
isImportant.Should().BeTrue();
}
[InlineData(5)]
[InlineData(10)]
[InlineData(50)]
[InlineData(100)]
[InlineData(501)]
[Theory]
public void NonRareContentShouldBeImportantInSomeCases(int machineCount)
{
// Using non random hash to make the test deterministic
var hash = VsoHashInfo.Instance.EmptyHash;
var machineIds = Enumerable.Range(1, machineCount).Select(v => (ushort)v).ToArray();
var clock = new MemoryClock();
// Creating an entry with 'machineCount' locations.
// But instead of using instance as is, we call Copy to serialize/deserialize the instance.
// In this case, we'll create different types at runtime instead of using just a specific type like ArrayMachineIdSet.
var machineIdSet = Copy(new ArrayMachineIdSet(machineIds));
var entry = ContentLocationEntry.Create(
locations: machineIdSet,
contentSize: 42,
lastAccessTimeUtc: clock.UtcNow,
creationTimeUtc: clock.UtcNow);
// Then checking all the "machines" (by creating MachineId for each id in machineIds)
// to figure out if the replica is important.
var importantMachineCount = machineIds.Select(
machineId => EffectiveLastAccessTimeProvider.IsImportantReplica(
hash,
entry,
new MachineId(machineId),
Configuration.DesiredReplicaRetention)).Count(important => important);
// We should get roughly 'configuration.DesiredReplicaRetention' important replicas.
// The number is not exact, because when we're computing the importance we're computing a hash of the first bytes of the hash
// plus the machine id. So it is possible that we can be slightly off here.
var desiredReplicaCount = Configuration.DesiredReplicaRetention;
importantMachineCount.Should().BeInRange(desiredReplicaCount - 2, desiredReplicaCount + 3);
}
[Fact]
public void ImportantReplicaMovesTheAgeBucket()
{
var nonImportant = EffectiveLastAccessTimeProvider.GetEffectiveLastAccessTime(
Configuration,
age: TimeSpan.FromHours(2),
replicaCount: 100,
42,
isImportantReplica: false,
logInverseMachineRisk: 0);
var important = EffectiveLastAccessTimeProvider.GetEffectiveLastAccessTime(
Configuration,
age: TimeSpan.FromHours(2),
replicaCount: 100,
42,
isImportantReplica: true,
logInverseMachineRisk: 0);
important.Should().BeLessThan(nonImportant, "Important replica should be consider to be younger, then non-important one.");
}
[Fact]
public void TestContentEviction()
{
var clock = new MemoryClock();
var entries = new List<ContentLocationEntry>();
entries.Add(
ContentLocationEntry.Create(
locations: CreateWithLocationCount(1), // Should be important
contentSize: 42,
lastAccessTimeUtc: clock.UtcNow - TimeSpan.FromHours(2),
creationTimeUtc: null));
entries.Add(
ContentLocationEntry.Create(
locations: CreateWithLocationCount(100), // Maybe important with 3% chance
contentSize: 42,
lastAccessTimeUtc: clock.UtcNow - TimeSpan.FromHours(2),
creationTimeUtc: null));
entries.Add(
ContentLocationEntry.Create(
locations: CreateWithLocationCount(100), // Maybe important with 3% chance
contentSize: 42,
lastAccessTimeUtc: clock.UtcNow - TimeSpan.FromHours(2),
creationTimeUtc: null));
var mock = new EffectiveLastAccessTimeProviderMock();
var hashes = new [] {ContentHash.Random(), ContentHash.Random(), ContentHash.Random()};
mock.Map = new Dictionary<ContentHash, ContentLocationEntry>()
{
[hashes[0]] = entries[0],
[hashes[1]] = entries[1],
[hashes[2]] = entries[2],
};
var provider = new EffectiveLastAccessTimeProvider(Configuration, clock, mock);
var context = new OperationContext(new Context(Logger));
// A given machine id index is higher then the max number of locations used in this test.
// This will prevent the provider to consider non-important locations randomly important
var input = hashes.Select(hash => new ContentHashWithLastAccessTime(hash, mock.Map[hash].LastAccessTimeUtc.ToDateTime())).ToList();
var result = provider.GetEffectiveLastAccessTimes(context, new MachineId(1024), input).ShouldBeSuccess();
var output = result.Value.ToList();
output.Sort(ContentEvictionInfo.AgeBucketingPrecedenceComparer.Instance);
// We know that the first hash should be the last one, because this is only important hash in the list.
output[output.Count - 1].ContentHash.Should().Be(hashes[0]);
}
private MachineIdSet CreateWithLocationCount(int locationCount)
{
var machineIds = Enumerable.Range(1, locationCount).Select(v => (ushort)v).ToArray();
return new ArrayMachineIdSet(machineIds);
}
public class EffectiveLastAccessTimeProviderMock : IContentResolver
{
public Dictionary<ContentHash, ContentLocationEntry> Map { get; set; }
/// <inheritdoc />
public (ContentInfo info, ContentLocationEntry entry) GetContentInfo(OperationContext context, ContentHash hash)
{
return (default, Map[hash]);
}
}
private static MachineIdSet Copy(MachineIdSet source)
{
using (var memoryStream = new MemoryStream())
{
using (var writer = BuildXLWriter.Create(memoryStream, leaveOpen: true))
{
source.Serialize(writer);
}
memoryStream.Position = 0;
using (var reader = BuildXLReader.Create(memoryStream))
{
return MachineIdSet.Deserialize(reader);
}
}
}
/// <inheritdoc />
public EffectiveLastAccessTimeProviderTests(ITestOutputHelper output = null)
: base(TestGlobal.Logger, output)
{
}
}
}

99
Public/Src/Cache/ContentStore/Interfaces/Stores/ContentEvictionInfo.cs Normal file
Просмотреть файл

@ -0,0 +1,99 @@
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
using System;
using System.Collections.Generic;
using BuildXL.Cache.ContentStore.Hashing;
using BuildXL.Cache.ContentStore.UtilitiesCore.Internal;
namespace BuildXL.Cache.ContentStore.Interfaces.Stores
{
/// <summary>
/// A wrapper for content hashes used by eviction logic.
/// </summary>
public readonly struct ContentEvictionInfo
{
/// <nodoc />
public ContentHash ContentHash { get; }
/// <nodoc/>
public int ReplicaCount { get; }
/// <nodoc/>
public long Size { get; }
/// <summary>
/// Cost of the content that affects an eviction order for the content within the same age bucket.
/// </summary>
public long Cost => Size * ReplicaCount;
/// <summary>
/// Indicates whether this replica is considered important and thus retention should be prioritized
/// </summary>
public bool IsImportantReplica { get; }
/// <summary>
/// Age of the content based on the last access time.
/// </summary>
/// <remarks>
/// The last access time is the file last access time or the "distributed" last access time obtained from local location store.
/// </remarks>
public TimeSpan Age { get; }
/// <summary>
/// An effective age of the content that is computed based on content importance or other metrics like content evictability.
/// </summary>
public TimeSpan EffectiveAge { get; }
/// <nodoc />
public ContentEvictionInfo(
ContentHash contentHash,
TimeSpan age,
TimeSpan effectiveAge,
int replicaCount,
long size,
bool isImportantReplica)
{
ContentHash = contentHash;
Age = age;
EffectiveAge = effectiveAge;
ReplicaCount = replicaCount;
Size = size;
IsImportantReplica = isImportantReplica;
}
/// <inheritdoc />
public override string ToString()
{
return $"[ContentHash={ContentHash.ToShortString()} Age={Age} EffectiveAge={EffectiveAge} Cost={ReplicaCount}*{Size} IsImportantReplica={IsImportantReplica}]";
}
/// <summary>
/// Object comparer for <see cref="ContentEvictionInfo"/>.
/// </summary>
/// <remarks>
/// The comparison is done by EffectiveAge, then by Importance and then by Cost.
/// When 'UseTieredDistributedEviction' configuration option is true, then EffectiveAge property is "rounded" to buckets boundaries.
/// For instance, all the content that is younger then 30 minutes old would have an EffectiveAge equals to 30min.
/// This allows us to sort by EffectiveAge and if it is the same, sort by some other properties (like importance).
/// </remarks>
public class AgeBucketingPrecedenceComparer : IComparer<ContentEvictionInfo>
{
/// <nodoc />
public static readonly AgeBucketingPrecedenceComparer Instance = new AgeBucketingPrecedenceComparer();
/// <inheritdoc />
public int Compare(ContentEvictionInfo x, ContentEvictionInfo y)
{
if (!CollectionUtilities.IsCompareEquals(x.EffectiveAge, y.EffectiveAge, out var compareResult, greatestFirst: true)
|| !CollectionUtilities.IsCompareEquals(x.IsImportantReplica, y.IsImportantReplica, out compareResult)
|| !CollectionUtilities.IsCompareEquals(x.Cost, y.Cost, out compareResult, greatestFirst: true))
{
return compareResult;
}
return 0;
}
}
}
}

15
Public/Src/Cache/ContentStore/Library/Stores/FileSystemContentStore.cs
Просмотреть файл

@ -209,6 +209,21 @@ namespace BuildXL.Cache.ContentStore.Stores
return Store.Contains(hash);
}
/// <inheritdoc />
public bool TryGetContentInfo(ContentHash hash, out ContentInfo info)
{
if (Store.TryGetFileInfo(hash, out var fileInfo))
{
info = new ContentInfo(hash, fileInfo.FileSize, DateTime.FromFileTimeUtc(fileInfo.LastAccessedFileTimeUtc));
return true;
}
else
{
info = default;
return false;
}
}
/// <inheritdoc />
public Task<OpenStreamResult> StreamContentAsync(Context context, ContentHash contentHash)
{

7
Public/Src/Cache/ContentStore/Library/Stores/ILocalContentStore.cs
Просмотреть файл

@ -25,6 +25,11 @@ namespace BuildXL.Cache.ContentStore.Stores
/// Gets whether the local content store contains the content specified by the hash
/// </summary>
bool Contains(ContentHash hash);
/// <summary>
/// Gets the information about the content hash if present
/// </summary>
bool TryGetContentInfo(ContentHash hash, out ContentInfo info);
}
/// <summary>
@ -45,7 +50,7 @@ namespace BuildXL.Cache.ContentStore.Stores
/// <summary>
/// Computes content hashes with effective last access time sorted in LRU manner.
/// </summary>
IEnumerable<ContentHashWithLastAccessTimeAndReplicaCount> GetHashesInEvictionOrder(
IEnumerable<ContentEvictionInfo> GetHashesInEvictionOrder(
Context context,
IReadOnlyList<ContentHashWithLastAccessTimeAndReplicaCount> contentHashesWithInfo);
}

7
Public/Src/Cache/ContentStore/Library/Stores/QuotaManagement/DiskFreePercentRule.cs
Просмотреть файл

@ -30,14 +30,11 @@ namespace BuildXL.Cache.ContentStore.Stores
/// </summary>
public DiskFreePercentRule(
DiskFreePercentQuota quota,
EvictAsync evictAsync,
IAbsFileSystem fileSystem,
AbsolutePath rootPath,
DistributedEvictionSettings distributedEvictionSettings = null)
: base(evictAsync, OnlyUnlinkedValue, distributedEvictionSettings)
AbsolutePath rootPath)
: base(OnlyUnlinkedValue)
{
Contract.Requires(quota != null);
Contract.Requires(evictAsync != null);
Contract.Requires(fileSystem != null);
Contract.Requires(rootPath != null);

7
Public/Src/Cache/ContentStore/Library/Stores/QuotaManagement/ElasticSizeRule.cs
Просмотреть файл

@ -72,17 +72,14 @@ namespace BuildXL.Cache.ContentStore.Stores
public ElasticSizeRule(
int? historyWindowSize,
MaxSizeQuota initialElasticSize,
EvictAsync evictAsync,
Func<long> getCurrentSizeFunc,
Func<int, PinSizeHistory.ReadHistoryResult> getPinnedSizeHistoryFunc,
IAbsFileSystem fileSystem,
AbsolutePath rootPath,
double? calibrationCoefficient = default(double?),
DistributedEvictionSettings distributedEvictionSettings = null)
: base(evictAsync, OnlyUnlinkedValue, distributedEvictionSettings)
double? calibrationCoefficient = default(double?))
: base(OnlyUnlinkedValue)
{
Contract.Requires(!historyWindowSize.HasValue || historyWindowSize.Value >= 0);
Contract.Requires(evictAsync != null);
Contract.Requires(getCurrentSizeFunc != null);
Contract.Requires(getPinnedSizeHistoryFunc != null);
Contract.Requires(fileSystem != null);

5
Public/Src/Cache/ContentStore/Library/Stores/QuotaManagement/IQuotaRule.cs
Просмотреть файл

@ -38,11 +38,6 @@ namespace BuildXL.Cache.ContentStore.Stores
/// </summary>
BoolResult IsInsideTargetLimit(long reserveSize = 0);
/// <summary>
/// Purge content in LRU-ed order according to limits.
/// </summary>
Task<PurgeResult> PurgeAsync(Context context, long reserveSize, IReadOnlyList<ContentHashWithLastAccessTimeAndReplicaCount> contentHashesWithInfo, CancellationToken token);
/// <summary>
/// Gets a value indicating whether quota can be calibrated.
/// </summary>

5
Public/Src/Cache/ContentStore/Library/Stores/QuotaManagement/MaxSizeRule.cs
Просмотреть файл

@ -25,11 +25,10 @@ namespace BuildXL.Cache.ContentStore.Stores
/// <summary>
/// Initializes a new instance of the <see cref="MaxSizeRule"/> class.
/// </summary>
public MaxSizeRule(MaxSizeQuota quota, EvictAsync evictAsync, Func<long> getCurrentSizeFunc, DistributedEvictionSettings distributedEvictionSettings = null)
: base(evictAsync, OnlyUnlinkedValue, distributedEvictionSettings)
public MaxSizeRule(MaxSizeQuota quota, Func<long> getCurrentSizeFunc)
: base(OnlyUnlinkedValue)
{
Contract.Requires(quota != null);
Contract.Requires(evictAsync != null);
Contract.Requires(getCurrentSizeFunc != null);
_quota = quota;

8
Public/Src/Cache/ContentStore/Library/Stores/QuotaManagement/QuotaKeeper.cs
Просмотреть файл

@ -235,24 +235,22 @@ namespace BuildXL.Cache.ContentStore.Stores
var elasticSizeRule = new ElasticSizeRule(
configuration.HistoryWindowSize,
configuration.InitialElasticSize,
EvictContentAsync,
() => CurrentSize,
store.ReadPinSizeHistory,
fileSystem,
store.RootPath,
distributedEvictionSettings: distributedEvictionSettings);
store.RootPath);
rules.Add(elasticSizeRule);
}
else
{
if (configuration.MaxSizeQuota != null)
{
rules.Add(new MaxSizeRule(configuration.MaxSizeQuota, EvictContentAsync, () => CurrentSize, distributedEvictionSettings));
rules.Add(new MaxSizeRule(configuration.MaxSizeQuota, () => CurrentSize));
}
if (configuration.DiskFreePercentQuota != null)
{
rules.Add(new DiskFreePercentRule(configuration.DiskFreePercentQuota, EvictContentAsync, fileSystem, store.RootPath, distributedEvictionSettings));
rules.Add(new DiskFreePercentRule(configuration.DiskFreePercentQuota, fileSystem, store.RootPath));
}
}

5
Public/Src/Cache/ContentStore/Library/Stores/QuotaManagement/QuotaKeeperPurger.cs
Просмотреть файл

@ -119,7 +119,8 @@ namespace BuildXL.Cache.ContentStore.Stores
break;
}
var evictionResult = await _quotaKeeper.EvictContentAsync(_context, contentHashInfo, rule.GetOnlyUnlinked());
var contentHashWithLastAccessTimeAndReplicaCount = ToContentHashListWithLastAccessTimeAndReplicaCount(contentHashInfo);
var evictionResult = await _quotaKeeper.EvictContentAsync(_context, contentHashWithLastAccessTimeAndReplicaCount, rule.GetOnlyUnlinked());
if (!evictionResult)
{
return evictionResult;
@ -142,6 +143,8 @@ namespace BuildXL.Cache.ContentStore.Stores
return BoolResult.Success;
}
private static ContentHashWithLastAccessTimeAndReplicaCount ToContentHashListWithLastAccessTimeAndReplicaCount(ContentEvictionInfo contentHashInfo) => new ContentHashWithLastAccessTimeAndReplicaCount(contentHashInfo.ContentHash, DateTime.UtcNow - contentHashInfo.Age, contentHashInfo.ReplicaCount, safeToEvict: true, effectiveLastAccessTime: DateTime.UtcNow - contentHashInfo.EffectiveAge);
/// <summary>
/// Evict hashes in LRU-ed order determined by remote last-access times.
/// </summary>

398
Public/Src/Cache/ContentStore/Library/Stores/QuotaManagement/QuotaRule.cs
Просмотреть файл

@ -1,19 +1,9 @@
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
using System;
using System.Collections.Generic;
using System.Diagnostics.ContractsLight;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using BuildXL.Cache.ContentStore.Hashing;
using BuildXL.Cache.ContentStore.Interfaces.Results;
using BuildXL.Cache.ContentStore.Interfaces.Sessions;
using BuildXL.Cache.ContentStore.Interfaces.Stores;
using BuildXL.Cache.ContentStore.Interfaces.Tracing;
using BuildXL.Cache.ContentStore.Tracing;
using BuildXL.Cache.ContentStore.Utils;
namespace BuildXL.Cache.ContentStore.Stores
{
@ -27,9 +17,6 @@ namespace BuildXL.Cache.ContentStore.Stores
/// </summary>
protected ContentStoreQuota _quota;
private readonly DistributedEvictionSettings _distributedEvictionSettings;
private readonly EvictAsync _evictAsync;
/// <inheritdoc />
public bool OnlyUnlinked { get; }
@ -39,10 +26,8 @@ namespace BuildXL.Cache.ContentStore.Stores
/// <summary>
/// Initializes a new instance of the <see cref="QuotaRule" /> class.
/// </summary>
protected QuotaRule(EvictAsync evictAsync, bool onlyUnlinked, DistributedEvictionSettings distributedEvictionSettings = null)
protected QuotaRule(bool onlyUnlinked)
{
_distributedEvictionSettings = distributedEvictionSettings;
_evictAsync = evictAsync;
OnlyUnlinked = onlyUnlinked;
}
@ -51,30 +36,6 @@ namespace BuildXL.Cache.ContentStore.Stores
/// </summary>
public abstract BoolResult IsInsideLimit(long limit, long reserveCount);
/// <inheritdoc />
public Task<PurgeResult> PurgeAsync(
Context context,
long reserveSize,
IReadOnlyList<ContentHashWithLastAccessTimeAndReplicaCount> contentHashesWithInfo,
CancellationToken token)
{
if (_distributedEvictionSettings != null)
{
Contract.Assert(_distributedEvictionSettings.IsInitialized);
if (_distributedEvictionSettings.DistributedStore?.CanComputeLru == true)
{
return EvictDistributedWithDistributedStoreAsync(context, contentHashesWithInfo, reserveSize, token);
}
return EvictDistributedAsync(context, contentHashesWithInfo, reserveSize, token);
}
else
{
return EvictLocalAsync(context, contentHashesWithInfo, reserveSize, token);
}
}
/// <inheritdoc />
public virtual BoolResult IsInsideHardLimit(long reserveSize = 0)
{
@ -116,362 +77,5 @@ namespace BuildXL.Cache.ContentStore.Stores
// Do nothing.
}
}
private async Task<PurgeResult> EvictLocalAsync(
Context context,
IReadOnlyList<ContentHashWithLastAccessTimeAndReplicaCount> contentHashesWithInfo,
long reserveSize,
CancellationToken cts)
{
var result = new PurgeResult();
foreach (var contentHashInfo in contentHashesWithInfo)
{
if (StopPurging(reserveSize, cts, out _))
{
break;
}
var r = await _evictAsync(context, contentHashInfo, OnlyUnlinked);
if (!r.Succeeded)
{
var errorResult = new PurgeResult(r);
errorResult.Merge(result);
return errorResult;
}
result.Merge(r);
}
return result;
}
private bool StopPurging(long reserveSize, CancellationToken cts, out string stopReason)
{
if (cts.IsCancellationRequested)
{
stopReason = "cancellation requested";
return true;
}
if (IsInsideTargetLimit(reserveSize).Succeeded)
{
stopReason = "inside target limit";
return true;
}
stopReason = null;
return false;
}
/// <summary>
/// Evict hashes in LRU-ed order determined by remote last-access times.
/// </summary>
/// <param name="context">Context.</param>
/// <param name="hashesToPurge">Hashes in LRU-ed order based on local last-access time.</param>
/// <param name="reserveSize">Reserve size.</param>
/// <param name="cts">Cancellation token source.</param>
private async Task<PurgeResult> EvictDistributedWithDistributedStoreAsync(
Context context,
IReadOnlyList<ContentHashWithLastAccessTimeAndReplicaCount> hashesToPurge,
long reserveSize,
CancellationToken cts)
{
var result = new PurgeResult(reserveSize, hashesToPurge.Count, _quota.ToString());
var evictedContent = new List<ContentHash>();
var distributedStore = _distributedEvictionSettings.DistributedStore;
foreach (var contentHashInfo in distributedStore.GetHashesInEvictionOrder(context, hashesToPurge))
{
if (StopPurging(reserveSize, cts, out var stopReason))
{
result.StopReason = stopReason;
break;
}
var r = await _evictAsync(context, contentHashInfo, OnlyUnlinked);
if (!r.Succeeded)
{
var errorResult = new PurgeResult(r);
errorResult.Merge(result);
return errorResult;
}
if (r.SuccessfullyEvictedHash)
{
evictedContent.Add(contentHashInfo.ContentHash);
}
result.Merge(r);
}
var unregisterResult = await distributedStore.UnregisterAsync(context, evictedContent, cts);
if (!unregisterResult)
{
var errorResult = new PurgeResult(unregisterResult);
errorResult.Merge(result);
return errorResult;
}
return result;
}
/// <summary>
/// Evict hashes in LRU-ed order determined by remote last-access times.
/// </summary>
/// <param name="context">Context.</param>
/// <param name="hashesToPurge">Hashes in LRU-ed order based on local last-access time.</param>
/// <param name="reserveSize">Reserve size.</param>
/// <param name="cts">Cancellation token source.</param>
private async Task<PurgeResult> EvictDistributedAsync(
Context context,
IReadOnlyList<ContentHashWithLastAccessTimeAndReplicaCount> hashesToPurge,
long reserveSize,
CancellationToken cts)
{
// Track hashes for final update.
// Item1 marks whether content was removed from Redis because it was safe to evict.
// Item2 is the last-access time as seen by the data center.
var unpurgedHashes = new Dictionary<ContentHash, Tuple<bool, DateTime>>();
var evictedHashes = new List<ContentHash>();
// Purge all unpinned content where local last-access time is in sync with remote last-access time.
var purgeInfo = await AttemptPurgeAsync(context, hashesToPurge, cts, reserveSize, unpurgedHashes, evictedHashes);
if (!purgeInfo.finishedPurging)
{
return purgeInfo.purgeResult; // Purging encountered an error.
}
// Update unused hashes during eviction in the data center and locally.
foreach (var contentHashWithAccessTime in unpurgedHashes)
{
var unregisteredFromRedis = contentHashWithAccessTime.Value.Item1;
var remoteLastAccessTime = contentHashWithAccessTime.Value.Item2;
if (unregisteredFromRedis)
{
// Re-register hash in the content tracker for use
_distributedEvictionSettings.ReregisterHashQueue.Enqueue(contentHashWithAccessTime.Key);
}
// Update local last-access time with remote last-access time for future use.
// Don't update when remote last-access time is DateTime.MinValue because that means the hash has aged out of the content tracker.
if (remoteLastAccessTime != DateTime.MinValue)
{
// Last-access time will only be updated if it is more recent than the locally saved one
await _distributedEvictionSettings
.UpdateContentWithLastAccessTimeAsync(contentHashWithAccessTime.Key, contentHashWithAccessTime.Value.Item2);
}
}
if (_distributedEvictionSettings.DistributedStore != null)
{
var unregisterResult = await _distributedEvictionSettings.DistributedStore.UnregisterAsync(context, evictedHashes, cts);
if (!unregisterResult)
{
return new PurgeResult(unregisterResult);
}
}
return purgeInfo.purgeResult;
}
/// <summary>
/// Attempts to evict hashes in hashesToPurge until the reserveSize is met or all hashes with in-sync local and remote last-access times have been evicted.
/// </summary>
/// <param name="context">Context.</param>
/// <param name="hashesToPurge">Hashes sorted in LRU-ed order.</param>
/// <param name="cts">Cancellation token source.</param>
/// <param name="reserveSize">Reserve size.</param>
/// <param name="unpurgedHashes">Hashes that were checked in the data center but not evicted.</param>
/// <param name="evictedHashes">list of evicted hashes</param>
private async Task<(bool finishedPurging, PurgeResult purgeResult)> AttemptPurgeAsync(
Context context,
IReadOnlyList<ContentHashWithLastAccessTimeAndReplicaCount> hashesToPurge,
CancellationToken cts,
long reserveSize,
Dictionary<ContentHash, Tuple<bool, DateTime>> unpurgedHashes,
List<ContentHash> evictedHashes)
{
var finalResult = new PurgeResult();
var finishedPurging = false;
var trimOrGetLastAccessTimeAsync = _distributedEvictionSettings.TrimOrGetLastAccessTimeAsync;
var batchSize = _distributedEvictionSettings.LocationStoreBatchSize;
var pinAndSizeChecker = _distributedEvictionSettings.PinnedSizeChecker;
var replicaCreditInMinutes = _distributedEvictionSettings.ReplicaCreditInMinutes;
var hashQueue = CreatePriorityQueue(hashesToPurge, replicaCreditInMinutes);
while (!finishedPurging && hashQueue.Count > 0)
{
var contentHashesWithInfo = GetLruBatch(hashQueue, batchSize);
var unpinnedHashes = GetUnpinnedHashesAndCompilePinnedSize(context, contentHashesWithInfo, pinAndSizeChecker, finalResult);
if (!unpinnedHashes.Any())
{
continue; // No hashes in this batch are able to evicted because they're all pinned
}
// If unpurgedHashes contains hash, it was checked once in the data center. We relax the replica restriction on retry
var unpinnedHashesWithCheck = unpinnedHashes.Select(hash => Tuple.Create(hash, !unpurgedHashes.ContainsKey(hash.ContentHash))).ToList();
// Unregister hashes that can be safely evicted and get distributed last-access time for the rest
var contentHashesInfoRemoteResult =
await trimOrGetLastAccessTimeAsync(context, unpinnedHashesWithCheck, cts, UrgencyHint.High);
if (!contentHashesInfoRemoteResult.Succeeded)
{
var errorResult = new PurgeResult(contentHashesInfoRemoteResult);
errorResult.Merge(finalResult);
return (finishedPurging, errorResult);
}
var purgeInfo = await ProcessHashesForEvictionAsync(
contentHashesInfoRemoteResult.Data,
reserveSize,
cts,
context,
finalResult,
unpurgedHashes,
hashQueue,
evictedHashes);
if (purgeInfo.purgeResult != null)
{
return purgeInfo; // Purging encountered an error.
}
finishedPurging = purgeInfo.finishedPurging;
}
return (finishedPurging, finalResult);
}
// Get up to a page of records to delete, prioritizing the least-wanted files.
private IEnumerable<ContentHashWithLastAccessTimeAndReplicaCount> GetLruBatch(
PriorityQueue<ContentHashWithLastAccessTimeAndReplicaCount> hashQueue,
int batchSize)
{
var candidates = new List<ContentHashWithLastAccessTimeAndReplicaCount>(batchSize);
while (candidates.Count < batchSize && hashQueue.Count > 0)
{
var candidate = hashQueue.Top;
hashQueue.Pop();
candidates.Add(candidate);
}
return candidates;
}
private IList<ContentHashWithLastAccessTimeAndReplicaCount> GetUnpinnedHashesAndCompilePinnedSize(
Context context,
IEnumerable<ContentHashWithLastAccessTimeAndReplicaCount> hashesToPurge,
PinnedSizeChecker pinnedSizeChecker,
PurgeResult purgeResult)
{
long totalPinnedSize = 0; // Compile aggregate pinnedSize for the fail faster case
var unpinnedHashes = new List<ContentHashWithLastAccessTimeAndReplicaCount>();
foreach (var hashInfo in hashesToPurge)
{
var pinnedSize = pinnedSizeChecker(context, hashInfo.ContentHash);
if (pinnedSize >= 0)
{
totalPinnedSize += pinnedSize;
}
else
{
unpinnedHashes.Add(hashInfo);
}
}
purgeResult.MergePinnedSize(totalPinnedSize);
return unpinnedHashes;
}
private async Task<(bool finishedPurging, PurgeResult purgeResult)> ProcessHashesForEvictionAsync(
IList<ContentHashWithLastAccessTimeAndReplicaCount> contentHashesWithRemoteInfo,
long reserveSize,
CancellationToken cts,
Context context,
PurgeResult finalResult,
Dictionary<ContentHash, Tuple<bool, DateTime>> unpurgedHashes,
PriorityQueue<ContentHashWithLastAccessTimeAndReplicaCount> hashQueue,
List<ContentHash> evictedHashes)
{
bool finishedPurging = false;
foreach (var contentHashWithRemoteInfo in contentHashesWithRemoteInfo)
{
if (StopPurging(reserveSize, cts, out _))
{
finishedPurging = true;
}
var trackHash = true;
if (!finishedPurging)
{
// If not done purging and locations is negative, safe to evict immediately because contentHash has either:
// 1) Aged out of content tracker
// 2) Has matching last-access time (implying that the hash's last-access time is in sync with the datacenter)
if (contentHashWithRemoteInfo.SafeToEvict)
{
var evictResult = await _evictAsync(
context,
contentHashWithRemoteInfo,
OnlyUnlinked);
if (!evictResult.Succeeded)
{
var errorResult = new PurgeResult(evictResult);
errorResult.Merge(finalResult);
return (finishedPurging, errorResult);
}
finalResult.Merge(evictResult);
// SLIGHT HACK: Only want to keep track of hashes that unsuccessfully evicted and were unpinned at eviction time.
// We can determine that it was unpinned by PinnedSize, which is pinned bytes encountered during eviction.
trackHash = !evictResult.SuccessfullyEvictedHash && evictResult.PinnedSize == 0;
}
else
{
hashQueue.Push(contentHashWithRemoteInfo);
}
}
if (trackHash)
{
unpurgedHashes[contentHashWithRemoteInfo.ContentHash] = Tuple.Create(contentHashWithRemoteInfo.SafeToEvict, contentHashWithRemoteInfo.LastAccessTime);
}
else
{
// Don't track hash to update with remote last-access time because it was evicted
unpurgedHashes.Remove(contentHashWithRemoteInfo.ContentHash);
evictedHashes.Add(contentHashWithRemoteInfo.ContentHash);
}
}
return (finishedPurging, (PurgeResult)null);
}
private PriorityQueue<ContentHashWithLastAccessTimeAndReplicaCount> CreatePriorityQueue(
IReadOnlyList<ContentHashWithLastAccessTimeAndReplicaCount> hashesToPurge,
int replicaCreditInMinutes)
{
var hashQueue = new PriorityQueue<ContentHashWithLastAccessTimeAndReplicaCount>(
hashesToPurge.Count,
new ContentHashWithLastAccessTimeAndReplicaCount.ByLastAccessTime(replicaCreditInMinutes));
foreach (var hashInfo in hashesToPurge)
{
hashQueue.Push(hashInfo);
}
return hashQueue;
}
}
}

1
Public/Src/Cache/ContentStore/Test/Stores/DiskFreePercentRuleTests.cs
Просмотреть файл

@ -85,7 +85,6 @@ namespace ContentStoreTest.Stores
var quota = new DiskFreePercentQuota(Hard, Soft);
return new DiskFreePercentRule(
quota,
(context, contentHashInfo, onlyUnlinked) => Task.FromResult(evictResult ?? new EvictResult("error")),
mock,
new AbsolutePath(dummyPath));
}

7
Public/Src/Cache/ContentStore/Test/Stores/ElasticSizeRuleTests.cs
Просмотреть файл

@ -164,7 +164,6 @@ namespace ContentStoreTest.Stores
return new ElasticSizeRule(
ElasticSizeRule.DefaultHistoryWindowSize,
initialQuota,
(context, contentHashInfo, onlyUnlinked) => Task.FromResult(evictResult ?? new EvictResult("error")),
() => currentSize,
windowSize => new PinSizeHistory.ReadHistoryResult(new long[0], DateTime.UtcNow.Ticks),
FileSystem,
@ -186,7 +185,6 @@ namespace ContentStoreTest.Stores
return new ElasticSizeRule(
windowSize,
initialQuota,
(context, contentHashInfo, onlyUnlinked) => Task.FromResult(new EvictResult("error")),
currentSizeFunc,
readHistoryFunc,
FileSystem,
@ -233,11 +231,6 @@ namespace ContentStoreTest.Stores
return Rule.IsInsideTargetLimit(reserveSize);
}
public Task<PurgeResult> PurgeAsync(Context context, long reserveSize, IReadOnlyList<ContentHashWithLastAccessTimeAndReplicaCount> contentHashesWithInfo, CancellationToken token)
{
return Rule.PurgeAsync(context, reserveSize, contentHashesWithInfo, token);
}
public bool CanBeCalibrated => Rule.CanBeCalibrated;
public Task<CalibrateResult> CalibrateAsync()

1
Public/Src/Cache/ContentStore/Test/Stores/MaxSizeRuleTests.cs
Просмотреть файл

@ -65,7 +65,6 @@ namespace ContentStoreTest.Stores
{
return new MaxSizeRule(
new MaxSizeQuota(Hard, Soft),
(context, contentHashInfo, onlyUnlinked) => Task.FromResult(evictResult ?? new EvictResult("error")),
() => currentSize);
}
}

2
Public/Src/Cache/ContentStore/Test/Stores/MockDistributedLocationStore.cs
Просмотреть файл

@ -27,7 +27,7 @@ namespace ContentStoreTest.Stores
}
/// <inheritdoc />
public IEnumerable<ContentHashWithLastAccessTimeAndReplicaCount> GetHashesInEvictionOrder(Context context, IReadOnlyList<ContentHashWithLastAccessTimeAndReplicaCount> contentHashesWithInfo)
public IEnumerable<ContentEvictionInfo> GetHashesInEvictionOrder(Context context, IReadOnlyList<ContentHashWithLastAccessTimeAndReplicaCount> contentHashesWithInfo)
{
yield break;
}

44
Public/Src/Cache/ContentStore/Test/Stores/QuotaRuleTests.cs
Просмотреть файл

@ -30,50 +30,6 @@ namespace ContentStoreTest.Stores
{
}
[Theory]
[InlineData(true, true)]
[InlineData(true, false)]
[InlineData(false, true)]
[InlineData(false, false)]
public async Task NoPurgingIf(bool withinTargetQuota, bool canceled)
{
var evictResult = new EvictResult(10, 20, 30, lastAccessTime: DateTime.Now, effectiveLastAccessTime: null, successfullyEvictedHash: true, replicaCount: 1);
var rule = CreateRule(withinTargetQuota ? SizeWithinTargetQuota : SizeBeyondTargetQuota, evictResult);
using (var cts = new CancellationTokenSource())
{
if (canceled)
{
cts.Cancel();
}
var result = await rule.PurgeAsync(new Context(Logger), 1, new[] { new ContentHashWithLastAccessTimeAndReplicaCount(ContentHash.Random(), DateTime.UtcNow) }, cts.Token);
// No purging if within quota, there's an active sensitive session, or cancellation has been requested.
if (withinTargetQuota || canceled)
{
AssertNoPurging(result);
}
else
{
AssertPurged(result, evictResult);
}
}
}
[Fact]
public async Task ErrorPropagated()
{
var evictResult = new EvictResult($"{nameof(ErrorPropagated)} test error.");
var rule = CreateRule(SizeBeyondTargetQuota, evictResult);
using (var cts = new CancellationTokenSource())
{
var result = await rule.PurgeAsync(new Context(Logger), 10, new[] { new ContentHashWithLastAccessTimeAndReplicaCount(ContentHash.Random(), DateTime.UtcNow) }, cts.Token);
result.ShouldBeError(evictResult.ErrorMessage);
}
}
private static void AssertNoPurging(PurgeResult result)
{
result.EvictedFiles.Should().Be(0);

16
Public/Src/Cache/ContentStore/UtilitiesCore/CollectionUtilities.cs
Просмотреть файл

@ -1,6 +1,7 @@
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
using System;
using System.Collections.Generic;
namespace BuildXL.Cache.ContentStore.UtilitiesCore.Internal
@ -26,5 +27,20 @@ namespace BuildXL.Cache.ContentStore.UtilitiesCore.Internal
public static readonly T[] EmptyArray = new T[] { };
}
/// <summary>
/// Compare two operands and returns true if two instances are equivalent.
/// </summary>
public static bool IsCompareEquals<T>(T x1, T x2, out int compareResult, bool greatestFirst = false)
where T : IComparable<T>
{
compareResult = x1.CompareTo(x2);
if (greatestFirst)
{
compareResult = -compareResult;
}
return compareResult == 0;
}
}
}

2
Public/Src/Cache/MemoizationStore/DistributedTest/Metadata/RedisMetadataCacheFactoryTests.cs
Просмотреть файл

@ -2,6 +2,7 @@
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
using System;
using System.Diagnostics.ContractsLight;
using System.Threading.Tasks;
using BuildXL.Cache.ContentStore.Distributed.Redis.Credentials;
using BuildXL.Cache.ContentStore.FileSystem;
@ -163,6 +164,7 @@ namespace BuildXL.Cache.MemoizationStore.DistributedTest.Metadata
private IConnectionStringProvider CreateMockProvider(string connectionString)
{
Contract.RequiresNotNullOrEmpty(connectionString);
var mockProvider = new TestConnectionStringProvider(connectionString);
return mockProvider;
}

2
Public/Src/Utilities/Collections/MinMaxHeap.cs
Просмотреть файл

@ -48,7 +48,7 @@ namespace BuildXL.Utilities.Collections
/// </summary>
/// <param name="capacity">The maximum capacity of the priority queue. This value must be positive.</param>
/// <param name="comparer">The comparer used to determine item ordering. This value must be non-null.</param>
public MinMaxHeap(int capacity, Comparer<T> comparer)
public MinMaxHeap(int capacity, IComparer<T> comparer)
{
Contract.Requires(capacity >= 1);
Contract.RequiresNotNull(comparer);