Merged PR 774198: Compact GC database on restore and before creating the last checkpoint

Compact GC database on restore and before creating the last checkpoint

Public/Src/Cache/LifetimeManager/Library/BlobLifetimeManager.cs

@@ -162,6 +162,10 @@ namespace BuildXL.Cache.BlobLifetimeManager.Library
                     {
                         await checkpointManager.RestoreCheckpointAsync(context, checkpointStateResult.Value!).ThrowIfFailure();
                         db = checkpointable.GetDatabase();
+
+                        // The checkpoint might not have been compacted when uploaded, so we should do it here.
+                        // This could happen if a checkpoint was created mid-run and GC didn't run to completion.
+                        db.Compact(context).ThrowIfFailure();
                     }
                     else
                     {
@@ -220,6 +224,10 @@ namespace BuildXL.Cache.BlobLifetimeManager.Library
                         {
                             context.Token.ThrowIfCancellationRequested();
 
+                            // Make sure to compact the database before creating a checkpoint to ensure we are being optimal
+                            // with the space we are using.
+                            db.Compact(context).ThrowIfFailure();
+
                             await checkpointManager.CreateCheckpointAsync(
                                 context,
                                 new EventSequencePoint(clock.UtcNow),

Public/Src/Cache/LifetimeManager/Library/RocksDbLifetimeDatabase.cs

@@ -12,7 +12,6 @@ using BuildXL.Cache.ContentStore.Distributed.Blob;
 using BuildXL.Cache.ContentStore.Distributed.NuCache;
 using BuildXL.Cache.ContentStore.Hashing;
 using BuildXL.Cache.ContentStore.Interfaces.FileSystem;
-using BuildXL.Cache.ContentStore.Interfaces.Logging;
 using BuildXL.Cache.ContentStore.Interfaces.Results;
 using BuildXL.Cache.ContentStore.Interfaces.Time;
 using BuildXL.Cache.ContentStore.Tracing;
@@ -217,6 +216,27 @@ namespace BuildXL.Cache.BlobLifetimeManager.Library
             return db;
         }
 
+        public BoolResult Compact(OperationContext context)
+        {
+            Dictionary<string, ColumnFamilyStats>? statsBeforeCompaction = null;
+            Dictionary<string, ColumnFamilyStats>? statsAfterCompaction = null;
+            return context.PerformOperation(
+                Tracer,
+                () =>
+                {
+                    statsBeforeCompaction = _db.GetStatisticsByColumnFamily(context);
+
+                    // By specifying no start and no limit, we compact the entire database.
+                    _db.CompactRange(start: (byte[]?)null, limit: null);
+
+                    statsAfterCompaction = _db.GetStatisticsByColumnFamily(context);
+
+                    return BoolResult.Success;
+                },
+                messageFactory: _ => $"Before compaction: {string.Join(", ", statsBeforeCompaction?.Select(kvp => $"{kvp.Key}=[{kvp.Value}]") ?? [])}. " +
+                    $"After compaction: {string.Join(", ", statsAfterCompaction?.Select(kvp => $"{kvp.Key}=[{kvp.Value}]") ?? [])}");
+        }
+
         public IAccessor GetAccessor(BlobNamespaceId namespaceId)
         {
             return new Accessor(this, namespaceId);

Public/Src/Cache/LifetimeManager/Library/RocksDbUtilities.cs

@@ -0,0 +1,50 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+using System;
+using System.Collections.Generic;
+using BuildXL.Cache.ContentStore.Tracing;
+using BuildXL.Cache.ContentStore.Tracing.Internal;
+using RocksDbSharp;
+
+namespace BuildXL.Cache.BlobLifetimeManager.Library
+{
+    public record ColumnFamilyStats(long? TotalLiveContentSize, long? TotalLiveFileSize, long? TotalFileSize);
+
+    internal static class RocksDbUtilities
+    {
+        private static readonly Tracer Tracer = new Tracer(nameof(RocksDbUtilities));
+
+        private static readonly string LiveDataSizeBytes = "rocksdb.estimate-live-data-size";
+        private static readonly string LiveFileSizeBytes = "rocksdb.live-sst-files-size";
+        private static readonly string TotalFileSizeBytes = "rocksdb.total-sst-files-size";
+
+        public static Dictionary<string, ColumnFamilyStats> GetStatisticsByColumnFamily(this RocksDb db, OperationContext context)
+        {
+            var result = new Dictionary<string, ColumnFamilyStats>();
+            foreach (KeyValuePair<string, ColumnFamilyHandle> kvp in db.GetColumnFamilyNames())
+            {
+                var liveContentSize = db.GetLongProperty(context, LiveDataSizeBytes, kvp.Key, kvp.Value);
+                var liveFileSize = db.GetLongProperty(context, LiveFileSizeBytes, kvp.Key, kvp.Value);
+                var totalFileSize = db.GetLongProperty(context, TotalFileSizeBytes, kvp.Key, kvp.Value);
+
+                result[kvp.Key] = new ColumnFamilyStats(liveContentSize, liveFileSize, totalFileSize);
+            }
+
+            return result;
+        }
+
+        private static long? GetLongProperty(this RocksDb db, OperationContext context, string propertyName, string columnFamilyName, ColumnFamilyHandle cfh)
+        {
+            try
+            {
+                return long.Parse(db.GetProperty(propertyName, cfh));
+            }
+            catch (Exception exception)
+            {
+                Tracer.Warning(context, exception, $"Error retrieving or parsing property '{propertyName}' for column '{columnFamilyName}'.");
+                return null;
+            }
+        }
+    }
+}

cg/nuget/cgmanifest.json

@@ -2661,7 +2661,7 @@
         "Type": "NuGet",
         "NuGet": {
           "Name": "RocksDbNative",
-          "Version": "8.1.1-20230829.3"
+          "Version": "8.1.1-20240321.8"
         }
       }
     },
@@ -2670,7 +2670,7 @@
         "Type": "NuGet",
         "NuGet": {
           "Name": "RocksDbSharp",
-          "Version": "8.1.1-20230829.3"
+          "Version": "8.1.1-20240321.8"
         }
       }
     },

config.dsc

@@ -169,8 +169,8 @@ config({
                 { id: "Microsoft.Windows.ProjFS", version: "1.2.19351.1" },
 
                 // RocksDb
-                { id: "RocksDbSharp", version: "8.1.1-20230829.3", alias: "RocksDbSharpSigned" },
-                { id: "RocksDbNative", version: "8.1.1-20230829.3" },
+                { id: "RocksDbSharp", version: "8.1.1-20240321.8", alias: "RocksDbSharpSigned" },
+                { id: "RocksDbNative", version: "8.1.1-20240321.8" },
 
                 { id: "JsonDiffPatch.Net", version: "2.1.0" },