Merged PR 532938: Add redis server for mac tests

This PR enables us to run redis server for mac tests.

However, tests have not been enabled yet, since they require tweaking.

This is a build in mac that runs with tests enabled, and everything passes: https://dev.azure.com/mseng/Domino/_build/results?buildId=11412761&view=results

Related work items: #1669645

Public/Src/Cache/ContentStore/DistributedTest/BuildXL.Cache.ContentStore.Distributed.Test.dsc

@@ -52,7 +52,14 @@ namespace DistributedTest {
             importFrom("WindowsAzure.Storage").pkg,
         ],
         runtimeContent: [
-            ...importFrom("Redis-64").Contents.all.contents,
+            {
+                subfolder: r`redisServer`,
+                contents: [
+                    ...BuildXLSdk.isTargetRuntimeOsx 
+                        ? importFrom("Redis-osx-x64").Contents.all.contents 
+                        : importFrom("Redis-64").Contents.all.contents,
+                ]
+            },
         ],
     });
 }

Public/Src/Cache/ContentStore/DistributedTest/ContentLocation/LocalLocationStoreDistributedContentTests.cs

@@ -53,7 +53,6 @@ namespace ContentStoreTest.Distributed.Sessions
     [Trait("Category", "Integration")]
     [Trait("Category", "LongRunningTest")]
     [Collection("Redis-based tests")]
-    [TestClassIfSupported(requiresWindowsBasedOperatingSystem: true)] // needs local redis-server.exe
     public class LocalLocationStoreDistributedContentTests : DistributedContentTests
     {
         private readonly LocalRedisFixture _redis;

Public/Src/Cache/ContentStore/DistributedTest/ContentLocation/RedisContentLocationStoreTests.cs

@@ -12,7 +12,6 @@ using BuildXL.Cache.ContentStore.Distributed;
 using BuildXL.Cache.ContentStore.Distributed.NuCache;
 using BuildXL.Cache.ContentStore.Distributed.Redis;
 using BuildXL.Cache.ContentStore.Distributed.Redis.Credentials;
-using BuildXL.Cache.ContentStore.FileSystem;
 using BuildXL.Cache.ContentStore.Tracing.Internal;
 using BuildXL.Cache.ContentStore.Interfaces.Distributed;
 using BuildXL.Cache.ContentStore.Interfaces.FileSystem;
@@ -31,16 +30,17 @@ using BuildXL.Cache.ContentStore.InterfacesTest;
 using StackExchange.Redis;
 using Xunit;
 using Xunit.Abstractions;
+using BuildXL.Cache.ContentStore.Interfaces.Utils;
+using System.IO;
 using Test.BuildXL.TestUtilities.Xunit;
 
 namespace ContentStoreTest.Distributed.ContentLocation
 {
-    [TestClassIfSupported(requiresWindowsBasedOperatingSystem: true)] // needs local redis-server.exe
     public abstract class RedisContentLocationStoreTests : TestWithOutput
     {
         private const string DefaultKeySpace = RedisContentLocationStoreFactory.DefaultKeySpace;
 
-        private static readonly AbsolutePath DefaultTempRoot = new AbsolutePath(@"Z:\TempRoot");
+        private static readonly AbsolutePath DefaultTempRoot = new AbsolutePath(XunitBuildXLTest.X("/Z/TempRoot"));
         private readonly IContentHasher _contentHasher = HashInfoLookup.Find(HashType.SHA256).CreateContentHasher();
         protected readonly MemoryClock _clock = new MemoryClock();
 
@@ -836,6 +836,8 @@ namespace ContentStoreTest.Distributed.ContentLocation
             _ = CreatePaths(1);
             var context = new Context(TestGlobal.Logger);
 
+            var path = new AbsolutePath(XunitBuildXLTest.X("/D/Dumps/CacheDump"));
+
             var storeFactory = new RedisContentLocationStoreFactory(
                 new EnvironmentConnectionStringProvider("TestConnectionString"),
                 new EnvironmentConnectionStringProvider("TestConnectionString2"),
@@ -844,9 +846,8 @@ namespace ContentStoreTest.Distributed.ContentLocation
                 "DM_S1CBPrefix", /* NOTE: This value may need to be changed if configured prefix is different for target environment. Find by using slowlog get 10 in redis console and find common prefix of commands */
                 new RedisContentLocationStoreConfiguration()
                 {
-                    Database = new RocksDbContentLocationDatabaseConfiguration(new AbsolutePath(@"D:\Dumps\CacheDump2"))
-                }
-                );
+                    Database = new RocksDbContentLocationDatabaseConfiguration(path)
+                });
 
             var r = await storeFactory.StartupAsync(context);
             r.ShouldBeSuccess();
@@ -872,6 +873,8 @@ namespace ContentStoreTest.Distributed.ContentLocation
             _ = CreatePaths(1);
             var context = new Context(TestGlobal.Logger);
 
+            var path = new AbsolutePath(XunitBuildXLTest.X("/D/Dumps/CacheDump"));
+
             var storeFactory = new RedisContentLocationStoreFactory(
                 new EnvironmentConnectionStringProvider("TestConnectionString"),
                 new EnvironmentConnectionStringProvider("TestConnectionString2"),
@@ -880,7 +883,7 @@ namespace ContentStoreTest.Distributed.ContentLocation
                 "DM_S1CBPrefix", /* NOTE: This value may need to be changed if configured prefix is different for target environment. Find by using slowlog get 10 in redis console and find common prefix of commands */
                 new RedisContentLocationStoreConfiguration()
                 {
-                    Database = new RocksDbContentLocationDatabaseConfiguration(new AbsolutePath(@"D:\Dumps\CacheDump2"))
+                    Database = new RocksDbContentLocationDatabaseConfiguration(path)
                 }
                 );
 
@@ -1087,7 +1090,7 @@ namespace ContentStoreTest.Distributed.ContentLocation
 
         protected string[] CreatePaths(int replicaCount)
         {
-            string pathPrefix = @"Z:\Temp";
+            string pathPrefix = XunitBuildXLTest.X("/Z/Temp");
             string[] paths = new string[replicaCount];
 
             for (int i = 0; i < replicaCount; i++)

Public/Src/Cache/ContentStore/DistributedTest/Redis/LocalRedisProcessDatabase.cs

@@ -16,6 +16,7 @@ using BuildXL.Cache.ContentStore.Interfaces.Time;
 using ContentStoreTest.Extensions;
 using StackExchange.Redis;
 using StackExchange.Redis.KeyspaceIsolation;
+using BuildXL.Cache.ContentStore.Interfaces.Utils;
 
 namespace ContentStoreTest.Distributed.Redis
 {
@@ -270,11 +271,13 @@ namespace ContentStoreTest.Distributed.Redis
                 throw;
             }
 
-            string redisServerPath = Path.GetFullPath("redis-server.exe");
+            var redisName = OperatingSystemHelper.IsWindowsOS ? "redis-server.exe" : "redis-server";
+
+            string redisServerPath = Path.GetFullPath(Path.Combine("redisServer", redisName));
 
             if (!File.Exists(redisServerPath))
             {
-                throw new InvalidOperationException("Could not find redis-server.exe at " + redisServerPath);
+                throw new InvalidOperationException($"Could not find {redisName} at {redisServerPath}");
             }
 
             int portNumber = 0;
@@ -306,7 +309,7 @@ port {portNumber}";
                 }
 
                 const bool createNoWindow = true;
-                _process = new ProcessUtility(redisServerPath, args, createNoWindow);
+                _process = new ProcessUtility(redisServerPath, args, createNoWindow, workingDirectory: Path.GetDirectoryName(redisServerPath));
 
                 _process.Start();
 

Public/Src/Cache/ContentStore/DistributedTest/Sessions/DistributedContentSessionTests.cs

@@ -25,7 +25,6 @@ namespace ContentStoreTest.Distributed.Sessions
 {
     [Collection("Redis-based tests")]
     [Trait("Category", "LongRunningTest")]
-    [TestClassIfSupported(requiresWindowsBasedOperatingSystem: true)] // needs local redis-server.exe
     public class DistributedContentSessionTests : ContentSessionTests
     {
         private readonly LocalRedisFixture _redis;

Public/Src/Cache/ContentStore/DistributedTest/Sessions/PinBetterDistributedContentSessionTests.cs

@@ -25,7 +25,6 @@ namespace ContentStoreTest.Distributed.Sessions
 {
     [Collection("Redis-based tests")]
     [Trait("Category", "LongRunningTest")]
-    [TestClassIfSupported(requiresWindowsBasedOperatingSystem: true)] // needs local redis-server.exe
     public class PinBetterDistributedContentSessionTests : ContentSessionTests
     {
         private readonly LocalRedisFixture _redis;

Public/Src/Cache/ContentStore/DistributedTest/Sessions/RedisDistributedContentTests.cs

@@ -33,7 +33,6 @@ namespace ContentStoreTest.Distributed.Sessions
     [Trait("Category", "Integration")]
     [Trait("Category", "LongRunningTest")]
     [Collection("Redis-based tests")]
-    [TestClassIfSupported(requiresWindowsBasedOperatingSystem: true)] // needs local redis-server.exe
     public class RedisDistributedContentTests : DistributedContentTests
     {
         private readonly LocalRedisFixture _redis;

Public/Src/Cache/ContentStore/DistributedTest/Stores/GrpcCopyContentTests.cs

@@ -29,7 +29,6 @@ using Xunit;
 
 namespace ContentStoreTest.Distributed.Stores
 {
-    [TestClassIfSupported(requiresWindowsBasedOperatingSystem: true)] // needs local redis-server.exe
     public class GrpcCopyContentTests : TestBase
     {
         private const int FileSize = 1000;

Public/Src/Cache/ContentStore/Library/Utils/ProcessUtility.cs

@@ -28,7 +28,8 @@ namespace BuildXL.Cache.ContentStore.Utils
         /// <param name="fileName">Application file name with which to start the process.</param>
         /// <param name="args">Arguments passed to the process.</param>
         /// <param name="createNoWindow">Whether the process shoudl not open a window.</param>
-        public ProcessUtility(string fileName, string args, bool createNoWindow)
+        /// <param name="workingDirectory">Working directory for the process</param>
+        public ProcessUtility(string fileName, string args, bool createNoWindow, string workingDirectory = null)
         {
             _createNoWindow = createNoWindow;
             _process = new Process();
@@ -43,6 +44,11 @@ namespace BuildXL.Cache.ContentStore.Utils
                 // ReSharper restore ConditionIsAlwaysTrueOrFalse
             };
 
+            if (workingDirectory != null)
+            {
+                _process.StartInfo.WorkingDirectory = workingDirectory;
+            }
+
             if (_createNoWindow)
             {
                 _process.OutputDataReceived += (sender, e) =>

Public/Src/Cache/MemoizationStore/DistributedTest/BuildXL.Cache.MemoizationStore.Distributed.Test.dsc

@@ -40,7 +40,14 @@ namespace DistributedTest {
             importFrom("System.Interactive.Async").pkg,
         ],
         runtimeContent: [
-            ...importFrom("Redis-64").Contents.all.contents,
+            {
+                subfolder: r`redisServer`,
+                contents: [
+                    ...BuildXLSdk.isTargetRuntimeOsx 
+                        ? importFrom("Redis-osx-x64").Contents.all.contents 
+                        : importFrom("Redis-64").Contents.all.contents,
+                ]
+            },
         ],
     });
 }

Public/Src/Cache/MemoizationStore/VstsTest/BuildXL.Cache.MemoizationStore.Vsts.Test.dsc

@@ -41,7 +41,14 @@ namespace VstsTest {
             importFrom("Newtonsoft.Json").pkg,
         ]},
         runtimeContent: [
-            ...importFrom("Redis-64").Contents.all.contents,
+            {
+                subfolder: r`redisServer`,
+                contents: [
+                    ...BuildXLSdk.isTargetRuntimeOsx 
+                        ? importFrom("Redis-osx-x64").Contents.all.contents 
+                        : importFrom("Redis-64").Contents.all.contents,
+                ]
+            },
             ...addIf(BuildXLSdk.isFullFramework,
                 importFrom("Microsoft.VisualStudio.Services.BlobStore.Client").pkg
             ),

cg/nuget/cgmanifest.json

@@ -2629,6 +2629,15 @@
         }
       }
     },
+    {
+      "Component": {
+        "Type": "NuGet",
+        "NuGet": {
+          "Name": "Redis-osx-x64",
+          "Version": "1.0.0"
+        }
+      }
+    },
     {
       "Component": {
         "Type": "NuGet",

config.dsc

@@ -215,7 +215,8 @@ config({
                     dependentPackageIdsToSkip: ["System.Threading.Tasks.Extensions"] },
                 { id: "System.Interactive.Async", version: "3.1.1" },
                 { id: "TransientFaultHandling.Core", version: "5.1.1209.1" },
-                { id: "Redis-64", version: "3.0.503" },
+                { id: "Redis-64", version: "3.0.503", osSkip: [ "macOS" ] },
+                { id: "Redis-osx-x64", version: "1.0.0", osSkip: [ "win" ] },
 
                 // Testing
                 { id: "System.Security.Cryptography.ProtectedData", version: "4.4.0"},