[macOS Catalina] Data partition filtering and file access adjustments (#655)

* Full support for macOS Catalina (e.g. data partition path filtering, test adjustments) and sandbox update to support CI and build scenarios on beta 4 and upwards.

Public/Src/Deployment/Tests.Osx/TestProj/tests/helpers.dsc

@@ -81,6 +81,7 @@ function runSingleXunitInstance(args: Arguments): DerivedFile[] {
             dependencies: [
                 ...globR(d`${args.testAssembly.parent}`, "*"),
                 f`/bin/sh`,
+                f`/bin/bash`,
                 f`/bin/ls`,
                 f`/bin/cat`,
             ],

Public/Src/Engine/Dll/MountsTable.cs

@@ -162,6 +162,7 @@ namespace BuildXL.Engine
             else
             {
                 table.AddStaticSystemMount("Applications", MacPaths.Applications, trackSourceFileChanges: true);
+                table.AddStaticSystemMount("Bin", MacPaths.Bin, trackSourceFileChanges: true);
                 table.AddStaticSystemMount("UsrBin", MacPaths.UsrBin, trackSourceFileChanges: true);
                 table.AddStaticSystemMount("UsrInclude", MacPaths.UsrInclude, trackSourceFileChanges: true);
                 table.AddStaticSystemMount("UsrLib", MacPaths.UsrLib, trackSourceFileChanges: true);

Public/Src/Engine/Processes/SandboxedKextConnection.cs

@@ -67,7 +67,7 @@ Use the the following command to load/reload the sandbox kernel extension and fi
         /// Until some automation for kernel extension building and deployment is in place, this number has to be kept in sync with the 'CFBundleVersion'
         /// inside the Info.plist file of the kernel extension code base. BuildXL will not work if a version mismatch is detected!
         /// </summary>
-        public const string RequiredKextVersionNumber = "1.94.99";
+        public const string RequiredKextVersionNumber = "1.97.99";
 
         /// <summary>
         /// See TN2420 (https://developer.apple.com/library/archive/technotes/tn2420/_index.html) on how versioning numbers are formatted in the Apple ecosystem

Public/Src/Engine/Scheduler/Scheduler.cs

@@ -1265,18 +1265,18 @@ namespace BuildXL.Scheduler
             m_serviceManager.Start(loggingContext, OperationTracker);
             m_apiServer?.Start(loggingContext);
             m_chooseWorkerCpu = new ChooseWorkerCpu(
-                loggingContext, 
-                m_configuration.Schedule.MaxChooseWorkerCpu, 
+                loggingContext,
+                m_configuration.Schedule.MaxChooseWorkerCpu,
                 m_workers,
-                m_pipQueue, 
-                PipGraph, 
+                m_pipQueue,
+                PipGraph,
                 m_fileContentManager);
 
             m_chooseWorkerCacheLookup = new ChooseWorkerCacheLookup(
-                loggingContext, 
-                m_configuration.Schedule.MaxChooseWorkerCacheLookup, 
-                m_configuration.Distribution.DistributeCacheLookups, 
-                m_workers, 
+                loggingContext,
+                m_configuration.Schedule.MaxChooseWorkerCacheLookup,
+                m_configuration.Distribution.DistributeCacheLookups,
+                m_workers,
                 m_pipQueue);
 
             ExecutionLog?.DominoInvocation(new DominoInvocationEventData(m_configuration));
@@ -2790,11 +2790,11 @@ namespace BuildXL.Scheduler
             ushort cpuUsageInPercent = m_scheduleConfiguration.UseHistoricalCpuUsageInfo ? RunningTimeTable[m_pipTable.GetPipSemiStableHash(pipId)].ProcessorsInPercents : (ushort)0;
 
             var runnablePip = RunnablePip.Create(
-                m_executePhaseLoggingContext, 
-                this, 
-                pipId, 
-                pipType, 
-                priority ?? GetPipPriority(pipId), 
+                m_executePhaseLoggingContext,
+                this,
+                pipId,
+                pipType,
+                priority ?? GetPipPriority(pipId),
                 m_executePipFunc,
                 cpuUsageInPercent);
 
@@ -2973,7 +2973,7 @@ namespace BuildXL.Scheduler
             if (runnablePip.AcquiredResourceWorker != null)
             {
                 // These steps run on the chosen worker so don't release the resources until they are completed.
-                // MaterializeOutputs can be also run on the workers; but we can release resources before that. 
+                // MaterializeOutputs can be also run on the workers; but we can release resources before that.
                 if (nextStep != PipExecutionStep.CacheLookup &&
                     nextStep != PipExecutionStep.ExecuteNonProcessPip &&
                     nextStep != PipExecutionStep.ExecuteProcess &&
@@ -4739,6 +4739,9 @@ namespace BuildXL.Scheduler
                             {
                                 ReportQueueSizeMB = m_configuration.Sandbox.KextReportQueueSizeMb,
                                 EnableReportBatching = m_configuration.Sandbox.KextEnableReportBatching,
+#if PLATFORM_OSX
+                                EnableCatalinaDataPartitionFiltering = OperatingSystemHelper.IsMacOSCatalinaOrHigher,
+#endif
                                 ResourceThresholds = new Sandbox.ResourceThresholds
                                 {
                                     CpuUsageBlockPercent = m_configuration.Sandbox.KextThrottleCpuUsageBlockThresholdPercent,

Public/Src/Engine/UnitTests/FingerprintStore/FingerprintStoreTests.cs

@@ -149,7 +149,7 @@ namespace Test.BuildXL.FingerprintStore
 
             var counters = testHooks.FingerprintStoreTestHooks.Counters;
             XAssert.AreEqual(1, counters.GetCounterValue(FingerprintStoreCounters.NumPathSetEntriesPut));
-            XAssert.AreEqual(1, counters.GetCounterValue(FingerprintStoreCounters.NumDirectoryMembershipEntriesPut));
+            XAssert.IsTrue(counters.GetCounterValue(FingerprintStoreCounters.NumDirectoryMembershipEntriesPut) >= 1);
         }
 
         /// <summary>
@@ -244,7 +244,7 @@ namespace Test.BuildXL.FingerprintStore
             // Make sure there are unique puts for each path set and directory membership
             // The FingerprintStore will not double-put the same content hash
             XAssert.AreEqual(2, counters.GetCounterValue(FingerprintStoreCounters.NumPathSetEntriesPut));
-            XAssert.AreEqual(2, counters.GetCounterValue(FingerprintStoreCounters.NumDirectoryMembershipEntriesPut));
+            XAssert.IsTrue(counters.GetCounterValue(FingerprintStoreCounters.NumDirectoryMembershipEntriesPut) >= 2);
         }
 
         /// <summary>
@@ -330,7 +330,7 @@ namespace Test.BuildXL.FingerprintStore
             // Pip unique output hash entry
             XAssert.AreEqual(1, testHooks.FingerprintStoreTestHooks.Counters.GetCounterValue(FingerprintStoreCounters.NumPipUniqueOutputHashEntriesGarbageCollected));
             // 1 pathset entry, 1 directory membership fingerprint entry
-            XAssert.AreEqual(2, testHooks.FingerprintStoreTestHooks.Counters.GetCounterValue(FingerprintStoreCounters.NumContentHashEntriesGarbageCollected));
+            XAssert.IsTrue(testHooks.FingerprintStoreTestHooks.Counters.GetCounterValue(FingerprintStoreCounters.NumContentHashEntriesGarbageCollected) >= 2);
 
             FingerprintStoreSession(ResultToStoreDirectory(build2), store =>
             {

Public/Src/Sandbox/MacOs/Sandbox/Src/BuildXLSandbox.cpp

@@ -21,6 +21,7 @@ static KextConfig sDefaultConfig =
 {
     .reportQueueSizeMB    = kSharedDataQueueSizeDefault,
     .enableReportBatching = false,
+    .enableCatalinaDataPartitionFiltering = false,
     .resourceThresholds   =
     {
         .cpuUsageBlock     = 0,

Public/Src/Sandbox/MacOs/Sandbox/Src/BuildXLSandbox.hpp

@@ -100,6 +100,8 @@ public:
     void stop(IOService *provider) override;
 
     void Configure(const KextConfig *config);
+    inline const KextConfig GetConfig() { return config_; }
+
     UInt32 GetReportQueueEntryCount();
 
     IOReturn AllocateNewClient(pid_t clientPid);

Public/Src/Sandbox/MacOs/Sandbox/Src/BuildXLSandboxClient.cpp

@@ -222,7 +222,7 @@ IOReturn BuildXLSandboxClient::sDebugCheck(BuildXLSandboxClient *target, void *r
 
 IOReturn BuildXLSandboxClient::sConfigure(BuildXLSandboxClient *target, void *reference, IOExternalMethodArguments *arguments)
 {
-    KextConfig *config = (KextConfig*)arguments->structureInput;
+    KextConfig *config = (KextConfig *) arguments->structureInput;
     target->sandbox_->Configure(config);
     return kIOReturnSuccess;
 }

Public/Src/Sandbox/MacOs/Sandbox/Src/BuildXLSandboxShared.hpp

@@ -190,6 +190,7 @@ typedef struct {
     uint reportQueueSizeMB;
     bool enableReportBatching;
     ResourceThresholds resourceThresholds;
+    bool enableCatalinaDataPartitionFiltering;
 } KextConfig;
 
 #define kMaxReportedPips 30

Public/Src/Sandbox/MacOs/Sandbox/Src/CLI/SandboxMonitor/main.cpp

@@ -320,7 +320,8 @@ int main(int argc, const char * argv[])
                    << endl
                    << endl;
             output << "Config     :: "
-                   << "Report Queue Size: " << kextCfg->reportQueueSizeMB << " MB"
+                   << "Catalina Data Partition filtering: " << (kextCfg->enableCatalinaDataPartitionFiltering ? "YES" : "NO")
+                   << ", Report Queue Size: " << kextCfg->reportQueueSizeMB << " MB"
                    << endl;
             output << "Thresholds :: "
                    << "Min Available RAM: " << thresholds->minAvailableRamMB << " MB"

Public/Src/Sandbox/MacOs/Sandbox/Src/Kauth/AccessHandler.cpp

@@ -205,6 +205,22 @@ static bool VNodeMatchesPath(vnode_t vp, vfs_context_t ctx, const char *path)
     return result;
 }
 
+const char* AccessHandler::IgnoreCatalinaDataPartitionPrefix(const char* path)
+{
+    if (!sandbox_->GetConfig().enableCatalinaDataPartitionFiltering)
+    {
+        return path;
+    }
+
+    const char *marker = path;
+    if (strprefix(marker, kCatalinaDataPartitionPrefix))
+    {
+        marker += kAdjustedCatalinaPrefixLength;
+    }
+
+    return marker;
+}
+
 bool AccessHandler::CheckAccess(vnode_t vp,
                                 vfs_context_t ctx,
                                 CheckFunc checker,
@@ -225,7 +241,8 @@ bool AccessHandler::CheckAccess(vnode_t vp,
     {
         // update policy and check again
         sandbox_->Counters()->numHardLinkRetries++;
-        *policy = PolicyForPath(lastLookupPath);
+
+        *policy = PolicyForPath(IgnoreCatalinaDataPartitionPrefix(lastLookupPath));
         checker(*policy, isDir, result);
         return true;
     }
@@ -245,7 +262,7 @@ AccessCheckResult AccessHandler::CheckAndReportInternal(FileOperation operation,
     Stopwatch stopwatch;
 
     // 1: check operation against given policy
-    PolicyResult policy = PolicyForPath(path);
+    PolicyResult policy = PolicyForPath(IgnoreCatalinaDataPartitionPrefix(path));
     AccessCheckResult result = AccessCheckResult::Invalid();
     if (vp != nullptr && ctx != nullptr)
     {

Public/Src/Sandbox/MacOs/Sandbox/Src/Kauth/AccessHandler.hpp

@@ -45,6 +45,10 @@ private:
         }
     }
 
+    const char *IgnoreCatalinaDataPartitionPrefix(const char* path);
+    const char *kCatalinaDataPartitionPrefix = "/System/Volumes/Data/";
+    const size_t kAdjustedCatalinaPrefixLength = strlen("/System/Volumes/Data");
+
 protected:
 
     BuildXLSandbox* GetSandbox()   const { return sandbox_; }

Public/Src/Sandbox/MacOs/Sandbox/Src/Resources/Info.plist

@@ -17,7 +17,7 @@
 	<key>CFBundleShortVersionString</key>
 	<string>1.0</string>
 	<key>CFBundleVersion</key>
-	<string>1.94.99</string>
+	<string>1.97.99</string>
 	<key>IOKitPersonalities</key>
 	<dict>
 		<key>BuildXLSandbox</key>

Public/Src/Tools/SandboxExec/Program.cs

@@ -175,7 +175,10 @@ namespace BuildXL.SandboxExec
                         KextConfig = new Sandbox.KextConfig
                         {
                             ReportQueueSizeMB = m_options.ReportQueueSizeMB,
-                            EnableReportBatching = m_options.EnableReportBatching
+                            EnableReportBatching = m_options.EnableReportBatching,
+#if PLATFORM_OSX
+                            EnableCatalinaDataPartitionFiltering = OperatingSystemHelper.IsMacOSCatalinaOrHigher
+#endif
                         },
                     })
                 : null;

Public/Src/Utilities/Interop/MacOS/Sandbox.cs

@@ -139,6 +139,9 @@ namespace BuildXL.Interop.MacOS
 
             /// <nodoc />
             public ResourceThresholds ResourceThresholds;
+            
+            /// <nodoc />
+            public bool EnableCatalinaDataPartitionFiltering;
         }
 
         /// <nodoc />

Public/Src/Utilities/UnitTests/Storage/FileContentTableUnitTests.cs

@@ -518,11 +518,12 @@ namespace Test.BuildXL.Storage
             Contract.Requires(table != null);
 
             using (
-                FileStream fs = File.Open(
+                FileStream fs = FileUtilities.CreateFileStream(
                     path.ToString(m_pathTable),
                     FileMode.Open,
                     strict ? FileAccess.ReadWrite : FileAccess.Read,
-                    FileShare.Read | FileShare.Delete))
+                    FileShare.Read | FileShare.Delete,
+                    FileOptions.Asynchronous, true))
             {
                 var info = table.RecordContentHash(fs, hash, strict: strict);
                 XAssert.AreEqual(hash, info.FileContentInfo.Hash);

Public/Src/Utilities/UnitTests/TestUtilities.XUnit/XunitBuildXLTest.cs

@@ -13,6 +13,7 @@ using BuildXL.Processes;
 using BuildXL.Utilities;
 using BuildXL.Utilities.Tracing;
 using Xunit.Abstractions;
+using static BuildXL.Interop.MacOS.Sandbox;
 
 namespace Test.BuildXL.TestUtilities.Xunit
 {
@@ -36,7 +37,13 @@ namespace Test.BuildXL.TestUtilities.Xunit
                         FailureCallback = (status, description) =>
                         {
                             XAssert.Fail($"Kernel extension failed.  Status: {status}.  Description: {description}");
-                        }
+                        },
+#if PLATFORM_OSX                        
+                        KextConfig = new KextConfig
+                        {
+                            EnableCatalinaDataPartitionFiltering = OperatingSystemHelper.IsMacOSCatalinaOrHigher
+                        }   
+#endif
                     })
                 : null);
 

Public/Src/Utilities/Utilities/OperatingSystemHelper.cs

@@ -68,6 +68,15 @@ namespace BuildXL.Utilities
         /// </summary>
         public static readonly bool IsMacOS = RuntimeInformation.IsOSPlatform(OSPlatform.OSX);
 
+        private static readonly Version CurrentMacOSVersion = GetOSVersionMacOS();
+
+#if PLATFORM_OSX
+        /// <summary>
+        /// Indicates if Catalina (10.15) or a higher macOS version is running on the host
+        /// </summary>
+        public static readonly bool IsMacOSCatalinaOrHigher = CurrentMacOSVersion.Major >= 10 && CurrentMacOSVersion.Minor >= 15;
+#endif
+
         private static readonly Tuple<string, string> ProcessorNameAndIdentifierMacOS =
             IsMacOS ? GetProcessorNameAndIdentifierMacOS() : Tuple.Create(String.Empty, String.Empty);
 
@@ -90,7 +99,7 @@ namespace BuildXL.Utilities
 #if FEATURE_CORECLR
             else if (IsMacOS)
             {
-                return GetOSVersionMacOS();
+                return string.Format("macOS {0}.{1}.{2}", CurrentMacOSVersion.Major, CurrentMacOSVersion.Minor, CurrentMacOSVersion.Build);
             }
 #endif
             // Extend this once we start supporting Linux etc.
@@ -322,7 +331,7 @@ namespace BuildXL.Utilities
                 {
                     result = "4.5";
                 }
-                
+
                 // This code should never execute. A non-null release key should mean
                 // that 4.5 or later is installed.
                 return result != null;
@@ -330,9 +339,9 @@ namespace BuildXL.Utilities
         }
 
         #region macOS Helpers
-#if FEATURE_CORECLR
 
-        private static string GetOSVersionMacOS()
+#if FEATURE_CORECLR
+        private static Version GetOSVersionMacOS()
         {
             try
             {
@@ -349,11 +358,7 @@ namespace BuildXL.Utilities
                                 string versionString = stringElement.Value;
                                 if (versionString != null)
                                 {
-                                    Version version = Version.Parse(versionString);
-                                    if (version.Major > 10 || (version.Major == 10 && version.Minor >= 13))
-                                    {
-                                        return string.Format("macOS High Sierra Version {0}.{1}.{2}", version.Major, version.Minor, version.Build);
-                                    }
+                                    return Version.Parse(versionString);
                                 }
                             }
                         }
@@ -366,7 +371,8 @@ namespace BuildXL.Utilities
             }
 #pragma warning restore ERP022
 
-            return String.Empty;
+            // Fallback is version 0.0
+            return new Version();
         }
 
         // This could potentially be replaced by a C wrapper querying the system information, the sysctl is just more convinient currently

config.microsoftInternal.dsc

@@ -14,7 +14,7 @@ export const pkgs = isMicrosoftInternal ? [
     { id: "BuildXL.DeviceMap", version: "0.0.1" },
 
     // Runtime dependencies used for macOS deployments
-    { id: "runtime.osx-x64.BuildXL", version: "1.96.99" },
+    { id: "runtime.osx-x64.BuildXL", version: "1.97.99" },
     { id: "Aria.Cpp.SDK", version: "8.5.6" },
 
     { id: "CB.QTest", version: "19.7.18.221046" },