Allow macOS sandbox implementation to be configurable (#785)

* Make macOS sandbox type configurable

Public/Src/Deployment/NugetPackages.dsc

@@ -134,7 +134,7 @@ namespace NugetPackages {
         targetRuntime: "win-x64"
     };
 
-    const xldblibrary = pack({
+    const xldblibrary = !canBuildAllPackagesOnThisHost ? undefined : pack({
         id: `${packageNamePrefix}.Xldb`,
         deployment: {
             contents: [ 
@@ -182,11 +182,11 @@ namespace NugetPackages {
                 cacheLibraries,
                 cacheInterfaces,
                 cacheHashing,
+                xldblibrary,
             ]),
             sdks,
             ...addIf(!BuildXLSdk.Flags.genVSSolution, osxX64, toolsOrchestrator),
             toolsSandBoxExec,
-            xldblibrary,
         ]
     };
 

Public/Src/Deployment/tools.dsc

@@ -16,16 +16,11 @@ namespace Tools {
 
         export const deployment : Deployment.Definition = {
             contents: [
-                importFrom("BuildXL.Tools").Xldb.Analyzer.withQualifier({
-                    configuration: qualifier.configuration,
-                    targetFramework: "netcoreapp3.0",
-                    targetRuntime: "win-x64"
-                }).exe,
+                importFrom("BuildXL.Tools").Xldb.Analyzer.exe,
             ]
         };
     
-        @@public
-        export const deployed = BuildXLSdk.DeploymentHelpers.deploy({
+        const deployed = BuildXLSdk.DeploymentHelpers.deploy({
             definition: deployment,
             targetLocation: r`${qualifier.configuration}/tools/XldbAnalyzer/${qualifier.targetRuntime}`,
         });
@@ -71,4 +66,4 @@ namespace Tools {
             targetLocation: r`${qualifier.configuration}/tools/Orchestrator/${qualifier.targetRuntime}`
         });
     }
-}
+}

Public/Src/Engine/Processes/SandboxedProcessPipExecutor.cs

@@ -3935,7 +3935,7 @@ namespace BuildXL.Processes
                         m_pip.SemiStableHash,
                         m_pip.GetDescription(m_context),
                         numErrors,
-                        Environment.NewLine + string.Join(Environment.NewLine, unexpectedSurvivingChildProcesses.Select(p => p.Path)));
+                        Environment.NewLine + string.Join(Environment.NewLine, unexpectedSurvivingChildProcesses.Select(p => $"{p.Path} ({p.ProcessId})")));
             }
 
             return numErrors;

Public/Src/Engine/Processes/Tracing/Log.cs

@@ -522,7 +522,7 @@ namespace BuildXL.Processes.Tracing
             Keywords = (int)(Keywords.UserMessage | Keywords.UserError),
             EventTask = (int)Tasks.PipExecutor,
             Message = EventConstants.PipPrefix + "Unexpected child processes survived: {2} process(es){3}")]
-        public abstract void PipProcessChildrenSurvivedError(LoggingContext context, long pipSemiStableHash, string pipDescription, int count,  string paths);
+        public abstract void PipProcessChildrenSurvivedError(LoggingContext context, long pipSemiStableHash, string pipDescription, int count, string paths);
 
         [GeneratedEvent(
             (int)LogEventId.PipProcessChildrenSurvivedTooMany,

Public/Src/Engine/Scheduler/Scheduler.cs

@@ -2354,8 +2354,8 @@ namespace BuildXL.Scheduler
                     bool hasMaterializationErrorHappened = m_executePhaseLoggingContext.ErrorsLoggedById.Contains((ushort)EventId.PipMaterializeDependenciesFromCacheFailure)
                         || m_executePhaseLoggingContext.ErrorsLoggedById.Contains((ushort)EventId.PipMaterializeDependenciesFailureUnrelatedToCache);
 
-                    // Early terminate the build if 
-                    // (1) StopOnFirstError is enabled or 
+                    // Early terminate the build if
+                    // (1) StopOnFirstError is enabled or
                     // (2) a materialization error is occurred in a distributed build.
                     bool earlyTerminate = m_scheduleConfiguration.StopOnFirstError || (hasMaterializationErrorHappened && IsDistributedMaster);
 
@@ -4759,10 +4759,20 @@ namespace BuildXL.Scheduler
                                 }
                             }
                         };
-                        
-                        sandboxConnection = OperatingSystemHelper.IsMacOSCatalinaOrHigher 
-                            ? ((ISandboxConnection) new SandboxConnectionES()) 
-                            : ((ISandboxConnection) new SandboxConnectionKext(config));
+
+                        switch (m_configuration.Sandbox.UnsafeSandboxConfiguration.SandboxKind)
+                        {
+                            case SandboxKind.MacOsEndpointSecurity:
+                            {
+                                sandboxConnection = (ISandboxConnection) new SandboxConnectionES();
+                                break;
+                            }
+                            default:
+                            {
+                                sandboxConnection = (ISandboxConnection) new SandboxConnectionKext(config);
+                                break;
+                            }
+                        }
 
                         if (m_performanceAggregator != null && config.KextConfig.Value.ResourceThresholds.IsProcessThrottlingEnabled())
                         {
@@ -4775,7 +4785,7 @@ namespace BuildXL.Scheduler
                                     : Convert.ToUInt32(Math.Round(availableRam));
                                 sandboxConnection.NotifyUsage(cpuUsageBasisPoints, availableRamMB);
                             };
-                        }   
+                        }
                     }
 
                     SandboxConnection = sandboxConnection;
@@ -4859,7 +4869,7 @@ namespace BuildXL.Scheduler
             if (m_configuration.Schedule.InputChanges.IsValid)
             {
                 inputChangeList = InputChangeList.CreateFromFile(
-                    loggingContext, 
+                    loggingContext,
                     m_configuration.Schedule.InputChanges.ToString(Context.PathTable),
                     m_configuration.Layout.SourceDirectory.ToString(Context.PathTable),
                     DirectoryTranslator);
@@ -6411,7 +6421,7 @@ namespace BuildXL.Scheduler
 
             if (cancelQueue)
             {
-                // We cancel the queue for more aggressive but still graceful cancellation. 
+                // We cancel the queue for more aggressive but still graceful cancellation.
                 m_pipQueue.Cancel();
             }
         }

Public/Src/Tools/SandboxExec/Program.cs

@@ -45,8 +45,9 @@ namespace BuildXL.SandboxExec
                     enableReportBatching: false,
                     reportQueueSizeMB: 1024,
                     enableTelemetry: true,
-                    processTimeout: (int)s_defaultProcessTimeOut,
-                    trackDirectoryCreation: false);
+                    processTimeout: (int) s_defaultProcessTimeOut,
+                    trackDirectoryCreation: false,
+                    useEndpointSecuritySandbox: false);
 
             /// <summary>
             /// When set to true, the output contains long instead of short description of reported accesses.
@@ -83,8 +84,13 @@ namespace BuildXL.SandboxExec
             /// </summary>
             public bool TrackDirectoryCreation { get; }
 
+            /// <summary>
+            /// When set, sandboxing is done using EndpointSecurity instead of the kernel extension
+            /// </summary>
+            public bool UseEndpointSecuritySandbox { get; }
+
             /// <nodoc />
-            public Options(bool verbose, bool logToStdOut, uint reportQueueSizeMB, bool enableTelemetry, int processTimeout, bool trackDirectoryCreation, bool enableReportBatching)
+            public Options(bool verbose, bool logToStdOut, uint reportQueueSizeMB, bool enableTelemetry, int processTimeout, bool trackDirectoryCreation, bool enableReportBatching, bool useEndpointSecuritySandbox)
             {
                 Verbose = verbose;
                 LogToStdOut = logToStdOut;
@@ -93,6 +99,7 @@ namespace BuildXL.SandboxExec
                 EnableTelemetry = enableTelemetry;
                 ProcessTimeout = processTimeout;
                 TrackDirectoryCreation = trackDirectoryCreation;
+                UseEndpointSecuritySandbox = useEndpointSecuritySandbox;
             }
         }
 
@@ -122,6 +129,9 @@ namespace BuildXL.SandboxExec
             /// <nodoc />
             public bool TrackDirectoryCreation;
 
+            /// <nodoc />
+            public bool UseEndpointSecuritySandbox;
+
             /// <nodoc />
             public OptionsBuilder() { }
 
@@ -135,10 +145,11 @@ namespace BuildXL.SandboxExec
                 EnableTelemetry = opts.EnableTelemetry;
                 ProcessTimeout = opts.ProcessTimeout;
                 TrackDirectoryCreation = opts.TrackDirectoryCreation;
+                UseEndpointSecuritySandbox = opts.UseEndpointSecuritySandbox;
             }
 
             /// <nodoc />
-            public Options Finish() => new Options(Verbose, LogToStdOut, ReportQueueSizeMB, EnableTelemetry, ProcessTimeout, TrackDirectoryCreation, EnableReportBatching);
+            public Options Finish() => new Options(Verbose, LogToStdOut, ReportQueueSizeMB, EnableTelemetry, ProcessTimeout, TrackDirectoryCreation, EnableReportBatching, UseEndpointSecuritySandbox);
         }
 
         private readonly Options m_options;
@@ -163,12 +174,12 @@ namespace BuildXL.SandboxExec
             m_options = options;
             s_crashCollector = OperatingSystemHelper.IsUnixOS ? new CrashCollectorMacOS(new[] { CrashType.SandboxExec, CrashType.Kernel }) : null;
             m_sandboxConnection = OperatingSystemHelper.IsUnixOS
-                ? 
+                ?
 #if PLATFORM_OSX
-                OperatingSystemHelper.IsMacOSCatalinaOrHigher 
-                    ? (ISandboxConnection) new SandboxConnectionES() 
-                    : 
-#endif                
+                m_options.UseEndpointSecuritySandbox
+                    ? (ISandboxConnection) new SandboxConnectionES()
+                    :
+#endif
                     (ISandboxConnection) new SandboxConnectionKext(
                         new SandboxConnectionKext.Config
                         {
@@ -233,7 +244,7 @@ namespace BuildXL.SandboxExec
             if (procArgs.Length < 1)
             {
                 var macOSUsageDescription = OperatingSystemHelper.IsUnixOS ? $" [/{ArgReportQueueSizeMB}:<1-1024>] [/{ArgEnableReportBatching}[+,-]]" : "";
-                PrintToStderr($"Usage: SandboxExec [[/{ArgVerbose}[+,-]] [/{ArgLogToStdOut}[+,-]] [/{ArgProcessTimeout}:seconds] [/{ArgTrackDirectoryCreation}] [/{ArgEnableStatistics}[+,-]]{macOSUsageDescription} --] executable [arg1 arg2 ...]");
+                PrintToStderr($"Usage: SandboxExec [[/{ArgVerbose}[+,-]] [/{ArgLogToStdOut}[+,-]] [/{ArgProcessTimeout}:seconds] [/{ArgTrackDirectoryCreation}] [/{ArgEnableStatistics}[+,-]] [/{ArgUseEndpointSecuritySandbox}[+,-]]{macOSUsageDescription} --] executable [arg1 arg2 ...]");
                 return 1;
             }
 
@@ -453,6 +464,7 @@ namespace BuildXL.SandboxExec
         private const string ArgEnableStatistics = "enableStatistics";
         private const string ArgProcessTimeout = "processTimeout";
         private const string ArgTrackDirectoryCreation = "trackDirectoryCreation";
+        private const string ArgUseEndpointSecuritySandbox = "useEndpointSecuritySandbox";
 
         private static Options ParseOptions(string[] toolArgs)
         {
@@ -496,6 +508,10 @@ namespace BuildXL.SandboxExec
                     case "d":
                         opts.TrackDirectoryCreation = CommandLineUtilities.ParseBooleanOption(opt);
                         break;
+                    case ArgUseEndpointSecuritySandbox:
+                    case "e":
+                        opts.UseEndpointSecuritySandbox = CommandLineUtilities.ParseBooleanOption(opt);
+                        break;
                     default:
                         throw new InvalidArgumentException($"Unrecognized option {opt.Name}");
                 }

Public/Src/Tools/Xldb.Proto/EnumsAndHelperStructs.proto

@@ -14,15 +14,15 @@ option csharp_namespace = "BuildXL.Xldb.Proto";
 // Enums
 //
 
-// Note that enum values use C++ scoping rules, meaning that enum values are siblings of their type, 
-// not children of it. Therefore, enum values must be unique within "BuildXL.Execution.Analyzer", not 
+// Note that enum values use C++ scoping rules, meaning that enum values are siblings of their type,
+// not children of it. Therefore, enum values must be unique within "BuildXL.Execution.Analyzer", not
 // just within a particular enum. If there are conflicting enum values, append the name of the enum in front.
 // ie: ChooseWorkerCpu ---> PipExecutionStep_ChooseWorkerCpu
 
 enum ExecutionEventId{
-    // Since we are using ExecutionEventId as in the key and protobuf 
-    // does not serialize default values, the pattern matching was bugging out. 
-    // A solution is to have an "invalid" enum constant take position 0. 
+    // Since we are using ExecutionEventId as in the key and protobuf
+    // does not serialize default values, the pattern matching was bugging out.
+    // A solution is to have an "invalid" enum constant take position 0.
     ExecutionEventId_Invalid = 0;
 
     FileArtifactContentDecided = 1;
@@ -31,7 +31,7 @@ enum ExecutionEventId{
     DirectoryMembershipHashed = 4;
 
     // Deprecated in favor of ProcessFingerprintComputation (enum 10)
-    ObservedInputs = 5; 
+    ObservedInputs = 5;
 
     ProcessExecutionMonitoringReported = 6;
     ExecutionEventId_ExtraEventDataReported = 7;
@@ -203,7 +203,7 @@ enum RequestedAccess{
     Enumerate = 8;
     EnumerationProbe = 16;
     // Read | Write = 1 | 2 = 3
-    ReadWrite = 3; 
+    ReadWrite = 3;
     // Read | Write | Probe | Enumerate | EnumerationProbe
     All = 31;
 }
@@ -296,6 +296,7 @@ enum SandboxKind{
     WinDetours = 2;
     MaxOsKext = 3;
     MacOsKextIgnoreFileAccesses = 4;
+    MacOsEndpointSecurity = 5;
 }
 
 enum DoubleWritePolicy{
@@ -522,7 +523,7 @@ message FileArtifact{
 }
 
 message FileContentInfo{
-    // [IsKnownExistenceFlag | Existence | IsKnownLengthFlag | Length]; 
+    // [IsKnownExistenceFlag | Existence | IsKnownLengthFlag | Length];
     int64 LengthAndExistence = 1;
 
     ContentHash Hash = 2;
@@ -532,19 +533,19 @@ message PipExecutionPerformance{
     int32 PipExecutionLevel = 1;
 
     google.protobuf.Timestamp ExecutionStart = 2;
-    
+
     google.protobuf.Timestamp ExecutionStop = 3;
 }
 
 message ProcessPipExecutionPerformance{
     google.protobuf.Duration ProcessExecutionTime = 1;
-    
+
     IOTypeCounters ReadCounters = 2;
 
     IOTypeCounters WriteCounters = 3;
 
     IOTypeCounters OtherCounters = 4;
-    
+
     google.protobuf.Duration UserTime = 5;
 
     google.protobuf.Duration KernelTime = 6;
@@ -772,7 +773,7 @@ message DirectoryFingerprint{
 
 message DirectoryOutput{
     DirectoryArtifact DirectoryArtifact = 1;
-    
+
     repeated FileArtifact fileArtifactArray = 2;
 }
 
@@ -786,7 +787,7 @@ message DirectoryArtifact{
 
 message FileOrDirectoryArtifact{
     FileArtifact FileArtifact = 1;
-    
+
     DirectoryArtifact DirectoryArtifact = 2;
 
     bool IsFile = 3;
@@ -802,7 +803,7 @@ message Pip{
     NodeId NodeId = 3;
 
     repeated Edge IncomingEdges = 4;
-    
+
     repeated Edge OutgoingEdges = 5;
 }
 
@@ -903,7 +904,7 @@ message SealDirectory{
     SealDirectoryKind Kind = 2;
 
     repeated string Patterns = 3;
-    
+
     AbsolutePath DirectoryRoot = 4;
 
     repeated FileArtifact Contents = 5;
@@ -967,7 +968,7 @@ message StandardInput{
 
 message EnvironmentVariable{
     string Name = 1;
-    
+
     PipData Value = 2;
 
     bool IsPassThrough = 3;

Public/Src/Utilities/Configuration/SandboxKind.cs

@@ -31,6 +31,11 @@ namespace BuildXL.Utilities.Configuration
         /// <summary>
         /// Like <see cref="MacOsKext"/> except that it gnores all reported file accesses.
         /// </summary>
-        MacOsKextIgnoreFileAccesses
+        MacOsKextIgnoreFileAccesses,
+
+        /// <summary>
+        /// macOs-specifc: Using the EndpointSecurity subsystem for sandboxing (available from 10.15+)
+        /// </summary>
+        MacOsEndpointSecurity
     }
 }

Public/Src/Utilities/UnitTests/TestUtilities.XUnit/XunitBuildXLTest.cs

@@ -28,15 +28,20 @@ namespace Test.BuildXL.TestUtilities.Xunit
     public abstract class XunitBuildXLTest : BuildXLTestBase, IDisposable
     {
         private static readonly Lazy<ISandboxConnection> s_sandboxConnection =  new Lazy<ISandboxConnection>(() =>
-            OperatingSystemHelper.IsUnixOS
+        {
 #if PLATFORM_OSX
-                ? OperatingSystemHelper.IsMacOSCatalinaOrHigher 
-                    ? (ISandboxConnection) new SandboxConnectionES() 
-                    : new SandboxConnectionKext(
-#else
-                    ? new SandboxConnectionKext(
+            var useEndpointSecuritySandboxEnv = Environment.GetEnvironmentVariable("BUILDXL_MACOS_ES_SANDBOX");
+            var useEndpointSecuritySandbox = !string.IsNullOrWhiteSpace(useEndpointSecuritySandboxEnv);
 #endif
-                        skipDisposingForTests: true, 
+            return OperatingSystemHelper.IsUnixOS
+#if PLATFORM_OSX
+                ? useEndpointSecuritySandbox
+                    ? (ISandboxConnection) new SandboxConnectionES()
+                    : (ISandboxConnection) new SandboxConnectionKext(
+#else
+                    ? (ISandboxConnection) new SandboxConnectionKext(
+#endif
+                        skipDisposingForTests: true,
                         config: new SandboxConnectionKext.Config
                         {
                             MeasureCpuTimes = true,
@@ -44,14 +49,16 @@ namespace Test.BuildXL.TestUtilities.Xunit
                             {
                                 XAssert.Fail($"Kernel extension failed.  Status: {status}.  Description: {description}");
                             },
-    #if PLATFORM_OSX                        
+    #if PLATFORM_OSX
                             KextConfig = new KextConfig
                             {
                                 EnableCatalinaDataPartitionFiltering = OperatingSystemHelper.IsMacOSCatalinaOrHigher
-                            }   
+                            }
     #endif
-                        })
-                : null);
+                        }
+                    )
+                    : null;
+        });
 
         /// <summary>
         /// Returns a static kernel connection object. Unit tests would spam the kernel extension if they need sandboxing, so we
@@ -105,7 +112,7 @@ namespace Test.BuildXL.TestUtilities.Xunit
         }
 
         /// <summary>
-        /// Value returned by <see cref="DiscoverCurrentlyExecutingXunitTestMethodFQN"/> when it cannot discover 
+        /// Value returned by <see cref="DiscoverCurrentlyExecutingXunitTestMethodFQN"/> when it cannot discover
         /// the currently executing XUnit method.
         /// </summary>
         protected const string UnknownXunitMethod = "<unknown>";
@@ -158,7 +165,7 @@ namespace Test.BuildXL.TestUtilities.Xunit
                     // Omit Windows type driver letter format e.g. /c
                     return unixPath.Substring(2);
                 }
-                
+
                 return unixPath;
             }
 

bxl.sh

@@ -48,7 +48,17 @@ function setMinimal() {
 }
 
 function setInternal() {
-    arg_Positional+=(/sandboxKind:macOsKext "/p:[Sdk.BuildXL]microsoftInternal=1")
+    arg_Positional+=("/p:[Sdk.BuildXL]microsoftInternal=1")
+    
+    for arg in "$@" 
+    do
+        to_lower=`printf '%s\n' "$arg" | awk '{ print tolower($0) }'`
+        if [[ " $to_lower " == *"endpointsecurity"* ]]; then
+            return
+        fi
+    done
+    
+    arg_Positional+=(/sandboxKind:macOsKext)
 }
 
 function compileWithBxl() {
@@ -123,7 +133,7 @@ if [[ -n "$arg_DeployDev" || -n "$arg_Minimal" ]]; then
 fi
 
 if [[ -n "$arg_Internal" ]]; then
-    setInternal
+    setInternal $@
 fi
 
 if [[ -n "$arg_UseDev" ]]; then