Allow macOS sandbox implementation to be configurable (#785)

* Make macOS sandbox type configurable

Public/Src/Deployment/NugetPackages.dsc

@@ -134,7 +134,7 @@ namespace NugetPackages {
         targetRuntime: "win-x64"
     };
 
-    const xldblibrary = pack({
+    const xldblibrary = !canBuildAllPackagesOnThisHost ? undefined : pack({
         id: `${packageNamePrefix}.Xldb`,
         deployment: {
             contents: [ 
@@ -182,11 +182,11 @@ namespace NugetPackages {
                 cacheLibraries,
                 cacheInterfaces,
                 cacheHashing,
+                xldblibrary,
             ]),
             sdks,
             ...addIf(!BuildXLSdk.Flags.genVSSolution, osxX64, toolsOrchestrator),
             toolsSandBoxExec,
-            xldblibrary,
         ]
     };
 

Public/Src/Deployment/tools.dsc

@@ -16,16 +16,11 @@ namespace Tools {
 
         export const deployment : Deployment.Definition = {
             contents: [
-                importFrom("BuildXL.Tools").Xldb.Analyzer.withQualifier({
-                    configuration: qualifier.configuration,
-                    targetFramework: "netcoreapp3.0",
-                    targetRuntime: "win-x64"
-                }).exe,
+                importFrom("BuildXL.Tools").Xldb.Analyzer.exe,
             ]
         };
     
-        @@public
-        export const deployed = BuildXLSdk.DeploymentHelpers.deploy({
+        const deployed = BuildXLSdk.DeploymentHelpers.deploy({
             definition: deployment,
             targetLocation: r`${qualifier.configuration}/tools/XldbAnalyzer/${qualifier.targetRuntime}`,
         });

Public/Src/Engine/Processes/SandboxedProcessPipExecutor.cs

@@ -3935,7 +3935,7 @@ namespace BuildXL.Processes
                         m_pip.SemiStableHash,
                         m_pip.GetDescription(m_context),
                         numErrors,
-                        Environment.NewLine + string.Join(Environment.NewLine, unexpectedSurvivingChildProcesses.Select(p => p.Path)));
+                        Environment.NewLine + string.Join(Environment.NewLine, unexpectedSurvivingChildProcesses.Select(p => $"{p.Path} ({p.ProcessId})")));
             }
 
             return numErrors;

Public/Src/Engine/Scheduler/Scheduler.cs

@@ -4760,9 +4760,19 @@ namespace BuildXL.Scheduler
                             }
                         };
 
-                        sandboxConnection = OperatingSystemHelper.IsMacOSCatalinaOrHigher 
-                            ? ((ISandboxConnection) new SandboxConnectionES()) 
-                            : ((ISandboxConnection) new SandboxConnectionKext(config));
+                        switch (m_configuration.Sandbox.UnsafeSandboxConfiguration.SandboxKind)
+                        {
+                            case SandboxKind.MacOsEndpointSecurity:
+                            {
+                                sandboxConnection = (ISandboxConnection) new SandboxConnectionES();
+                                break;
+                            }
+                            default:
+                            {
+                                sandboxConnection = (ISandboxConnection) new SandboxConnectionKext(config);
+                                break;
+                            }
+                        }
 
                         if (m_performanceAggregator != null && config.KextConfig.Value.ResourceThresholds.IsProcessThrottlingEnabled())
                         {

Public/Src/Tools/SandboxExec/Program.cs

@@ -45,8 +45,9 @@ namespace BuildXL.SandboxExec
                     enableReportBatching: false,
                     reportQueueSizeMB: 1024,
                     enableTelemetry: true,
-                    processTimeout: (int)s_defaultProcessTimeOut,
-                    trackDirectoryCreation: false);
+                    processTimeout: (int) s_defaultProcessTimeOut,
+                    trackDirectoryCreation: false,
+                    useEndpointSecuritySandbox: false);
 
             /// <summary>
             /// When set to true, the output contains long instead of short description of reported accesses.
@@ -83,8 +84,13 @@ namespace BuildXL.SandboxExec
             /// </summary>
             public bool TrackDirectoryCreation { get; }
 
+            /// <summary>
+            /// When set, sandboxing is done using EndpointSecurity instead of the kernel extension
+            /// </summary>
+            public bool UseEndpointSecuritySandbox { get; }
+
             /// <nodoc />
-            public Options(bool verbose, bool logToStdOut, uint reportQueueSizeMB, bool enableTelemetry, int processTimeout, bool trackDirectoryCreation, bool enableReportBatching)
+            public Options(bool verbose, bool logToStdOut, uint reportQueueSizeMB, bool enableTelemetry, int processTimeout, bool trackDirectoryCreation, bool enableReportBatching, bool useEndpointSecuritySandbox)
             {
                 Verbose = verbose;
                 LogToStdOut = logToStdOut;
@@ -93,6 +99,7 @@ namespace BuildXL.SandboxExec
                 EnableTelemetry = enableTelemetry;
                 ProcessTimeout = processTimeout;
                 TrackDirectoryCreation = trackDirectoryCreation;
+                UseEndpointSecuritySandbox = useEndpointSecuritySandbox;
             }
         }
 
@@ -122,6 +129,9 @@ namespace BuildXL.SandboxExec
             /// <nodoc />
             public bool TrackDirectoryCreation;
 
+            /// <nodoc />
+            public bool UseEndpointSecuritySandbox;
+
             /// <nodoc />
             public OptionsBuilder() { }
 
@@ -135,10 +145,11 @@ namespace BuildXL.SandboxExec
                 EnableTelemetry = opts.EnableTelemetry;
                 ProcessTimeout = opts.ProcessTimeout;
                 TrackDirectoryCreation = opts.TrackDirectoryCreation;
+                UseEndpointSecuritySandbox = opts.UseEndpointSecuritySandbox;
             }
 
             /// <nodoc />
-            public Options Finish() => new Options(Verbose, LogToStdOut, ReportQueueSizeMB, EnableTelemetry, ProcessTimeout, TrackDirectoryCreation, EnableReportBatching);
+            public Options Finish() => new Options(Verbose, LogToStdOut, ReportQueueSizeMB, EnableTelemetry, ProcessTimeout, TrackDirectoryCreation, EnableReportBatching, UseEndpointSecuritySandbox);
         }
 
         private readonly Options m_options;
@@ -165,7 +176,7 @@ namespace BuildXL.SandboxExec
             m_sandboxConnection = OperatingSystemHelper.IsUnixOS
                 ?
 #if PLATFORM_OSX
-                OperatingSystemHelper.IsMacOSCatalinaOrHigher 
+                m_options.UseEndpointSecuritySandbox
                     ? (ISandboxConnection) new SandboxConnectionES()
                     :
 #endif
@@ -233,7 +244,7 @@ namespace BuildXL.SandboxExec
             if (procArgs.Length < 1)
             {
                 var macOSUsageDescription = OperatingSystemHelper.IsUnixOS ? $" [/{ArgReportQueueSizeMB}:<1-1024>] [/{ArgEnableReportBatching}[+,-]]" : "";
-                PrintToStderr($"Usage: SandboxExec [[/{ArgVerbose}[+,-]] [/{ArgLogToStdOut}[+,-]] [/{ArgProcessTimeout}:seconds] [/{ArgTrackDirectoryCreation}] [/{ArgEnableStatistics}[+,-]]{macOSUsageDescription} --] executable [arg1 arg2 ...]");
+                PrintToStderr($"Usage: SandboxExec [[/{ArgVerbose}[+,-]] [/{ArgLogToStdOut}[+,-]] [/{ArgProcessTimeout}:seconds] [/{ArgTrackDirectoryCreation}] [/{ArgEnableStatistics}[+,-]] [/{ArgUseEndpointSecuritySandbox}[+,-]]{macOSUsageDescription} --] executable [arg1 arg2 ...]");
                 return 1;
             }
 
@@ -453,6 +464,7 @@ namespace BuildXL.SandboxExec
         private const string ArgEnableStatistics = "enableStatistics";
         private const string ArgProcessTimeout = "processTimeout";
         private const string ArgTrackDirectoryCreation = "trackDirectoryCreation";
+        private const string ArgUseEndpointSecuritySandbox = "useEndpointSecuritySandbox";
 
         private static Options ParseOptions(string[] toolArgs)
         {
@@ -496,6 +508,10 @@ namespace BuildXL.SandboxExec
                     case "d":
                         opts.TrackDirectoryCreation = CommandLineUtilities.ParseBooleanOption(opt);
                         break;
+                    case ArgUseEndpointSecuritySandbox:
+                    case "e":
+                        opts.UseEndpointSecuritySandbox = CommandLineUtilities.ParseBooleanOption(opt);
+                        break;
                     default:
                         throw new InvalidArgumentException($"Unrecognized option {opt.Name}");
                 }

Public/Src/Tools/Xldb.Proto/EnumsAndHelperStructs.proto

@@ -296,6 +296,7 @@ enum SandboxKind{
     WinDetours = 2;
     MaxOsKext = 3;
     MacOsKextIgnoreFileAccesses = 4;
+    MacOsEndpointSecurity = 5;
 }
 
 enum DoubleWritePolicy{

Public/Src/Utilities/Configuration/SandboxKind.cs

@@ -31,6 +31,11 @@ namespace BuildXL.Utilities.Configuration
         /// <summary>
         /// Like <see cref="MacOsKext"/> except that it gnores all reported file accesses.
         /// </summary>
-        MacOsKextIgnoreFileAccesses
+        MacOsKextIgnoreFileAccesses,
+
+        /// <summary>
+        /// macOs-specifc: Using the EndpointSecurity subsystem for sandboxing (available from 10.15+)
+        /// </summary>
+        MacOsEndpointSecurity
     }
 }

Public/Src/Utilities/UnitTests/TestUtilities.XUnit/XunitBuildXLTest.cs

@@ -28,13 +28,18 @@ namespace Test.BuildXL.TestUtilities.Xunit
     public abstract class XunitBuildXLTest : BuildXLTestBase, IDisposable
     {
         private static readonly Lazy<ISandboxConnection> s_sandboxConnection =  new Lazy<ISandboxConnection>(() =>
-            OperatingSystemHelper.IsUnixOS
+        {
 #if PLATFORM_OSX
-                ? OperatingSystemHelper.IsMacOSCatalinaOrHigher 
+            var useEndpointSecuritySandboxEnv = Environment.GetEnvironmentVariable("BUILDXL_MACOS_ES_SANDBOX");
+            var useEndpointSecuritySandbox = !string.IsNullOrWhiteSpace(useEndpointSecuritySandboxEnv);
+#endif
+            return OperatingSystemHelper.IsUnixOS
+#if PLATFORM_OSX
+                ? useEndpointSecuritySandbox
                     ? (ISandboxConnection) new SandboxConnectionES()
-                    : new SandboxConnectionKext(
+                    : (ISandboxConnection) new SandboxConnectionKext(
 #else
-                    ? new SandboxConnectionKext(
+                    ? (ISandboxConnection) new SandboxConnectionKext(
 #endif
                         skipDisposingForTests: true,
                         config: new SandboxConnectionKext.Config
@@ -50,8 +55,10 @@ namespace Test.BuildXL.TestUtilities.Xunit
                                 EnableCatalinaDataPartitionFiltering = OperatingSystemHelper.IsMacOSCatalinaOrHigher
                             }
     #endif
-                        })
-                : null);
+                        }
+                    )
+                    : null;
+        });
 
         /// <summary>
         /// Returns a static kernel connection object. Unit tests would spam the kernel extension if they need sandboxing, so we

bxl.sh

@@ -48,7 +48,17 @@ function setMinimal() {
 }
 
 function setInternal() {
-    arg_Positional+=(/sandboxKind:macOsKext "/p:[Sdk.BuildXL]microsoftInternal=1")
+    arg_Positional+=("/p:[Sdk.BuildXL]microsoftInternal=1")
+    
+    for arg in "$@" 
+    do
+        to_lower=`printf '%s\n' "$arg" | awk '{ print tolower($0) }'`
+        if [[ " $to_lower " == *"endpointsecurity"* ]]; then
+            return
+        fi
+    done
+    
+    arg_Positional+=(/sandboxKind:macOsKext)
 }
 
 function compileWithBxl() {
@@ -123,7 +133,7 @@ if [[ -n "$arg_DeployDev" || -n "$arg_Minimal" ]]; then
 fi
 
 if [[ -n "$arg_Internal" ]]; then
-    setInternal
+    setInternal $@
 fi
 
 if [[ -n "$arg_UseDev" ]]; then