Execute process pips externally via sandboxed process executor tool (#159)

This change introduces a so-called sandboxed process executor tool that takes a sandboxed process info as an input and outputs a sandboxed process result containing details of file accesses.

The tool will be used to run process pips that require admin privilege, and the tool will run inside a VM. Traditionally, BuildXL in SandboxedProcessPipExecutor will create a detoured child process and communicate with the child process by means of pipes. For process pips that require admin, SandboxedProcessPipExecutor will (1) serialize sandboxed process info, (2) launch the sandboxed process executor tool, and (3) deserialize sandboxed process result produce in (2).

The sandboxed process executor tool will either replace QuickBuild's Tracker.exe or be called by QuickBuild's Tracker.exe. The cutting layer allows the two scenarios to be done, but the latter is the easiest.

Private/Wdg/ExecutionLogSdk/ProcessDescriptor.cs

@@ -5,6 +5,7 @@ using System;
 using System.Collections.Generic;
 using System.Diagnostics.CodeAnalysis;
 using BuildXL.Pips;
+using BuildXL.Native.IO;
 
 namespace Tool.ExecutionLogSdk
 {

Private/Wdg/ExecutionLogSdk/ProcessInstanceDescriptor.cs

@@ -4,6 +4,7 @@
 using System;
 using System.Diagnostics.CodeAnalysis;
 using BuildXL.Pips;
+using BuildXL.Native.IO;
 
 namespace Tool.ExecutionLogSdk
 {

Public/Sdk/Public/Transformers/Transformer.Execute.dsc

@@ -114,6 +114,9 @@ namespace Transformer {
         /** Set outputs to remain writable */
         keepOutputsWritable?: boolean;
 
+        /** Privilege level required by this process to execute. */
+        privilegeLevel?: "standard" | "admin";
+
         /** Whether this process should run in an isolated container (i.e. filesystem isolation)
          * When running in a container, the isolation level can be controlled by 'containerIsolationLevel' field.
          * Note: this is an experimental feature for now, use at your own risk 

Public/Sdk/SelfHost/BuildXL/BuildXLSdk.dsc

@@ -131,6 +131,13 @@ namespace Flags {
     */
     @@public
     export const isQTestEnabled = isMicrosoftInternal && Environment.getFlag("[Sdk.BuildXL]useQTest");
+
+    /** 
+     * Whether we are generating VS solution. 
+     * We are using this flag to filter out some deployment items that can cause race in the generated VS project files.
+     */
+    @@public
+    export const genVSSolution = Environment.getFlag("[Sdk.BuildXL]GenerateVSSolution");
 }
 
 @@public

Public/Src/App/Bxl/Args.cs

@@ -203,6 +203,9 @@ namespace BuildXL
                             "additionalConfigFile",
                             "ac",
                             opt => ParsePathOption(opt, pathTable, startupConfiguration.AdditionalConfigFiles)),
+                        OptionHandlerFactory.CreateOption(
+                            "adminRequiredProcessExecutionMode",
+                            opt => sandboxConfiguration.AdminRequiredProcessExecutionMode = CommandLineUtilities.ParseEnumOption<AdminRequiredProcessExecutionMode>(opt)),
                         OptionHandlerFactory.CreateBoolOption(
                             "allowFetchingCachedGraphFromContentCache",
                             sign => cacheConfiguration.AllowFetchingCachedGraphFromContentCache = sign),
@@ -1215,7 +1218,15 @@ namespace BuildXL
                     // profile redirection only happens on Windows
                     layoutConfiguration.RedirectedUserProfileJunctionRoot = AbsolutePath.Invalid;
                 }
-                
+
+                if (OperatingSystemHelper.IsUnixOS)
+                {
+                    // TODO: Non Windows OS doesn't support admin-required process external execution mode.
+                    if (sandboxConfiguration.AdminRequiredProcessExecutionMode != AdminRequiredProcessExecutionMode.Internal)
+                    {
+                        throw CommandLineUtilities.Error(Strings.Args_AdminRequiredProcessExecutionMode_NotSupportedOnNonWindows, sandboxConfiguration.AdminRequiredProcessExecutionMode.ToString());
+                    }
+                }
 
                 // Disable reuseEngineState (enabled by default) in case of /server- or /cacheGraph- (even if /reuseEngineState+ is passed)
                 if (configuration.Server == ServerMode.Disabled || !cacheConfiguration.CacheGraph)

Public/Src/App/Bxl/HelpText.cs

@@ -750,6 +750,11 @@ namespace BuildXL
                 "/useFileContentTable[+|-]",
                 Strings.HelpText_DisplayHelp_UseFileContentTable,
                 HelpLevel.Verbose);
+
+            hw.WriteOption(
+                "/adminRequiredProcessExecutionMode:<mode>",
+                Strings.HelpText_DisplayHelp_AdminRequiredProcessExecutionMode,
+                HelpLevel.Verbose);
             #endregion
 
             hw.WriteBanner(

Public/Src/App/Bxl/Strings.resx

@@ -997,7 +997,13 @@ Example: ad2d42d2ec5d2ca0c0b7ad65402d07c7ef40b91e</value>
   <data name="HelpText_DisplayHelp_RunInContainerAndAllowDoubleWrites" xml:space="preserve">
     <value>Configures the default for all scheduled pips to run in a container with output isolation, allowing double writes to occur. Individual pips can override this setting. This is an unsafe option.</value>
   </data>
+  <data name="HelpText_DisplayHelp_AdminRequiredProcessExecutionMode" xml:space="preserve">
+    <value>Mode for running processes that required admin privilege. Allowed values are 'Internal' (BuildXL starts the process as an immediate child process), 'ExternalTool' (BuildXL starts a sandboxed process executor tool as a child process, and in turn the tool starts the admin-required process as its child process), 'ExternalVM' (BuildXL sends command to VM to execute a sandboxed process executor tool in VM, and in turn the tool starts the admin-required process as its child process) . For Internal and ExternalTool, the process will be run with the same elevation level as BuildXL. For ExternalVM, the process will be run with the elevation level in the VM. Defaults to 'Internal'.</value>
+  </data>
   <data name="HelpText_DisplayHelp_AdditionalConfigFile" xml:space="preserve">
     <value>Additional configuration files that can contain module definitions (short form: /ac)</value>
   </data>
+  <data name="Args_AdminRequiredProcessExecutionMode_NotSupportedOnNonWindows" xml:space="preserve">
+    <value>Admin-required process execution mode '{0}' is not supported on non-Windows OS</value>
+  </data>
 </root>

Public/Src/Deployment/NugetPackages.dsc

@@ -120,8 +120,7 @@ namespace NugetPackages {
         contents: [
             ...addIfLazy(Context.getCurrentHost().os === "win", () => [
                 net472,
-                net461,
-                winX64
+                ...(BuildXLSdk.Flags.genVSSolution ? [] : [net461, winX64])
             ]),
             osxX64,
             sdks,

Public/Src/Deployment/buildXL.dsc

@@ -35,14 +35,16 @@ namespace BuildXL {
                             importFrom("BuildXL.Explorer").App.app.appFolder
                         ]
                     },
-                    {
-                        subfolder: r`bxp-server`,
-                        contents: [
-                            importFrom("BuildXL.Explorer").Server.withQualifier(
-                                Object.merge<BuildXLSdk.NetCoreAppQualifier>(qualifier, {targetFramework: "netcoreapp2.2"})
-                            ).exe
-                        ]
-                    },
+                    ...(BuildXLSdk.Flags.genVSSolution
+                        ? []
+                        : [ {
+                                subfolder: r`bxp-server`,
+                                    contents: [
+                                    importFrom("BuildXL.Explorer").Server.withQualifier(
+                                        Object.merge<BuildXLSdk.NetCoreAppQualifier>(qualifier, {targetFramework: "netcoreapp2.2"})
+                                    ).exe
+                                ]
+                            } ] ),
                     {
                         subfolder: r`MsBuildGraphBuilder`,
                         contents: qualifier.targetFramework === "netcoreapp2.2" ? [] : [
@@ -64,7 +66,12 @@ namespace BuildXL {
                             importFrom("BuildXL.Tools").CMakeRunner.exe,
                         ]
                     },
-                    
+                    {
+                        subfolder: r`SandboxedProcessExecutor`,
+                        contents: [
+                            importFrom("BuildXL.Tools").SandboxedProcessExecutor.exe,
+                        ]
+                    },
                 ]
             }])
         ]

Public/Src/Engine/Processes/AsyncProcessExecutor.cs

@@ -0,0 +1,237 @@
+// Copyright (c) Microsoft. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+using System;
+using System.ComponentModel;
+using System.Diagnostics;
+using System.Diagnostics.ContractsLight;
+using System.Threading.Tasks;
+using BuildXL.Interop;
+using BuildXL.Utilities;
+using BuildXL.Utilities.Tasks;
+
+namespace BuildXL.Processes
+{
+    /// <summary>
+    /// Executes process asynchronously.
+    /// </summary>
+    public class AsyncProcessExecutor : IDisposable
+    {
+        private readonly TaskSourceSlim<Unit> m_processExitedTcs = TaskSourceSlim.Create<Unit>();
+        private readonly TaskSourceSlim<Unit> m_stdoutFlushedTcs = TaskSourceSlim.Create<Unit>();
+        private readonly TaskSourceSlim<Unit> m_stderrFlushedTcs = TaskSourceSlim.Create<Unit>();
+
+        /// <summary>
+        /// Underlying process
+        /// </summary>
+        public Process Process { get; private set; }
+
+        /// <summary>
+        /// Start time.
+        /// </summary>
+        public DateTime StartTime { get; private set; }
+
+        /// <summary>
+        /// Exit time.
+        /// </summary>
+        public DateTime ExitTime { get; private set; }
+
+        /// <summary>
+        /// Flag indicating if the process was killed.
+        /// </summary>
+        public bool Killed { get; private set; }
+
+        /// <summary>
+        /// Flag indicating if the process timed out.
+        /// </summary>
+        public bool TimedOut { get; private set; }
+
+        /// <summary>
+        /// Task that completes once this process dies.
+        /// </summary>
+        private Task WhenExited => m_processExitedTcs.Task;
+
+        /// <summary>
+        /// Checks if process exit is completed.
+        /// </summary>
+        public bool ExitCompleted => WhenExited.IsCompleted;
+
+        /// <summary>
+        /// Checks if standard out flush is completed.
+        /// </summary>
+        public bool StdOutCompleted => m_stdoutFlushedTcs.Task.IsCompleted;
+
+        /// <summary>
+        /// Checks if standard error flush is completed.
+        /// </summary>
+        public bool StdErrCompleted => m_stderrFlushedTcs.Task.IsCompleted;
+
+        /// <summary>
+        /// Timeout.
+        /// </summary>
+        private readonly TimeSpan m_timeout;
+
+        /// <summary>
+        /// Sandboxed process info for provenence purpose.
+        /// </summary>
+        private readonly SandboxedProcessInfo m_sandboxedProcessInfo;
+
+        private int m_processId = -1;
+
+        /// <summary>
+        /// Process id.
+        /// </summary>
+        public int ProcessId => m_processId != -1 ? m_processId : (m_processId = Process.Id);
+
+        /// <summary>
+        /// Gets active peak memory usage.
+        /// </summary>
+        public ulong? GetActivePeakMemoryUsage()
+        {
+            try
+            {
+                if (Process == null || Process.HasExited)
+                {
+                    return null;
+                }
+
+                return Dispatch.GetActivePeakMemoryUsage(Process.Handle, ProcessId);
+            }
+#pragma warning disable ERP022 // Unobserved exception in generic exception handler
+            catch
+            {
+                return null;
+            }
+#pragma warning restore ERP022 // Unobserved exception in generic exception handler
+        }
+
+        /// <summary>
+        /// Creates an instance of <see cref="AsyncProcessExecutor"/>.
+        /// </summary>
+        public AsyncProcessExecutor(
+            Process process, 
+            TimeSpan timeout,
+            Action<string> outputBuilder = null,
+            Action<string> errorBuilder = null,
+            SandboxedProcessInfo sandboxedProcessInfo = null)
+        {
+            Contract.Requires(process != null);
+
+            Process = process;
+            Process.Exited += (sender, e) => m_processExitedTcs.TrySetResult(Unit.Void);
+
+            if (outputBuilder != null)
+            {
+                process.OutputDataReceived += (sender, e) => FeedOutputBuilder(m_stdoutFlushedTcs, e.Data, outputBuilder);
+            }
+
+            if (errorBuilder != null)
+            {
+                process.ErrorDataReceived += (sender, e) => FeedOutputBuilder(m_stderrFlushedTcs, e.Data, errorBuilder);
+            }
+
+            m_timeout = timeout;
+            m_sandboxedProcessInfo = sandboxedProcessInfo;
+        }
+
+        /// <summary>
+        /// Starts process.
+        /// </summary>
+        public void Start()
+        {
+            if (!System.IO.File.Exists(Process.StartInfo.FileName))
+            {
+                ThrowBuildXLException($"Process creation failed: File '{Process.StartInfo.FileName}' not found", new Win32Exception(0x2));
+            }
+
+            try
+            {
+                Process.Start();
+            }
+            catch (Win32Exception e)
+            {
+                ThrowBuildXLException($"Failed to start process '{Process.StartInfo.FileName}'", e);
+            }
+
+            Process.BeginOutputReadLine();
+            Process.BeginErrorReadLine();
+            StartTime = DateTime.UtcNow;
+        }
+
+        /// <summary>
+        /// Waits for process to exit or to get killed due to timed out.
+        /// </summary>
+        public async Task WaitForExitAsync(Func<Task> getProcessReport = null)
+        {
+            var finishedTask = await Task.WhenAny(Task.Delay(m_timeout), WhenExited);
+            ExitTime = DateTime.UtcNow;
+
+            var timedOut = finishedTask != WhenExited;
+            if (timedOut)
+            {
+                TimedOut = true;
+                await KillAsync();
+            }
+
+            if (getProcessReport != null)
+            {
+                await getProcessReport();
+            }
+
+            await Task.WhenAll(m_stdoutFlushedTcs.Task, m_stderrFlushedTcs.Task);
+        }
+
+        /// <summary>
+        /// Kills process.
+        /// </summary>
+        public Task KillAsync()
+        {
+            Contract.Requires(Process != null);
+
+            try
+            {
+                if (!Process.HasExited)
+                {
+                    Process.Kill();
+                }
+            }
+            catch (Exception e) when (e is Win32Exception || e is InvalidOperationException)
+            {
+                // thrown if the process doesn't exist (e.g., because it has already completed on its own)
+            }
+
+            m_stdoutFlushedTcs.TrySetResult(Unit.Void);
+            m_stderrFlushedTcs.TrySetResult(Unit.Void);
+            Killed = true;
+
+            return WhenExited;
+        }
+
+        /// <inheritdoc />
+        public void Dispose()
+        {
+            Process?.Dispose();
+        }
+
+        private void ThrowBuildXLException(string message, Exception inner = null)
+        {
+            string description = m_sandboxedProcessInfo == null ? string.Empty : $"[Pip{m_sandboxedProcessInfo.PipSemiStableHash:X16} -- {m_sandboxedProcessInfo.PipDescription}] ";
+            throw new BuildXLException($"{description}{message}", inner);
+        }
+
+        private static void FeedOutputBuilder(TaskSourceSlim<Unit> signalCompletion, string line, Action<string> eat)
+        {
+            if (signalCompletion.Task.IsCompleted)
+            {
+                return;
+            }
+
+            eat(line);
+
+            if (line == null)
+            {
+                signalCompletion.TrySetResult(Unit.Void);
+            }
+        }
+    }
+}

Public/Src/Engine/Processes/DetoursEventListener.cs

@@ -135,7 +135,7 @@ namespace BuildXL.Processes
             TimeSpan kernelTime,
             TimeSpan userTime,
             uint exitCode,
-            BuildXL.Pips.IOCounters ioCounters,
+            Native.IO.IOCounters ioCounters,
             uint parentProcessId);
 
         /// <summary>

Public/Src/Engine/Processes/ExpandedRegexDescriptor.cs

@@ -0,0 +1,102 @@
+// Copyright (c) Microsoft. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+using System.Diagnostics.ContractsLight;
+using System.Text.RegularExpressions;
+using BuildXL.Utilities;
+
+namespace BuildXL.Processes
+{
+    /// <summary>
+    /// Regex descriptor.
+    /// </summary>
+    public class ExpandedRegexDescriptor
+    {
+        /// <summary>
+        /// Pattern.
+        /// </summary>
+        public string Pattern { get; }
+
+        /// <summary>
+        /// Regex option.
+        /// </summary>
+        public RegexOptions Options { get; }
+
+        /// <summary>
+        /// Creates an instance of <see cref="ExpandedRegexDescriptor"/>.
+        /// </summary>
+        public ExpandedRegexDescriptor(string pattern, RegexOptions options)
+        {
+            Contract.Requires(pattern != null);
+
+            Pattern = pattern;
+            Options = options;
+        }
+
+        /// <summary>
+        /// Serializes this instance to a given <paramref name="writer"/>.
+        /// </summary>
+        public void Serialize(BuildXLWriter writer)
+        {
+            Contract.Requires(writer != null);
+
+            writer.Write(Pattern);
+            writer.Write((uint)Options);
+        }
+
+        /// <summary>
+        /// Deserializes an instance of <see cref="ExpandedRegexDescriptor"/>.
+        /// </summary>
+        public static ExpandedRegexDescriptor Deserialize(BuildXLReader reader)
+        {
+            Contract.Requires(reader != null);
+
+            return new ExpandedRegexDescriptor(pattern: reader.ReadString(), options: (RegexOptions)reader.ReadUInt32());
+        }
+
+        /// <inheritdoc />
+        public override bool Equals(object obj)
+        {
+            return !(obj is null) && (ReferenceEquals(this, obj) || ((obj is ExpandedRegexDescriptor descriptor) && Equals(descriptor)));
+        }
+
+        /// <summary>
+        /// Checks for equality.
+        /// </summary>
+        public bool Equals(ExpandedRegexDescriptor descriptor)
+        {
+            return !(descriptor is null)
+                && (ReferenceEquals(this, descriptor)
+                    || (string.Equals(Pattern, descriptor.Pattern) && Options == descriptor.Options));
+        }
+
+        /// <summary>
+        /// Checks for equality.
+        /// </summary>
+        public static bool operator ==(ExpandedRegexDescriptor descriptor1, ExpandedRegexDescriptor descriptor2)
+        {
+            if (ReferenceEquals(descriptor1, descriptor2))
+            {
+                return true;
+            }
+
+            if (descriptor1 is null)
+            {
+                return false;
+            }
+
+            return descriptor1.Equals(descriptor2);
+        }
+
+        /// <summary>
+        /// Checks for disequality.
+        /// </summary>
+        public static bool operator !=(ExpandedRegexDescriptor descriptor1, ExpandedRegexDescriptor descriptor2) => !(descriptor1 == descriptor2);
+
+        /// <inheritdoc />
+        public override int GetHashCode()
+        {
+            return HashCodeHelper.Combine(Pattern.GetHashCode(), (int)Options);
+        }
+    }
+}

Public/Src/Engine/Processes/ExternalSandboxedProcess.cs

@@ -0,0 +1,156 @@
+// Copyright (c) Microsoft. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+using System;
+using System.Diagnostics.ContractsLight;
+using System.IO;
+using System.Threading.Tasks;
+using BuildXL.Native.IO;
+using BuildXL.Utilities;
+
+namespace BuildXL.Processes
+{
+    /// <summary>
+    /// Class representing external execution of sandboxed process.
+    /// </summary>
+    public abstract class ExternalSandboxedProcess : ISandboxedProcess
+    {
+        /// <summary>
+        /// Sanboxed process info.
+        /// </summary>
+        protected SandboxedProcessInfo SandboxedProcessInfo { get; private set; }
+
+        /// <summary>
+        /// Creates an instance of <see cref="ExternalSandboxedProcess"/>.
+        /// </summary>
+        protected ExternalSandboxedProcess(SandboxedProcessInfo sandboxedProcessInfo)
+        {
+            Contract.Requires(sandboxedProcessInfo != null);
+
+            SandboxedProcessInfo = sandboxedProcessInfo;
+        }
+
+        /// <inheritdoc />
+        public abstract int ProcessId { get; }
+
+        /// <inheritdoc />
+        public abstract void Dispose();
+
+        /// <inheritdoc />
+        public abstract string GetAccessedFileName(ReportedFileAccess reportedFileAccess);
+
+        /// <inheritdoc />
+        public abstract ulong? GetActivePeakMemoryUsage();
+
+        /// <inheritdoc />
+        public abstract long GetDetoursMaxHeapSize();
+
+        /// <inheritdoc />
+        public abstract int GetLastMessageCount();
+
+        /// <inheritdoc />
+        public abstract Task<SandboxedProcessResult> GetResultAsync();
+
+        /// <inheritdoc />
+        public abstract Task KillAsync();
+
+        /// <inheritdoc />
+        public abstract void Start();
+
+        /// <summary>
+        /// Throws an instance of <see cref="BuildXLException"/>.
+        /// </summary>
+        protected void ThrowBuildXLException(string message, Exception inner = null)
+        {
+            throw new BuildXLException($"[Pip{SandboxedProcessInfo.PipSemiStableHash:X16} -- {SandboxedProcessInfo.PipDescription}] {message}", inner);
+        }
+
+        /// <summary>
+        /// Gets the file to which sandboxed process info will be written.
+        /// </summary>
+        protected string GetSandboxedProcessInfoFile() => Path.Combine(GetOutputDirectory(), $"SandboxedProcessInfo-Pip{SandboxedProcessInfo.PipSemiStableHash:X16}");
+
+        /// <summary>
+        /// Gets the file in which sandboxed process result will be available.
+        /// </summary>
+        protected string GetSandboxedProcessResultsFile() => Path.Combine(GetOutputDirectory(), $"SandboxedProcessResult-Pip{SandboxedProcessInfo.PipSemiStableHash:X16}");
+
+        /// <summary>
+        /// Gets the output directory for starting process externally.
+        /// </summary>
+        /// <remarks>
+        /// This output directory can be the place where BuildXL puts the serialization result of sandboxed process info and
+        /// the deserialization result of sandboxed process result. This output directory can also contain the dump of the external process
+        /// when it gets killed.
+        /// </remarks>
+        protected string GetOutputDirectory() => Path.GetDirectoryName(SandboxedProcessInfo.FileStorage.GetFileName(SandboxedProcessFile.StandardOutput));
+
+        /// <summary>
+        /// Gets the standard output path for the external executor; not the detoured process.
+        /// </summary>
+        protected string GetStdOutPath(string hint) => Path.Combine(GetOutputDirectory(), $"{hint ?? string.Empty}-Pip{SandboxedProcessInfo.PipSemiStableHash:X16}.out");
+
+        /// <summary>
+        /// Gets the standard error path for the external executor; not the detoured process.
+        /// </summary>
+        protected string GetStdErrPath(string hint) => Path.Combine(GetOutputDirectory(), $"{hint ?? string.Empty}-Pip{SandboxedProcessInfo.PipSemiStableHash:X16}.err");
+
+        /// <summary>
+        /// Standard output for the external executor.
+        /// </summary>
+        public abstract string StdOut { get; }
+
+        /// <summary>
+        /// Standard error for the external executor.
+        /// </summary>
+        public abstract string StdErr { get; }
+
+        /// <summary>
+        /// Gets the exit code of the external executor.
+        /// </summary>
+        public abstract int? ExitCode { get; }
+
+        /// <summary>
+        /// Serializes sandboxed process info to file.
+        /// </summary>
+        protected void SerializeSandboxedProcessInfoToFile()
+        {
+            string file = GetSandboxedProcessInfoFile();
+            FileUtilities.CreateDirectory(Path.GetDirectoryName(file));
+
+            try
+            {
+                using (FileStream stream = File.OpenWrite(file))
+                {
+                    SandboxedProcessInfo.Serialize(stream);
+                }
+            }
+            catch (IOException ioException)
+            {
+                ThrowBuildXLException($"Failed to serialize sandboxed process info '{file}'", ioException);
+            }
+        }
+
+        /// <summary>
+        /// Deserializes sandboxed process result from file.
+        /// </summary>
+        /// <returns></returns>
+        protected SandboxedProcessResult DeserializeSandboxedProcessResultFromFile()
+        {
+            string file = GetSandboxedProcessResultsFile();
+
+            try
+            {
+                using (FileStream stream = File.OpenRead(file))
+                {
+                    return SandboxedProcessResult.Deserialize(stream);
+                }
+            }
+            catch (IOException ioException)
+            {
+                ThrowBuildXLException($"Failed to deserialize sandboxed process result '{file}'", ioException);
+                return null;
+            }
+        }
+    }
+}

Public/Src/Engine/Processes/ExternalToolSandboxedProcess.cs

@@ -0,0 +1,210 @@
+// Copyright (c) Microsoft. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Diagnostics.ContractsLight;
+using System.IO;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace BuildXL.Processes
+{
+    /// <summary>
+    /// Sanboxed process that will be executed by an external tool.
+    /// </summary>
+    public class ExternalToolSandboxedProcess : ExternalSandboxedProcess
+    {
+        private static readonly ISet<ReportedFileAccess> s_emptyFileAccessesSet = new HashSet<ReportedFileAccess>();
+
+        /// <summary>
+        /// Relative path to the default tool.
+        /// </summary>
+        public const string DefaultToolRelativePath = @"tools\SandboxedProcessExecutor\SandboxedProcessExecutor.exe";
+
+        private readonly string m_toolPath;
+
+        private readonly StringBuilder m_output = new StringBuilder();
+        private readonly StringBuilder m_error = new StringBuilder();
+
+        private AsyncProcessExecutor m_processExecutor;
+        private Exception m_dumpCreationException;
+
+        /// <summary>
+        /// Creates an instance of <see cref="ExternalToolSandboxedProcess"/>.
+        /// </summary>
+        public ExternalToolSandboxedProcess(SandboxedProcessInfo sandboxedProcessInfo, string toolPath)
+            : base(sandboxedProcessInfo)
+        {
+            Contract.Requires(!string.IsNullOrWhiteSpace(toolPath));
+            m_toolPath = toolPath;
+        }
+
+        private int m_processId = -1;
+
+        /// <inheritdoc />
+        public override int ProcessId => m_processId != -1 ? m_processId : (m_processId = Process?.Id ?? -1);
+
+        /// <summary>
+        /// Underlying managed <see cref="Process"/> object.
+        /// </summary>
+        public Process Process => m_processExecutor?.Process;
+
+        /// <inheritdoc />
+        public override string StdOut => m_processExecutor?.StdOutCompleted ?? false ? m_error.ToString() : string.Empty;
+
+        /// <inheritdoc />
+        public override string StdErr => m_processExecutor?.StdErrCompleted ?? false ? m_error.ToString() : string.Empty;
+
+        /// <inheritdoc />
+        public override int? ExitCode => m_processExecutor.ExitCompleted ? Process?.ExitCode : default;
+
+        /// <inheritdoc />
+        public override void Dispose()
+        {
+            m_processExecutor?.Dispose();
+        }
+
+        /// <inheritdoc />
+        public override string GetAccessedFileName(ReportedFileAccess reportedFileAccess) => null;
+
+        /// <inheritdoc />
+        public override ulong? GetActivePeakMemoryUsage() => m_processExecutor?.GetActivePeakMemoryUsage();
+
+        /// <inheritdoc />
+        public override long GetDetoursMaxHeapSize() => 0;
+
+        /// <inheritdoc />
+        public override int GetLastMessageCount() => 0;
+
+        /// <inheritdoc />
+        public override async Task<SandboxedProcessResult> GetResultAsync()
+        {
+            Contract.Requires(m_processExecutor != null);
+
+            await m_processExecutor.WaitForExitAsync();
+
+            if (m_processExecutor.TimedOut || m_processExecutor.Killed)
+            {
+                // If timed out/killed, then sandboxed process result may have not been deserialized yet.
+                return CreateResultForFailure();
+            }
+
+            if (Process.ExitCode != 0)
+            {
+                return CreateResultForFailure();
+            }
+
+            return DeserializeSandboxedProcessResultFromFile();
+        }
+
+        /// <inheritdoc />
+        public override Task KillAsync()
+        {
+            Contract.Requires(m_processExecutor != null);
+
+            ProcessDumper.TryDumpProcessAndChildren(ProcessId, GetOutputDirectory(), out m_dumpCreationException);
+
+            return m_processExecutor.KillAsync();
+        }
+
+        /// <inheritdoc />
+        public override void Start()
+        {
+            Setup();
+            m_processExecutor.Start();
+        }
+
+        private string CreateArguments() => $"/sandboxedProcessInfo:{GetSandboxedProcessInfoFile()} /sandboxedProcessResult:{GetSandboxedProcessResultsFile()}";
+
+        private void Setup()
+        {
+            SerializeSandboxedProcessInfoToFile();
+
+            var process = new Process
+            {
+                StartInfo = new ProcessStartInfo
+                {
+                    FileName = m_toolPath,
+                    Arguments = CreateArguments(),
+                    WorkingDirectory = SandboxedProcessInfo.WorkingDirectory,
+                    RedirectStandardError = true,
+                    RedirectStandardOutput = true,
+                    UseShellExecute = false,
+                    CreateNoWindow = true
+                },
+
+                EnableRaisingEvents = true
+            };
+
+            m_processExecutor = new AsyncProcessExecutor(
+                process,
+                TimeSpan.FromMilliseconds(-1), // Timeout should only be applied to the process that the external tool executes.
+                line => AppendLineIfNotNull(m_output, line),
+                line => AppendLineIfNotNull(m_error, line),
+                SandboxedProcessInfo);
+        }
+
+        private void AppendLineIfNotNull(StringBuilder sb, string line)
+        {
+            if (line != null)
+            {
+                sb.AppendLine(line);
+            }
+        }
+
+        /// <summary>
+        /// Starts process asynchronously.
+        /// </summary>
+        public static Task<ISandboxedProcess> StartAsync(SandboxedProcessInfo info, string toolPath)
+        {
+            return Task.Factory.StartNew(() =>
+            {
+                ISandboxedProcess process = new ExternalToolSandboxedProcess(info, toolPath);
+
+                try
+                {
+                    process.Start();
+                }
+                catch
+                {
+                    process?.Dispose();
+                    throw;
+                }
+
+                return process;
+            });
+        }
+
+        private SandboxedProcessResult CreateResultForFailure()
+        {
+            string output = m_output.ToString();
+            string error = m_error.ToString();
+            string hint = Path.GetFileNameWithoutExtension(m_toolPath);
+            var standardFiles = new SandboxedProcessStandardFiles(GetStdOutPath(hint), GetStdErrPath(hint));
+            var storage = new StandardFileStorage(standardFiles);
+
+            return new SandboxedProcessResult
+            {
+                ExitCode = m_processExecutor.TimedOut ? ExitCodes.Timeout : Process.ExitCode,
+                Killed = m_processExecutor.Killed,
+                TimedOut = m_processExecutor.TimedOut,
+                HasDetoursInjectionFailures = false,
+                StandardOutput = new SandboxedProcessOutput(output.Length, output, null, Console.OutputEncoding, storage, SandboxedProcessFile.StandardOutput, null),
+                StandardError = new SandboxedProcessOutput(error.Length, error, null, Console.OutputEncoding, storage, SandboxedProcessFile.StandardError, null),
+                HasReadWriteToReadFileAccessRequest = false,
+                AllUnexpectedFileAccesses = s_emptyFileAccessesSet,
+                FileAccesses = s_emptyFileAccessesSet,
+                DetouringStatuses = new ProcessDetouringStatusData[0],
+                ExplicitlyReportedFileAccesses = s_emptyFileAccessesSet,
+                Processes = new ReportedProcess[0],
+                MessageProcessingFailure = null,
+                DumpCreationException = m_dumpCreationException,
+                DumpFileDirectory = GetOutputDirectory(),
+                PrimaryProcessTimes = new ProcessTimes(0, 0, 0, 0),
+                SurvivingChildProcesses = new ReportedProcess[0],
+            };
+        }
+    }
+}

Public/Src/Engine/Processes/FileAccessManifest.cs

@@ -29,6 +29,21 @@ namespace BuildXL.Processes
         /// </summary>
         private readonly Node m_rootNode;
 
+        /// <summary>
+        /// File access manifest flags.
+        /// </summary>
+        private FileAccessManifestFlag m_fileAccessManifestFlag;
+
+        /// <summary>
+        /// Name of semaphore for message count.
+        /// </summary>
+        private string m_messageCountSemaphoreName;
+
+        /// <summary>
+        /// Sealed manifest tree.
+        /// </summary>
+        private byte[] m_sealedManifestTreeBlock;
+
         /// <summary>
         /// Creates an empty instance.
         /// </summary>
@@ -62,7 +77,6 @@ namespace BuildXL.Processes
             LogProcessData = false;
             IgnoreGetFinalPathNameByHandle = true;
             LogProcessDetouringStatus = false;
-            AllowInternalDetoursErrorNotificationFile = false;
             HardExitOnErrorInDetours = false;
             CheckDetoursMessageCount = false;
             InternalDetoursErrorNotificationFile = string.Empty;
@@ -71,11 +85,39 @@ namespace BuildXL.Processes
             EnforceAccessPoliciesOnDirectoryCreation = false;
         }
 
+        private bool GetFlag(FileAccessManifestFlag flag) => (m_fileAccessManifestFlag & flag) != 0;
+
+        /// <summary>
+        /// Gets file access manifest flag.
+        /// </summary>
+        internal FileAccessManifestFlag Flag => m_fileAccessManifestFlag;
+
+        private void SetFlag(FileAccessManifestFlag flag, bool value)
+        {
+            if (value)
+            {
+                m_fileAccessManifestFlag |= flag;
+            }
+            else
+            {
+                m_fileAccessManifestFlag &= ~flag;
+            }
+        }
+
+        /// <summary>
+        /// Flag indicating if the manifest tree block is sealed.
+        /// </summary>
+        public bool IsManifestTreeBlockSealed => m_sealedManifestTreeBlock != null;
+
         /// <summary>
         /// If true, then the detoured file access functions will write diagnostic messages
         /// to stderr when access to files is prevented.
         /// </summary>
-        public bool EnableDiagnosticMessages { get; set; }
+        public bool EnableDiagnosticMessages
+        {
+            get => GetFlag(FileAccessManifestFlag.DiagnosticMessagesEnabled);
+            set => SetFlag(FileAccessManifestFlag.DiagnosticMessagesEnabled, value);
+        }
 
         /// <summary>
         /// If true, then detoured file access functions will treat invalid file access as an
@@ -83,174 +125,309 @@ namespace BuildXL.Processes
         /// If false, then detoured file access functions will treat invalid file access as a
         /// warning, but will still allow the file access to occur.
         /// </summary>
-        public bool FailUnexpectedFileAccesses { get; set; }
+        public bool FailUnexpectedFileAccesses
+        {
+            get => GetFlag(FileAccessManifestFlag.FailUnexpectedFileAccesses);
+            set => SetFlag(FileAccessManifestFlag.FailUnexpectedFileAccesses, value);
+        }
 
         /// <summary>
         /// If true, the detoured process won't be downgrading CreateDirectory call to read-only probe
         /// in cases when the directory already exists.
         /// </summary>
-        public bool EnforceAccessPoliciesOnDirectoryCreation { get; set; }
+        public bool EnforceAccessPoliciesOnDirectoryCreation
+        {
+            get => GetFlag(FileAccessManifestFlag.EnforceAccessPoliciesOnDirectoryCreation);
+            set => SetFlag(FileAccessManifestFlag.EnforceAccessPoliciesOnDirectoryCreation, value);
+        }
 
         /// <summary>
         /// If true, then the detoured process will execute the Win32 DebugBreak() function
         /// when the process attempts to access a file that is outside of its permitted scope.
         /// </summary>
-        public bool BreakOnUnexpectedAccess { get; set; }
+        public bool BreakOnUnexpectedAccess
+        {
+            get => GetFlag(FileAccessManifestFlag.BreakOnAccessDenied);
+            set => SetFlag(FileAccessManifestFlag.BreakOnAccessDenied, value);
+        }
 
         /// <summary>
         /// If true, then all file accesses will be recorded
         /// </summary>
-        public bool ReportFileAccesses { get; set; }
+        public bool ReportFileAccesses
+        {
+            get => GetFlag(FileAccessManifestFlag.ReportFileAccesses);
+            set => SetFlag(FileAccessManifestFlag.ReportFileAccesses, value);
+        }
 
         /// <summary>
         /// If true, then all unexpected file accesses will be recorded
         /// </summary>
-        public bool ReportUnexpectedFileAccesses { get; set; }
+        public bool ReportUnexpectedFileAccesses
+        {
+            get => GetFlag(FileAccessManifestFlag.ReportUnexpectedFileAccesses);
+            set => SetFlag(FileAccessManifestFlag.ReportUnexpectedFileAccesses, value);
+        }
 
         /// <summary>
         /// If true, then nested processes will be monitored just like the main process
         /// </summary>
-        public bool MonitorChildProcesses { get; set; }
+        public bool MonitorChildProcesses
+        {
+            get => GetFlag(FileAccessManifestFlag.MonitorChildProcesses);
+            set => SetFlag(FileAccessManifestFlag.MonitorChildProcesses, value);
+        }
 
         /// <summary>
         /// Monitor files opened for read by NtCreateFile
         /// TODO: This should be a temporary hack until we fix all places broken by the NtCreateFile monitoring
         /// </summary>
-        public bool MonitorNtCreateFile { get; set; }
+        public bool MonitorNtCreateFile
+        {
+            get => GetFlag(FileAccessManifestFlag.MonitorNtCreateFile);
+            set => SetFlag(FileAccessManifestFlag.MonitorNtCreateFile, value);
+        }
 
         /// <summary>
         /// Monitor files opened for read by ZwCreateOpenQueryFile
         /// </summary>
-        public bool MonitorZwCreateOpenQueryFile { get; set; }
+        public bool MonitorZwCreateOpenQueryFile
+        {
+            get => GetFlag(FileAccessManifestFlag.MonitorZwCreateOpenQueryFile);
+            set => SetFlag(FileAccessManifestFlag.MonitorZwCreateOpenQueryFile, value);
+        }
 
         /// <summary>
         /// If true, force read only for requested read-write access so long as the tool is allowed to read.
         /// </summary>
-        public bool ForceReadOnlyForRequestedReadWrite { get; set; }
+        public bool ForceReadOnlyForRequestedReadWrite
+        {
+            get => GetFlag(FileAccessManifestFlag.ForceReadOnlyForRequestedReadWrite);
+            set => SetFlag(FileAccessManifestFlag.ForceReadOnlyForRequestedReadWrite, value);
+        }
 
         /// <summary>
         /// If true, allows detouring the ZwRenameFileInformation API.
         /// </summary>
-        public bool IgnoreZwRenameFileInformation { get; set; }
+        public bool IgnoreZwRenameFileInformation
+        {
+            get => GetFlag(FileAccessManifestFlag.IgnoreZwRenameFileInformation);
+            set => SetFlag(FileAccessManifestFlag.IgnoreZwRenameFileInformation, value);
+        }
 
         /// <summary>
         /// If true, allows detouring the ZwOtherFileInformation API.
         /// </summary>
-        public bool IgnoreZwOtherFileInformation { get; set; }
+        public bool IgnoreZwOtherFileInformation
+        {
+            get => GetFlag(FileAccessManifestFlag.IgnoreZwOtherFileInformation);
+            set => SetFlag(FileAccessManifestFlag.IgnoreZwOtherFileInformation, value);
+        }
 
         /// <summary>
         /// If true, allows following symlinks for Non CreateFile APIs.
         /// </summary>
-        public bool IgnoreNonCreateFileReparsePoints { get; set; }
+        public bool IgnoreNonCreateFileReparsePoints
+        {
+            get => GetFlag(FileAccessManifestFlag.IgnoreNonCreateFileReparsePoints);
+            set => SetFlag(FileAccessManifestFlag.IgnoreNonCreateFileReparsePoints, value);
+        }
 
         /// <summary>
         /// If true, allows detouring the SetFileInformationByHandle API.
         /// </summary>
-        public bool IgnoreSetFileInformationByHandle { get; set; }
+        public bool IgnoreSetFileInformationByHandle
+        {
+            get => GetFlag(FileAccessManifestFlag.IgnoreSetFileInformationByHandle);
+            set => SetFlag(FileAccessManifestFlag.IgnoreSetFileInformationByHandle, value);
+        }
 
         /// <summary>
         /// If true, allows following reparse points without enforcing any file access rules.
         /// </summary>
-        public bool IgnoreReparsePoints { get; set; }
+        public bool IgnoreReparsePoints
+        {
+            get => GetFlag(FileAccessManifestFlag.IgnoreReparsePoints);
+            set => SetFlag(FileAccessManifestFlag.IgnoreReparsePoints, value);
+        }
 
         /// <summary>
         /// If true, allows enforcing file access rules for preloaded (statically) loaded Dlls.
         /// </summary>
-        public bool IgnorePreloadedDlls { get; set; }
+        public bool IgnorePreloadedDlls
+        {
+            get => GetFlag(FileAccessManifestFlag.IgnorePreloadedDlls);
+            set => SetFlag(FileAccessManifestFlag.IgnorePreloadedDlls, value);
+        }
 
         /// <summary>
         /// If true, disables detouring any file access APIs, which will lead to incorrect builds.
         /// </summary>
-        public bool DisableDetours { get; set; }
+        public bool DisableDetours
+        {
+            get => GetFlag(FileAccessManifestFlag.DisableDetours);
+            set => SetFlag(FileAccessManifestFlag.DisableDetours, value);
+        }
 
         /// <summary>
         /// If true, ignore certain files that are accessed as a side-effect of code coverage collection
         /// </summary>
-        public bool IgnoreCodeCoverage { get; set; }
+        public bool IgnoreCodeCoverage
+        {
+            get => GetFlag(FileAccessManifestFlag.IgnoreCodeCoverage);
+            set => SetFlag(FileAccessManifestFlag.IgnoreCodeCoverage, value);
+        }
 
         /// <summary>
         /// If true, the command line arguments of all processes (including child processes) launched by the pip will be reported
         /// </summary>
-        public bool ReportProcessArgs { get; set; }
+        public bool ReportProcessArgs
+        {
+            get => GetFlag(FileAccessManifestFlag.ReportProcessArgs);
+            set => SetFlag(FileAccessManifestFlag.ReportProcessArgs, value);
+        }
 
         /// <summary>
         /// If true, all file reads will get a consistent timestamp. When false, actual timestamps will be allowed to flow
         /// through, as long as they are newer than the static <see cref="WellKnownTimestamps.NewInputTimestamp"/> used for enforcing rewrite order.
         /// </summary>
-        public bool NormalizeReadTimestamps { get; set; }
+        public bool NormalizeReadTimestamps
+        {
+            get => GetFlag(FileAccessManifestFlag.NormalizeReadTimestamps);
+            set => SetFlag(FileAccessManifestFlag.NormalizeReadTimestamps, value);
+        }
 
         /// <summary>
         /// Whether BuildXL will use larger NtClose preallocated list.
         /// </summary>
-        public bool UseLargeNtClosePreallocatedList { get; set; }
+        public bool UseLargeNtClosePreallocatedList
+        {
+            get => GetFlag(FileAccessManifestFlag.UseLargeNtClosePreallocatedList);
+            set => SetFlag(FileAccessManifestFlag.UseLargeNtClosePreallocatedList, value);
+        }
 
         /// <summary>
         /// Whether BuildXL will use extra thread to drain NtClose handle List or clean the cache directly.
         /// </summary>
-        public bool UseExtraThreadToDrainNtClose { get; set; }
+        public bool UseExtraThreadToDrainNtClose
+        {
+            get => GetFlag(FileAccessManifestFlag.UseExtraThreadToDrainNtClose);
+            set => SetFlag(FileAccessManifestFlag.UseExtraThreadToDrainNtClose, value);
+        }
 
         /// <summary>
         /// Determines whether BuildXL will collect proccess execution data, kernel/user mode time and IO counts.
         /// </summary>
-        public bool LogProcessData { get; set; }
+        public bool LogProcessData
+        {
+            get => GetFlag(FileAccessManifestFlag.LogProcessData);
+            set => SetFlag(FileAccessManifestFlag.LogProcessData, value);
+        }
 
         /// <summary>
         /// If true, don't detour the GetFinalPathNameByHandle API.
         /// </summary>
-        public bool IgnoreGetFinalPathNameByHandle { get; set; }
+        public bool IgnoreGetFinalPathNameByHandle
+        {
+            get => GetFlag(FileAccessManifestFlag.IgnoreGetFinalPathNameByHandle);
+            set => SetFlag(FileAccessManifestFlag.IgnoreGetFinalPathNameByHandle, value);
+        }
 
         /// <summary>
         /// If true it enables logging of Detouring status.
         /// </summary>
-        public bool LogProcessDetouringStatus { get; set; }
+        public bool LogProcessDetouringStatus
+        {
+            get => GetFlag(FileAccessManifestFlag.LogProcessDetouringStatus);
+            set => SetFlag(FileAccessManifestFlag.LogProcessDetouringStatus, value);
+        }
 
         /// <summary>
         /// If true it enables hard exiting on error with special exit code.
         /// </summary>
-        public bool HardExitOnErrorInDetours { get; set; }
+        public bool HardExitOnErrorInDetours
+        {
+            get => GetFlag(FileAccessManifestFlag.HardExitOnErrorInDetours);
+            set => SetFlag(FileAccessManifestFlag.HardExitOnErrorInDetours, value);
+        }
 
         /// <summary>
         /// If true it enables logging of Detouring status.
         /// </summary>
-        public bool CheckDetoursMessageCount { get; set; }
+        public bool CheckDetoursMessageCount
+        {
+            get => GetFlag(FileAccessManifestFlag.CheckDetoursMessageCount);
+            set => SetFlag(FileAccessManifestFlag.CheckDetoursMessageCount, value);
+        }
+
+        /// <summary>
+        /// True whan the sandbox is integrated in QBuild. False otherwise.
+        /// </summary>
+        /// <remarks>Note: this is only an option that is set programmatically. Not controlled by a command line option.</remarks>
+        public bool QBuildIntegrated
+        {
+            get => GetFlag(FileAccessManifestFlag.QBuildIntegrated);
+            set => SetFlag(FileAccessManifestFlag.QBuildIntegrated, value);
+        }
 
         /// <summary>
         /// A location for a file where Detours to log failure messages.
         /// </summary>
         public string InternalDetoursErrorNotificationFile { get; set; }
 
-        /// <summary>
-        /// If true it enables logging of Detouring internal errors to an external file.
-        /// </summary>
-        public bool AllowInternalDetoursErrorNotificationFile { get; set; }
-
         /// <summary>
         /// The semaphore that keeps count to the sent and received messages.
         /// </summary>
-        public System.Threading.Semaphore MessageCountSemaphore { get; set; }
+        public System.Threading.Semaphore MessageCountSemaphore { get; private set; }
 
         /// <summary>
         /// Directory translator.
         /// </summary>
         public DirectoryTranslator DirectoryTranslator { get; }
 
-        /// <summary>
-        /// True whan the sandbox is integrated in QBuild. False otherwise.
-        /// </summary>
-        /// <remarks>Note: this is only an option that is set programmatically. Not controlled by a command line option.</remarks>
-        public bool QBuildIntegrated { get; set; }
-
         /// <summary>
         /// The pip's unique id.
         /// Right now it is the Pip.SemiStableHash.
         /// </summary>
         public long PipId { get; set; }
 
+        /// <summary>
+        /// Sets message count semaphore.
+        /// </summary>
+        public bool SetMessageCountSemaphore(string semaphoreName)
+        {
+            Contract.Requires(!string.IsNullOrEmpty(semaphoreName));
+
+            UnsetMessageCountSemaphore();
+            m_messageCountSemaphoreName = semaphoreName;
+            MessageCountSemaphore = new System.Threading.Semaphore(0, int.MaxValue, semaphoreName, out bool newlyCreated);
+
+            return newlyCreated;
+        }
+
+        /// <summary>
+        /// Unset message count semaphore.
+        /// </summary>
+        public void UnsetMessageCountSemaphore()
+        {
+            if (MessageCountSemaphore == null)
+            {
+                Contract.Assert(m_messageCountSemaphoreName == null);
+                return;
+            }
+
+            MessageCountSemaphore.Dispose();
+            MessageCountSemaphore = null;
+            m_messageCountSemaphoreName = null;
+        }
+
         /// <summary>
         /// Adds a policy to an entire scope
         /// </summary>
         public void AddScope(AbsolutePath path, FileAccessPolicy mask, FileAccessPolicy values)
         {
+            Contract.Requires(!IsManifestTreeBlockSealed);
+
             if (!path.IsValid)
             {
                 // Note that AbsolutePath.Invalid is allowable (representing the root scope).
@@ -266,6 +443,7 @@ namespace BuildXL.Processes
         /// </summary>
         public void AddPath(AbsolutePath path, FileAccessPolicy mask, FileAccessPolicy values, Usn? expectedUsn = null)
         {
+            Contract.Requires(!IsManifestTreeBlockSealed);
             Contract.Requires(path != AbsolutePath.Invalid);
 
             m_rootNode.AddNodeWithScope(this, path, new FileAccessScope(mask, values), expectedUsn ?? ReportedFileAccess.NoUsn);
@@ -281,26 +459,63 @@ namespace BuildXL.Processes
             }
         }
 
+        private static string ReadChars(BinaryReader reader)
+        {
+            uint length = reader.ReadUInt32();
+            if (length == 0)
+            {
+                // TODO: Make ReadChars and WriteChars symmetric.
+                // Note that there's asymmetry between WriteChars and ReadChars.
+                // WriteChars has existed for long time and is used to serialize string to Detours.
+                // ReadChars is introduced as a way to reuse WriteChars when BuildXL has to serialize/deserialize
+                // manifest to files. We can make WriteChars and ReadChars symmetric, as well as, optimize the encoding.
+                // However, such changes entails changing Detours code.
+                return null;
+            }
+
+            char[] chars = new char[length];
+
+            for (int i = 0; i < chars.Length; ++i)
+            {
+                chars[i] = reader.ReadChar();
+            }
+
+            return new string(chars);
+        }
+
         private static void WriteInjectionTimeoutBlock(BinaryWriter writer, uint timeoutInMins)
         {
             writer.Write((uint)timeoutInMins);
         }
 
+        private const uint ErrorDumpLocationCheckedCode = 0xABCDEF03;
+        private const uint TranslationPathStringCheckedCode = 0xABCDEF02;
+        private const uint FlagsCheckedCode = 0xF1A6B10C; // Flag block
+        private const uint PipIdCheckedCode = 0xF1A6B10E;
+
         [SuppressMessage("Microsoft.Globalization", "CA1308:NormalizeStringsToUppercase", Justification = "Architecture strings are USASCII")]
-        private static void WriteErrorDumpLocation(
-            BinaryWriter writer, string internalDetoursErrorNotificationFile)
+        private static void WriteErrorDumpLocation(BinaryWriter writer, string internalDetoursErrorNotificationFile)
         {
 #if DEBUG
-            writer.Write((uint)0xABCDEF03); // "ABCDEF03"
+            writer.Write(ErrorDumpLocationCheckedCode);
 #endif
             WriteChars(writer, internalDetoursErrorNotificationFile);
         }
 
+        private static string ReadErrorDumpLocation(BinaryReader reader)
+        {
+#if DEBUG
+            uint code = reader.ReadUInt32();
+            Contract.Assert(ErrorDumpLocationCheckedCode == code);
+#endif
+            return ReadChars(reader);
+        }
+
         [SuppressMessage("Microsoft.Globalization", "CA1308:NormalizeStringsToUppercase", Justification = "Architecture strings are USASCII")]
         private static void WriteTranslationPathStrings(BinaryWriter writer, DirectoryTranslator translatePaths)
         {
 #if DEBUG
-            writer.Write(0xABCDEF02); // "ABCDEF02"
+            writer.Write(TranslationPathStringCheckedCode);
 #endif
 
             // Write the number of translation paths.
@@ -319,6 +534,34 @@ namespace BuildXL.Processes
             }
         }
 
+        private static DirectoryTranslator ReadTranslationPathStrings(BinaryReader reader)
+        {
+#if DEBUG
+            uint code = reader.ReadUInt32();
+            Contract.Assert(TranslationPathStringCheckedCode == code);
+#endif
+
+            uint length = reader.ReadUInt32();
+
+            if (length == 0)
+            {
+                return null;
+            }
+
+            DirectoryTranslator directoryTranslator = new DirectoryTranslator();
+
+            for (int i = 0; i < length; ++i)
+            {
+                string source = ReadChars(reader);
+                string target = ReadChars(reader);
+                directoryTranslator.AddTranslation(source, target);
+            }
+
+            directoryTranslator.Seal();
+
+            return directoryTranslator;
+        }
+
         [SuppressMessage("Microsoft.Naming", "CA2204:LiteralsShouldBeSpelledCorrectly")]
         private static void WriteDebugFlagBlock(BinaryWriter writer, ref bool debugFlagsMatch)
         {
@@ -342,45 +585,25 @@ namespace BuildXL.Processes
 #endif
         }
 
-        private void WriteFlagsBlock(BinaryWriter writer)
+        private static void WriteFlagsBlock(BinaryWriter writer, FileAccessManifestFlag flags)
         {
 #if DEBUG
-            writer.Write((uint)0xF1A6B10C); // "flag block"
+            writer.Write(FlagsCheckedCode);
 #endif
 
-            FileAccessManifestFlag flags =
-                (FailUnexpectedFileAccesses ? FileAccessManifestFlag.FailUnexpectedFileAccesses : 0)
-                | (BreakOnUnexpectedAccess ? FileAccessManifestFlag.BreakOnAccessDenied : 0)
-                | (EnableDiagnosticMessages ? FileAccessManifestFlag.DiagnosticMessagesEnabled : 0)
-                | (ReportFileAccesses ? FileAccessManifestFlag.ReportFileAccesses : 0)
-                | (ReportUnexpectedFileAccesses ? FileAccessManifestFlag.ReportUnexpectedFileAccesses : 0)
-                | (MonitorChildProcesses ? FileAccessManifestFlag.MonitorChildProcesses : 0)
-                | (MonitorNtCreateFile ? FileAccessManifestFlag.MonitorNtCreateFile : 0)
-                | (MonitorZwCreateOpenQueryFile ? FileAccessManifestFlag.MonitorZwCreateOpenQueryFile : 0)
-                | (IgnoreCodeCoverage ? FileAccessManifestFlag.IgnoreCodeCoverage : 0)
-                | (ReportProcessArgs ? FileAccessManifestFlag.ReportProcessArgs : 0)
-                | (ForceReadOnlyForRequestedReadWrite ? FileAccessManifestFlag.ForceReadOnlyForRequestedReadWrite : 0)
-                | (IgnoreReparsePoints ? FileAccessManifestFlag.IgnoreReparsePoints : 0)
-                | (IgnorePreloadedDlls ? FileAccessManifestFlag.IgnorePreloadedDlls : 0)
-                | (NormalizeReadTimestamps ? FileAccessManifestFlag.NormalizeReadTimestamps : 0)
-                | (IgnoreZwRenameFileInformation ? FileAccessManifestFlag.IgnoreZwRenameFileInformation : 0)
-                | (IgnoreZwOtherFileInformation ? FileAccessManifestFlag.IgnoreZwOtherFileInformation : 0)
-                | (IgnoreNonCreateFileReparsePoints ? FileAccessManifestFlag.IgnoreNonCreateFileReparsePoints : 0)
-                | (IgnoreSetFileInformationByHandle ? FileAccessManifestFlag.IgnoreSetFileInformationByHandle : 0)
-                | (UseLargeNtClosePreallocatedList ? FileAccessManifestFlag.UseLargeNtClosePreallocatedList : 0)
-                | (UseExtraThreadToDrainNtClose ? FileAccessManifestFlag.UseExtraThreadToDrainNtClose : 0)
-                | (DisableDetours ? FileAccessManifestFlag.DisableDetours : 0)
-                | (LogProcessData ? FileAccessManifestFlag.LogProcessData : 0)
-                | (IgnoreGetFinalPathNameByHandle ? FileAccessManifestFlag.IgnoreGetFinalPathNameByHandle : 0)
-                | (LogProcessDetouringStatus ? FileAccessManifestFlag.LogProcessDetouringStatus : 0)
-                | (HardExitOnErrorInDetours ? FileAccessManifestFlag.HardExitOnErrorInDetours : 0)
-                | (CheckDetoursMessageCount ? FileAccessManifestFlag.CheckDetoursMessageCount : 0)
-                | (QBuildIntegrated ? FileAccessManifestFlag.QBuildIntegrated : 0)
-                | (EnforceAccessPoliciesOnDirectoryCreation ? FileAccessManifestFlag.EnforceAccessPoliciesOnDirectoryCreation : 0);
-
             writer.Write((uint)flags);
         }
 
+        private static FileAccessManifestFlag ReadFlagsBlock(BinaryReader reader)
+        {
+#if DEBUG
+            uint code = reader.ReadUInt32();
+            Contract.Assert(FlagsCheckedCode == code);
+#endif
+
+            return (FileAccessManifestFlag)reader.ReadUInt32();
+        }
+
         // Allow for future expansions with 64 more flags (in case they become needed))
         private static void WriteExtraFlagsBlock(BinaryWriter writer, FileAccessManifestExtraFlag extraFlags)
         {
@@ -393,13 +616,25 @@ namespace BuildXL.Processes
         private static void WritePipId(BinaryWriter writer, long pipId)
         {
 #if DEBUG
-            writer.Write((uint)0xF1A6B10E); // "extra flags block"
+            writer.Write(PipIdCheckedCode);
             writer.Write(0); // Padding. Needed to keep the data properly aligned for the C/C++ compiler.
 #endif
             // The PipId is needed for reporting purposes on non Windows OSs. Write it here.
             writer.Write(pipId);
         }
 
+        private static long ReadPipId(BinaryReader reader)
+        {
+#if DEBUG
+            uint code = reader.ReadUInt32();
+            Contract.Assert(PipIdCheckedCode == code);
+
+            int zero = reader.ReadInt32();
+            Contract.Assert(0 == zero);
+#endif
+            return reader.ReadInt64();
+        }
+
         private static void WriteReportBlock(BinaryWriter writer, FileAccessSetup setup)
         {
 #if DEBUG
@@ -485,7 +720,14 @@ namespace BuildXL.Processes
 
         private void WriteManifestTreeBlock(BinaryWriter writer)
         {
-            m_rootNode.InternalSerialize(default(NormalizedPathString), writer);
+            if (m_sealedManifestTreeBlock != null)
+            {
+                writer.Write(m_sealedManifestTreeBlock);
+            }
+            else
+            {
+                m_rootNode.InternalSerialize(default(NormalizedPathString), writer);
+            }
         }
 
         /// <summary>
@@ -504,7 +746,7 @@ namespace BuildXL.Processes
                 WriteInjectionTimeoutBlock(writer, timeoutMins);
                 WriteTranslationPathStrings(writer, DirectoryTranslator);
                 WriteErrorDumpLocation(writer, InternalDetoursErrorNotificationFile);
-                WriteFlagsBlock(writer);
+                WriteFlagsBlock(writer, m_fileAccessManifestFlag);
                 WriteExtraFlagsBlock(writer, FileAccessManifestExtraFlag.None);
                 WritePipId(writer, PipId);
                 WriteReportBlock(writer, setup);
@@ -515,6 +757,61 @@ namespace BuildXL.Processes
             }
         }
 
+        /// <summary>
+        /// Serializes this manifest.
+        /// </summary>
+        public void Serialize(Stream stream)
+        {
+            Contract.Requires(stream != null);
+
+            using (var writer = new BinaryWriter(stream, Encoding.Unicode, true))
+            {
+                WriteTranslationPathStrings(writer, DirectoryTranslator);
+                WriteErrorDumpLocation(writer, InternalDetoursErrorNotificationFile);
+                WriteFlagsBlock(writer, m_fileAccessManifestFlag);
+                WritePipId(writer, PipId);
+                WriteChars(writer, m_messageCountSemaphoreName);
+
+                // The manifest tree block has to be serialized the last.
+                WriteManifestTreeBlock(writer);
+            }
+        }
+
+        /// <summary>
+        /// Deserialize an instance of <see cref="FileAccessManifest"/> from stream.
+        /// </summary>
+        public static FileAccessManifest Deserialize(Stream stream)
+        {
+            Contract.Requires(stream != null);
+
+            using (var reader = new BinaryReader(stream, Encoding.Unicode, true))
+            {
+                DirectoryTranslator directoryTranslator = ReadTranslationPathStrings(reader);
+                string internalDetoursErrorNotificationFile = ReadErrorDumpLocation(reader);
+                FileAccessManifestFlag fileAccessManifestFlag = ReadFlagsBlock(reader);
+                long pipId = ReadPipId(reader);
+                string messageCountSemaphoreName = ReadChars(reader);
+
+                byte[] sealedManifestTreeBlock;
+
+                // TODO: Check perf. a) if this is a good buffer size, b) if the buffers should be pooled (now they are just allocated and thrown away)
+                using (MemoryStream ms = new MemoryStream(4096))
+                {
+                    stream.CopyTo(ms);
+                    sealedManifestTreeBlock = ms.ToArray();
+                }
+
+                return new FileAccessManifest(new PathTable(), directoryTranslator)
+                {
+                    InternalDetoursErrorNotificationFile = internalDetoursErrorNotificationFile,
+                    PipId = pipId,
+                    m_fileAccessManifestFlag = fileAccessManifestFlag,
+                    m_sealedManifestTreeBlock = sealedManifestTreeBlock,
+                    m_messageCountSemaphoreName = messageCountSemaphoreName
+                };
+            }
+        }
+
         /// <summary>
         /// Return the raw byte array encoding of the tree-structured FileAccessManifest.
         /// </summary>
@@ -524,6 +821,12 @@ namespace BuildXL.Processes
         [SuppressMessage("Microsoft.Usage", "CA2202:Do not dispose objects multiple times")]
         public byte[] GetManifestTreeBytes()
         {
+            if (IsManifestTreeBlockSealed)
+            {
+                Contract.Assert(m_sealedManifestTreeBlock != null);
+                return m_sealedManifestTreeBlock;
+            }
+
             // start with 4 KB of memory (one page), which will expand as necessary
             // stream will be disposed by the BinaryWriter when it goes out of scope
             using (var stream = new MemoryStream(4096))
@@ -548,7 +851,7 @@ namespace BuildXL.Processes
 
         // Keep this in sync with the C++ version declared in DataTypes.h
         [Flags]
-        private enum FileAccessManifestFlag
+        internal enum FileAccessManifestFlag
         {
             None = 0,
             BreakOnAccessDenied = 0x1,

Public/Src/Engine/Processes/JobObject.cs

@@ -83,73 +83,6 @@ namespace BuildXL.Processes
 
         #region Accounting types
 
-        /// <summary>
-        /// Contains I/O accounting information for a process or a job object, for a particular type of IO (e.g. read or write).
-        /// These counters include all operations performed by all processes ever associated with the job.
-        /// </summary>
-        [SuppressMessage("Microsoft.Performance", "CA1815:OverrideEqualsAndOperatorEqualsOnValueTypes")]
-        public struct IOTypeCounters
-        {
-            /// <summary>
-            /// Number of operations performed (independent of size).
-            /// </summary>
-            public ulong OperationCount;
-
-            /// <summary>
-            /// Total bytes transferred (regardless of the number of operations used to transfer them).
-            /// </summary>
-            public ulong TransferCount;
-        }
-
-        /// <summary>
-        /// Contains I/O accounting information for a process or a job object.
-        /// These counters include all operations performed by all processes ever associated with the job.
-        /// </summary>
-        [SuppressMessage("Microsoft.Performance", "CA1815:OverrideEqualsAndOperatorEqualsOnValueTypes")]
-        public struct IOCounters
-        {
-            /// <summary>
-            /// Counters for read operations.
-            /// </summary>
-            public IOTypeCounters ReadCounters;
-
-            /// <summary>
-            /// Counters for write operations.
-            /// </summary>
-            public IOTypeCounters WriteCounters;
-
-            /// <summary>
-            /// Counters for other operations (not classified as either read or write).
-            /// </summary>
-            public IOTypeCounters OtherCounters;
-
-            internal IOCounters(IO_COUNTERS nativeCounters)
-            {
-                ReadCounters.OperationCount = nativeCounters.ReadOperationCount;
-                ReadCounters.TransferCount = nativeCounters.ReadTransferCount;
-
-                WriteCounters.OperationCount = nativeCounters.WriteOperationCount;
-                WriteCounters.TransferCount = nativeCounters.WriteTransferCount;
-
-                OtherCounters.OperationCount = nativeCounters.OtherOperationCount;
-                OtherCounters.TransferCount = nativeCounters.OtherTransferCount;
-            }
-
-            /// <summary>
-            /// Computes the aggregate I/O performed (sum of the read, write, and other counters).
-            /// </summary>
-            [SuppressMessage("Microsoft.Design", "CA1024:UsePropertiesWhereAppropriate")]
-            [Pure]
-            public IOTypeCounters GetAggregateIO()
-            {
-                return new IOTypeCounters()
-                       {
-                           OperationCount = ReadCounters.OperationCount + WriteCounters.OperationCount + OtherCounters.OperationCount,
-                           TransferCount = ReadCounters.TransferCount + WriteCounters.TransferCount + OtherCounters.TransferCount,
-                       };
-            }
-        }
-
         /// <summary>
         /// Accounting information for resources used by the job so far.
         /// </summary>
@@ -184,6 +117,29 @@ namespace BuildXL.Processes
             /// Number of processes started within or added to the job. This includes both running and already-terminated processes, if any.
             /// </summary>
             public uint NumberOfProcesses;
+
+            /// <nodoc />
+            public void Serialize(BuildXLWriter writer)
+            {
+                IO.Serialize(writer);
+                writer.Write(UserTime);
+                writer.Write(KernelTime);
+                writer.Write(PeakMemoryUsage);
+                writer.Write(NumberOfProcesses);
+            }
+
+            /// <nodoc />
+            public static AccountingInformation Deserialize(BuildXLReader reader)
+            {
+                return new AccountingInformation()
+                {
+                    IO = IOCounters.Deserialize(reader),
+                    UserTime = reader.ReadTimeSpan(),
+                    KernelTime = reader.ReadTimeSpan(),
+                    PeakMemoryUsage = reader.ReadUInt64(),
+                    NumberOfProcesses = reader.ReadUInt32()
+                };
+            }
         }
 
         #endregion

Public/Src/Engine/Processes/ProcessDetouringStatusData.cs

@@ -1,7 +1,6 @@
 // Copyright (c) Microsoft. All rights reserved.
 // Licensed under the MIT license. See LICENSE file in the project root for full license information.
 
-using System.Collections.Generic;
 using BuildXL.Utilities;
 
 namespace BuildXL.Processes
@@ -114,41 +113,8 @@ namespace BuildXL.Processes
             CreateProcessStatusReturn = createProcessStatusReturn;
         }
 
-        /// <summary>
-        /// Deserialize result from reader
-        /// </summary>
-        public static IReadOnlyCollection<ProcessDetouringStatusData> Deserialize(BuildXLReader reader)
-        {
-            int processDetouringStatusStatusesCount = reader.ReadInt32Compact();
-            var processDetouringStatuses = new ProcessDetouringStatusData[processDetouringStatusStatusesCount];
-            for (int i = 0; i < processDetouringStatusStatusesCount; i++)
-            {
-                processDetouringStatuses[i] = ReadReportedProcessDetouringStatusData(reader);
-            }
-
-            return processDetouringStatuses;
-        }
-
-        /// <summary>
-        /// Serialize result to writer
-        /// </summary>
-        public static void Serialize(BuildXLWriter writer, IReadOnlyCollection<ProcessDetouringStatusData> processDetouringStatuses)
-        {
-            if (processDetouringStatuses == null)
-            {
-                writer.WriteCompact(0);
-            }
-            else
-            {
-                writer.WriteCompact(processDetouringStatuses.Count);
-                foreach (var processDetouringData in processDetouringStatuses)
-                {
-                    WriteReportedProcessDetouringStatus(writer, processDetouringData);
-                }
-            }
-        }
-
-        private static ProcessDetouringStatusData ReadReportedProcessDetouringStatusData(BuildXLReader reader)
+        /// <nodoc />
+        public static ProcessDetouringStatusData Deserialize(BuildXLReader reader)
         {
             return new ProcessDetouringStatusData(
                 processId: reader.ReadUInt64(),
@@ -165,20 +131,21 @@ namespace BuildXL.Processes
                 createProcessStatusReturn: reader.ReadUInt32());
         }
 
-        private static void WriteReportedProcessDetouringStatus(BuildXLWriter writer, ProcessDetouringStatusData detouringData)
+        /// <nodoc />
+        public void Serialize(BuildXLWriter writer)
         {
-            writer.Write(detouringData.ProcessId);
-            writer.Write(detouringData.ReportStatus);
-            writer.Write(detouringData.ProcessName);
-            writer.Write(detouringData.StartApplicationName);
-            writer.Write(detouringData.StartCommandLine);
-            writer.Write(detouringData.NeedsInjection);
-            writer.Write(detouringData.Job);
-            writer.Write(detouringData.DisableDetours);
-            writer.Write(detouringData.CreationFlags);
-            writer.Write(detouringData.Detoured);
-            writer.Write(detouringData.Error);
-            writer.Write(detouringData.CreateProcessStatusReturn);
+            writer.Write(ProcessId);
+            writer.Write(ReportStatus);
+            writer.Write(ProcessName);
+            writer.Write(StartApplicationName);
+            writer.Write(StartCommandLine);
+            writer.Write(NeedsInjection);
+            writer.Write(Job);
+            writer.Write(DisableDetours);
+            writer.Write(CreationFlags);
+            writer.Write(Detoured);
+            writer.Write(Error);
+            writer.Write(CreateProcessStatusReturn);
         }
     }
 }

Public/Src/Engine/Processes/ProcessTimes.cs

@@ -3,6 +3,7 @@
 
 using System;
 using System.Diagnostics.ContractsLight;
+using System.IO;
 
 namespace BuildXL.Processes
 {
@@ -124,5 +125,22 @@ namespace BuildXL.Processes
         /// Total time between creation and exit.
         /// </summary>
         public TimeSpan TotalWallClockTime => StartTimeUtc < ExitTimeUtc ? ExitTimeUtc - StartTimeUtc : TimeSpan.Zero;
+
+        /// <nodoc />
+        public void Serialize(BinaryWriter writer)
+        {
+            writer.Write(m_create);
+            writer.Write(m_exit);
+            writer.Write(m_kernel);
+            writer.Write(m_user);
+        }
+
+        /// <nodoc />
+        public static ProcessTimes Deserialize(BinaryReader reader) 
+            => new ProcessTimes(
+                creation: reader.ReadInt64(), 
+                exit:     reader.ReadInt64(), 
+                kernel:   reader.ReadInt64(), 
+                user:     reader.ReadInt64());
     }
 }

Public/Src/Engine/Processes/ReportedFileAccess.cs

@@ -2,6 +2,7 @@
 // Licensed under the MIT license. See LICENSE file in the project root for full license information.
 
 using System;
+using System.Collections.Generic;
 using System.Diagnostics.CodeAnalysis;
 using System.Diagnostics.ContractsLight;
 using System.Globalization;
@@ -621,6 +622,72 @@ namespace BuildXL.Processes
                 enumeratePattern);
         }
 
+        /// <nodoc />
+        public void Serialize(
+            BuildXLWriter writer,
+            Dictionary<ReportedProcess, int> processMap,
+            Action<BuildXLWriter, AbsolutePath> writePath)
+        {
+            writer.Write((byte)Operation);
+
+            if (processMap != null && processMap.TryGetValue(Process, out int index))
+            {
+                writer.WriteCompact(index);
+            }
+            else
+            {
+                Process.Serialize(writer);
+            }
+
+            writer.WriteCompact((int)RequestedAccess);
+            writer.WriteCompact((int)Status);
+            writer.Write(ExplicitlyReported);
+            writer.Write(Error);
+            writer.Write(Usn.Value);
+            writer.Write((uint)DesiredAccess);
+            writer.Write((uint)ShareMode);
+            writer.Write((uint)CreationDisposition);
+            writer.Write((uint)FlagsAndAttributes);
+
+            if (writePath != null)
+            {
+                writePath(writer, ManifestPath);
+            }
+            else
+            {
+                writer.Write(ManifestPath);
+            }
+
+            writer.WriteNullableString(Path);
+            writer.WriteNullableString(EnumeratePattern);
+        }
+
+        /// <nodoc />
+        public static ReportedFileAccess Deserialize(
+            BuildXLReader reader, 
+            IReadOnlyList<ReportedProcess> processes, 
+            Func<BuildXLReader, AbsolutePath> readPath)
+        {
+            return new ReportedFileAccess(
+                operation: (ReportedFileOperation)reader.ReadByte(),
+                process: processes != null ? processes[reader.ReadInt32Compact()] : ReportedProcess.Deserialize(reader),
+                requestedAccess: (RequestedAccess)reader.ReadInt32Compact(),
+                status: (FileAccessStatus)reader.ReadInt32Compact(),
+                explicitlyReported: reader.ReadBoolean(),
+                error: reader.ReadUInt32(),
+                // In general if process is executed externally, e.g., in VM, the obtained USN cannot be translated to the host.
+                // However, for our low-privilege build, we are going to map the host volumes to the VM, and thus the USN
+                // can still be used.
+                usn: new Usn(reader.ReadUInt64()),
+                desiredAccess: (DesiredAccess)reader.ReadUInt32(),
+                shareMode: (ShareMode)reader.ReadUInt32(),
+                creationDisposition: (CreationDisposition)reader.ReadUInt32(),
+                flagsAndAttributes: (FlagsAndAttributes)reader.ReadUInt32(),
+                manifestPath: readPath != null ? readPath(reader) : reader.ReadAbsolutePath(),
+                path: reader.ReadNullableString(),
+                enumeratePatttern: reader.ReadNullableString());
+        }
+
         /// <inherit />
         public override bool Equals(object obj)
         {

Public/Src/Engine/Processes/ReportedProcess.cs

@@ -3,7 +3,8 @@
 
 using System;
 using System.Diagnostics.ContractsLight;
-using BuildXL.Pips;
+using BuildXL.Native.IO;
+using BuildXL.Utilities;
 
 namespace BuildXL.Processes
 {
@@ -89,5 +90,40 @@ namespace BuildXL.Processes
             : this(processId, path, string.Empty)
         {
         }
+
+        /// <nodoc />
+        public void Serialize(BuildXLWriter writer)
+        {
+            writer.Write(ProcessId);
+            writer.Write(Path);
+            writer.Write(ProcessArgs);
+            writer.Write(CreationTime);
+            writer.Write(ExitTime);
+            writer.Write(KernelTime);
+            writer.Write(UserTime);
+            IOCounters.Serialize(writer);
+            writer.Write(ExitCode);
+            writer.Write(ParentProcessId);
+        }
+
+        /// <nodoc />
+        public static ReportedProcess Deserialize(BuildXLReader reader)
+        {
+            var reportedProcess = new ReportedProcess(
+                processId: reader.ReadUInt32(), 
+                path:      reader.ReadString(), 
+                args:      reader.ReadString())
+            {
+                CreationTime    = reader.ReadDateTime(),
+                ExitTime        = reader.ReadDateTime(),
+                KernelTime      = reader.ReadTimeSpan(),
+                UserTime        = reader.ReadTimeSpan(),
+                IOCounters      = IOCounters.Deserialize(reader),
+                ExitCode        = reader.ReadUInt32(),
+                ParentProcessId = reader.ReadUInt32()
+            };
+
+            return reportedProcess;
+        }
     }
 }

Public/Src/Engine/Processes/SandboxObserverDescriptor.cs

@@ -0,0 +1,101 @@
+// Copyright (c) Microsoft. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+using System.Diagnostics.ContractsLight;
+using BuildXL.Utilities;
+
+namespace BuildXL.Processes
+{
+    /// <summary>
+    /// Descriptor for observer.
+    /// </summary>
+    public class SandboxObserverDescriptor
+    {
+        /// <summary>
+        /// Warning regex.
+        /// </summary>
+        public ExpandedRegexDescriptor WarningRegex { get; set; }
+
+        /// <summary>
+        /// Logs output to console.
+        /// </summary>
+        public bool LogOutputToConsole { get; set; }
+
+        /// <summary>
+        /// Logs error to console.
+        /// </summary>
+        public bool LogErrorToConsole { get; set; }
+
+        /// <summary>
+        /// Serializes this instance to a given <paramref name="writer"/>.
+        /// </summary>
+        public void Serialize(BuildXLWriter writer)
+        {
+            Contract.Requires(writer != null);
+
+            writer.Write(WarningRegex, (w, v) => v.Serialize(w));
+            writer.Write(LogOutputToConsole);
+            writer.Write(LogErrorToConsole);
+        }
+
+        /// <summary>
+        /// Deserializes an instance of <see cref="SandboxObserverDescriptor"/>.
+        /// </summary>
+        public static SandboxObserverDescriptor Deserialize(BuildXLReader reader)
+        {
+            Contract.Requires(reader != null);
+
+            return new SandboxObserverDescriptor()
+            {
+                WarningRegex = reader.ReadNullable(r => ExpandedRegexDescriptor.Deserialize(r)),
+                LogOutputToConsole = reader.ReadBoolean(),
+                LogErrorToConsole = reader.ReadBoolean()
+            };
+        }
+
+        /// <inheritdoc />
+        public override bool Equals(object obj)
+        {
+            return !(obj is null) && (ReferenceEquals(this, obj) || ((obj is SandboxObserverDescriptor descriptor) && Equals(descriptor)));
+        }
+
+        /// <summary>
+        /// Checks for equality.
+        /// </summary>
+        public bool Equals(SandboxObserverDescriptor descriptor)
+        {
+            return !(descriptor is null)
+                && (ReferenceEquals(this, descriptor)
+                    || (LogErrorToConsole == descriptor.LogErrorToConsole && LogOutputToConsole == descriptor.LogOutputToConsole && WarningRegex == descriptor.WarningRegex));
+        }
+
+        /// <summary>
+        /// Checks for equality.
+        /// </summary>
+        public static bool operator ==(SandboxObserverDescriptor descriptor1, SandboxObserverDescriptor descriptor2)
+        {
+            if (ReferenceEquals(descriptor1, descriptor2))
+            {
+                return true;
+            }
+
+            if (descriptor1 is null)
+            {
+                return false;
+            }
+
+            return descriptor1.Equals(descriptor2);
+        }
+
+        /// <summary>
+        /// Checks for disequality.
+        /// </summary>
+        public static bool operator !=(SandboxObserverDescriptor descriptor1, SandboxObserverDescriptor descriptor2) => !(descriptor1 == descriptor2);
+
+        /// <inheritdoc />
+        public override int GetHashCode()
+        {
+            return HashCodeHelper.Combine(WarningRegex.GetHashCode(), LogErrorToConsole ? 1 : 0, LogOutputToConsole ? 1 : 0);
+        }
+    }
+}

Public/Src/Engine/Processes/SandboxedProcess.MacKext.cs

@@ -329,7 +329,7 @@ namespace BuildXL.Processes
             Process.StandardInput.Close();
         }
 
-        internal override void FeedStdErr(SandboxedProcessOutputBuilder builder, TaskSourceSlim<Unit> tsc, string line)
+        internal override void FeedStdErr(SandboxedProcessOutputBuilder builder, string line)
         {
             if (line == null) // designates EOF
             {
@@ -337,17 +337,17 @@ namespace BuildXL.Processes
                 m_cpuTimes = ExtractCpuTimes(m_lastStdErrLine, out string unprocessedFragment);
 
                 // feed whatever wasn't consumed
-                FeedOutputBuilder(builder, tsc, unprocessedFragment);
+                FeedOutputBuilder(builder, unprocessedFragment);
 
                 // feed EOF
-                FeedOutputBuilder(builder, tsc, null);
+                FeedOutputBuilder(builder, null);
             }
             else
             {
                 // feed previous line (if any)
                 if (m_lastStdErrLine != null)
                 {
-                    FeedOutputBuilder(builder, tsc, m_lastStdErrLine);
+                    FeedOutputBuilder(builder, m_lastStdErrLine);
                 }
 
                 // update previous line

Public/Src/Engine/Processes/SandboxedProcessInfo.cs

@@ -13,6 +13,8 @@ using BuildXL.Utilities;
 using BuildXL.Utilities.Configuration;
 using BuildXL.Utilities.Instrumentation.Common;
 using static BuildXL.Utilities.BuildParameters;
+using System.Linq;
+using System.Text.RegularExpressions;
 
 namespace BuildXL.Processes
 {
@@ -80,9 +82,9 @@ namespace BuildXL.Processes
             bool disableConHostSharing,
             bool testRetries = false,
             LoggingContext loggingContext = null,
-            IDetoursEventListener detourseEventListener = null,
+            IDetoursEventListener detoursEventListener = null,
             IKextConnection sandboxedKextConnection = null)
-            : this(new PathTable(), fileStorage, fileName, disableConHostSharing, testRetries, loggingContext, detourseEventListener, sandboxedKextConnection)
+            : this(new PathTable(), fileStorage, fileName, disableConHostSharing, testRetries, loggingContext, detoursEventListener, sandboxedKextConnection)
         {
         }
 
@@ -365,5 +367,151 @@ namespace BuildXL.Processes
         /// Notify this delegate once process id becomes available.
         /// </summary>
         public Action<int> ProcessIdListener { get; set; }
+
+        /// <summary>
+        /// Standard output and error for sandboxed process.
+        /// </summary>
+        /// <remarks>
+        /// This instance of <see cref="SandboxedProcessStandardFiles"/> is used as an alternative to <see cref="FileStorage"/>.
+        /// </remarks>
+        public SandboxedProcessStandardFiles SandboxedProcessStandardFiles { get; set; }
+
+        /// <summary>
+        /// Info about the source of standard input.
+        /// </summary>
+        /// <remarks>
+        /// This instance of <see cref="StandardInputInfo"/> is used as a serialized version of <see cref="StandardInputReader"/>.
+        /// </remarks>
+        public StandardInputInfo StandardInputSourceInfo { get; set; }
+
+        /// <summary>
+        /// Observer descriptor.
+        /// </summary>
+        /// <remarks>
+        /// This instance of <see cref="SandboxObserverDescriptor"/> is used as a serialized version of <see cref="StandardOutputObserver"/> and <see cref="StandardErrorObserver"/>.
+        /// </remarks>
+        public SandboxObserverDescriptor StandardObserverDescriptor { get; set; }
+
+        #region Serialization
+
+        /// <nodoc />
+        public void Serialize(Stream stream)
+        {
+            using (var writer = new BuildXLWriter(false, stream, true, true))
+            {
+                writer.WriteNullableString(m_arguments);
+                writer.WriteNullableString(m_commandLine);
+                writer.Write(DisableConHostSharing);
+                writer.WriteNullableString(FileName);
+                writer.Write(StandardInputEncoding, (w, v) => w.Write(v));
+                writer.Write(StandardOutputEncoding, (w, v) => w.Write(v));
+                writer.Write(StandardErrorEncoding, (w, v) => w.Write(v));
+                writer.WriteNullableString(WorkingDirectory);
+                writer.Write(
+                    EnvironmentVariables,
+                    (w, v) => w.WriteReadOnlyList(
+                        v.ToDictionary().ToList(),
+                        (w2, kvp) =>
+                        {
+                            w2.Write(kvp.Key);
+                            w2.Write(kvp.Value);
+                        }));
+                writer.Write(
+                    AllowedSurvivingChildProcessNames,
+                    (w, v) => w.WriteReadOnlyList(v, (w2, v2) => w2.Write(v2)));
+                writer.Write(MaxLengthInMemory);
+                writer.Write(Timeout, (w, v) => w.Write(v));
+                writer.Write(NestedProcessTerminationTimeout);
+                writer.Write(PipSemiStableHash);
+                writer.WriteNullableString(TimeoutDumpDirectory);
+                writer.Write((byte)SandboxKind);
+                writer.WriteNullableString(PipDescription);
+
+                if (SandboxedProcessStandardFiles == null)
+                {
+                    SandboxedProcessStandardFiles.From(FileStorage).Serialize(writer);
+                }
+                else
+                {
+                    SandboxedProcessStandardFiles.Serialize(writer);
+                }
+
+                writer.Write(StandardInputSourceInfo, (w, v) => v.Serialize(w));
+                writer.Write(StandardObserverDescriptor, (w, v) => v.Serialize(w));
+
+                // File access manifest should be serialize the last.
+                writer.Write(FileAccessManifest, (w, v) => FileAccessManifest.Serialize(stream));
+            }
+        }
+
+        /// <nodoc />
+        public static SandboxedProcessInfo Deserialize(Stream stream, LoggingContext loggingContext, IDetoursEventListener detoursEventListener)
+        {
+            using (var reader = new BuildXLReader(false, stream, true))
+            {
+                string arguments = reader.ReadNullableString();
+                string commandLine = reader.ReadNullableString();
+                bool disableConHostSharing = reader.ReadBoolean();
+                string fileName = reader.ReadNullableString();
+                Encoding standardInputEncoding = reader.ReadNullable(r => r.ReadEncoding());
+                Encoding standardOutputEncoding = reader.ReadNullable(r => r.ReadEncoding());
+                Encoding standardErrorEncoding = reader.ReadNullable(r => r.ReadEncoding());
+                string workingDirectory = reader.ReadNullableString();
+                IBuildParameters buildParameters = null;
+                var envVars = reader.ReadNullable(r => r.ReadReadOnlyList(r2 => new KeyValuePair<string, string>(r2.ReadString(), r2.ReadString())));
+                if (envVars != null)
+                {
+
+                    buildParameters = BuildParameters.GetFactory().PopulateFromDictionary(envVars.ToDictionary(kvp => kvp.Key, kvp => kvp.Value));
+                }
+
+                string[] allowedSurvivingChildNames = reader.ReadNullable(r => r.ReadReadOnlyList(r2 => r2.ReadString()))?.ToArray();
+                int maxLengthInMemory = reader.ReadInt32();
+                TimeSpan? timeout = reader.ReadNullableStruct(r => r.ReadTimeSpan());
+                TimeSpan nestedProcessTerminationTimeout = reader.ReadTimeSpan();
+                long pipSemiStableHash = reader.ReadInt64();
+                string timeoutDumpDirectory = reader.ReadNullableString();
+                SandboxKind sandboxKind = (SandboxKind)reader.ReadByte();
+                string pipDescription = reader.ReadNullableString();
+                SandboxedProcessStandardFiles sandboxedProcessStandardFiles = SandboxedProcessStandardFiles.Deserialize(reader);
+                StandardInputInfo standardInputSourceInfo = reader.ReadNullable(r => StandardInputInfo.Deserialize(r));
+                SandboxObserverDescriptor standardObserverDescriptor = reader.ReadNullable(r => SandboxObserverDescriptor.Deserialize(r));
+
+                FileAccessManifest fam = reader.ReadNullable(r => FileAccessManifest.Deserialize(stream));
+
+                return new SandboxedProcessInfo(
+                    new PathTable(),
+                    new StandardFileStorage(sandboxedProcessStandardFiles),
+                    fileName,
+                    fam,
+                    disableConHostSharing,
+                    // TODO: serialize/deserialize container configuration.
+                    containerConfiguration: ContainerConfiguration.DisabledIsolation,
+                    loggingContext: loggingContext,
+                    detoursEventListener: detoursEventListener)
+                {
+                    m_arguments = arguments,
+                    m_commandLine = commandLine,
+                    StandardInputEncoding = standardInputEncoding,
+                    StandardOutputEncoding = standardOutputEncoding,
+                    StandardErrorEncoding = standardErrorEncoding,
+                    WorkingDirectory = workingDirectory,
+                    EnvironmentVariables = buildParameters,
+                    AllowedSurvivingChildProcessNames = allowedSurvivingChildNames,
+                    MaxLengthInMemory = maxLengthInMemory,
+                    Timeout = timeout,
+                    NestedProcessTerminationTimeout = nestedProcessTerminationTimeout,
+                    PipSemiStableHash = pipSemiStableHash,
+                    TimeoutDumpDirectory = timeoutDumpDirectory,
+                    SandboxKind = sandboxKind,
+                    PipDescription = pipDescription,
+                    SandboxedProcessStandardFiles = sandboxedProcessStandardFiles,
+                    StandardInputSourceInfo = standardInputSourceInfo,
+                    StandardObserverDescriptor = standardObserverDescriptor
+                };
+            }
+        }
+
+        #endregion
     }
-}
+}

Public/Src/Engine/Processes/SandboxedProcessOutput.cs

@@ -63,6 +63,52 @@ namespace BuildXL.Processes
             m_exception = exception;
         }
 
+        /// <summary>
+        /// Serializes this instance to a given <paramref name="writer"/>.
+        /// </summary>
+        public void Serialize(BuildXLWriter writer)
+        {
+            writer.Write(m_length);
+            writer.WriteNullableString(m_value);
+            writer.WriteNullableString(m_fileName);
+            writer.Write(m_encoding);
+            writer.Write(m_fileStorage, (w, v) => SandboxedProcessStandardFiles.From(v).Serialize(w));
+            writer.WriteCompact((uint)m_file);
+            writer.Write(m_exception, (w, v) =>
+            {
+                w.WriteNullableString(v.Message);
+                w.WriteCompact((uint)v.RootCause);
+            });
+        }
+
+        /// <summary>
+        /// Deserializes an instance of <see cref="SandboxedProcessOutput"/>.
+        /// </summary>
+        public static SandboxedProcessOutput Deserialize(BuildXLReader reader)
+        {
+            long length = reader.ReadInt64();
+            string value = reader.ReadNullableString();
+            string fileName = reader.ReadNullableString();
+            Encoding encoding = reader.ReadEncoding();
+            SandboxedProcessStandardFiles standardFiles = reader.ReadNullable(r => SandboxedProcessStandardFiles.Deserialize(r));
+            ISandboxedProcessFileStorage fileStorage = null;
+            if (standardFiles != null)
+            {
+                fileStorage = new StandardFileStorage(standardFiles);
+            }
+            SandboxedProcessFile file = (SandboxedProcessFile)reader.ReadUInt32Compact();
+            BuildXLException exception = reader.ReadNullable(r => new BuildXLException(r.ReadNullableString(), (ExceptionRootCause)r.ReadUInt32Compact()));
+
+            return new SandboxedProcessOutput(
+                length,
+                value,
+                fileName,
+                encoding,
+                fileStorage,
+                file,
+                exception);
+        }
+
         /// <summary>
         /// The encoding used when saving the file
         /// </summary>

Public/Src/Engine/Processes/SandboxedProcessPipExecutionResult.cs

@@ -155,6 +155,27 @@ namespace BuildXL.Processes
                 containerConfiguration: containerConfiguration);
         }
 
+        internal static SandboxedProcessPipExecutionResult MismatchedMessageCountFailure(SandboxedProcessPipExecutionResult result)
+            => new SandboxedProcessPipExecutionResult(
+                   SandboxedProcessPipExecutionStatus.MismatchedMessageCount,
+                   result.ObservedFileAccesses,
+                   result.SharedDynamicDirectoryWriteAccesses,
+                   result.EncodedStandardOutput,
+                   result.EncodedStandardError,
+                   result.NumberOfWarnings,
+                   result.UnexpectedFileAccesses,
+                   result.PrimaryProcessTimes,
+                   result.JobAccountingInformation,
+                   result.NumberOfProcessLaunchRetries,
+                   result.ExitCode,
+                   result.SandboxPrepMs,
+                   result.ProcessSandboxedProcessResultMs,
+                   result.ProcessStartTimeMs,
+                   result.AllReportedFileAccesses,
+                   result.DetouringStatuses,
+                   result.MaxDetoursHeapSizeInBytes,
+                   result.ContainerConfiguration);
+
         /// <summary>
         /// Indicates if the pip succeeded.
         /// </summary>

Public/Src/Engine/Processes/SandboxedProcessPipExecutor.cs

@@ -58,43 +58,6 @@ namespace BuildXL.Processes
         // Compute if WCI and Bind are available on this machine
         private static readonly bool s_isIsolationSupported = ProcessUtilities.IsWciAndBindFiltersAvailable();
 
-        private readonly struct ExpandedRegexDescriptor : IEquatable<ExpandedRegexDescriptor>
-        {
-            public readonly string Pattern;
-            public readonly RegexOptions Options;
-
-            public ExpandedRegexDescriptor(StringTable stringTable, RegexDescriptor descriptor)
-            {
-                Pattern = stringTable.GetString(descriptor.Pattern);
-                Options = descriptor.Options;
-            }
-
-            public bool Equals(ExpandedRegexDescriptor other)
-            {
-                return Pattern == other.Pattern && Options == other.Options;
-            }
-
-            public override int GetHashCode()
-            {
-                return HashCodeHelper.Combine(Pattern.GetHashCode(), (int)Options);
-            }
-
-            public override bool Equals(object obj)
-            {
-                return obj is ExpandedRegexDescriptor && Equals((ExpandedRegexDescriptor)obj);
-            }
-
-            public static bool operator ==(ExpandedRegexDescriptor left, ExpandedRegexDescriptor right)
-            {
-                return left.Equals(right);
-            }
-
-            public static bool operator !=(ExpandedRegexDescriptor left, ExpandedRegexDescriptor right)
-            {
-                return !(left == right);
-            }
-        }
-
         /// <summary>
         /// The maximum number that this executor will try to launch the given process pip.
         /// </summary>
@@ -280,14 +243,14 @@ namespace BuildXL.Processes
 
             if (pip.WarningRegex.IsValid)
             {
-                var expandedDescriptor = new ExpandedRegexDescriptor(context.StringTable, pip.WarningRegex);
+                var expandedDescriptor = new ExpandedRegexDescriptor(pip.WarningRegex.Pattern.ToString(context.StringTable), pip.WarningRegex.Options);
                 m_warningRegexTask = GetRegexAsync(expandedDescriptor);
                 m_warningRegexIsDefault = RegexDescriptor.IsDefault(expandedDescriptor.Pattern, expandedDescriptor.Options);
             }
 
             if (pip.ErrorRegex.IsValid)
             {
-                var expandedDescriptor = new ExpandedRegexDescriptor(context.StringTable, pip.ErrorRegex);
+                var expandedDescriptor = new ExpandedRegexDescriptor(pip.ErrorRegex.Pattern.ToString(context.StringTable), pip.ErrorRegex.Options);
                 m_errorRegexTask = GetRegexAsync(expandedDescriptor);
             }
 
@@ -606,7 +569,7 @@ namespace BuildXL.Processes
         {
             try
             {
-                System.Diagnostics.Stopwatch sandboxPrepTime = System.Diagnostics.Stopwatch.StartNew();
+                var sandboxPrepTime = System.Diagnostics.Stopwatch.StartNew();
                 var environmentVariables = m_pipEnvironment.GetEffectiveEnvironmentVariables(m_pip, m_pipDataRenderer);
 
                 if (!PrepareWorkingDirectory())
@@ -651,191 +614,44 @@ namespace BuildXL.Processes
 
                     string executable = m_pip.Executable.Path.ToString(m_pathTable);
                     string arguments = m_pip.Arguments.ToString(m_pipDataRenderer);
-
-                    Action<string> observer = m_warningRegexTask == null ? null : (Action<string>)Observe;
-
                     m_timeout = GetEffectiveTimeout(m_pip.Timeout, m_sandboxConfig.DefaultTimeout, m_sandboxConfig.TimeoutMultiplier);
 
-                    using (Stream standardInputStream = TryOpenStandardInputStream(out bool openStandardInputStreamSuccess))
+                    SandboxedProcessInfo info = new SandboxedProcessInfo(
+                        m_pathTable,
+                        this,
+                        executable,
+                        m_fileAccessManifest,
+                        m_disableConHostSharing,
+                        m_containerConfiguration,
+                        m_pip.TestRetries,
+                        m_loggingContext,
+                        sandboxedKextConnection: sandboxedKextConnection)
                     {
-                        if (!openStandardInputStreamSuccess)
-                        {
-                            return SandboxedProcessPipExecutionResult.PreparationFailure();
-                        }
+                        Arguments = arguments,
+                        WorkingDirectory = m_workingDirectory,
+                        RootMappings = m_rootMappings,
+                        EnvironmentVariables = environmentVariables,
+                        Timeout = m_timeout,
+                        PipSemiStableHash = m_pip.SemiStableHash,
+                        PipDescription = m_pip.GetDescription(m_context),
+                        ProcessIdListener = m_processIdListener,
+                        TimeoutDumpDirectory = ComputePipTimeoutDumpDirectory(m_sandboxConfig, m_pip, m_pathTable),
+                        SandboxKind = m_sandboxConfig.UnsafeSandboxConfiguration.SandboxKind,
+                        AllowedSurvivingChildProcessNames = m_pip.AllowedSurvivingChildProcessNames.Select(n => n.ToString(m_pathTable.StringTable)).ToArray(),
+                        NestedProcessTerminationTimeout = m_pip.NestedProcessTerminationTimeout ?? SandboxedProcessInfo.DefaultNestedProcessTerminationTimeout,
+                    };
 
-                        using (StreamReader standardInputReader = standardInputStream == null ? null : new StreamReader(standardInputStream, CharUtilities.Utf8NoBomNoThrow))
-                        {
-                            var info =
-                                new SandboxedProcessInfo(
-                                    m_pathTable,
-                                    this,
-                                    executable,
-                                    m_fileAccessManifest,
-                                    m_disableConHostSharing,
-                                    m_containerConfiguration,
-                                    m_pip.TestRetries,
-                                    m_loggingContext,
-                                    sandboxedKextConnection: sandboxedKextConnection)
-                                {
-                                    Arguments = arguments,
-                                    WorkingDirectory = m_workingDirectory,
-                                    StandardInputReader = standardInputReader,
-                                    StandardInputEncoding = standardInputReader?.CurrentEncoding,
-
-                                    // MaxLengthInMemory, TODO: We could let the Process Pip configure this
-                                    // BufferSize, TODO: We could let the Process Pip configure this
-                                    StandardErrorObserver = observer,
-                                    StandardOutputObserver = observer,
-                                    RootMappings = m_rootMappings,
-                                    EnvironmentVariables = environmentVariables,
-                                    Timeout = m_timeout,
-                                    PipSemiStableHash = m_pip.SemiStableHash,
-                                    PipDescription = m_pip.GetDescription(m_context),
-                                    ProcessIdListener = m_processIdListener,
-                                    TimeoutDumpDirectory = ComputePipTimeoutDumpDirectory(m_sandboxConfig, m_pip, m_pathTable),
-                                    SandboxKind = m_sandboxConfig.UnsafeSandboxConfiguration.SandboxKind,
-                                    AllowedSurvivingChildProcessNames = m_pip.AllowedSurvivingChildProcessNames.Select(n => n.ToString(m_pathTable.StringTable)).ToArray(),
-                                    NestedProcessTerminationTimeout = m_pip.NestedProcessTerminationTimeout ?? SandboxedProcessInfo.DefaultNestedProcessTerminationTimeout,
-                                };
-
-                            if (info.GetCommandLine().Length > SandboxedProcessInfo.MaxCommandLineLength)
-                            {
-                                LogCommandLineTooLong(info);
-                                return SandboxedProcessPipExecutionResult.PreparationFailure();
-                            }
-
-                            if (!await TryInitializeWarningRegexAsync())
-                            {
-                                return SandboxedProcessPipExecutionResult.PreparationFailure();
-                            }
-
-                            sandboxPrepTime.Stop();
-                            ISandboxedProcess process = null;
-
-                            // Sometimes the injection of Detours fails with error ERROR_PARTIAL_COPY (0x12b)
-                            // This is random failure, not consistent at all and it seems to be in the lower levels of
-                            // Detours. If we get such error attempt running the process up to RetryStartCount times
-                            // before bailing out and reporting an error.
-                            int processLaunchRetryCount = 0;
-                            long maxDetoursHeapSize = 0L;
-                            bool shouldRelaunchProcess = true;
-
-                            while (shouldRelaunchProcess)
-                            {
-                                try
-                                {
-                                    shouldRelaunchProcess = false;
-
-                                    // If the process should be run in a container, we (verbose) log the remapping information
-                                    if (m_containerConfiguration.IsIsolationEnabled)
-                                    {
-                                        // If the process was specified to run in a container but isolation is not supported, then we bail out
-                                        // before even trying to run the process
-                                        if (!s_isIsolationSupported)
-                                        {
-                                            Tracing.Logger.Log.PipSpecifiedToRunInContainerButIsolationIsNotSupported(m_loggingContext, m_pip.SemiStableHash, m_pip.GetDescription(m_context));
-                                            return SandboxedProcessPipExecutionResult.PreparationFailure(processLaunchRetryCount, (int)EventId.PipSpecifiedToRunInContainerButIsolationIsNotSupported, maxDetoursHeapSize: maxDetoursHeapSize);
-                                        }
-
-                                        Tracing.Logger.Log.PipInContainerStarting(m_loggingContext, m_pip.SemiStableHash, m_pip.GetDescription(m_context), m_containerConfiguration.ToDisplayString());
-                                    }
-
-                                    process = await SandboxedProcessFactory.StartAsync(info, forceSandboxing: false);
-
-                                    // If the process started in a container, the setup of it is ready at this point, so we (verbose) log it
-                                    if (m_containerConfiguration.IsIsolationEnabled)
-                                    {
-                                        Tracing.Logger.Log.PipInContainerStarted(m_loggingContext, m_pip.SemiStableHash, m_pip.GetDescription(m_context));
-                                    }
-                                }
-                                catch (BuildXLException ex)
-                                {
-                                    if (ex.LogEventErrorCode == NativeIOConstants.ErrorFileNotFound)
-                                    {
-                                        LocationData location = m_pip.Provenance.Token;
-                                        string specFile = location.Path.ToString(m_pathTable);
-
-                                        Tracing.Logger.Log.PipProcessStartFailed(m_loggingContext, m_pip.SemiStableHash, m_pip.GetDescription(m_context), 2,
-                                            string.Format(CultureInfo.InvariantCulture, "File '{0}' was not found on disk. The tool is referred in '{1}({2})'.", executable, specFile, location.Position));
-                                    }
-                                    else if (ex.LogEventErrorCode == NativeIOConstants.ErrorPartialCopy && (processLaunchRetryCount < ProcessLaunchRetryCountMax))
-                                    {
-                                        processLaunchRetryCount++;
-                                        shouldRelaunchProcess = true;
-                                        Tracing.Logger.Log.RetryStartPipDueToErrorPartialCopyDuringDetours(
-                                            m_loggingContext,
-                                            m_pip.SemiStableHash,
-                                            m_pip.GetDescription(m_context),
-                                            ex.LogEventErrorCode,
-                                            processLaunchRetryCount);
-
-                                        // We are about to retry a process execution.
-                                        // Make sure we wait for the process to end. This way the reporting messages get flushed.
-                                        if (process != null)
-                                        {
-                                            maxDetoursHeapSize = process.GetDetoursMaxHeapSize();
-
-                                            try
-                                            {
-                                                await process.GetResultAsync();
-                                            }
-                                            finally
-                                            {
-                                                process.Dispose();
-                                            }
-                                        }
-
-                                        continue;
-                                    }
-                                    else
-                                    {
-                                        // not all start failures map to Win32 error code, so we have a message here too
-                                        Tracing.Logger.Log.PipProcessStartFailed(
-                                            m_loggingContext,
-                                            m_pip.SemiStableHash,
-                                            m_pip.GetDescription(m_context),
-                                            ex.LogEventErrorCode,
-                                            ex.LogEventMessage);
-                                    }
-
-                                    return SandboxedProcessPipExecutionResult.PreparationFailure(processLaunchRetryCount, ex.LogEventErrorCode, maxDetoursHeapSize: maxDetoursHeapSize);
-                                }
-                            }
-
-                            using (var cancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, m_context.CancellationToken))
-                            {
-                                var cancellationTokenRegistration = cancellationTokenSource.Token.Register(() => process.KillAsync());
-
-                                SandboxedProcessResult result;
-
-                                int lastMessageCount;
-
-                                try
-                                {
-                                    m_activeProcess = process;
-                                    result = await process.GetResultAsync();
-                                    lastMessageCount = process.GetLastMessageCount();
-                                }
-                                finally
-                                {
-                                    m_activeProcess = null;
-                                    cancellationTokenRegistration.Dispose();
-                                    process.Dispose();
-                                }
-
-                                result.NumberOfProcessLaunchRetries = processLaunchRetryCount;
-                                SandboxedProcessPipExecutionResult executionResult =
-                                    await
-                                        ProcessSandboxedProcessResultAsync(
-                                            m_loggingContext,
-                                            result,
-                                            sandboxPrepTime.ElapsedMilliseconds,
-                                            cancellationTokenSource.Token,
-                                            process.GetDetoursMaxHeapSize(),
-                                            allInputPathsUnderSharedOpaques);
-                                return ValidateDetoursCommunication(executionResult, lastMessageCount);
-                            }
-                        }
+                    if (m_sandboxConfig.AdminRequiredProcessExecutionMode == AdminRequiredProcessExecutionMode.Internal
+                        || !m_pip.RequiresAdmin
+                        || m_processIdListener != null
+                        || m_containerConfiguration.IsIsolationEnabled
+                        || OperatingSystemHelper.IsUnixOS)
+                    {
+                        return await RunInternalAsync(info, allInputPathsUnderSharedOpaques, sandboxPrepTime, cancellationToken);
+                    }
+                    else
+                    {
+                        return await RunExternalAsync(info, allInputPathsUnderSharedOpaques, sandboxPrepTime, cancellationToken);
                     }
                 }
             }
@@ -843,19 +659,267 @@ namespace BuildXL.Processes
             {
                 Contract.Assert(m_fileAccessManifest != null);
 
-                // Always dispose the semaphore.
-                if (m_fileAccessManifest.MessageCountSemaphore != null)
+                m_fileAccessManifest.UnsetMessageCountSemaphore();
+            }
+        }
+
+        private async Task<SandboxedProcessPipExecutionResult> RunInternalAsync(
+            SandboxedProcessInfo info,
+            HashSet<AbsolutePath> allInputPathsUnderSharedOpaques,
+            System.Diagnostics.Stopwatch sandboxPrepTime,
+            CancellationToken cancellationToken = default)
+        {
+            using (Stream standardInputStream = TryOpenStandardInputStream(out bool openStandardInputStreamSuccess))
+            {
+                if (!openStandardInputStreamSuccess)
                 {
-                    m_fileAccessManifest.MessageCountSemaphore.Dispose();
-                    m_fileAccessManifest.MessageCountSemaphore = null;
+                    return SandboxedProcessPipExecutionResult.PreparationFailure();
                 }
+
+                using (StreamReader standardInputReader = standardInputStream == null ? null : new StreamReader(standardInputStream, CharUtilities.Utf8NoBomNoThrow))
+                {
+                    info.StandardInputReader = standardInputReader;
+                    info.StandardInputEncoding = standardInputReader?.CurrentEncoding;
+
+                    Action<string> observer = m_warningRegexTask == null ? null : (Action<string>)Observe;
+
+                    info.StandardOutputObserver = observer;
+                    info.StandardErrorObserver = observer;
+
+                    if (info.GetCommandLine().Length > SandboxedProcessInfo.MaxCommandLineLength)
+                    {
+                        LogCommandLineTooLong(info);
+                        return SandboxedProcessPipExecutionResult.PreparationFailure();
+                    }
+
+                    if (!await TryInitializeWarningRegexAsync())
+                    {
+                        return SandboxedProcessPipExecutionResult.PreparationFailure();
+                    }
+
+                    sandboxPrepTime.Stop();
+                    ISandboxedProcess process = null;
+
+                    // Sometimes the injection of Detours fails with error ERROR_PARTIAL_COPY (0x12b)
+                    // This is random failure, not consistent at all and it seems to be in the lower levels of
+                    // Detours. If we get such error attempt running the process up to RetryStartCount times
+                    // before bailing out and reporting an error.
+                    int processLaunchRetryCount = 0;
+                    long maxDetoursHeapSize = 0L;
+                    bool shouldRelaunchProcess = true;
+
+                    while (shouldRelaunchProcess)
+                    {
+                        try
+                        {
+                            shouldRelaunchProcess = false;
+
+                            // If the process should be run in a container, we (verbose) log the remapping information
+                            if (m_containerConfiguration.IsIsolationEnabled)
+                            {
+                                // If the process was specified to run in a container but isolation is not supported, then we bail out
+                                // before even trying to run the process
+                                if (!s_isIsolationSupported)
+                                {
+                                    Tracing.Logger.Log.PipSpecifiedToRunInContainerButIsolationIsNotSupported(m_loggingContext, m_pip.SemiStableHash, m_pip.GetDescription(m_context));
+                                    return SandboxedProcessPipExecutionResult.PreparationFailure(processLaunchRetryCount, (int)EventId.PipSpecifiedToRunInContainerButIsolationIsNotSupported, maxDetoursHeapSize: maxDetoursHeapSize);
+                                }
+
+                                Tracing.Logger.Log.PipInContainerStarting(m_loggingContext, m_pip.SemiStableHash, m_pip.GetDescription(m_context), m_containerConfiguration.ToDisplayString());
+                            }
+
+                            process = await SandboxedProcessFactory.StartAsync(info, forceSandboxing: false);
+
+                            // If the process started in a container, the setup of it is ready at this point, so we (verbose) log it
+                            if (m_containerConfiguration.IsIsolationEnabled)
+                            {
+                                Tracing.Logger.Log.PipInContainerStarted(m_loggingContext, m_pip.SemiStableHash, m_pip.GetDescription(m_context));
+                            }
+                        }
+                        catch (BuildXLException ex)
+                        {
+                            if (ex.LogEventErrorCode == NativeIOConstants.ErrorFileNotFound)
+                            {
+                                LocationData location = m_pip.Provenance.Token;
+                                string specFile = location.Path.ToString(m_pathTable);
+
+                                Tracing.Logger.Log.PipProcessStartFailed(m_loggingContext, m_pip.SemiStableHash, m_pip.GetDescription(m_context), 2,
+                                    string.Format(CultureInfo.InvariantCulture, "File '{0}' was not found on disk. The tool is referred in '{1}({2})'.", info.FileName, specFile, location.Position));
+                            }
+                            else if (ex.LogEventErrorCode == NativeIOConstants.ErrorPartialCopy && (processLaunchRetryCount < ProcessLaunchRetryCountMax))
+                            {
+                                processLaunchRetryCount++;
+                                shouldRelaunchProcess = true;
+                                Tracing.Logger.Log.RetryStartPipDueToErrorPartialCopyDuringDetours(
+                                    m_loggingContext,
+                                    m_pip.SemiStableHash,
+                                    m_pip.GetDescription(m_context),
+                                    ex.LogEventErrorCode,
+                                    processLaunchRetryCount);
+
+                                // We are about to retry a process execution.
+                                // Make sure we wait for the process to end. This way the reporting messages get flushed.
+                                if (process != null)
+                                {
+                                    maxDetoursHeapSize = process.GetDetoursMaxHeapSize();
+
+                                    try
+                                    {
+                                        await process.GetResultAsync();
+                                    }
+                                    finally
+                                    {
+                                        process.Dispose();
+                                    }
+                                }
+
+                                continue;
+                            }
+                            else
+                            {
+                                // not all start failures map to Win32 error code, so we have a message here too
+                                Tracing.Logger.Log.PipProcessStartFailed(
+                                    m_loggingContext,
+                                    m_pip.SemiStableHash,
+                                    m_pip.GetDescription(m_context),
+                                    ex.LogEventErrorCode,
+                                    ex.LogEventMessage);
+                            }
+
+                            return SandboxedProcessPipExecutionResult.PreparationFailure(processLaunchRetryCount, ex.LogEventErrorCode, maxDetoursHeapSize: maxDetoursHeapSize);
+                        }
+                    }
+
+                    return await GetAndProcessResultAsync(process, allInputPathsUnderSharedOpaques, sandboxPrepTime, cancellationToken);
+                }
+            }
+        }
+
+        private async Task<SandboxedProcessPipExecutionResult> RunExternalAsync(
+            SandboxedProcessInfo info,
+            HashSet<AbsolutePath> allInputPathsUnderSharedOpaques,
+            System.Diagnostics.Stopwatch sandboxPrepTime,
+            CancellationToken cancellationToken = default)
+        {
+            StandardInputInfo standardInputSource = m_pip.StandardInput.IsData
+                ? StandardInputInfo.CreateForData(m_pip.StandardInput.Data.ToString(m_context.PathTable))
+                : (m_pip.StandardInput.IsFile
+                    ? StandardInputInfo.CreateForFile(m_pip.StandardInput.File.Path.ToString(m_context.PathTable))
+                    : null);
+
+            info.StandardInputSourceInfo = standardInputSource;
+
+            if (m_pip.WarningRegex.IsValid)
+            {
+                var observerDescriptor = new SandboxObserverDescriptor
+                {
+                    WarningRegex = new ExpandedRegexDescriptor(m_pip.WarningRegex.Pattern.ToString(m_context.StringTable), m_pip.WarningRegex.Options)
+                };
+
+                info.StandardObserverDescriptor = observerDescriptor;
+            }
+
+            // Preparation should be finished.
+            sandboxPrepTime.Stop();
+            ISandboxedProcess process = null;
+
+            try
+            {
+                if (m_sandboxConfig.AdminRequiredProcessExecutionMode == AdminRequiredProcessExecutionMode.ExternalTool)
+                {
+                    string toolPath = Path.Combine(
+                        m_layoutConfiguration.BuildEngineDirectory.ToString(m_context.PathTable),
+                        ExternalToolSandboxedProcess.DefaultToolRelativePath);
+
+                    Tracing.Logger.Log.PipProcessStartExternalTool(m_loggingContext, m_pip.SemiStableHash, m_pip.GetDescription(m_context), toolPath);
+
+                    process = await ExternalToolSandboxedProcess.StartAsync(info, toolPath);
+                }
+                else
+                {
+                    throw new NotImplementedException();
+                }
+            }
+            catch (BuildXLException ex)
+            {
+                Tracing.Logger.Log.PipProcessStartFailed(
+                    m_loggingContext,
+                    m_pip.SemiStableHash,
+                    m_pip.GetDescription(m_context),
+                    ex.LogEventErrorCode,
+                    ex.LogEventMessage);
+
+                return SandboxedProcessPipExecutionResult.PreparationFailure(0, ex.LogEventErrorCode);
+            }
+
+            return await GetAndProcessResultAsync(process, allInputPathsUnderSharedOpaques, sandboxPrepTime, cancellationToken);
+        }
+
+        private async Task<SandboxedProcessPipExecutionResult> GetAndProcessResultAsync(
+            ISandboxedProcess process,
+            HashSet<AbsolutePath> allInputPathsUnderSharedOpaques,
+            System.Diagnostics.Stopwatch sandboxPrepTime,
+            CancellationToken cancellationToken)
+        {
+            using (var cancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, m_context.CancellationToken))
+            {
+                var cancellationTokenRegistration = cancellationTokenSource.Token.Register(() => process.KillAsync());
+
+                SandboxedProcessResult result;
+
+                int lastMessageCount = 0;
+                bool isMessageCountSemaphoreCreated = false;
+
+                try
+                {
+                    m_activeProcess = process;
+                    result = await process.GetResultAsync();
+                    lastMessageCount = process.GetLastMessageCount() + result.LastMessageCount;
+                    m_numWarnings += result.WarningCount;
+                    isMessageCountSemaphoreCreated = m_fileAccessManifest.MessageCountSemaphore != null || result.MessageCountSemaphoreCreated;
+
+                    if (process is ExternalSandboxedProcess externalSandboxedProcess)
+                    {
+                        Tracing.Logger.Log.PipProcessFinishedExternalTool(
+                            m_loggingContext,
+                            m_pip.SemiStableHash,
+                            m_pip.GetDescription(m_context),
+                            externalSandboxedProcess.ExitCode ?? -1,
+                            Environment.NewLine + "StdOut:" + Environment.NewLine + externalSandboxedProcess.StdOut,
+                            Environment.NewLine + "StdErr:" + Environment.NewLine + externalSandboxedProcess.StdErr);
+                    }
+                }
+                finally
+                {
+                    m_activeProcess = null;
+                    cancellationTokenRegistration.Dispose();
+                    process.Dispose();
+                }
+
+                SandboxedProcessPipExecutionResult executionResult =
+                    await
+                        ProcessSandboxedProcessResultAsync(
+                            m_loggingContext,
+                            result,
+                            sandboxPrepTime.ElapsedMilliseconds,
+                            cancellationTokenSource.Token,
+                            process.GetDetoursMaxHeapSize() + result.DetoursMaxHeapSize,
+                            allInputPathsUnderSharedOpaques);
+
+                return ValidateDetoursCommunication(
+                    executionResult,
+                    lastMessageCount,
+                    isMessageCountSemaphoreCreated);
             }
         }
 
         /// <summary>
         /// These various validations that the detours communication channel
         /// </summary>
-        private SandboxedProcessPipExecutionResult ValidateDetoursCommunication(SandboxedProcessPipExecutionResult result, int lastMessageCount)
+        private SandboxedProcessPipExecutionResult ValidateDetoursCommunication(
+            SandboxedProcessPipExecutionResult result, 
+            int lastMessageCount,
+            bool isMessageSemaphoreCountCreated)
         {
             // If we have a failure already, that could have cause some of the mismatch in message count of writing the side communication file.
             if (result.Status == SandboxedProcessPipExecutionStatus.Succeeded && !string.IsNullOrEmpty(m_detoursFailuresFile))
@@ -892,7 +956,7 @@ namespace BuildXL.Processes
                 // a pip running longer than the timeout (5 hours). The pip gets killed and in such cases the message count mismatch
                 // is legitimate.
                 // Report a counter mismatch only if there are no other errors.
-                if (result.Status == SandboxedProcessPipExecutionStatus.Succeeded && m_fileAccessManifest.MessageCountSemaphore != null)
+                if (result.Status == SandboxedProcessPipExecutionStatus.Succeeded && isMessageSemaphoreCountCreated)
                 {
                     if (lastMessageCount != 0)
                     {
@@ -902,25 +966,7 @@ namespace BuildXL.Processes
                             m_pip.GetDescription(m_context),
                             lastMessageCount);
 
-                        return new SandboxedProcessPipExecutionResult(
-                            SandboxedProcessPipExecutionStatus.MismatchedMessageCount,
-                            result.ObservedFileAccesses,
-                            result.SharedDynamicDirectoryWriteAccesses,
-                            result.EncodedStandardOutput,
-                            result.EncodedStandardError,
-                            result.NumberOfWarnings,
-                            result.UnexpectedFileAccesses,
-                            result.PrimaryProcessTimes,
-                            result.JobAccountingInformation,
-                            result.NumberOfProcessLaunchRetries,
-                            result.ExitCode,
-                            result.SandboxPrepMs,
-                            result.ProcessSandboxedProcessResultMs,
-                            result.ProcessStartTimeMs,
-                            result.AllReportedFileAccesses,
-                            result.DetouringStatuses,
-                            result.MaxDetoursHeapSizeInBytes,
-                            result.ContainerConfiguration);
+                        return SandboxedProcessPipExecutionResult.MismatchedMessageCountFailure(result);
                     }
                 }
             }
@@ -1459,7 +1505,7 @@ namespace BuildXL.Processes
             m_fileAccessManifest.ReportProcessArgs = m_sandboxConfig.LogProcesses;
             m_fileAccessManifest.EnforceAccessPoliciesOnDirectoryCreation = m_sandboxConfig.EnforceAccessPoliciesOnDirectoryCreation;
 
-            bool allowInternalErrorsLogging = m_fileAccessManifest.AllowInternalDetoursErrorNotificationFile = m_sandboxConfig.AllowInternalDetoursErrorNotificationFile;
+            bool allowInternalErrorsLogging = m_sandboxConfig.AllowInternalDetoursErrorNotificationFile;
             bool checkMessageCount = m_fileAccessManifest.CheckDetoursMessageCount = m_sandboxConfig.CheckDetoursMessageCount;
 
             if (allowInternalErrorsLogging || checkMessageCount)
@@ -1478,11 +1524,12 @@ namespace BuildXL.Processes
                     }
 
                     // TODO: named semaphores are not supported in NetStandard2.0
-                    if (checkMessageCount && !OperatingSystemHelper.IsUnixOS)
+                    if (m_sandboxConfig.AdminRequiredProcessExecutionMode == AdminRequiredProcessExecutionMode.Internal 
+                        && checkMessageCount 
+                        && !OperatingSystemHelper.IsUnixOS)
                     {
                         // Semaphore names don't allow '\\' chars.
-                        m_fileAccessManifest.MessageCountSemaphore = new Semaphore(0, int.MaxValue, m_detoursFailuresFile.Replace('\\', '_'), out bool newCreated);
-                        if (!newCreated)
+                        if (!m_fileAccessManifest.SetMessageCountSemaphore(m_detoursFailuresFile.Replace('\\', '_')))
                         {
                             Tracing.Logger.Log.LogMessageCountSemaphoreExists(loggingContext, m_pip.SemiStableHash, m_pip.GetDescription(m_context));
                             return false;

Public/Src/Engine/Processes/SandboxedProcessReports.cs

@@ -538,6 +538,12 @@ namespace BuildXL.Processes
                 return false;
             }
 
+            // Special case seen with vstest.console.exe
+            if (string.IsNullOrEmpty(path))
+            {
+                return true;
+            }
+
             // If there is a listener registered and notifications allowed, notify over the interface.
             if (m_detoursEventListener != null && (m_detoursEventListener.GetMessageHandlingFlags() & MessageHandlingFlags.FileAccessNotify) != 0)
             {
@@ -554,7 +560,7 @@ namespace BuildXL.Processes
                     shareMode,
                     creationDisposition,
                     flagsAndAttributes,
-                    path == null ? manifestPath.ToString(m_pathTable) : path,
+                    path,
                     processArgs);
             }
 
@@ -564,12 +570,6 @@ namespace BuildXL.Processes
                 return true;
             }
 
-            // Special case seen with vstest.console.exe
-            if (path.Length == 0)
-            {
-                return true;
-            }
-
             if (m_manifest.DirectoryTranslator != null)
             {
                 path = m_manifest.DirectoryTranslator.Translate(path);
@@ -700,7 +700,7 @@ namespace BuildXL.Processes
                 string line,
                 out uint processId,
                 out string processName,
-                out Pips.IOCounters ioCounters,
+                out IOCounters ioCounters,
                 out DateTime creationDateTime,
                 out DateTime exitDateTime,
                 out TimeSpan kernelTime,
@@ -793,10 +793,10 @@ namespace BuildXL.Processes
                     fileTime += userLowDateTime;
                     userTime = TimeSpan.FromTicks(fileTime);
 
-                    ioCounters = new BuildXL.Pips.IOCounters(
-                        new BuildXL.Pips.IOTypeCounters(readOperationCount, readTransferCount),
-                        new BuildXL.Pips.IOTypeCounters(writeOperationCount, writeTransferCount),
-                        new BuildXL.Pips.IOTypeCounters(otherOperationCount, otherTransferCount));
+                    ioCounters = new IOCounters(
+                        new IOTypeCounters(readOperationCount, readTransferCount),
+                        new IOTypeCounters(writeOperationCount, writeTransferCount),
+                        new IOTypeCounters(otherOperationCount, otherTransferCount));
                     return true;
                 }
 

Public/Src/Engine/Processes/SandboxedProcessResult.cs

@@ -3,6 +3,9 @@
 
 using System;
 using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using BuildXL.Utilities;
 
 namespace BuildXL.Processes
 {
@@ -119,11 +122,188 @@ namespace BuildXL.Processes
         /// This could happen if the child process is killed while writing a message in the pipe.
         /// If null there is no error, otherwise the Faiulure object contains string, describing the error.
         /// </summary>
-        public BuildXL.Utilities.Failure<string> MessageProcessingFailure { get; internal set; }
+        public Failure<string> MessageProcessingFailure { get; internal set; }
 
         /// <summary>
         /// Time (in ms.) spent for startiing the process.
         /// </summary>
         public long ProcessStartTime { get; internal set; }
+
+        /// <summary>
+        /// Number of warnings.
+        /// </summary>
+        public int WarningCount { get; set; }
+
+        /// <summary>
+        /// Maximum heap size of the sandboxed process.
+        /// </summary>
+        public long DetoursMaxHeapSize { get; set; }
+
+        /// <summary>
+        /// Differences in sent and received messages from sandboxed process.
+        /// </summary>
+        public int LastMessageCount { get; set; }
+
+        /// <summary>
+        /// Flag indicating if a semaphore is created for Detours message.
+        /// </summary>
+        public bool MessageCountSemaphoreCreated { get; set; }
+
+        /// <summary>
+        /// Serializes this instance to a given <paramref name="stream"/>.
+        /// </summary>
+        public void Serialize(Stream stream)
+        {
+            using (var writer = new BuildXLWriter(false, stream, true, true))
+            {
+                Serialize(writer);
+            }
+        }
+
+        /// <summary>
+        /// Serializes this instance to a given <paramref name="writer"/>.
+        /// </summary>
+        public void Serialize(BuildXLWriter writer)
+        {
+            writer.Write(ExitCode);
+            writer.Write(Killed);
+            writer.Write(TimedOut);
+            writer.Write(HasDetoursInjectionFailures);
+
+            var processMap = CreateAndSerializeProcessMap(writer);
+
+            writer.Write(SurvivingChildProcesses, (w, v) => w.WriteReadOnlyList(v.ToList(), (w2, v2) => w2.Write(processMap[v2])));
+            writer.Write(PrimaryProcessTimes, (w, v) => v.Serialize(w));
+            writer.Write(JobAccountingInformation, (w, v) => v.Serialize(w));
+            writer.Write(StandardOutput, (w, v) => v.Serialize(w));
+            writer.Write(StandardError, (w, v) => v.Serialize(w));
+            writer.Write(FileAccesses, (w, v) => w.WriteReadOnlyList(v.ToList(), (w2, v2) => v2.Serialize(writer, processMap, writePath: null)));
+            writer.Write(ExplicitlyReportedFileAccesses, (w, v) => w.WriteReadOnlyList(v.ToList(), (w2, v2) => v2.Serialize(writer, processMap, writePath: null)));
+            writer.Write(AllUnexpectedFileAccesses, (w, v) => w.WriteReadOnlyList(v.ToList(), (w2, v2) => v2.Serialize(writer, processMap, writePath: null)));
+            writer.Write(Processes, (w, v) => w.WriteReadOnlyList(v, (w2, v2) => w2.Write(processMap[v2])));
+            writer.Write(DetouringStatuses, (w, v) => w.WriteReadOnlyList(v, (w2, v2) => v2.Serialize(w2)));
+            writer.WriteNullableString(DumpFileDirectory);
+            writer.WriteNullableString(DumpCreationException?.Message);
+            writer.WriteNullableString(StandardInputException?.Message);
+            writer.Write(NumberOfProcessLaunchRetries);
+            writer.Write(HasReadWriteToReadFileAccessRequest);
+            writer.WriteNullableString(MessageProcessingFailure?.Describe());
+            writer.Write(ProcessStartTime);
+            writer.Write(WarningCount);
+            writer.Write(DetoursMaxHeapSize);
+            writer.Write(LastMessageCount);
+            writer.Write(MessageCountSemaphoreCreated);
+        }
+
+        /// <summary>
+        /// Deserializes an instance of <see cref="SandboxedProcessResult"/>.
+        /// </summary>
+        public static SandboxedProcessResult Deserialize(Stream stream)
+        {
+            using (var reader = new BuildXLReader(false, stream, true))
+            {
+                return Deserialize(reader);
+            }
+        }
+
+        /// <summary>
+        /// Deserializes an instance of <see cref="SandboxedProcessResult"/>.
+        /// </summary>
+        /// <param name="reader"></param>
+        /// <returns></returns>
+        public static SandboxedProcessResult Deserialize(BuildXLReader reader)
+        {
+            int exitCode = reader.ReadInt32();
+            bool killed = reader.ReadBoolean();
+            bool timedOut = reader.ReadBoolean();
+            bool hasDetoursInjectionFailures = reader.ReadBoolean();
+
+            IReadOnlyList<ReportedProcess> allReportedProcesses = reader.ReadReadOnlyList(r => ReportedProcess.Deserialize(r));
+            IReadOnlyList<ReportedProcess> survivingChildProcesses = reader.ReadNullable(r => r.ReadReadOnlyList(r2 => allReportedProcesses[r2.ReadInt32()]));
+            ProcessTimes primaryProcessTimes = reader.ReadNullable(r => ProcessTimes.Deserialize(r));
+            JobObject.AccountingInformation? jobAccountingInformation = reader.ReadNullableStruct(r => JobObject.AccountingInformation.Deserialize(r));
+            SandboxedProcessOutput standardOutput = reader.ReadNullable(r => SandboxedProcessOutput.Deserialize(r));
+            SandboxedProcessOutput standardError = reader.ReadNullable(r => SandboxedProcessOutput.Deserialize(r));
+            IReadOnlyList<ReportedFileAccess> fileAccesses = reader.ReadNullable(r => r.ReadReadOnlyList(r2 => ReportedFileAccess.Deserialize(r2, allReportedProcesses, readPath: null)));
+            IReadOnlyList<ReportedFileAccess> explicitlyReportedFileAccesses = reader.ReadNullable(r => r.ReadReadOnlyList(r2 => ReportedFileAccess.Deserialize(r2, allReportedProcesses, readPath: null)));
+            IReadOnlyList<ReportedFileAccess> allUnexpectedFileAccesses = reader.ReadNullable(r => r.ReadReadOnlyList(r2 => ReportedFileAccess.Deserialize(r2, allReportedProcesses, readPath: null)));
+            IReadOnlyList<ReportedProcess> processes = reader.ReadNullable(r => r.ReadReadOnlyList(r2 => allReportedProcesses[r2.ReadInt32()]));
+            IReadOnlyList<ProcessDetouringStatusData> detouringStatuses = reader.ReadNullable(r => r.ReadReadOnlyList(r2 => ProcessDetouringStatusData.Deserialize(r2)));
+            string dumpFileDirectory = reader.ReadNullableString();
+            string dumpCreationExceptionMessage = reader.ReadNullableString();
+            string standardInputExceptionMessage = reader.ReadNullableString();
+            int numberOfPRocessLaunchRetries = reader.ReadInt32();
+            bool hasReadWriteToReadFileAccessRequest = reader.ReadBoolean();
+            string messageProcessingFailureMessage = reader.ReadNullableString();
+            long processStartTime = reader.ReadInt64();
+            int warningCount = reader.ReadInt32();
+            long detoursMaxHeapSize = reader.ReadInt64();
+            int lastMessageCount = reader.ReadInt32();
+            bool messageCountSemaphoreCreated = reader.ReadBoolean();
+
+            return new SandboxedProcessResult()
+            {
+                ExitCode = exitCode,
+                Killed = killed,
+                TimedOut = timedOut,
+                HasDetoursInjectionFailures = hasDetoursInjectionFailures,
+                SurvivingChildProcesses = survivingChildProcesses,
+                PrimaryProcessTimes = primaryProcessTimes,
+                JobAccountingInformation = jobAccountingInformation,
+                StandardOutput = standardOutput,
+                StandardError = standardError,
+                FileAccesses = fileAccesses != null ? new HashSet<ReportedFileAccess>(fileAccesses) : null,
+                ExplicitlyReportedFileAccesses = explicitlyReportedFileAccesses != null ? new HashSet<ReportedFileAccess>(explicitlyReportedFileAccesses) : null,
+                AllUnexpectedFileAccesses = allUnexpectedFileAccesses != null ? new HashSet<ReportedFileAccess>(allUnexpectedFileAccesses) : null,
+                Processes = processes,
+                DetouringStatuses = detouringStatuses,
+                DumpFileDirectory = dumpFileDirectory,
+                DumpCreationException = dumpCreationExceptionMessage != null ? new Exception(dumpCreationExceptionMessage) : null,
+                StandardInputException = standardInputExceptionMessage != null ? new Exception(standardInputExceptionMessage) : null,
+                NumberOfProcessLaunchRetries = numberOfPRocessLaunchRetries,
+                HasReadWriteToReadFileAccessRequest = hasReadWriteToReadFileAccessRequest,
+                MessageProcessingFailure = messageProcessingFailureMessage != null ? new Failure<string>(messageProcessingFailureMessage) : null,
+                ProcessStartTime = processStartTime,
+                WarningCount = warningCount,
+                DetoursMaxHeapSize = detoursMaxHeapSize,
+                LastMessageCount = lastMessageCount,
+                MessageCountSemaphoreCreated = messageCountSemaphoreCreated
+            };
+        }
+
+        private Dictionary<ReportedProcess, int> CreateAndSerializeProcessMap(BuildXLWriter writer)
+        {
+            var processMap = new Dictionary<ReportedProcess, int>();
+
+            PopulateProcesses(Processes);
+            PopulateProcesses(SurvivingChildProcesses);
+            PopulateProcesses(FileAccesses?.Select(f => f.Process));
+            PopulateProcesses(ExplicitlyReportedFileAccesses?.Select(f => f.Process));
+            PopulateProcesses(AllUnexpectedFileAccesses?.Select(f => f.Process));
+
+            ReportedProcess[] processes = new ReportedProcess[processMap.Count];
+            foreach (var process in processMap)
+            {
+                processes[process.Value] = process.Key;
+            }
+
+            writer.WriteReadOnlyList(processes, (w, v) => v.Serialize(w));
+
+            return processMap;
+
+            void PopulateProcesses(IEnumerable<ReportedProcess> processesToPopulate)
+            {
+                if (processesToPopulate != null)
+                {
+                    foreach (var process in processesToPopulate)
+                    {
+                        if (!processMap.ContainsKey(process))
+                        {
+                            processMap.Add(process, processMap.Count);
+                        }
+                    }
+                }
+            }
+        }
     }
 }

Public/Src/Engine/Processes/SandboxedProcessStandardFiles.cs

@@ -0,0 +1,68 @@
+// Copyright (c) Microsoft. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+using System.Diagnostics.ContractsLight;
+using BuildXL.Utilities;
+
+namespace BuildXL.Processes
+{
+    /// <summary>
+    /// Files potentially created by the sandboxed process.
+    /// </summary>
+    public class SandboxedProcessStandardFiles
+    {
+        /// <summary>
+        /// Standard output.
+        /// </summary>
+        public string StandardOutput { get; }
+
+        /// <summary>
+        /// Standard error.
+        /// </summary>
+        public string StandardError { get; }
+
+        /// <summary>
+        /// Creates an instance of <see cref="SandboxedProcessFile"/>.
+        /// </summary>
+        public SandboxedProcessStandardFiles(string standardOutput, string standardError)
+        {
+            Contract.Requires(!string.IsNullOrWhiteSpace(standardOutput));
+            Contract.Requires(!string.IsNullOrWhiteSpace(standardError));
+
+            StandardOutput = standardOutput;
+            StandardError = standardError;
+        }
+
+        /// <summary>
+        /// Serializes this instance into the given <paramref name="writer"/>.
+        /// </summary>
+        public void Serialize(BuildXLWriter writer)
+        {
+            Contract.Requires(writer != null);
+
+            writer.Write(StandardOutput);
+            writer.Write(StandardError);
+        }
+
+        /// <summary>
+        /// Deserializes an instance of <see cref="SandboxedProcessStandardFiles"/>.
+        /// </summary>
+        public static SandboxedProcessStandardFiles Deserialize(BuildXLReader reader)
+        {
+            Contract.Requires(reader != null);
+
+            string output = reader.ReadString();
+            string error = reader.ReadString();
+
+            return new SandboxedProcessStandardFiles(output, error);
+        }
+
+        /// <summary>
+        /// Creates an instance of <see cref="SandboxedProcessStandardFiles"/> from <see cref="ISandboxedProcessFileStorage"/>.
+        /// </summary>
+        public static SandboxedProcessStandardFiles From(ISandboxedProcessFileStorage fileStorage) => 
+            new SandboxedProcessStandardFiles(
+                fileStorage.GetFileName(SandboxedProcessFile.StandardOutput), 
+                fileStorage.GetFileName(SandboxedProcessFile.StandardError));
+    }
+}

Public/Src/Engine/Processes/StandardFileStorage.cs

@@ -0,0 +1,39 @@
+// Copyright (c) Microsoft. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+using System.Diagnostics.ContractsLight;
+
+namespace BuildXL.Processes
+{
+    /// <summary>
+    /// Implementation of <see cref="ISandboxedProcessFileStorage"/> based off <see cref="SandboxedProcessStandardFiles"/>.
+    /// </summary>
+    public class StandardFileStorage : ISandboxedProcessFileStorage
+    {
+        private readonly SandboxedProcessStandardFiles m_sandboxedProcessStandardFiles;
+
+        /// <summary>
+        /// Create an instance of <see cref="StandardFileStorage"/>.
+        /// </summary>
+        public StandardFileStorage(SandboxedProcessStandardFiles sandboxedProcessStandardFiles)
+        {
+            Contract.Requires(sandboxedProcessStandardFiles != null);
+            m_sandboxedProcessStandardFiles = sandboxedProcessStandardFiles;
+        }
+
+        /// <inheritdoc />
+        public string GetFileName(SandboxedProcessFile file)
+        {
+            switch (file)
+            {
+                case SandboxedProcessFile.StandardError:
+                    return m_sandboxedProcessStandardFiles.StandardError;
+                case SandboxedProcessFile.StandardOutput:
+                    return m_sandboxedProcessStandardFiles.StandardOutput;
+                default:
+                    Contract.Assert(false);
+                    return null;
+            }
+        }
+    }
+}

Public/Src/Engine/Processes/StandardInputInfo.cs

@@ -0,0 +1,120 @@
+// Copyright (c) Microsoft. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+using System;
+using System.Diagnostics.ContractsLight;
+using BuildXL.Utilities;
+
+namespace BuildXL.Processes
+{
+    /// <summary>
+    /// Info about the source of standard input.
+    /// </summary>
+    public class StandardInputInfo : IEquatable<StandardInputInfo>
+    {
+        /// <summary>
+        /// File path.
+        /// </summary>
+        public string File { get; }
+
+        /// <summary>
+        /// Raw data.
+        /// </summary>
+        public string Data { get; }
+
+        private StandardInputInfo(string file, string data)
+        {
+            Contract.Requires((file != null && data == null) || (file == null && data != null));
+
+            File = file;
+            Data = data;
+        }
+
+        /// <summary>
+        /// Creates a standard input info where the source comes from a file.
+        /// </summary>
+        public static StandardInputInfo CreateForFile(string file)
+        {
+            Contract.Requires(!string.IsNullOrEmpty(file));
+
+            return new StandardInputInfo(file, null);
+        }
+
+        /// <summary>
+        /// Creates a standard input info where the source comes from raw data.
+        /// </summary>
+        public static StandardInputInfo CreateForData(string data)
+        {
+            Contract.Requires(data != null);
+
+            return new StandardInputInfo(null, data);
+        }
+
+        /// <summary>
+        /// Serializes this instance to a given <paramref name="writer"/>.
+        /// </summary>
+        public void Serialize(BuildXLWriter writer)
+        {
+            Contract.Requires(writer != null);
+
+            writer.WriteNullableString(File);
+            writer.WriteNullableString(Data);
+        }
+
+        /// <summary>
+        /// Deserializes an instance of <see cref="StandardInputInfo"/>
+        /// </summary>
+        public static StandardInputInfo Deserialize(BuildXLReader reader)
+        {
+            Contract.Requires(reader != null);
+
+            return new StandardInputInfo(file: reader.ReadNullableString(), data: reader.ReadNullableString());
+        }
+
+        /// <inheritdoc />
+        public override bool Equals(object obj)
+        {
+            return !(obj is null) && (ReferenceEquals(this, obj) || ((obj is StandardInputInfo info) && Equals(info)));
+        }
+
+        /// <summary>
+        /// Checks for equality.
+        /// </summary>
+        public bool Equals(StandardInputInfo standardInputInfo)
+        {
+            return !(standardInputInfo is null)
+                && (ReferenceEquals(this, standardInputInfo)
+                    || (string.Equals(File, standardInputInfo.File, StringComparison.OrdinalIgnoreCase)
+                        && string.Equals(Data, standardInputInfo.Data)));
+        }
+
+        /// <summary>
+        /// Checks for equality.
+        /// </summary>
+        public static bool operator ==(StandardInputInfo info1, StandardInputInfo info2)
+        {
+            if (ReferenceEquals(info1, info2))
+            {
+                return true;
+            }
+
+            if (info1 is null)
+            {
+                return false;
+            }
+
+            return info1.Equals(info2);
+        }
+
+        /// <summary>
+        /// Checks for disequality.
+        /// </summary>
+        public static bool operator !=(StandardInputInfo info1, StandardInputInfo info2) => !(info1 == info2);
+
+        /// <inheritdoc />
+        public override int GetHashCode()
+        {
+            return HashCodeHelper.Combine(File != null ? File.GetHashCode() : -1, Data != null ? Data.GetHashCode() : -1);
+        }
+    }
+}

Public/Src/Engine/Processes/Tracing/Log.cs

@@ -882,5 +882,23 @@ namespace BuildXL.Processes.Tracing
             EventTask = (int)Events.Tasks.PipExecutor,
             Message = Events.PipPrefix + "Process was specified to run in a container, but this capability is not available on this machine.")]
         public abstract void PipSpecifiedToRunInContainerButIsolationIsNotSupported(LoggingContext context, long pipSemiStableHash, string pipDescription);
+
+        [GeneratedEvent(
+            (int) EventId.PipProcessStartExternalTool,
+            EventGenerators = EventGenerators.LocalOnly,
+            EventLevel = Level.Verbose,
+            Keywords = (int)Events.Keywords.UserMessage,
+            EventTask = (int)Events.Tasks.PipExecutor,
+            Message = Events.PipPrefix + "Process execution via external tool '{tool}' starts")]
+        public abstract void PipProcessStartExternalTool(LoggingContext context, long pipSemiStableHash, string pipDescription, string tool);
+
+        [GeneratedEvent(
+            (int)EventId.PipProcessFinishedExternalTool,
+            EventGenerators = EventGenerators.LocalOnly,
+            EventLevel = Level.Verbose,
+            Keywords = (int)Events.Keywords.UserMessage,
+            EventTask = (int)Events.Tasks.PipExecutor,
+            Message = Events.PipPrefix + "Process execution via external tool finished with the tool's exit code {exitCode}:{stdOut}{stdErr}")]
+        public abstract void PipProcessFinishedExternalTool(LoggingContext context, long pipSemiStableHash, string pipDescription, int exitCode, string stdOut, string stdErr);
     }
 }

Public/Src/Engine/Processes/UnSandboxedProcess.cs

@@ -2,18 +2,16 @@
 // Licensed under the MIT license. See LICENSE file in the project root for full license information.
 
 using System;
-using System.Collections.Generic;
 using System.ComponentModel;
+using System.Collections.Generic;
 using System.Diagnostics;
 using System.Diagnostics.ContractsLight;
 using System.Linq;
 using System.Threading.Tasks;
 using BuildXL.Interop;
-using BuildXL.Native.IO;
 using BuildXL.Utilities;
-using BuildXL.Utilities.Tasks;
-using JetBrains.Annotations;
 using static BuildXL.Utilities.FormattableStringEx;
+using JetBrains.Annotations;
 #if FEATURE_SAFE_PROCESS_HANDLE
 using Microsoft.Win32.SafeHandles;
 #else
@@ -32,18 +30,13 @@ namespace BuildXL.Processes
     {
         private static readonly ISet<ReportedFileAccess> s_emptyFileAccessesSet = new HashSet<ReportedFileAccess>();
 
-        private readonly TaskSourceSlim<Unit> m_processExitedTcs = TaskSourceSlim.Create<Unit>();
-        private readonly TaskSourceSlim<Unit> m_stdoutFlushedTcs = TaskSourceSlim.Create<Unit>();
-        private readonly TaskSourceSlim<Unit> m_stderrFlushedTcs = TaskSourceSlim.Create<Unit>();
         private readonly SandboxedProcessOutputBuilder m_output;
         private readonly SandboxedProcessOutputBuilder m_error;
 
-        private DateTime m_startTime;
-        private DateTime m_exitTime;
         private DateTime m_reportsReceivedTime;
 
+        private AsyncProcessExecutor m_processExecutor;
         private Exception m_dumpCreationException;
-        private bool m_killed = false;
 
         internal class CpuTimes
         {
@@ -62,7 +55,7 @@ namespace BuildXL.Processes
         /// <summary>
         /// Indicates if the process has been force killed during execution.
         /// </summary>
-        protected virtual bool Killed => m_killed;
+        protected virtual bool Killed => m_processExecutor?.Killed ?? false;
 
         /// <summary>
         /// Process information associated with this process.
@@ -72,12 +65,7 @@ namespace BuildXL.Processes
         /// <summary>
         /// Underlying managed <see cref="Process"/> object.
         /// </summary>
-        protected Process Process { get; private set; }
-
-        /// <summary>
-        /// Task that completes once this process dies.
-        /// </summary>
-        protected Task WhenExited => m_processExitedTcs.Task;
+        protected Process Process => m_processExecutor?.Process;
 
         /// <summary>
         /// Returns the path table from the supplied <see cref="SandboxedProcessInfo"/>.
@@ -125,7 +113,7 @@ namespace BuildXL.Processes
         protected TimeSpan CurrentRunningTime()
         {
             Contract.Requires(Started);
-            return DateTime.UtcNow.Subtract(m_startTime);
+            return DateTime.UtcNow.Subtract(m_processExecutor.StartTime);
         }
 
         /// <inheritdoc />
@@ -134,19 +122,8 @@ namespace BuildXL.Processes
             Contract.Requires(!Started, "Process was already started.  Cannot start process more than once.");
 
             CreateAndSetUpProcess();
+            m_processExecutor.Start();
 
-            try
-            {
-                Process.Start();
-            }
-            catch (Win32Exception e)
-            {
-                // Can't use helper because when this throws the standard streams haven't been redirected and closing the streams fail...
-                throw new BuildXLException($"[Pip{ProcessInfo.PipSemiStableHash} -- {ProcessInfo.PipDescription}] Failed to launch process: {ProcessInfo.FileName} {ProcessInfo.Arguments}", e);
-            }
-
-            Process.BeginOutputReadLine();
-            Process.BeginErrorReadLine();
             SetProcessStartedExecuting();
         }
 
@@ -158,18 +135,21 @@ namespace BuildXL.Processes
         /// <inheritdoc />
         public virtual void Dispose()
         {
-            var lifetime = DateTime.UtcNow - m_startTime;
+            var startTime = m_processExecutor?.StartTime ?? DateTime.UtcNow;
+            var exitTime = m_processExecutor?.ExitTime ?? DateTime.UtcNow;
+
+            var lifetime = DateTime.UtcNow - startTime;
             var cpuTimes = GetCpuTimes();
             LogProcessState(
                 $"Process Times: " +
-                $"started = {m_startTime}, " +
-                $"exited = {m_exitTime} (since start = {ToSeconds(m_exitTime - m_startTime)}s), " +
-                $"received reports = {m_reportsReceivedTime} (since start = {ToSeconds(m_reportsReceivedTime - m_startTime)}s), " +
+                $"started = {startTime}, " +
+                $"exited = {exitTime} (since start = {ToSeconds(exitTime - startTime)}s), " +
+                $"received reports = {m_reportsReceivedTime} (since start = {ToSeconds(m_reportsReceivedTime - startTime)}s), " +
                 $"life time = {ToSeconds(lifetime)}s, " +
                 $"user time = {ToSeconds(cpuTimes.User)}s, " +
                 $"system time = {ToSeconds(cpuTimes.System)}s");
             SandboxedProcessFactory.Counters.AddToCounter(SandboxedProcessFactory.SandboxedProcessCounters.SandboxedProcessLifeTimeMs, (long)lifetime.TotalMilliseconds);
-            Process?.Dispose();
+            m_processExecutor?.Dispose();
 
             string ToSeconds(TimeSpan ts)
             {
@@ -211,37 +191,25 @@ namespace BuildXL.Processes
         {
             Contract.Requires(Started);
 
-            // 1: wait until process exits or process timeout is reached
-            LogProcessState("Waiting for process to exit");
-            var finishedTask = await Task.WhenAny(Task.Delay(ProcessInfo.Timeout.Value), WhenExited);
-            m_exitTime = DateTime.UtcNow;
+            SandboxedProcessReports reports = null;
 
-            // 2: kill process if timed out
-            var timedOut = finishedTask != WhenExited;
-            if (timedOut)
-            {
-                LogProcessState($"Process timed out after {ProcessInfo.Timeout.Value}; it will be forcefully killed.");
-                await KillAsync();
-            }
-
-            // 3: wait for reports to be all received
-            LogProcessState("Waiting for reports to be received");
-            var reports = await GetReportsAsync();
-            m_reportsReceivedTime = DateTime.UtcNow;
-            reports?.Freeze();
-
-            // 4: wait for stdout and stderr to be flushed
-            LogProcessState("Waiting for stdout and stderr to be flushed");
-            await Task.WhenAll(m_stdoutFlushedTcs.Task, m_stderrFlushedTcs.Task);
+            await m_processExecutor.WaitForExitAsync(
+                async () => 
+                {
+                    LogProcessState("Waiting for reports to be received");
+                    reports = await GetReportsAsync();
+                    m_reportsReceivedTime = DateTime.UtcNow;
+                    reports?.Freeze();
+                });
 
             var reportFileAccesses = ProcessInfo.FileAccessManifest?.ReportFileAccesses == true;
             var fileAccesses = reportFileAccesses ? (reports?.FileAccesses ?? s_emptyFileAccessesSet) : null;
 
             return new SandboxedProcessResult
             {
-                ExitCode                            = timedOut ? ExitCodes.Timeout : Process.ExitCode,
+                ExitCode                            = m_processExecutor.TimedOut ? ExitCodes.Timeout : Process.ExitCode,
                 Killed                              = Killed,
-                TimedOut                            = timedOut,
+                TimedOut                            = m_processExecutor.TimedOut,
                 HasDetoursInjectionFailures         = HasSandboxFailures,
                 StandardOutput                      = m_output.Freeze(),
                 StandardError                       = m_error.Freeze(),
@@ -277,41 +245,21 @@ namespace BuildXL.Processes
 
             ProcessDumper.TryDumpProcessAndChildren(ProcessId, ProcessInfo.TimeoutDumpDirectory, out m_dumpCreationException);
 
-            try
-            {
-                if (!Process.HasExited)
-                {
-                    Process.Kill();
-                }
-            }
-            catch (Exception e) when (e is Win32Exception || e is InvalidOperationException)
-            {
-                // thrown if the process doesn't exist (e.g., because it has already completed on its own)
-            }
-
-            m_stdoutFlushedTcs.TrySetResult(Unit.Void);
-            m_stderrFlushedTcs.TrySetResult(Unit.Void);
-            m_killed = true;
-            return WhenExited;
+            return m_processExecutor.KillAsync();
         }
 
         /// <summary>
         /// Mutates <see cref="Process"/>.
         /// </summary>
-        protected Process CreateAndSetUpProcess()
+        protected void CreateAndSetUpProcess()
         {
             Contract.Requires(Process == null);
 
-#if !PLATFORM_OSX
-            if (!FileUtilities.FileExistsNoFollow(ProcessInfo.FileName))
-            {
-                ThrowFileDoesNotExist();
-            }
-#else
+#if PLATFORM_OSX
             var mode = GetFilePermissionsForFilePath(ProcessInfo.FileName, followSymlink: false);
             if (mode < 0)
             {
-                ThrowFileDoesNotExist();
+                ThrowBuildXLException($"Process creation failed: File '{ProcessInfo.FileName}' not found", new Win32Exception(0x2));
             }
 
             var filePermissions = checked((FilePermissions)mode);
@@ -322,47 +270,45 @@ namespace BuildXL.Processes
             }
 #endif
 
-            Process = new Process();
-            Process.StartInfo = new ProcessStartInfo
+            var process = new Process
             {
-                FileName = ProcessInfo.FileName,
-                Arguments = ProcessInfo.Arguments,
-                WorkingDirectory = ProcessInfo.WorkingDirectory,
-                StandardErrorEncoding = m_output.Encoding,
-                StandardOutputEncoding = m_error.Encoding,
-                RedirectStandardError = true,
-                RedirectStandardOutput = true,
-                UseShellExecute = false,
-                CreateNoWindow = true
+                StartInfo = new ProcessStartInfo
+                {
+                    FileName = ProcessInfo.FileName,
+                    Arguments = ProcessInfo.Arguments,
+                    WorkingDirectory = ProcessInfo.WorkingDirectory,
+                    StandardErrorEncoding = m_output.Encoding,
+                    StandardOutputEncoding = m_error.Encoding,
+                    RedirectStandardError = true,
+                    RedirectStandardOutput = true,
+                    UseShellExecute = false,
+                    CreateNoWindow = true
+                },
+                EnableRaisingEvents = true
             };
 
-            Process.StartInfo.EnvironmentVariables.Clear();
+            process.StartInfo.EnvironmentVariables.Clear();
             if (ProcessInfo.EnvironmentVariables != null)
             {
                 foreach (var envKvp in ProcessInfo.EnvironmentVariables.ToDictionary())
                 {
-                    Process.StartInfo.EnvironmentVariables[envKvp.Key] = envKvp.Value;
+                    process.StartInfo.EnvironmentVariables[envKvp.Key] = envKvp.Value;
                 }
             }
 
-            Process.EnableRaisingEvents = true;
-            Process.OutputDataReceived += (sender, e) => FeedStdOut(m_output, m_stdoutFlushedTcs, e.Data);
-            Process.ErrorDataReceived  += (sender, e) => FeedStdErr(m_error, m_stderrFlushedTcs, e.Data);
-            Process.Exited             += (sender, e) => m_processExitedTcs.TrySetResult(Unit.Void);
-
-            return Process;
-
-            void ThrowFileDoesNotExist()
-            {
-                ThrowCouldNotStartProcess(I($"File '{ProcessInfo.FileName}' not found"), new Win32Exception(0x2));
-            }
+            m_processExecutor = new AsyncProcessExecutor(
+                process,
+                ProcessInfo.Timeout ?? TimeSpan.FromMinutes(10),
+                line => FeedStdOut(m_output, line),
+                line => FeedStdErr(m_error, line),
+                ProcessInfo);
         }
 
-        internal virtual void FeedStdOut(SandboxedProcessOutputBuilder b, TaskSourceSlim<Unit> tsc, string line)
-            => FeedOutputBuilder(b, tsc, line);
+        internal virtual void FeedStdOut(SandboxedProcessOutputBuilder b, string line)
+            => FeedOutputBuilder(b, line);
 
-        internal virtual void FeedStdErr(SandboxedProcessOutputBuilder b, TaskSourceSlim<Unit> tsc, string line)
-            => FeedOutputBuilder(b, tsc, line);
+        internal virtual void FeedStdErr(SandboxedProcessOutputBuilder b, string line)
+            => FeedOutputBuilder(b, line);
 
         /// <summary>
         /// Throws a <see cref="BuildXLException"/> with pip hash and description prefixed to a supplied <paramref name="reason"/>.
@@ -393,21 +339,16 @@ namespace BuildXL.Processes
         {
             if (ProcessInfo.LoggingContext != null)
             {
-                string fullMessage = I($"Exited: {m_processExitedTcs.Task.IsCompleted}, StdOut: {m_stdoutFlushedTcs.Task.IsCompleted}, StdErr: {m_stderrFlushedTcs.Task.IsCompleted}, Reports: {ReportsCompleted()} :: {message}");
+                string fullMessage = I($"Exited: {m_processExecutor?.ExitCompleted ?? false}, StdOut: {m_processExecutor?.StdOutCompleted ?? false}, StdErr: {m_processExecutor?.StdErrCompleted ?? false}, Reports: {ReportsCompleted()} :: {message}");
                 Tracing.Logger.Log.LogDetoursDebugMessage(ProcessInfo.LoggingContext, ProcessInfo.PipSemiStableHash, ProcessInfo.PipDescription, fullMessage);
             }
         }
 
         /// <summary>
-        ///   - sets <see cref="m_startTime"/> to current time
-        ///   - notifies the process id listener
+        /// Notifies the process id listener.
         /// </summary>
-        /// <remarks>
-        /// Mutates <see cref="m_startTime"/>.
-        /// </remarks>
         protected void SetProcessStartedExecuting()
         {
-            m_startTime = DateTime.UtcNow;
             ProcessInfo.ProcessIdListener?.Invoke(ProcessId);
         }
 
@@ -423,17 +364,11 @@ namespace BuildXL.Processes
         /// </summary>
         internal virtual Task<SandboxedProcessReports> GetReportsAsync() => Task.FromResult<SandboxedProcessReports>(null);
 
-        internal static void FeedOutputBuilder(SandboxedProcessOutputBuilder output, TaskSourceSlim<Unit> signalCompletion, string line)
+        internal static void FeedOutputBuilder(SandboxedProcessOutputBuilder output, string line)
         {
-            if (signalCompletion.Task.IsCompleted)
+            if (line != null)
             {
-                return;
-            }
-
-            output.AppendLine(line);
-            if (line == null)
-            {
-                signalCompletion.TrySetResult(Unit.Void);
+                output.AppendLine(line);
             }
         }
 
@@ -447,15 +382,15 @@ namespace BuildXL.Processes
 
         private ProcessTimes GetProcessTimes()
         {
-            if (m_startTime == default(DateTime) || m_exitTime == default(DateTime))
+            if (m_processExecutor.StartTime == default || m_processExecutor.ExitTime == default)
             {
                 return new ProcessTimes(0, 0, 0, 0);
             }
 
             var cpuTimes = GetCpuTimes();
             return new ProcessTimes(
-                creation: m_startTime.ToFileTimeUtc(),
-                exit: m_exitTime.ToFileTimeUtc(),
+                creation: m_processExecutor.StartTime.ToFileTimeUtc(),
+                exit: m_processExecutor.ExitTime.ToFileTimeUtc(),
                 kernel: cpuTimes.System.Ticks,
                 user: cpuTimes.User.Ticks);
         }

Public/Src/Engine/Scheduler/Distribution/ExecutionResultSerializer.cs

@@ -398,21 +398,7 @@ namespace BuildXL.Scheduler.Distribution
             ReportedProcess[] processes = new ReportedProcess[count];
             for (int i = 0; i < count; i++)
             {
-                processes[i] = new ReportedProcess(
-                    reader.ReadUInt32(),
-                    reader.ReadString(),
-                    reader.ReadString());
-
-                processes[i].CreationTime = reader.ReadDateTime();
-                processes[i].ExitTime = reader.ReadDateTime();
-                processes[i].KernelTime = reader.ReadTimeSpan();
-                processes[i].UserTime = reader.ReadTimeSpan();
-                IOTypeCounters readCounters = new IOTypeCounters(reader.ReadUInt64(), reader.ReadUInt64());
-                IOTypeCounters writeCounters = new IOTypeCounters(reader.ReadUInt64(), reader.ReadUInt64());
-                IOTypeCounters otherCounters = new IOTypeCounters(reader.ReadUInt64(), reader.ReadUInt64());
-                processes[i].IOCounters = new IOCounters(readCounters, writeCounters, otherCounters);
-                processes[i].ExitCode = reader.ReadUInt32();
-                processes[i].ParentProcessId = reader.ReadUInt32();
+                processes[i] = ReportedProcess.Deserialize(reader);
             }
 
             return processes;
@@ -429,41 +415,13 @@ namespace BuildXL.Scheduler.Distribution
 
             for (int i = 0; i < processes.Length; i++)
             {
-                writer.Write(processes[i].ProcessId);
-                writer.Write(processes[i].Path);
-                writer.Write(processes[i].ProcessArgs);
-                writer.Write(processes[i].CreationTime);
-                writer.Write(processes[i].ExitTime);
-                writer.Write(processes[i].KernelTime);
-                writer.Write(processes[i].UserTime);
-                writer.Write(processes[i].IOCounters.ReadCounters.OperationCount);
-                writer.Write(processes[i].IOCounters.ReadCounters.TransferCount);
-                writer.Write(processes[i].IOCounters.WriteCounters.OperationCount);
-                writer.Write(processes[i].IOCounters.WriteCounters.TransferCount);
-                writer.Write(processes[i].IOCounters.OtherCounters.OperationCount);
-                writer.Write(processes[i].IOCounters.OtherCounters.TransferCount);
-                writer.Write(processes[i].ExitCode);
-                writer.Write(processes[i].ParentProcessId);
+                processes[i].Serialize(writer);
             }
         }
 
         private static ReportedFileAccess ReadReportedFileAccess(BuildXLReader reader, ReportedProcess[] processes, Func<BuildXLReader, AbsolutePath> readPath)
         {
-            return new ReportedFileAccess(
-                operation: (ReportedFileOperation)reader.ReadByte(),
-                process: processes[reader.ReadInt32Compact()],
-                requestedAccess: (RequestedAccess)reader.ReadInt32Compact(),
-                status: (FileAccessStatus)reader.ReadInt32Compact(),
-                explicitlyReported: reader.ReadBoolean(),
-                error: reader.ReadUInt32(),
-                usn: new Usn(reader.ReadUInt64()),
-                desiredAccess: (DesiredAccess)reader.ReadUInt32(),
-                shareMode: (ShareMode)reader.ReadUInt32(),
-                creationDisposition: (CreationDisposition)reader.ReadUInt32(),
-                flagsAndAttributes: (FlagsAndAttributes)reader.ReadUInt32(),
-                manifestPath: readPath(reader),
-                path: ReadNullableString(reader),
-                enumeratePatttern: ReadNullableString(reader));
+            return ReportedFileAccess.Deserialize(reader, processes, readPath);
         }
 
         private static void WriteReportedFileAccess(
@@ -472,20 +430,7 @@ namespace BuildXL.Scheduler.Distribution
             Dictionary<ReportedProcess, int> processMap,
             Action<BuildXLWriter, AbsolutePath> writePath)
         {
-            writer.Write((byte) reportedFileAccess.Operation);
-            writer.WriteCompact(processMap[reportedFileAccess.Process]);
-            writer.WriteCompact((int) reportedFileAccess.RequestedAccess);
-            writer.WriteCompact((int) reportedFileAccess.Status);
-            writer.Write(reportedFileAccess.ExplicitlyReported);
-            writer.Write(reportedFileAccess.Error);
-            writer.Write(reportedFileAccess.Usn.Value);
-            writer.Write((uint) reportedFileAccess.DesiredAccess);
-            writer.Write((uint) reportedFileAccess.ShareMode);
-            writer.Write((uint) reportedFileAccess.CreationDisposition);
-            writer.Write((uint) reportedFileAccess.FlagsAndAttributes);
-            writePath(writer, reportedFileAccess.ManifestPath);
-            WriteNullableString(writer, reportedFileAccess.Path);
-            WriteNullableString(writer, reportedFileAccess.EnumeratePattern);
+            reportedFileAccess.Serialize(writer, processMap, writePath);
         }
 
         private static string ReadNullableString(BuildXLReader reader)

Public/Src/Engine/Scheduler/ExecutionResult.cs

@@ -621,7 +621,7 @@ namespace BuildXL.Scheduler
                             fingerprint: m_weakFingerprint?.Hash ?? FingerprintUtilities.ZeroFingerprint,
                             processExecutionTime: wallClockTime,
                             fileMonitoringViolations: ConvertFileMonitoringViolationCounters(m_unsealedState.UnexpectedFileAccessCounters.Value),
-                            ioCounters: ConvertIOCounters(jobAccounting.IO),
+                            ioCounters: jobAccounting.IO,
                             userTime: jobAccounting.UserTime,
                             kernelTime: jobAccounting.KernelTime,
                             peakMemoryUsage: jobAccounting.PeakMemoryUsage,
@@ -653,19 +653,6 @@ namespace BuildXL.Scheduler
             return new ReadOnlyDictionary<AbsolutePath, IReadOnlyCollection<AbsolutePath>>(sharedDynamicAccesses);
         }
 
-        private static IOTypeCounters ConvertIOCounters(JobObject.IOTypeCounters counters)
-        {
-            return new IOTypeCounters(operationCount: counters.OperationCount, transferCount: counters.TransferCount);
-        }
-
-        private static IOCounters ConvertIOCounters(JobObject.IOCounters counters)
-        {
-            return new IOCounters(
-                readCounters: ConvertIOCounters(counters.ReadCounters),
-                writeCounters: ConvertIOCounters(counters.WriteCounters),
-                otherCounters: ConvertIOCounters(counters.OtherCounters));
-        }
-
         private static FileMonitoringViolationCounters ConvertFileMonitoringViolationCounters(UnexpectedFileAccessCounters counters)
         {
             return new FileMonitoringViolationCounters(

Public/Src/Engine/Scheduler/Fingerprints/PipFingerprinter.cs

@@ -272,6 +272,12 @@ namespace BuildXL.Scheduler.Fingerprints
             }
 
             fingerprinter.Add("DoubleWritePolicy", (byte)process.DoubleWritePolicy);
+
+            if (process.RequiresAdmin)
+            {
+                fingerprinter.Add("RequiresAdmin", 1);
+            }
+            
             fingerprinter.Add("NeedsToRunInContainer", process.NeedsToRunInContainer ? 1 : 0);
             fingerprinter.Add("ContainerIsolationLevel", (byte) process.ContainerIsolationLevel);
 

Public/Src/Engine/Scheduler/Fingerprints/PipFingerprintingVersion.cs

@@ -31,7 +31,8 @@ namespace BuildXL.Scheduler.Fingerprints
         /// 55: Changed FileMaterializationInfo/FileContentInfo bond serialization
         /// 56: Added NeedsToRunInContainer, ContainerIsolationLevel and DoubleWritePolicy
         /// 57: Fixed enumeration in StoreContentForProcessAndCreateCacheEntryAsync
+        /// 58: Added RequiresAdmin field into the process pip.
         /// </remarks>
-        TwoPhaseV2 = 57,
+        TwoPhaseV2 = 58,
     }
 }

Public/Src/Engine/Scheduler/RunnablePip.cs

@@ -13,6 +13,7 @@ using BuildXL.Storage;
 using BuildXL.Utilities.Configuration;
 using BuildXL.Utilities.Instrumentation.Common;
 using static BuildXL.Utilities.FormattableStringEx;
+using BuildXL.Native.IO;
 
 namespace BuildXL.Scheduler
 {

Public/Src/Engine/Scheduler/Tracing/ExecutionLog.Events.cs

@@ -15,6 +15,7 @@ using BuildXL.Utilities;
 using BuildXL.Utilities.Collections;
 using BuildXL.Utilities.Configuration;
 using BuildXL.Utilities.Tracing;
+using System.Linq;
 
 #pragma warning disable SA1649 // File name must match first type name
 
@@ -914,7 +915,9 @@ namespace BuildXL.Scheduler.Tracing
                 ReportedFileAccesses,
                 WhitelistedReportedFileAccesses,
                 ReportedProcesses);
-            ProcessDetouringStatusData.Serialize(writer, ProcessDetouringStatuses);
+            writer.Write(
+                ProcessDetouringStatuses, 
+                (w, v) => w.WriteReadOnlyList(v.ToList(), (w2, v2) => v2.Serialize(w2)));
         }
 
         /// <inheritdoc />
@@ -929,7 +932,7 @@ namespace BuildXL.Scheduler.Tracing
                 out reportedFileAccesses,
                 out whitelistedReportedFileAccesses,
                 out reportedProcesses);
-            ProcessDetouringStatuses = ProcessDetouringStatusData.Deserialize(reader);
+            ProcessDetouringStatuses = reader.ReadNullable(r => r.ReadReadOnlyList(r2 => ProcessDetouringStatusData.Deserialize(r2)));
             ReportedProcesses = reportedProcesses;
             ReportedFileAccesses = reportedFileAccesses;
             WhitelistedReportedFileAccesses = whitelistedReportedFileAccesses;

Public/Src/Engine/UnitTests/Processes.Detours/FileAccessManifestTreeTest.cs

@@ -4,6 +4,7 @@
 using System;
 using System.Collections.Generic;
 using System.IO;
+using System.Linq;
 using BuildXL.Native.IO;
 using BuildXL.Processes;
 using BuildXL.Utilities;
@@ -107,12 +108,14 @@ namespace Test.BuildXL.Processes.Detours
         /// This test exercises the ability of the manifest to tolerate trailing
         /// backslashes for directory leaves.
         /// </summary>
-        [Fact]
-        public void TestTrailingSeparatorsManifestTest()
+        [Theory]
+        [InlineData(true)]
+        [InlineData(false)]
+        public void TestTrailingSeparatorsManifestTest(bool serializeManifest)
         {
             var pt = new PathTable();
             var fam =
-                new FileAccessManifest(pt)
+                new FileAccessManifest(pt, CreateDirectoryTranslator())
                 {
                     FailUnexpectedFileAccesses = false,
                     IgnoreCodeCoverage = false,
@@ -131,7 +134,7 @@ namespace Test.BuildXL.Processes.Detours
             vac.AddScopeCheck(@"C:\Windows\System32", sys32, FileAccessPolicy.AllowReadAlways);
             vac.AddScopeCheck(@"C:\Windows\System32\", sys32, FileAccessPolicy.AllowReadAlways);
 
-            TestManifestRetrieval(vac.DataItems, fam);
+            ValidationDataCreator.TestManifestRetrieval(vac.DataItems, fam, serializeManifest);
         }
 
         /// <summary>
@@ -139,12 +142,14 @@ namespace Test.BuildXL.Processes.Detours
         /// completely disjoint paths and scopes. Also tests case-insensitive matching
         /// by using lower and upper case in paths to find. No paths added have trailing '\\'.
         /// </summary>
-        [Fact]
-        public void SimpleRecordManifestTest()
+        [Theory]
+        [InlineData(true)]
+        [InlineData(false)]
+        public void SimpleRecordManifestTest(bool serializeManifest)
         {
             var pt = new PathTable();
             var fam =
-                new FileAccessManifest(pt)
+                new FileAccessManifest(pt, CreateDirectoryTranslator())
                 {
                     FailUnexpectedFileAccesses = false,
                     IgnoreCodeCoverage = false,
@@ -177,15 +182,17 @@ namespace Test.BuildXL.Processes.Detours
                 windows,
                 FileAccessPolicy.Deny);
 
-            TestManifestRetrieval(vac.DataItems, fam);
+            ValidationDataCreator.TestManifestRetrieval(vac.DataItems, fam, serializeManifest);
         }
 
-        [Fact]
-        public void TestLinkExeManifestExample()
+        [Theory]
+        [InlineData(true)]
+        [InlineData(false)]
+        public void TestLinkExeManifestExample(bool serializeManifest)
         {
             var pt = new PathTable();
             var fam =
-                new FileAccessManifest(pt)
+                new FileAccessManifest(pt, CreateDirectoryTranslator())
                 {
                     FailUnexpectedFileAccesses = false,
                     IgnoreCodeCoverage = false,
@@ -214,15 +221,17 @@ namespace Test.BuildXL.Processes.Detours
             vac.AddPath(@"E:\dev\BuildXL\Out\Objects\Debug-X64\Detours-lib\cl_3\modules.obj", FileAccessPolicy.AllowReadAlways);
             vac.AddPath(@"E:\dev\BuildXL\Out\Objects\Debug-X64\Detours-lib\Link\Detours.lib", FileAccessPolicy.AllowReadAlways);
 
-            TestManifestRetrieval(vac.DataItems, fam);
+            ValidationDataCreator.TestManifestRetrieval(vac.DataItems, fam, serializeManifest);
         }
 
-        [Fact]
-        public void TestStyleCopCmdExeManifestExample()
+        [Theory]
+        [InlineData(true)]
+        [InlineData(false)]
+        public void TestStyleCopCmdExeManifestExample(bool serializeManifest)
         {
             var pt = new PathTable();
             var fam =
-                new FileAccessManifest(pt)
+                new FileAccessManifest(pt, CreateDirectoryTranslator())
                 {
                     FailUnexpectedFileAccesses = false,
                     IgnoreCodeCoverage = false,
@@ -283,15 +292,17 @@ namespace Test.BuildXL.Processes.Detours
             vac.AddPath(@"E:\dev\BuildXL\src\BuildXL.Processes\SandboxedProcessResult.cs", FileAccessPolicy.AllowReadAlways);
             vac.AddPath(@"E:\dev\BuildXL\src\BuildXL.Processes\SandboxedProcessTimes.cs", FileAccessPolicy.AllowReadAlways);
 
-            TestManifestRetrieval(vac.DataItems, fam);
+            ValidationDataCreator.TestManifestRetrieval(vac.DataItems, fam, serializeManifest);
         }
 
-        [Fact]
-        public void TestCscExeTestRunnerGenManifestExample()
+        [Theory]
+        [InlineData(true)]
+        [InlineData(false)]
+        public void TestCscExeTestRunnerGenManifestExample(bool serializeManifest)
         {
             var pt = new PathTable();
             var fam =
-                new FileAccessManifest(pt)
+                new FileAccessManifest(pt, CreateDirectoryTranslator())
                 {
                     FailUnexpectedFileAccesses = false,
                     IgnoreCodeCoverage = false,
@@ -332,15 +343,17 @@ namespace Test.BuildXL.Processes.Detours
             vac.AddPath(@"E:\dev\BuildXL\src\Test.BuildXLRunnerGen\Properties\AssemblyInfo.cs", FileAccessPolicy.AllowReadAlways);
             vac.AddPath(@"E:\dev\BuildXL\src\Test.BuildXLRunnerGen\SymbolTableBuilderTests.cs", FileAccessPolicy.AllowReadAlways);
 
-            TestManifestRetrieval(vac.DataItems, fam);
+            ValidationDataCreator.TestManifestRetrieval(vac.DataItems, fam, serializeManifest);
         }
 
-        [Fact]
-        public void TestGenManifestExample()
+        [Theory]
+        [InlineData(true)]
+        [InlineData(false)]
+        public void TestGenManifestExample(bool serializeManifest)
         {
             var pt = new PathTable();
             var fam =
-                new FileAccessManifest(pt)
+                new FileAccessManifest(pt, CreateDirectoryTranslator())
                 {
                     FailUnexpectedFileAccesses = false,
                     IgnoreCodeCoverage = false,
@@ -530,7 +543,7 @@ namespace Test.BuildXL.Processes.Detours
             vac.AddPath(@"E:\dev\BuildXL\src\BuildXL.Transformers\Values\IVersion.cs", FileAccessPolicy.AllowReadAlways);
             vac.AddPath(@"E:\dev\BuildXL\src\BuildXL.Transformers\Values\StaticDirectory.cs", FileAccessPolicy.AllowReadAlways);
 
-            TestManifestRetrieval(vac.DataItems, fam);
+            ValidationDataCreator.TestManifestRetrieval(vac.DataItems, fam, serializeManifest);
         }
 
         /// <summary>
@@ -538,7 +551,7 @@ namespace Test.BuildXL.Processes.Detours
         /// which has 1 project and 100 C# files. (Access pattern made by csc.exe.)
         /// This is intended to be a stress test.
         /// </summary>
-        private void TestSolutionMockupManifestTest(int numFiles)
+        private void TestSolutionMockupManifestTest(int numFiles, bool serializeManifest = false)
         {
             var pt = new PathTable();
             var fam =
@@ -577,13 +590,15 @@ namespace Test.BuildXL.Processes.Detours
 
             vac.AddScope(@"E:\dev\BuildXL\TestSolution\Project0\Properties\AssemblyInfo.cs", FileAccessPolicy.AllowReadAlways);
 
-            TestManifestRetrieval(vac.DataItems, fam);
+            ValidationDataCreator.TestManifestRetrieval(vac.DataItems, fam, serializeManifest);
         }
 
-        [Fact]
-        public void TestSolution100()
+        [Theory]
+        [InlineData(true)]
+        [InlineData(false)]
+        public void TestSolution100(bool serializeManifest)
         {
-            TestSolutionMockupManifestTest(100);
+            TestSolutionMockupManifestTest(100, serializeManifest);
         }
 
         [Fact]
@@ -604,144 +619,14 @@ namespace Test.BuildXL.Processes.Detours
             TestSolutionMockupManifestTest(10000);
         }
 
-        private static void TestManifestRetrieval(IEnumerable<ValidationData> validationData, FileAccessManifest fam)
+        private DirectoryTranslator CreateDirectoryTranslator()
         {
-            foreach (var line in fam.Describe())
-            {
-                Console.WriteLine(line);
-            }
+            var translator = new DirectoryTranslator();
+            translator.AddTranslation(@"E:\", @"C:\");
+            translator.AddTranslation(@"D:\el\io", @"D:\sh\io");
 
-            byte[] manifestTreeBytes = fam.GetManifestTreeBytes();
-
-            foreach (ValidationData dataItem in validationData)
-            {
-                uint nodePolicy;
-                uint conePolicy;
-                uint pathId;
-                Usn expectedUsn;
-
-                bool success =
-                    global::BuildXL.Native.Processes.Windows.ProcessUtilitiesWin.FindFileAccessPolicyInTree(
-                        manifestTreeBytes,
-                        dataItem.Path,
-                        new UIntPtr((uint)dataItem.Path.Length),
-                        out conePolicy,
-                        out nodePolicy,
-                        out pathId,
-                        out expectedUsn);
-
-                XAssert.IsTrue(success, "Unable to find path in manifest");
-                XAssert.AreEqual(
-                    unchecked((uint)dataItem.PathId),
-                    pathId,
-                    "PathId for '{0}' did not match", dataItem.Path);
-
-                if (dataItem.NodePolicy.HasValue)
-                {
-                    XAssert.AreEqual(
-                        unchecked((uint)dataItem.NodePolicy.Value),
-                        nodePolicy,
-                        "Policy for '{0}' did not match", dataItem.Path);
-                }
-
-                if (dataItem.ConePolicy.HasValue)
-                {
-                    XAssert.AreEqual(
-                        unchecked((uint)dataItem.ConePolicy.Value),
-                        conePolicy,
-                        "Policy for '{0}' did not match", dataItem.Path);
-                }
-
-                XAssert.AreEqual(
-                    dataItem.ExpectedUsn,
-                    expectedUsn,
-                    "Usn for '{0}' did not match", dataItem.Path);
-            }
-        }
-
-        private struct ValidationData
-        {
-            public string Path { get; set; }
-
-            public FileAccessPolicy? NodePolicy { get; set; }
-
-            public FileAccessPolicy? ConePolicy { get; set; }
-
-            public int PathId { get; set; }
-
-            public Usn ExpectedUsn { get; set; }
-        }
-
-        private class ValidationDataCreator
-        {
-            private readonly FileAccessManifest m_manifest;
-            private readonly PathTable m_pathTable;
-
-            public List<ValidationData> DataItems { get; private set; }
-
-            public ValidationDataCreator(FileAccessManifest manifest, PathTable pathTable)
-            {
-                m_manifest = manifest;
-                m_pathTable = pathTable;
-                DataItems = new List<ValidationData>();
-            }
-
-            public AbsolutePath AddScope(
-                string path,
-                FileAccessPolicy values,
-                FileAccessPolicy mask = FileAccessPolicy.Deny,
-                FileAccessPolicy basePolicy = FileAccessPolicy.Deny)
-            {
-                AbsolutePath scopeAbsolutePath = AbsolutePath.Create(m_pathTable, path);
-                var dataItem =
-                    new ValidationData
-                    {
-                        Path = path,
-                        PathId = scopeAbsolutePath.Value.Value,
-                        NodePolicy = (basePolicy & mask) | values,
-                        ConePolicy = null,
-                        ExpectedUsn = ReportedFileAccess.NoUsn
-                    };
-
-                DataItems.Add(dataItem);
-                m_manifest.AddScope(scopeAbsolutePath, mask, values);
-
-                return scopeAbsolutePath;
-            }
-
-            public void AddPath(
-                string path,
-                FileAccessPolicy policy,
-                FileAccessPolicy? expectedEffectivePolicy = null,
-                Usn? expectedUsn = null)
-            {
-                AbsolutePath absolutePath = AbsolutePath.Create(m_pathTable, path);
-                var dataItem =
-                    new ValidationData
-                    {
-                        Path = path,
-                        PathId = absolutePath.Value.Value,
-                        ConePolicy = null,
-                        NodePolicy = expectedEffectivePolicy ?? policy,
-                        ExpectedUsn = expectedUsn ?? ReportedFileAccess.NoUsn
-                    };
-
-                DataItems.Add(dataItem);
-                m_manifest.AddPath(absolutePath, values: policy, mask: FileAccessPolicy.MaskNothing, expectedUsn: expectedUsn);
-            }
-
-            public void AddScopeCheck(string path, AbsolutePath scopePath, FileAccessPolicy policy)
-            {
-                DataItems.Add(
-                    new ValidationData
-                    {
-                        Path = path,
-                        PathId = scopePath.Value.Value,
-                        ConePolicy = policy,
-                        NodePolicy = null,
-                        ExpectedUsn = ReportedFileAccess.NoUsn
-                    });
-            }
+            translator.Seal();
+            return translator;
         }
     }
 }

Public/Src/Engine/UnitTests/Processes.Detours/SandboxedProcessInfoTest.cs

@@ -0,0 +1,135 @@
+// Copyright (c) Microsoft. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+using System;
+using System.Collections.Generic;
+using System.IO;
+using BuildXL.Processes;
+using BuildXL.Utilities;
+using Test.BuildXL.TestUtilities.Xunit;
+using Xunit;
+using Xunit.Abstractions;
+using static BuildXL.Utilities.BuildParameters;
+
+namespace Test.BuildXL.Processes.Detours
+{
+    public class SandboxedProcessInfoTest : XunitBuildXLTest
+    {
+        public SandboxedProcessInfoTest(ITestOutputHelper output)
+            : base(output)
+        {
+        }
+
+        [Fact]
+        public void SerializeSandboxedProcessInfo()
+        {
+            var pt = new PathTable();
+            var fam =
+                new FileAccessManifest(pt, CreateDirectoryTranslator())
+                {
+                    FailUnexpectedFileAccesses = false,
+                    IgnoreCodeCoverage = false,
+                    ReportFileAccesses = false,
+                    ReportUnexpectedFileAccesses = false,
+                    MonitorChildProcesses = false
+                };
+
+            var vac = new ValidationDataCreator(fam, pt);
+            vac.AddScope(A("C", "Users", "AppData"), FileAccessPolicy.AllowAll);
+            vac.AddPath(A("C", "Source", "source.txt"), FileAccessPolicy.AllowReadAlways);
+            vac.AddPath(A("C", "Out", "out.txt"), FileAccessPolicy.AllowAll);
+
+            var standardFiles = new SandboxedProcessStandardFiles(A("C", "pip", "pip.out"), A("C", "pip", "pip.err"));
+            var envVars = new Dictionary<string, string>()
+            {
+                ["Var1"] = "Val1",
+                ["Var2"] = "Val2",
+            };
+            IBuildParameters buildParameters = BuildParameters.GetFactory().PopulateFromDictionary(envVars);
+
+            SandboxedProcessInfo info = new SandboxedProcessInfo(
+                pt,
+                new StandardFileStorage(standardFiles),
+                A("C", "tool", "tool.exe"),
+                fam,
+                true,
+                null)
+            {
+                Arguments = @"/arg1:val1 /arg2:val2",
+                WorkingDirectory = A("C", "Source"),
+                EnvironmentVariables = buildParameters,
+                Timeout = TimeSpan.FromMinutes(10),
+                PipSemiStableHash = 0x12345678,
+                PipDescription = nameof(SerializeSandboxedProcessInfo),
+                ProcessIdListener = null,
+                TimeoutDumpDirectory = A("C", "Timeout"),
+                SandboxKind = global::BuildXL.Utilities.Configuration.SandboxKind.Default,
+                AllowedSurvivingChildProcessNames = new[] { "conhost.exe", "mspdbsrv.exe" },
+                NestedProcessTerminationTimeout = SandboxedProcessInfo.DefaultNestedProcessTerminationTimeout,
+                StandardInputSourceInfo = StandardInputInfo.CreateForData("Data"),
+                StandardObserverDescriptor = new SandboxObserverDescriptor()
+                {
+                    WarningRegex = new ExpandedRegexDescriptor("*warn", System.Text.RegularExpressions.RegexOptions.Compiled)
+                },
+            };
+
+            // Serialize and deserialize.
+            SandboxedProcessInfo readInfo = null;
+
+            using (var stream = new MemoryStream())
+            {
+                info.Serialize(stream);
+                stream.Position = 0;
+                readInfo = SandboxedProcessInfo.Deserialize(
+                    stream,
+                    new global::BuildXL.Utilities.Instrumentation.Common.LoggingContext("Test"),
+                    null);
+            }
+
+            // Verify.
+            XAssert.AreEqual(info.FileName, readInfo.FileName);
+            XAssert.AreEqual(info.Arguments, readInfo.Arguments);
+            XAssert.AreEqual(info.WorkingDirectory, readInfo.WorkingDirectory);
+            var readEnvVars = readInfo.EnvironmentVariables.ToDictionary();
+            XAssert.AreEqual(envVars.Count, readEnvVars.Count);
+            foreach (var kvp in envVars)
+            {
+                XAssert.AreEqual(kvp.Value, readEnvVars[kvp.Key]);
+            }
+
+            XAssert.AreEqual(info.Timeout, readInfo.Timeout);
+            XAssert.AreEqual(info.PipSemiStableHash, readInfo.PipSemiStableHash);
+            XAssert.AreEqual(info.PipDescription, readInfo.PipDescription);
+            XAssert.AreEqual(info.ProcessIdListener, readInfo.ProcessIdListener);
+            XAssert.AreEqual(info.TimeoutDumpDirectory, readInfo.TimeoutDumpDirectory);
+            XAssert.AreEqual(info.SandboxKind, readInfo.SandboxKind);
+
+            XAssert.AreEqual(info.AllowedSurvivingChildProcessNames.Length, readInfo.AllowedSurvivingChildProcessNames.Length);
+            for (int i = 0; i < info.AllowedSurvivingChildProcessNames.Length; ++i)
+            {
+                XAssert.AreEqual(info.AllowedSurvivingChildProcessNames[i], readInfo.AllowedSurvivingChildProcessNames[i]);
+            }
+
+            XAssert.AreEqual(info.NestedProcessTerminationTimeout, readInfo.NestedProcessTerminationTimeout);
+            XAssert.AreEqual(info.StandardInputSourceInfo, readInfo.StandardInputSourceInfo);
+            XAssert.IsNotNull(readInfo.SandboxedProcessStandardFiles);
+            XAssert.AreEqual(standardFiles.StandardOutput, readInfo.SandboxedProcessStandardFiles.StandardOutput);
+            XAssert.AreEqual(standardFiles.StandardError, readInfo.SandboxedProcessStandardFiles.StandardError);
+            XAssert.AreEqual(standardFiles.StandardOutput, readInfo.FileStorage.GetFileName(SandboxedProcessFile.StandardOutput));
+            XAssert.AreEqual(standardFiles.StandardError, readInfo.FileStorage.GetFileName(SandboxedProcessFile.StandardError));
+            XAssert.IsFalse(readInfo.ContainerConfiguration.IsIsolationEnabled);
+
+            ValidationDataCreator.TestManifestRetrieval(vac.DataItems, readInfo.FileAccessManifest, false);
+        }
+
+        private DirectoryTranslator CreateDirectoryTranslator()
+        {
+            var translator = new DirectoryTranslator();
+            translator.AddTranslation(@"E:\", @"C:\");
+            translator.AddTranslation(@"D:\el\io", @"D:\sh\io");
+
+            translator.Seal();
+            return translator;
+        }
+    }
+}

Public/Src/Engine/UnitTests/Processes.Detours/Test.BuildXL.Processes.Detours.dsc

@@ -24,7 +24,9 @@ namespace Processes.Detours {
                 f`PipExecutorDetoursTest.cs`,
                 f`SandboxedProcessPipExecutorTest.cs`,
                 f`SandboxedProcessPipExecutorWindowsCallTest.cs`,
+                f`ValidationDataCreator.cs`,
                 f`FileAccessManifestTreeTest.cs`,
+                f`SandboxedProcessInfoTest.cs`,
             ],
             references: [
                 EngineTestUtilities.dll,

Public/Src/Engine/UnitTests/Processes.Detours/ValidationDataCreator.cs

@@ -0,0 +1,200 @@
+// Copyright (c) Microsoft. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using BuildXL.Native.IO;
+using BuildXL.Processes;
+using BuildXL.Utilities;
+using Test.BuildXL.TestUtilities.Xunit;
+
+namespace Test.BuildXL.Processes.Detours
+{
+    /// <summary>
+    /// Class for validating file access manifest.
+    /// </summary>
+    internal class ValidationDataCreator
+    {
+        private readonly FileAccessManifest m_manifest;
+        private readonly PathTable m_pathTable;
+
+        public List<ValidationData> DataItems { get; private set; }
+
+        /// <summary>
+        /// Creates an instance of <see cref="ValidationDataCreator"/>.
+        /// </summary>
+        public ValidationDataCreator(FileAccessManifest manifest, PathTable pathTable)
+        {
+            m_manifest = manifest;
+            m_pathTable = pathTable;
+            DataItems = new List<ValidationData>();
+        }
+
+        /// <summary>
+        /// Adds scope policy.
+        /// </summary>
+        public AbsolutePath AddScope(
+            string path,
+            FileAccessPolicy values,
+            FileAccessPolicy mask = FileAccessPolicy.Deny,
+            FileAccessPolicy basePolicy = FileAccessPolicy.Deny)
+        {
+            AbsolutePath scopeAbsolutePath = AbsolutePath.Create(m_pathTable, path);
+            var dataItem =
+                new ValidationData
+                {
+                    Path = path,
+                    PathId = scopeAbsolutePath.Value.Value,
+                    NodePolicy = (basePolicy & mask) | values,
+                    ConePolicy = null,
+                    ExpectedUsn = ReportedFileAccess.NoUsn
+                };
+
+            DataItems.Add(dataItem);
+            m_manifest.AddScope(scopeAbsolutePath, mask, values);
+
+            return scopeAbsolutePath;
+        }
+
+        /// <summary>
+        /// Adds path policy.
+        /// </summary>
+        public void AddPath(
+            string path,
+            FileAccessPolicy policy,
+            FileAccessPolicy? expectedEffectivePolicy = null,
+            Usn? expectedUsn = null)
+        {
+            AbsolutePath absolutePath = AbsolutePath.Create(m_pathTable, path);
+            var dataItem =
+                new ValidationData
+                {
+                    Path = path,
+                    PathId = absolutePath.Value.Value,
+                    ConePolicy = null,
+                    NodePolicy = expectedEffectivePolicy ?? policy,
+                    ExpectedUsn = expectedUsn ?? ReportedFileAccess.NoUsn
+                };
+
+            DataItems.Add(dataItem);
+            m_manifest.AddPath(absolutePath, values: policy, mask: FileAccessPolicy.MaskNothing, expectedUsn: expectedUsn);
+        }
+
+        /// <summary>
+        /// Adds check for scope policy without adding the policy to the file access manifest.
+        /// </summary>
+        public void AddScopeCheck(string path, AbsolutePath scopePath, FileAccessPolicy policy)
+        {
+            DataItems.Add(
+                new ValidationData
+                {
+                    Path = path,
+                    PathId = scopePath.Value.Value,
+                    ConePolicy = policy,
+                    NodePolicy = null,
+                    ExpectedUsn = ReportedFileAccess.NoUsn
+                });
+        }
+
+        /// <summary>
+        /// Tests file access manifest.
+        /// </summary>
+        public static void TestManifestRetrieval(IEnumerable<ValidationData> validationData, FileAccessManifest fam, bool serializeManifest)
+        {
+            foreach (var line in fam.Describe())
+            {
+                Console.WriteLine(line);
+            }
+
+            if (serializeManifest)
+            {
+                var writtenFlag = fam.Flag;
+                var writtenDirectoryTranslator = fam.DirectoryTranslator;
+
+                using (var stream = new MemoryStream())
+                {
+                    fam.Serialize(stream);
+                    stream.Position = 0;
+                    fam = FileAccessManifest.Deserialize(stream);
+                }
+
+                XAssert.AreEqual(writtenFlag, fam.Flag);
+                XAssert.AreEqual(writtenDirectoryTranslator == null, fam.DirectoryTranslator == null);
+                if (writtenDirectoryTranslator != null)
+                {
+                    XAssert.AreEqual(writtenDirectoryTranslator.Count, fam.DirectoryTranslator.Count);
+                    var writtenTranslations = writtenDirectoryTranslator.Translations.ToArray();
+                    var readTranslations = fam.DirectoryTranslator.Translations.ToArray();
+
+                    for (int i = 0; i < writtenTranslations.Length; ++i)
+                    {
+                        XAssert.AreEqual(writtenTranslations[i].SourcePath, readTranslations[i].SourcePath);
+                        XAssert.AreEqual(writtenTranslations[i].TargetPath, readTranslations[i].TargetPath);
+                    }
+                }
+            }
+
+            byte[] manifestTreeBytes = fam.GetManifestTreeBytes();
+
+            foreach (ValidationData dataItem in validationData)
+            {
+                uint nodePolicy;
+                uint conePolicy;
+                uint pathId;
+                Usn expectedUsn;
+
+                bool success =
+                    global::BuildXL.Native.Processes.Windows.ProcessUtilitiesWin.FindFileAccessPolicyInTree(
+                        manifestTreeBytes,
+                        dataItem.Path,
+                        new UIntPtr((uint)dataItem.Path.Length),
+                        out conePolicy,
+                        out nodePolicy,
+                        out pathId,
+                        out expectedUsn);
+
+                XAssert.IsTrue(success, "Unable to find path in manifest");
+                XAssert.AreEqual(
+                    unchecked((uint)dataItem.PathId),
+                    pathId,
+                    "PathId for '{0}' did not match", dataItem.Path);
+
+                if (dataItem.NodePolicy.HasValue)
+                {
+                    XAssert.AreEqual(
+                        unchecked((uint)dataItem.NodePolicy.Value),
+                        nodePolicy,
+                        "Policy for '{0}' did not match", dataItem.Path);
+                }
+
+                if (dataItem.ConePolicy.HasValue)
+                {
+                    XAssert.AreEqual(
+                        unchecked((uint)dataItem.ConePolicy.Value),
+                        conePolicy,
+                        "Policy for '{0}' did not match", dataItem.Path);
+                }
+
+                XAssert.AreEqual(
+                    dataItem.ExpectedUsn,
+                    expectedUsn,
+                    "Usn for '{0}' did not match", dataItem.Path);
+            }
+        }
+
+        internal struct ValidationData
+        {
+            public string Path { get; set; }
+
+            public FileAccessPolicy? NodePolicy { get; set; }
+
+            public FileAccessPolicy? ConePolicy { get; set; }
+
+            public int PathId { get; set; }
+
+            public Usn ExpectedUsn { get; set; }
+        }
+    }
+}

Public/Src/Engine/UnitTests/Scheduler.IntegrationTest/ExternalToolExecutionTests.cs

@@ -0,0 +1,153 @@
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+using System;
+using System.IO;
+using BuildXL.Pips.Builders;
+using BuildXL.Pips.Operations;
+using BuildXL.Utilities;
+using BuildXL.Utilities.Tracing;
+using Test.BuildXL.Executables.TestProcess;
+using Test.BuildXL.Scheduler;
+using Test.BuildXL.TestUtilities.Xunit;
+using Xunit;
+using Xunit.Abstractions;
+
+namespace IntegrationTest.BuildXL.Scheduler
+{
+    [TestClassIfSupported(requiresWindowsBasedOperatingSystem: true)]
+    public class ExternalToolExecutionTests : SchedulerIntegrationTestBase
+    {
+        public ExternalToolExecutionTests(ITestOutputHelper output) : base(output)
+        {
+            Configuration.Sandbox.AdminRequiredProcessExecutionMode = global::BuildXL.Utilities.Configuration.AdminRequiredProcessExecutionMode.ExternalTool;
+        }
+
+        [Fact]
+        public void RunSingleProcessInExternalTool()
+        {
+            ProcessBuilder builder = CreatePipBuilder(new[] { Operation.ReadFile(CreateSourceFile()), Operation.WriteFile(CreateOutputFileArtifact()) });
+            builder.Options |= Process.Options.RequiresAdmin;
+            ProcessWithOutputs process = SchedulePipBuilder(builder);
+
+            RunScheduler().AssertSuccess();
+            RunScheduler().AssertCacheHit(process.Process.PipId);
+        }
+
+        [Fact]
+        public void RunMultipleProcessesInExternalTool()
+        {
+            for (int i = 0; i < 5; ++i)
+            {
+                ProcessBuilder builder = CreatePipBuilder(new[] { Operation.ReadFile(CreateSourceFile()), Operation.WriteFile(CreateOutputFileArtifact()) });
+                builder.Options |= Process.Options.RequiresAdmin;
+                ProcessWithOutputs process = SchedulePipBuilder(builder);
+            }
+
+            RunScheduler().AssertSuccess();
+        }
+
+        [Fact]
+        public void ExternalToolPreserveWarning()
+        {
+            ProcessBuilder builder = CreatePipBuilder(new[] {
+                Operation.ReadFile(CreateSourceFile()),
+                Operation.Echo("WARN this is a warning"),
+                Operation.WriteFile(CreateOutputFileArtifact()) });
+            builder.Options |= Process.Options.RequiresAdmin;
+            builder.WarningRegex = new RegexDescriptor(StringId.Create(Context.StringTable, @"^WARN"), System.Text.RegularExpressions.RegexOptions.IgnoreCase);
+
+            ProcessWithOutputs process = SchedulePipBuilder(builder);
+
+            ScheduleRunResult result = RunScheduler().AssertSuccess();
+            AssertWarningEventLogged(EventId.PipProcessWarning, count: 1);
+        }
+
+        [Fact]
+        public void ExternalToolRespectFailure()
+        {
+            ProcessBuilder builder = CreatePipBuilder(new[] {
+                Operation.ReadFile(CreateSourceFile()),
+                Operation.Fail(),
+                Operation.WriteFile(CreateOutputFileArtifact()) });
+            builder.Options |= Process.Options.RequiresAdmin;
+            ProcessWithOutputs process = SchedulePipBuilder(builder);
+
+            RunScheduler().AssertFailure();
+            AssertErrorEventLogged(EventId.PipProcessError, count: 1);
+        }
+
+        [Fact]
+        public void ExternalToolRespectFileAccessManifest()
+        {
+            ProcessBuilder builder = CreatePipBuilder(new[] { Operation.ReadFile(CreateSourceFile(), doNotInfer: true), Operation.WriteFile(CreateOutputFileArtifact()) });
+            builder.Options |= Process.Options.RequiresAdmin;
+            ProcessWithOutputs process = SchedulePipBuilder(builder);
+
+            RunScheduler().AssertFailure();
+            AssertWarningEventLogged(EventId.ProcessNotStoredToCacheDueToFileMonitoringViolations, count: 1);
+            AssertErrorEventLogged(EventId.FileMonitoringError, count: 1);
+        }
+
+        [Fact]
+        public void ExternalToolRecordsReportedFileAccesses()
+        {
+            FileArtifact sourceFile = CreateSourceFile();
+
+            SealDirectory sourceDirectory = CreateAndScheduleSealDirectory(sourceFile.Path.GetParent(Context.PathTable), SealDirectoryKind.SourceAllDirectories);
+            ProcessBuilder builder = CreatePipBuilder(new[] { Operation.ReadFile(sourceFile, doNotInfer: true), Operation.WriteFile(CreateOutputFileArtifact()) });
+            builder.AddInputDirectory(sourceDirectory.Directory);
+            builder.Options |= Process.Options.RequiresAdmin;
+            ProcessWithOutputs process = SchedulePipBuilder(builder);
+
+            RunScheduler().AssertSuccess();
+            RunScheduler().AssertCacheHit(process.Process.PipId);
+
+            File.WriteAllText(ArtifactToString(sourceFile), Guid.NewGuid().ToString());
+
+            RunScheduler().AssertCacheMiss(process.Process.PipId);
+        }
+
+        [Fact]
+        public void ExternalToolExecuteProcessReadingStdIn()
+        {
+            FileArtifact stdOut = CreateOutputFileArtifact();
+            ProcessBuilder builder = CreatePipBuilder(new[] { Operation.ReadStdIn() });
+            PipDataBuilder dataBuilder = new PipDataBuilder(Context.PathTable.StringTable);
+            dataBuilder.Add("Data0");
+            dataBuilder.Add("Data1");
+            dataBuilder.Add("Data2");
+            builder.StandardInput = global::BuildXL.Pips.StandardInput.CreateFromData(dataBuilder.ToPipData(Environment.NewLine, PipDataFragmentEscaping.NoEscaping));
+            builder.SetStandardOutputFile(stdOut.Path);
+            builder.Options |= Process.Options.RequiresAdmin;
+            ProcessWithOutputs process = SchedulePipBuilder(builder);
+
+            RunScheduler().AssertSuccess();
+
+            string[] output = File.ReadAllLines(ArtifactToString(stdOut));
+            string actualContent = string.Join(Environment.NewLine, output);
+
+            XAssert.AreEqual(3, output.Length, "Actual content: {0}{1}", Environment.NewLine, string.Join(Environment.NewLine, output));
+            for (int i = 0; i < 3; ++i)
+            {
+                XAssert.AreEqual("Data" + i, output[i], "Actual content: {0}", output[i]);
+            }
+        }
+
+        [Fact]
+        public void ExternalToolRespectTimeout()
+        {
+            ProcessBuilder builder = CreatePipBuilder(new[] {
+                Operation.ReadFile(CreateSourceFile()),
+                Operation.Block(),
+                Operation.WriteFile(CreateOutputFileArtifact()) });
+            builder.Timeout = TimeSpan.FromSeconds(1);
+            builder.Options |= Process.Options.RequiresAdmin;
+            
+            ProcessWithOutputs process = SchedulePipBuilder(builder);
+
+            RunScheduler().AssertFailure();
+            AssertErrorEventLogged(EventId.PipProcessTookTooLongError, count: 1);
+            AssertErrorEventLogged(EventId.PipProcessError, count: 1);
+        }
+    }
+}

Public/Src/Engine/UnitTests/Scheduler.IntegrationTest/IntegrationTest.BuildXL.Scheduler.dsc

@@ -54,7 +54,18 @@ namespace Scheduler.IntegrationTest {
             importFrom("BuildXL.Utilities.UnitTests").StorageTestUtilities.dll,
         ],
         runtimeContent: [
-            importFrom("BuildXL.Utilities.UnitTests").TestProcess.deploymentDefinition
+            importFrom("BuildXL.Utilities.UnitTests").TestProcess.deploymentDefinition,
+            {
+                subfolder: a`tools`,
+                contents: [
+                    {
+                        subfolder: a`SandboxedProcessExecutor`,
+                        contents: [
+                            importFrom("BuildXL.Tools").SandboxedProcessExecutor.exe
+                        ]
+                    }
+                ]
+            }
         ],
     });
 }

Public/Src/Engine/UnitTests/Scheduler/PipFingerprinterTests.cs

@@ -626,6 +626,7 @@ namespace Test.BuildXL.Scheduler
             bool outputsMustBeWritable = source.Vary(p => p.OutputsMustRemainWritable);
             bool allowUndeclaredSourceReads = source.Vary(p => p.AllowUndeclaredSourceReads);
             bool needsToRunInContainer = source.Vary(p => p.NeedsToRunInContainer);
+            bool requiresAdmin = source.Vary(p => p.RequiresAdmin);
             DoubleWritePolicy doubleWritePolicy = source.Vary(p => p.DoubleWritePolicy);
             ContainerIsolationLevel containerIsolationLevel = source.Vary(p => p.ContainerIsolationLevel);
             var uniqueRedirectedDirectoryRoot = source.Vary(p => p.UniqueRedirectedDirectoryRoot);
@@ -662,6 +663,11 @@ namespace Test.BuildXL.Scheduler
                 }
             }
 
+            if (requiresAdmin)
+            {
+                options |= Process.Options.RequiresAdmin;
+            }
+
             return new Process(
                 executable: executable,
                 workingDirectory: workingDirectory,

Public/Src/Engine/UnitTests/Scheduler/PipRunningTimeTableTests.cs

@@ -3,6 +3,7 @@
 
 using System;
 using System.IO;
+using BuildXL.Native.IO;
 using BuildXL.Pips;
 using BuildXL.Scheduler;
 using BuildXL.Storage;

Public/Src/Engine/UnitTests/Scheduler/ProcessExecutionResultSerializerTests.cs

@@ -16,6 +16,7 @@ using BuildXL.Cache.MemoizationStore.Interfaces.Sessions;
 using Test.BuildXL.TestUtilities.Xunit;
 using Xunit;
 using Xunit.Abstractions;
+using BuildXL.Native.IO;
 
 namespace Test.BuildXL.Scheduler
 {

Public/Src/FrontEnd/Script/Ambients/Transformers/AmbientTransformer.Process.cs

@@ -51,6 +51,12 @@ namespace BuildXL.FrontEnd.Script.Ambients.Transformers
             ["unsafeFirstDoubleWriteWins"] = DoubleWritePolicy.UnsafeFirstDoubleWriteWins,
         };
 
+        private static readonly Dictionary<string, bool> s_privilegeLevel = new Dictionary<string, bool>(StringComparer.Ordinal)
+        {
+            ["standard"] = false,
+            ["admin"]    = true,
+        };
+
         // these values must be kept in sync with the ones defined on the BuildXL Script side
         private static readonly Dictionary<string, Process.AbsentPathProbeInUndeclaredOpaquesMode> s_absentPathProbeModes = new Dictionary<string, Process.AbsentPathProbeInUndeclaredOpaquesMode>(StringComparer.Ordinal)
         {
@@ -89,6 +95,7 @@ namespace BuildXL.FrontEnd.Script.Ambients.Transformers
         private SymbolAtom m_executeDoubleWritePolicy;
         private SymbolAtom m_executeAllowUndeclaredSourceReads;
         private SymbolAtom m_executeKeepOutputsWritable;
+        private SymbolAtom m_privilegeLevel;
         private SymbolAtom m_disableCacheLookup;
         private SymbolAtom m_executeWarningRegex;
         private SymbolAtom m_executeErrorRegex;
@@ -221,6 +228,7 @@ namespace BuildXL.FrontEnd.Script.Ambients.Transformers
             m_executeAbsentPathProbeInUndeclaredOpaqueMode = Symbol("absentPathProbeInUndeclaredOpaquesMode");
 
             m_executeKeepOutputsWritable = Symbol("keepOutputsWritable");
+            m_privilegeLevel = Symbol("privilegeLevel");
             m_disableCacheLookup = Symbol("disableCacheLookup");
             m_executeTags = Symbol("tags");
             m_executeServiceShutdownCmd = Symbol("serviceShutdownCmd");
@@ -489,6 +497,13 @@ namespace BuildXL.FrontEnd.Script.Ambients.Transformers
                 processBuilder.Options |= Process.Options.OutputsMustRemainWritable;
             }
 
+            // Set outputs to remain writable.
+            var privilegeLevel = Converter.ExtractStringLiteral(obj, m_privilegeLevel, s_privilegeLevel.Keys, allowUndefined: true);
+            if (privilegeLevel != null && s_privilegeLevel.TryGetValue(privilegeLevel, out bool level) && level)
+            {
+                processBuilder.Options |= Process.Options.RequiresAdmin;
+            }
+
             var absentPathProbeMode = Converter.ExtractStringLiteral(obj, m_executeAbsentPathProbeInUndeclaredOpaqueMode, s_absentPathProbeModes.Keys, allowUndefined: true);
             if (absentPathProbeMode != null)
             {

Public/Src/FrontEnd/UnitTests/Libs/Sdk.Transformers.ts

@@ -168,25 +168,28 @@ namespace Transformer {
             /** Set outputs to remain writable */
             keepOutputsWritable?: boolean;
 
-			/** Whether this process should run in an isolated container (i.e. filesystem isolation)
-			 * When running in a container, the isolation level can be controlled by 'containerIsolationLevel' field.
-			 * Note: this is an experimental feature for now, use at your own risk
-			 * Default is globally controlled by the sandbox configuration
-			 * */
-			runInContainer?: boolean;
+            /** Privilege level required by this process to execute. */
+            privilegeLevel?: "standard" | "admin";
 
-			/**
-			 * Configures which inputs and outputs of this process should be isolated.
-			 * Default is globally controlled by the sandbox configuration
-			 * TODO: input isolation is not implemented
-			 */
-			containerIsolationLevel?: ContainerIsolationLevel;
+            /** Whether this process should run in an isolated container (i.e. filesystem isolation)
+             * When running in a container, the isolation level can be controlled by 'containerIsolationLevel' field.
+             * Note: this is an experimental feature for now, use at your own risk
+             * Default is globally controlled by the sandbox configuration
+             * */
+            runInContainer?: boolean;
 
-			/**
-			 * The policy to apply when a double write occurs.
-			 * Default is globally controlled by the sandbox configuration
-			 */
-			doubleWritePolicy?: DoubleWritePolicy;
+            /**
+             * Configures which inputs and outputs of this process should be isolated.
+             * Default is globally controlled by the sandbox configuration
+             * TODO: input isolation is not implemented
+             */
+            containerIsolationLevel?: ContainerIsolationLevel;
+
+            /**
+             * The policy to apply when a double write occurs.
+             * Default is globally controlled by the sandbox configuration
+             */
+            doubleWritePolicy?: DoubleWritePolicy;
         }
 
         @@public
@@ -325,7 +328,7 @@ namespace Transformer {
         return _PreludeAmbientHack_Transformer.sealPartialDirectory(rootOrArgs, files, tags, description);
     }
 
-	/** Creates a shared opaque directory whose content is the aggregation of a collection of shared opaque directories.
+    /** Creates a shared opaque directory whose content is the aggregation of a collection of shared opaque directories.
      * The provided root can be any arbitrary directory that is a common ancestor to all the provided directories
     */
     @@public
@@ -614,14 +617,14 @@ namespace Artifact {
         return createArtifact(value, ArtifactKind.output);
     }
 
-	/**
+    /**
      * Creates a shared opaque directory from a directory.
      * This is an unsafe feature, the current implementation is in a prototyping stage. Use at your own risk.
      * */
-	@@public
-	export function sharedOpaqueOutput(value: Directory): Artifact {
-		return createArtifact(value, ArtifactKind.sharedOpaque);
-	}
+    @@public
+    export function sharedOpaqueOutput(value: Directory): Artifact {
+        return createArtifact(value, ArtifactKind.sharedOpaque);
+    }
 
     /** Creates a list of output artifacts from a list of files.  */
     @@public

Public/Src/Pips/Dll/Builders/ProcessBuilder.cs

@@ -619,8 +619,7 @@ namespace BuildXL.Pips.Builders
                 doubleWritePolicy: DoubleWritePolicy,
                 containerIsolationLevel: ContainerIsolationLevel,
                 absentPathProbeMode: AbsentPathProbeUnderOpaquesMode,
-                weight: Weight
-            );
+                weight: Weight);
 
             return true;
         }

Public/Src/Pips/Dll/Operations/Process.Options.cs

@@ -90,6 +90,11 @@ namespace BuildXL.Pips.Operations
             /// Windows only, has no effect on other operating systems.
             /// </remarks>
             DependsOnWindowsProgramData = 1 << 10,
+
+            /// <summary>
+            /// Whether this process requires admin privilege.
+            /// </summary>
+            RequiresAdmin = 1 << 11
         }
     }
 }

Public/Src/Pips/Dll/Operations/Process.cs

@@ -563,6 +563,12 @@ namespace BuildXL.Pips.Operations
         [PipCaching(FingerprintingRole = FingerprintingRole.None)]
         public bool OutputsMustRemainWritable => (ProcessOptions & Options.OutputsMustRemainWritable) != 0;
 
+        /// <summary>
+        /// Whether this process requires admin privilege
+        /// </summary>
+        [PipCaching(FingerprintingRole = FingerprintingRole.Semantic)]
+        public bool RequiresAdmin => (ProcessOptions & Options.RequiresAdmin) != 0;
+
         /// <summary>
         /// Indicates the process may run without deleting prior outputs from a previous run.
         /// </summary>

Public/Src/Pips/Dll/Operations/RegexDescriptor.cs

@@ -16,30 +16,7 @@ namespace BuildXL.Pips.Operations
         /// <summary>
         /// Adapted from Microsoft.BUild.Utilities.Core / CanonicalError.cs
         /// </summary>
-        public const string DefaultWarningPattern =
-
-            // Beginning of line and any amount of whitespace.
-            @"^\s*"
-
-                // Match a [optional project number prefix 'ddd>'], single letter + colon + remaining filename, or
-                // string with no colon followed by a colon.
-            + @"((((((\d+>)?[a-zA-Z]?:[^:]*)|([^:]*))):)"
-
-                // Origin may also be empty. In this case there's no trailing colon.
-            + "|())"
-
-                // Match the empty string or a string without a colon that ends with a space
-            + "(()|([^:]*? ))"
-
-                // Match 'warning'.
-            + @"warning"
-
-                // Match anything starting with a space that's not a colon/space, followed by a colon.
-                // Error code is optional in which case "warning" can be followed immediately by a colon.
-            + @"( \s*([^: ]*))?\s*:"
-
-                // Whatever's left on this line, including colons.
-            + ".*$";
+        public const string DefaultWarningPattern = Warning.DefaultWarningPattern;
 
         private const RegexOptions DefaultOptions = RegexOptions.IgnoreCase;
 

Public/Src/Pips/Dll/PipExecutionPerformance.cs

@@ -6,6 +6,8 @@ using System.Diagnostics.ContractsLight;
 using BuildXL.Cache.MemoizationStore.Interfaces.Sessions;
 using BuildXL.Storage;
 using BuildXL.Utilities;
+using BuildXL.Native.IO;
+using System.IO;
 
 namespace BuildXL.Pips
 {
@@ -276,7 +278,7 @@ namespace BuildXL.Pips
 
             writer.Write(ProcessExecutionTime);
             WriteFileMonitoringViolationCounters(writer, FileMonitoringViolations);
-            WriteIOCounters(writer, IO);
+            IO.Serialize(writer);
             writer.Write(UserTime);
             writer.Write(KernelTime);
             writer.Write(PeakMemoryUsage);
@@ -289,7 +291,7 @@ namespace BuildXL.Pips
 
             TimeSpan processExecutionTime = reader.ReadTimeSpan();
             FileMonitoringViolationCounters fileMonitoringViolations = ReadFileMonitoringViolationCounters(reader);
-            IOCounters ioCounters = ReadIOCounters(reader);
+            IOCounters ioCounters = IOCounters.Deserialize(reader);
             TimeSpan userTime = reader.ReadTimeSpan();
             TimeSpan kernelTime = reader.ReadTimeSpan();
             ulong peakMemoryUsage = reader.ReadUInt64();
@@ -324,160 +326,6 @@ namespace BuildXL.Pips
             writer.WriteCompact((int)counters.NumFileAccessesWhitelistedButNotCacheable);
             writer.WriteCompact((int)counters.NumFileAccessesWhitelistedAndCacheable);
         }
-
-        private static IOCounters ReadIOCounters(BuildXLReader reader)
-        {
-            return new IOCounters(
-                readCounters: ReadIOTypeCounters(reader),
-                writeCounters: ReadIOTypeCounters(reader),
-                otherCounters: ReadIOTypeCounters(reader));
-        }
-
-        private static void WriteIOCounters(BuildXLWriter writer, in IOCounters ioCounters)
-        {
-            WriteIOTypeCounters(writer, ioCounters.ReadCounters);
-            WriteIOTypeCounters(writer, ioCounters.WriteCounters);
-            WriteIOTypeCounters(writer, ioCounters.OtherCounters);
-        }
-
-        private static IOTypeCounters ReadIOTypeCounters(BuildXLReader reader)
-        {
-            return new IOTypeCounters(
-                 operationCount: reader.ReadUInt64(),
-                 transferCount: reader.ReadUInt64());
-        }
-
-        private static void WriteIOTypeCounters(BuildXLWriter writer, in IOTypeCounters ioTypeCounters)
-        {
-            writer.Write(ioTypeCounters.OperationCount);
-            writer.Write(ioTypeCounters.TransferCount);
-        }
-    }
-
-    /// <summary>
-    /// Contains I/O accounting information for a process or process tree for a particular type of IO (e.g. read or write).
-    /// </summary>
-    [System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Performance", "CA1815:OverrideEqualsAndOperatorEqualsOnValueTypes")]
-    public readonly struct IOTypeCounters : IEquatable<IOTypeCounters>
-    {
-        /// <summary>
-        /// Number of operations performed (independent of size).
-        /// </summary>
-        public readonly ulong OperationCount;
-
-        /// <summary>
-        /// Total bytes transferred (regardless of the number of operations used to transfer them).
-        /// </summary>
-        public readonly ulong TransferCount;
-
-        /// <inheritdoc/>
-        public bool Equals(IOTypeCounters other)
-        {
-            return (OperationCount == other.OperationCount) && (TransferCount == other.TransferCount);
-        }
-
-        /// <nodoc />
-        public static bool operator !=(IOTypeCounters t1, IOTypeCounters t2)
-        {
-            return !t1.Equals(t2);
-        }
-
-        /// <nodoc />
-        public static bool operator ==(IOTypeCounters t1, IOTypeCounters t2)
-        {
-            return t1.Equals(t2);
-        }
-
-        /// <inheritdoc/>
-        public override bool Equals(object obj)
-        {
-            return (obj is IOTypeCounters) ? Equals((IOTypeCounters)obj) : false;
-        }
-
-        /// <inheritdoc/>
-        public override int GetHashCode()
-        {
-            return HashCodeHelper.Combine(OperationCount.GetHashCode(), TransferCount.GetHashCode());
-        }
-
-        /// <nodoc />
-        public IOTypeCounters(ulong operationCount, ulong transferCount)
-        {
-            OperationCount = operationCount;
-            TransferCount = transferCount;
-        }
-    }
-
-    /// <summary>
-    /// Contains I/O accounting information for a process or process tree.
-    /// </summary>
-    [System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Performance", "CA1815:OverrideEqualsAndOperatorEqualsOnValueTypes")]
-    public readonly struct IOCounters : IEquatable<IOCounters>
-    {
-        /// <summary>
-        /// Counters for read operations.
-        /// </summary>
-        public readonly IOTypeCounters ReadCounters;
-
-        /// <summary>
-        /// Counters for write operations.
-        /// </summary>
-        public readonly IOTypeCounters WriteCounters;
-
-        /// <summary>
-        /// Counters for other operations (not classified as either read or write).
-        /// </summary>
-        public readonly IOTypeCounters OtherCounters;
-
-        /// <inheritdoc/>
-        public bool Equals(IOCounters other)
-        {
-            return (ReadCounters == other.ReadCounters) && (WriteCounters == other.WriteCounters) && (OtherCounters == other.OtherCounters);
-        }
-
-        /// <nodoc/>
-        public static bool operator !=(IOCounters t1, IOCounters t2)
-        {
-            return !t1.Equals(t2);
-        }
-
-        /// <nodoc/>
-        public static bool operator ==(IOCounters t1, IOCounters t2)
-        {
-            return t1.Equals(t2);
-        }
-
-        /// <inheritdoc/>
-        public override bool Equals(object obj)
-        {
-            return (obj is IOCounters) ? Equals((IOCounters)obj) : false;
-        }
-
-        /// <inheritdoc/>
-        public override int GetHashCode()
-        {
-            return HashCodeHelper.Combine(ReadCounters.GetHashCode(), WriteCounters.GetHashCode(), OtherCounters.GetHashCode());
-        }
-
-        /// <nodoc />
-        public IOCounters(IOTypeCounters readCounters, IOTypeCounters writeCounters, IOTypeCounters otherCounters)
-        {
-            ReadCounters = readCounters;
-            WriteCounters = writeCounters;
-            OtherCounters = otherCounters;
-        }
-
-        /// <summary>
-        /// Computes the aggregate I/O performed (sum of the read, write, and other counters).
-        /// </summary>
-        [System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Design", "CA1024:UsePropertiesWhereAppropriate")]
-        [Pure]
-        public IOTypeCounters GetAggregateIO()
-        {
-            return new IOTypeCounters(
-                operationCount: ReadCounters.OperationCount + WriteCounters.OperationCount + OtherCounters.OperationCount,
-                transferCount: ReadCounters.TransferCount + WriteCounters.TransferCount + OtherCounters.TransferCount);
-        }
     }
 
     /// <summary>

Public/Src/Tools/Execution.Analyzer/Analyzers.Partner/DumpPipAnalyzer.cs

@@ -7,6 +7,7 @@ using System.Globalization;
 using System.IO;
 using System.Linq;
 using System.Xml.Linq;
+using BuildXL.Native.IO;
 using BuildXL.Pips;
 using BuildXL.Pips.Operations;
 using BuildXL.Processes;

Public/Src/Tools/SandboxedProcessExecutor/Args.cs

@@ -0,0 +1,84 @@
+// Copyright (c) Microsoft. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+using System;
+using System.Linq;
+using BuildXL.ToolSupport;
+
+namespace BuildXL.SandboxedProcessExecutor
+{
+    /// <summary>
+    /// Commmand line argument parser.
+    /// </summary>
+    internal sealed class Args : CommandLineUtilities
+    {
+        private static readonly string[] s_helpStrings = new[] { "?", "help" };
+        public bool Help;
+
+        /// <summary>
+        /// Initializes and parses command line arguments.
+        /// </summary>
+        public Args(string[] args)
+            : base(args)
+        {
+        }
+
+        /// <summary>
+        /// Processes command line arguments to construct an instance of <see cref="Configuration"/>.
+        /// </summary>
+        public bool TryProcess(out Configuration configuration)
+        {
+            configuration = new Configuration();
+
+            foreach (Option option in Options)
+            {
+                if (OptionEquals(option, "sandboxedProcessInfo") || OptionEquals(option, "i"))
+                {
+                    configuration.SandboxedProcessInfoInputFile = ParsePathOption(option);
+                }
+                else if (OptionEquals(option, "sandboxedProcessResult") || OptionEquals(option, "r"))
+                {
+                    configuration.SandboxedProcessResultOutputFile = ParsePathOption(option);
+                }
+                else if (OptionEquals(option, "enableTelemetry"))
+                {
+                    configuration.EnableTelemetry = ParseBooleanOption(option);
+                }
+                else if (s_helpStrings.Any(s => OptionEquals(option, s)))
+                {
+                    // If the analyzer was called with '/help' argument - print help and exit
+                    Help = true;
+                    WriteHelp();
+                    return true;
+                }
+                else
+                {
+                    Console.Error.WriteLine($"Option '{option.Name}' is not recognized");
+                    return false;
+                }
+            }
+
+            if (!configuration.Validate(out string errorConfiguration))
+            {
+                Console.Error.WriteLine($"Configuration error: {errorConfiguration}");
+                return false;
+            }
+
+            return true;
+        }
+
+        private bool OptionEquals(Option option, string name) => option.Name.Equals(name, StringComparison.OrdinalIgnoreCase);
+
+        private void WriteHelp()
+        {
+            HelpWriter writer = new HelpWriter();
+            writer.WriteBanner("Tool for executing and monitoring process in a sandbox");
+
+            writer.WriteLine("");
+            writer.WriteLine("/sandboxedProcessInfo:<file>   -- Sandboxed process info input file [short: /i]");
+            writer.WriteLine("/sandboxedProcessResult:<file> -- Sandboxed process result output file [short: /r]");
+            writer.WriteLine("/enableTelemetry[+/-]          -- Enable telemetry [default: false]");
+            writer.WriteLine("/help                          -- Print help message and exit [short: /?]");
+        }
+    }
+}

Public/Src/Tools/SandboxedProcessExecutor/Configuration.cs

@@ -0,0 +1,48 @@
+// Copyright (c) Microsoft. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+namespace BuildXL.SandboxedProcessExecutor
+{
+    /// <summary>
+    /// Configuration.
+    /// </summary>
+    internal sealed class Configuration
+    {
+        /// <summary>
+        /// Path to sandboxed process info input file.
+        /// </summary>
+        public string SandboxedProcessInfoInputFile { get; set; }
+
+        /// <summary>
+        /// Path to sandboxed process result output file.
+        /// </summary>
+        public string SandboxedProcessResultOutputFile { get; set; }
+
+        /// <summary>
+        /// Enables telemetry.
+        /// </summary>
+        public bool EnableTelemetry { get; set; }
+
+        /// <summary>
+        /// Validates configuration.
+        /// </summary>
+        public bool Validate(out string errorMessage)
+        {
+            errorMessage = null;
+
+            if (string.IsNullOrWhiteSpace(SandboxedProcessInfoInputFile))
+            {
+                errorMessage = "Missing sandboxed process info file";
+                return false;
+            }
+
+            if (string.IsNullOrWhiteSpace(SandboxedProcessResultOutputFile))
+            {
+                errorMessage = "Missing sandboxed process result file";
+                return false;
+            }
+
+            return true;
+        }
+    }
+}

Public/Src/Tools/SandboxedProcessExecutor/ConsoleLogger.cs

@@ -0,0 +1,71 @@
+// Copyright (c) Microsoft. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+using System;
+
+namespace BuildXL.SandboxedProcessExecutor
+{
+    /// <summary>
+    /// Simple console logger.
+    /// </summary>
+    internal class ConsoleLogger
+    {
+        private readonly object m_lock = new object();
+
+        /// <summary>
+        /// Logs informational message.
+        /// </summary>
+        public void LogInfo(string message)
+        {
+            LogMessage(message, MessageType.Info);
+        }
+
+        /// <summary>
+        /// Logs error message.
+        /// </summary>
+        public void LogError(string message)
+        {
+            LogMessage(message, MessageType.Error);
+        }
+
+        /// <summary>
+        /// Logs warning message.
+        /// </summary>
+        public void LogWarn(string message)
+        {
+            LogMessage(message, MessageType.Warn);
+        }
+
+        private void LogMessage(string message, MessageType type)
+        {
+            lock (m_lock)
+            {
+                ConsoleColor original = Console.ForegroundColor;
+                
+                switch (type)
+                {
+                    case MessageType.Error:
+                        Console.ForegroundColor = ConsoleColor.Red;
+                        Console.Error.WriteLine(message);
+                        break;
+                    case MessageType.Warn:
+                        Console.ForegroundColor = ConsoleColor.Yellow;
+                        Console.Out.WriteLine(message);
+                        break;
+                    case MessageType.Info:
+                        Console.Out.WriteLine(message);
+                        break;
+                }
+
+                Console.ForegroundColor = original;
+            }
+        }
+
+        private enum MessageType : byte
+        {
+            Info,
+            Warn,
+            Error
+        }
+    }
+}

Public/Src/Tools/SandboxedProcessExecutor/Executor.cs

@@ -0,0 +1,321 @@
+// Copyright (c) Microsoft. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+using System;
+using System.Diagnostics;
+using System.Diagnostics.ContractsLight;
+using System.IO;
+using System.Threading.Tasks;
+using BuildXL.Native.IO;
+using BuildXL.Processes;
+using BuildXL.SandboxedProcessExecutor.Tracing;
+using BuildXL.Utilities;
+using BuildXL.Utilities.Instrumentation.Common;
+using BuildXL.Utilities.Tracing;
+
+namespace BuildXL.SandboxedProcessExecutor
+{
+    internal sealed class Executor
+    {
+        private const int ProcessRelauchCountMax = 5;
+
+        private readonly Configuration m_configuration;
+        private readonly LoggingContext m_loggingContext = new LoggingContext("BuildXL.SandboxedProcessExecutor");
+        public readonly TrackingEventListener TrackingEventListener = new TrackingEventListener(Events.Log);
+        private readonly Stopwatch m_telemetryStopwatch = new Stopwatch();
+        private OutputErrorObserver m_outputErrorObserver;
+        private readonly ConsoleLogger m_logger = new ConsoleLogger();
+
+        /// <summary>
+        /// Creates an instance of <see cref="Executor"/>.
+        /// </summary>
+        public Executor(Configuration configuration)
+        {
+            Contract.Requires(configuration != null);
+
+            m_configuration = configuration;
+        }
+
+        public int Run()
+        {
+            AppDomain.CurrentDomain.UnhandledException +=
+                (sender, eventArgs) =>
+                {
+                    HandleUnhandledFailure(eventArgs.ExceptionObject as Exception);
+                };
+
+            TelemetryStartup();
+
+            ExitCode exitCode = RunInternal();
+
+            TelemetryShutdown();
+
+            return (int)exitCode;
+        }
+
+        private void HandleUnhandledFailure(Exception exception)
+        {
+            // Show the exception to the user
+            m_logger.LogError(exception.ToString());
+            
+            // Log the exception to telemetry
+            if (AriaV2StaticState.IsEnabled)
+            {
+                Logger.Log.SandboxedProcessExecutorCatastrophicFailure(m_loggingContext, exception.ToString());
+                TelemetryShutdown();
+            }
+
+            Environment.Exit((int)ExitCode.InternalError);
+        }
+
+        private void TelemetryStartup()
+        {
+            if (!Debugger.IsAttached && m_configuration.EnableTelemetry)
+            {
+                AriaV2StaticState.Enable(global::BuildXL.Tracing.AriaTenantToken.Key);
+                TrackingEventListener.RegisterEventSource(ETWLogger.Log);
+                m_telemetryStopwatch.Start();
+            }
+        }
+
+        private void TelemetryShutdown()
+        {
+            if (AriaV2StaticState.IsEnabled && m_configuration.EnableTelemetry)
+            {
+                m_telemetryStopwatch.Stop();
+                Logger.Log.SandboxedProcessExecutorInvoked(m_loggingContext, m_telemetryStopwatch.ElapsedMilliseconds, Environment.CommandLine);
+                AriaV2StaticState.TryShutDown(TimeSpan.FromSeconds(10), out var telemetryShutdownException);
+            }
+        }
+
+        internal ExitCode RunInternal()
+        {
+            if (!TryReadSandboxedProcessInfo(out SandboxedProcessInfo sandboxedProcessInfo))
+            {
+                return ExitCode.FailedReadInput;
+            }
+
+            if (!TryPrepareSandboxedProcess(sandboxedProcessInfo))
+            {
+                return ExitCode.FailedSandboxPreparation;
+            }
+
+            (ExitCode exitCode, SandboxedProcessResult result) executeResult = ExecuteAsync(sandboxedProcessInfo).GetAwaiter().GetResult();
+
+            if (executeResult.result != null)
+            {
+                if (!TryWriteSandboxedProcessResult(executeResult.result))
+                {
+                    return ExitCode.FailedWriteOutput;
+                }
+            }
+
+            return executeResult.exitCode;
+        }
+
+        private bool TryReadSandboxedProcessInfo(out SandboxedProcessInfo sandboxedProcessInfo)
+        {
+            sandboxedProcessInfo = null;
+            sandboxedProcessInfo = ExceptionUtilities.HandleRecoverableIOException(
+               () =>
+               {
+                   using (FileStream stream = File.OpenRead(Path.GetFullPath(m_configuration.SandboxedProcessInfoInputFile)))
+                   {
+                        // TODO: Custom DetoursEventListener?
+                        return SandboxedProcessInfo.Deserialize(stream, m_loggingContext, detoursEventListener: null);
+                   }
+               },
+               ex =>
+               {
+                   m_logger.LogError(ex.ToString());
+               });
+
+            return sandboxedProcessInfo != null;
+        }
+
+        private bool TryWriteSandboxedProcessResult(SandboxedProcessResult result)
+        {
+            Contract.Requires(result != null);
+
+            bool success = false;
+
+            ExceptionUtilities.HandleRecoverableIOException(
+                () =>
+                {
+                    using (FileStream stream = File.OpenWrite(Path.GetFullPath(m_configuration.SandboxedProcessResultOutputFile)))
+                    {
+                        result.Serialize(stream);
+                    }
+
+                    success = true;
+                },
+                ex =>
+                {
+                    m_logger.LogError(ex.ToString());
+                    success = false;
+                });
+
+            return success;
+        }
+
+        private bool TryPrepareSandboxedProcess(SandboxedProcessInfo info)
+        {
+            Contract.Requires(info != null);
+
+            FileAccessManifest fam = info.FileAccessManifest;
+
+            if (!string.IsNullOrEmpty(fam.InternalDetoursErrorNotificationFile))
+            {
+                Analysis.IgnoreResult(FileUtilities.TryDeleteFile(fam.InternalDetoursErrorNotificationFile));
+            }
+
+            if (fam.CheckDetoursMessageCount && !OperatingSystemHelper.IsUnixOS)
+            {
+                string semaphoreName = fam.InternalDetoursErrorNotificationFile.Replace('\\', '_');
+
+                if (!fam.SetMessageCountSemaphore(semaphoreName))
+                {
+                    m_logger.LogError($"Semaphore '{semaphoreName}' for counting Detours messages is already opened");
+                    return false;
+                }
+            }
+
+            if (info.GetCommandLine().Length > SandboxedProcessInfo.MaxCommandLineLength)
+            {
+                m_logger.LogError($"Process command line is longer than {SandboxedProcessInfo.MaxCommandLineLength} characters: {info.GetCommandLine().Length}");
+                return false;
+            }
+
+            m_outputErrorObserver = OutputErrorObserver.Create(m_logger, info);
+            info.StandardOutputObserver = m_outputErrorObserver.ObserveStandardOutputForWarning;
+            info.StandardErrorObserver = m_outputErrorObserver.ObserveStandardErrorForWarning;
+
+            return true;
+        }
+
+        private async Task<(ExitCode, SandboxedProcessResult)> ExecuteAsync(SandboxedProcessInfo info)
+        {
+            try
+            {
+                using (Stream standardInputStream = TryOpenStandardInputStream(info, out bool succeedInOpeningStdIn))
+                {
+                    if (!succeedInOpeningStdIn)
+                    {
+                        return (ExitCode.FailedSandboxPreparation, null);
+                    }
+
+                    using (StreamReader standardInputReader = standardInputStream == null ? null : new StreamReader(standardInputStream, CharUtilities.Utf8NoBomNoThrow))
+                    {
+                        info.StandardInputReader = standardInputReader;
+
+                        ISandboxedProcess process = await StartProcessAsync(info);
+
+                        if (process == null)
+                        {
+                            return (ExitCode.FailedStartProcess, null);
+                        }
+
+                        SandboxedProcessResult result = await process.GetResultAsync();
+
+                        // Patch result.
+                        result.WarningCount = m_outputErrorObserver.WarningCount;
+                        result.LastMessageCount = process.GetLastMessageCount();
+                        result.DetoursMaxHeapSize = process.GetDetoursMaxHeapSize();
+                        result.MessageCountSemaphoreCreated = info.FileAccessManifest.MessageCountSemaphore != null;
+
+                        return (ExitCode.Success, result);
+                    }
+                }
+            }
+            finally
+            {
+                info.FileAccessManifest.UnsetMessageCountSemaphore();
+            }
+        }
+
+        private async Task<ISandboxedProcess> StartProcessAsync(SandboxedProcessInfo info)
+        {
+            ISandboxedProcess process = null;
+            bool shouldRelaunchProcess = true;
+            int processRelaunchCount = 0;
+
+            while (shouldRelaunchProcess)
+            {
+                try
+                {
+                    shouldRelaunchProcess = false;
+                    process = await SandboxedProcessFactory.StartAsync(info, forceSandboxing: false);
+                }
+                catch (BuildXLException ex)
+                {
+                    if (ex.LogEventErrorCode == NativeIOConstants.ErrorFileNotFound)
+                    {
+                        m_logger.LogError($"Failed to start process: '{info.FileName}' not found");
+                        return null;
+                    }
+                    else if (ex.LogEventErrorCode == NativeIOConstants.ErrorPartialCopy && (processRelaunchCount < ProcessRelauchCountMax))
+                    {
+                        ++processRelaunchCount;
+                        shouldRelaunchProcess = true;
+
+                        m_logger.LogInfo($"Retry to start process for {processRelaunchCount} time(s) due to the following error: {ex.LogEventErrorCode}");
+
+                        // Ensure that process terminates before relaunching it.
+                        if (process != null)
+                        {
+                            try
+                            {
+                                await process.GetResultAsync();
+                            }
+                            finally
+                            {
+                                process.Dispose();
+                            }
+                        }
+                    }
+                    else
+                    {
+                        m_logger.LogError($"Failed to start process '{info.FileName}': {ex.LogEventMessage} ({ex.LogEventErrorCode})");
+                        return null;
+                    }
+                }
+            }
+
+            return process;
+        }
+
+        private Stream TryOpenStandardInputStream(SandboxedProcessInfo info, out bool success)
+        {
+            success = true;
+
+            if (info.StandardInputSourceInfo == null)
+            {
+                return null;
+            }
+
+            try
+            {
+                if (info.StandardInputSourceInfo.Data != null)
+                {
+                    return new MemoryStream(CharUtilities.Utf8NoBomNoThrow.GetBytes(info.StandardInputSourceInfo.Data));
+                }
+                else  
+                {
+                    Contract.Assert(info.StandardInputSourceInfo.File != null);
+
+                    return FileUtilities.CreateAsyncFileStream(
+                        info.StandardInputSourceInfo.File,
+                        FileMode.Open,
+                        FileAccess.Read,
+                        FileShare.Read | FileShare.Delete);
+                }
+            }
+            catch (BuildXLException ex)
+            {
+                m_logger.LogError($"Unable to open standard input stream: {ex.ToString()}");
+                success = false;
+                return null;
+            }
+        }
+    }
+}

Public/Src/Tools/SandboxedProcessExecutor/ExitCode.cs

@@ -0,0 +1,46 @@
+// Copyright (c) Microsoft. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+namespace BuildXL.SandboxedProcessExecutor
+{
+    /// <summary>
+    /// Exit status.
+    /// </summary>
+    public enum ExitCode : short
+    {
+        /// <summary>
+        /// Internal error.
+        /// </summary>
+        InternalError = -1,
+
+        /// <summary>
+        /// Success.
+        /// </summary>
+        Success = 0,
+
+        /// <summary>
+        /// Invalid argument.
+        /// </summary>
+        InvalidArgument = 1,
+
+        /// <summary>
+        /// Failed reading inputs.
+        /// </summary>
+        FailedReadInput = 2,
+
+        /// <summary>
+        /// Failed preparing sandboxed process.
+        /// </summary>
+        FailedSandboxPreparation = 3,
+
+        /// <summary>
+        /// Failed starting process.
+        /// </summary>
+        FailedStartProcess = 4,
+
+        /// <summary>
+        /// Failed writing output.
+        /// </summary>
+        FailedWriteOutput = 5,
+    }
+}

Public/Src/Tools/SandboxedProcessExecutor/OutputErrorObserver.cs

@@ -0,0 +1,129 @@
+// Copyright (c) Microsoft. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+using System;
+using System.Diagnostics.ContractsLight;
+using System.Text.RegularExpressions;
+using System.Threading;
+using BuildXL.Processes;
+using BuildXL.Utilities;
+
+namespace BuildXL.SandboxedProcessExecutor
+{
+    /// <summary>
+    /// Observer of standard output and standard error.
+    /// </summary>
+    internal class OutputErrorObserver
+    {
+        private readonly Regex m_warningRegex;
+        private readonly bool m_isDefaultWarningRegex;
+
+        private readonly bool m_logOutputToConsole;
+        private readonly bool m_logErrorToConsole;
+
+        private int m_warningCount;
+
+        private readonly ConsoleLogger m_logger;
+
+        /// <summary>
+        /// Total number of warnings.
+        /// </summary>
+        public int WarningCount => Volatile.Read(ref m_warningCount);
+
+        private OutputErrorObserver(ConsoleLogger logger, Regex warningRegex, bool isDefaultWarningRegex, bool logOutputToConsole, bool logErrorToConsole)
+        {
+            m_logger = logger;
+            m_warningRegex = warningRegex;
+            m_isDefaultWarningRegex = isDefaultWarningRegex;
+            m_logOutputToConsole = logOutputToConsole;
+            m_logErrorToConsole = logErrorToConsole;
+        }
+
+        private bool IsWarning(string line)
+        {
+            Contract.Requires(line != null);
+
+            if (m_warningRegex == null)
+            {
+                return false;
+            }
+
+            // An unusually long string causes pathologically slow Regex back-tracking.
+            // To avoid that, only scan the first 400 characters. That's enough for
+            // the longest possible prefix: MAX_PATH, plus a huge subcategory string, and an error location.
+            // After the regex is done, we can append the overflow.
+            if (line.Length > 400)
+            {
+                line = line.Substring(0, 400);
+            }
+
+            // If a tool has a large amount of output that isn't a warning (eg., "dir /s %hugetree%")
+            // the regex matching below may be slow. It's faster to pre-scan for "warning"
+            // and bail out if neither are present.
+            if (m_isDefaultWarningRegex &&
+                line.IndexOf("warning", StringComparison.OrdinalIgnoreCase) == -1)
+            {
+                return false;
+            }
+
+            return m_warningRegex.IsMatch(line);
+        }
+
+        /// <summary>
+        /// Creates an instance of <see cref="OutputErrorObserver"/>
+        /// </summary>
+        public static OutputErrorObserver Create(ConsoleLogger logger, SandboxedProcessInfo info)
+        {
+            Contract.Requires(info != null);
+
+            Regex warningRegex = info.StandardObserverDescriptor == null || info.StandardObserverDescriptor.WarningRegex == null
+                ? null
+                : CreateRegex(info.StandardObserverDescriptor.WarningRegex);
+            bool isDefaultWarningRegex = warningRegex == null ? false : IsDefaultWarningRegex(info.StandardObserverDescriptor.WarningRegex);
+            bool logOutputToConsole = info.StandardObserverDescriptor == null ? false : info.StandardObserverDescriptor.LogOutputToConsole;
+            bool logErrorToConsole = info.StandardObserverDescriptor == null ? false : info.StandardObserverDescriptor.LogErrorToConsole;
+
+            return new OutputErrorObserver(logger, warningRegex, isDefaultWarningRegex, logOutputToConsole, logErrorToConsole);
+        }
+
+        private static bool IsDefaultWarningRegex(ExpandedRegexDescriptor regexDescriptor) 
+            => string.Equals(regexDescriptor.Pattern, Warning.DefaultWarningPattern) && regexDescriptor.Options == RegexOptions.IgnoreCase;
+
+        private static Regex CreateRegex(ExpandedRegexDescriptor regexDescriptor) 
+            => new Regex(
+                    regexDescriptor.Pattern,
+                    regexDescriptor.Options | RegexOptions.Compiled | RegexOptions.CultureInvariant);
+
+        /// <summary>
+        /// Observes standard output for warning.
+        /// </summary>
+        public void ObserveStandardOutputForWarning(string outputLine)
+        {
+            if (IsWarning(outputLine))
+            {
+                Interlocked.Increment(ref m_warningCount);
+            }
+
+            if (m_logOutputToConsole)
+            {
+                m_logger.LogInfo(outputLine);
+            }
+        }
+
+        /// <summary>
+        /// Observes standard error for warning.
+        /// </summary>
+        public void ObserveStandardErrorForWarning(string errorLine)
+        {
+            if (IsWarning(errorLine))
+            {
+                Interlocked.Increment(ref m_warningCount);
+            }
+
+            if (m_logErrorToConsole)
+            {
+                m_logger.LogError(errorLine);
+            }
+        }
+    }
+}

Public/Src/Tools/SandboxedProcessExecutor/Program.cs

@@ -0,0 +1,55 @@
+// Copyright (c) Microsoft. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+using System;
+using BuildXL.Processes;
+using BuildXL.ToolSupport;
+
+namespace BuildXL.SandboxedProcessExecutor
+{
+    internal sealed class Program : ToolProgram<Args>
+    {
+        private Program()
+            : base("SandboxedProcessExecutor")
+        { }
+
+        public static int Main(string[] arguments)
+        {
+            try
+            {
+                return new Program().MainHandler(arguments);
+            }
+            catch (InvalidArgumentException e)
+            {
+                Console.Error.WriteLine("Execution error: " + (e.InnerException ?? e).Message);
+            }
+
+            return (int)ExitCode.InvalidArgument;
+        }
+
+        public override bool TryParse(string[] rawArgs, out Args arguments)
+        {
+            // If there is any exception with parsing arguments, allow the program to exit
+            arguments = new Args(rawArgs);
+            return true;
+        }
+
+        public override int Run(Args arguments)
+        {
+            bool success = arguments.TryProcess(out Configuration configuration);
+
+            if (!success)
+            {
+                return (int)ExitCode.InvalidArgument;
+            }
+
+            if (arguments.Help)
+            {
+                return (int)ExitCode.Success;
+            }
+
+            var exitCode = new Executor(configuration).Run();
+            return exitCode;
+        }
+    }
+}

Public/Src/Tools/SandboxedProcessExecutor/SandboxedProcessExecutor.dsc

@@ -0,0 +1,35 @@
+// Copyright (c) Microsoft. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+import { NetFx } from "Sdk.BuildXL";
+import * as Managed from "Sdk.Managed";
+import * as BuildXLSdk from "Sdk.BuildXL";
+import * as DetoursServices from "BuildXL.Sandbox.Windows";
+
+namespace SandboxedProcessExecutor {
+
+    export declare const qualifier: BuildXLSdk.DefaultQualifier;
+
+    @@public
+    export const exe = BuildXLSdk.executable({
+        generateLogs: true,
+        allowUnsafeBlocks: true,
+        assemblyName: "SandboxedProcessExecutor",
+        rootNamespace: "BuildXL.SandboxedProcessExecutor",
+        sources: globR(d`.`, "*.cs"),
+        references: [
+            importFrom("BuildXL.Utilities").dll,
+            importFrom("BuildXL.Utilities").Collections.dll,
+            importFrom("BuildXL.Utilities").Interop.dll,
+            importFrom("BuildXL.Utilities").Native.dll,
+            importFrom("BuildXL.Utilities").ToolSupport.dll,
+            importFrom("BuildXL.Utilities").Configuration.dll,
+            importFrom("BuildXL.Utilities.Instrumentation").Common.dll,
+            importFrom("BuildXL.Utilities.Instrumentation").Tracing.dll,
+            importFrom("BuildXL.Engine").Processes.dll,
+        ],
+        runtimeContent: [
+            DetoursServices.Deployment.definition,
+        ],
+    });
+}

Public/Src/Tools/SandboxedProcessExecutor/Tracing/Log.cs

@@ -0,0 +1,48 @@
+// Copyright (c) Microsoft. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+using System.Collections.Generic;
+using BuildXL.Tracing;
+using BuildXL.Utilities.Instrumentation.Common;
+using BuildXL.Utilities.Tracing;
+
+#if !FEATURE_MICROSOFT_DIAGNOSTICS_TRACING
+using System.Diagnostics.Tracing;
+#endif
+
+#pragma warning disable 1591
+
+namespace BuildXL.SandboxedProcessExecutor.Tracing
+{
+    /// <summary>
+    /// Logging for executor.
+    /// There are no log files, so messages for events with <see cref="EventGenerators.LocalOnly"/> will be lost.
+    /// </summary>
+    [EventKeywordsType(typeof(Events.Keywords))]
+    [EventTasksType(typeof(Events.Tasks))]
+    public abstract partial class Logger : LoggerBase
+    {
+        /// <summary>
+        /// Returns the logger instance
+        /// </summary>
+        public static Logger Log => m_log;
+
+        [GeneratedEvent(
+            (int)LogEventId.SandboxedProcessExecutorInvoked,
+            EventGenerators = EventGenerators.TelemetryOnly,
+            EventLevel = Level.Verbose,
+            Keywords = (int)Events.Keywords.UserMessage,
+            EventTask = (int)Events.Tasks.SandboxedProcessExecutor,
+            Message = "Invocation")]
+        public abstract void SandboxedProcessExecutorInvoked(LoggingContext context, long runtimeMs, string commandLine);
+
+        [GeneratedEvent(
+            (int)LogEventId.SandboxedProcessExecutorCatastrophicFailure,
+            EventGenerators = EventGenerators.TelemetryOnly,
+            EventLevel = Level.Verbose,
+            Keywords = (int)Events.Keywords.UserMessage,
+            EventTask = (int)Events.Tasks.SandboxedProcessExecutor,
+            Message = "Catastrophic failure")]
+        public abstract void SandboxedProcessExecutorCatastrophicFailure(LoggingContext context, string exceptionMessage);
+    }
+}

Public/Src/Tools/SandboxedProcessExecutor/Tracing/LogEventId.cs

@@ -0,0 +1,18 @@
+// Copyright (c) Microsoft. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+namespace BuildXL.SandboxedProcessExecutor.Tracing
+{
+    // disable warning regarding 'missing XML comments on public API'. We don't need docs for these values
+#pragma warning disable 1591
+
+    /// <summary>
+    /// Defines event IDs corresponding to events in <see cref="Logger" />.
+    /// </summary>
+    public enum LogEventId : ushort
+    {
+        // RESERVED TO [8700, 8799] (BuildXL.SandboxedProcessExecutor)
+        SandboxedProcessExecutorInvoked = 8700,
+        SandboxedProcessExecutorCatastrophicFailure = 8701,
+    }
+}

Public/Src/Utilities/Collections/HashCodeHelper.cs

@@ -4,7 +4,7 @@
 using System.Diagnostics.CodeAnalysis;
 using System.Diagnostics.ContractsLight;
 
-namespace BuildXL.Utilities
+namespace BuildXL.Utilities.Collections
 {
     /// <summary>
     /// This class provides some utility function to compute stable strong hash codes.

Public/Src/Utilities/Configuration/AdminRequiredProcessExecutionMode.cs

@@ -0,0 +1,29 @@
+// Copyright (c) Microsoft. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+namespace BuildXL.Utilities.Configuration
+{
+    /// <summary>
+    /// The verbosity level for logging
+    /// </summary>
+    public enum AdminRequiredProcessExecutionMode : byte
+    {
+        /// <summary>
+        /// The admin-required sandboxed process will be launched internally from BuildXL process.
+        /// </summary>
+        Internal,
+
+        /// <summary>
+        /// The admin-required sandboxed process will be launched from a separate sandboxed process executor tool.
+        /// </summary>
+        /// <remarks>
+        /// This mode is mainly used for testing purpose.
+        /// </remarks>
+        ExternalTool,
+
+        /// <summary>
+        /// The admin-required sandboxed process will be launched from a VM via a separate sandboxed process executor tool 
+        /// </summary>
+        ExternalVM,
+    }
+}

Public/Src/Utilities/Configuration/ISandboxConfiguration.cs

@@ -207,5 +207,10 @@ namespace BuildXL.Utilities.Configuration
         /// Container-related configuration
         /// </summary>
         ISandboxContainerConfiguration ContainerConfiguration { get; }
+
+        /// <summary>
+        /// Execution mode for processes that require admin privilege.
+        /// </summary>
+        AdminRequiredProcessExecutionMode AdminRequiredProcessExecutionMode { get; }
     }
 }

Public/Src/Utilities/Configuration/Mutable/SandboxConfiguration.cs

@@ -40,6 +40,7 @@ namespace BuildXL.Utilities.Configuration.Mutable
             KextThrottleCpuUsageWakeupThresholdPercent = 0; // no throttling by default
             KextThrottleMinAvailableRamMB = 0;              // no throttling by default
             ContainerConfiguration = new SandboxContainerConfiguration();
+            AdminRequiredProcessExecutionMode = AdminRequiredProcessExecutionMode.Internal;
         }
 
         /// <nodoc />
@@ -81,6 +82,7 @@ namespace BuildXL.Utilities.Configuration.Mutable
             KextThrottleCpuUsageWakeupThresholdPercent = template.KextThrottleCpuUsageWakeupThresholdPercent;
             KextThrottleMinAvailableRamMB = template.KextThrottleMinAvailableRamMB;
             ContainerConfiguration = new SandboxContainerConfiguration(template.ContainerConfiguration);
+            AdminRequiredProcessExecutionMode = template.AdminRequiredProcessExecutionMode;
         }
 
         /// <inheritdoc />
@@ -212,5 +214,8 @@ namespace BuildXL.Utilities.Configuration.Mutable
 
         /// <inheritdoc/>
         ISandboxContainerConfiguration ISandboxConfiguration.ContainerConfiguration => ContainerConfiguration;
+
+        /// <inheritdoc />
+        public AdminRequiredProcessExecutionMode AdminRequiredProcessExecutionMode { get; set; }
     }
 }

Public/Src/Utilities/Native/IO/IOCounters.cs

@@ -0,0 +1,180 @@
+// Copyright (c) Microsoft. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+using System;
+using System.Diagnostics.ContractsLight;
+using System.IO;
+using BuildXL.Native.Processes;
+using BuildXL.Utilities;
+
+namespace BuildXL.Native.IO
+{
+    /// <summary>
+    /// Contains I/O accounting information for a process or process tree for a particular type of IO (e.g. read or write).
+    /// </summary>
+    /// <remarks>
+    /// For job object, this structure contains I/O accounting information for a process or a job object, for a particular type of IO (e.g. read or write).
+    /// These counters include all operations performed by all processes ever associated with the job.
+    /// </remarks>
+    [System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Performance", "CA1815:OverrideEqualsAndOperatorEqualsOnValueTypes")]
+    public readonly struct IOTypeCounters : IEquatable<IOTypeCounters>
+    {
+        /// <summary>
+        /// Number of operations performed (independent of size).
+        /// </summary>
+        public readonly ulong OperationCount;
+
+        /// <summary>
+        /// Total bytes transferred (regardless of the number of operations used to transfer them).
+        /// </summary>
+        public readonly ulong TransferCount;
+
+        /// <inheritdoc/>
+        public bool Equals(IOTypeCounters other)
+        {
+            return (OperationCount == other.OperationCount) && (TransferCount == other.TransferCount);
+        }
+
+        /// <nodoc />
+        public static bool operator !=(IOTypeCounters t1, IOTypeCounters t2)
+        {
+            return !t1.Equals(t2);
+        }
+
+        /// <nodoc />
+        public static bool operator ==(IOTypeCounters t1, IOTypeCounters t2)
+        {
+            return t1.Equals(t2);
+        }
+
+        /// <inheritdoc/>
+        public override bool Equals(object obj)
+        {
+            return (obj is IOTypeCounters) ? Equals((IOTypeCounters)obj) : false;
+        }
+
+        /// <inheritdoc/>
+        public override int GetHashCode()
+        {
+            return HashCodeHelper.Combine(OperationCount.GetHashCode(), TransferCount.GetHashCode());
+        }
+
+        /// <nodoc />
+        public IOTypeCounters(ulong operationCount, ulong transferCount)
+        {
+            OperationCount = operationCount;
+            TransferCount = transferCount;
+        }
+
+        /// <nodoc />
+        public void Serialize(BinaryWriter writer)
+        {
+            writer.Write(OperationCount);
+            writer.Write(TransferCount);
+        }
+
+        /// <nodoc />
+        public static IOTypeCounters Deserialize(BinaryReader reader) => new IOTypeCounters(reader.ReadUInt64(), reader.ReadUInt64());
+    }
+
+    /// <summary>
+    /// Contains I/O accounting information for a process or process tree.
+    /// </summary>
+    /// <remarks>
+    /// For job object, this structure contains I/O accounting information for a process or a job object.
+    /// These counters include all operations performed by all processes ever associated with the job.
+    /// </remarks>
+    [System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Performance", "CA1815:OverrideEqualsAndOperatorEqualsOnValueTypes")]
+    public readonly struct IOCounters : IEquatable<IOCounters>
+    {
+        /// <summary>
+        /// Counters for read operations.
+        /// </summary>
+        public readonly IOTypeCounters ReadCounters;
+
+        /// <summary>
+        /// Counters for write operations.
+        /// </summary>
+        public readonly IOTypeCounters WriteCounters;
+
+        /// <summary>
+        /// Counters for other operations (not classified as either read or write).
+        /// </summary>
+        public readonly IOTypeCounters OtherCounters;
+
+        /// <summary>
+        /// Creates an instance of <see cref="IOCounters"/> from <see cref="IO_COUNTERS"/>.
+        /// </summary>
+        public IOCounters(IO_COUNTERS nativeCounters)
+        {
+            ReadCounters = new IOTypeCounters(nativeCounters.ReadOperationCount, nativeCounters.ReadTransferCount);
+            WriteCounters = new IOTypeCounters(nativeCounters.WriteOperationCount, nativeCounters.WriteTransferCount);
+            OtherCounters = new IOTypeCounters(nativeCounters.OtherOperationCount, nativeCounters.OtherTransferCount);
+        }
+
+        /// <inheritdoc/>
+        public bool Equals(IOCounters other)
+        {
+            return (ReadCounters == other.ReadCounters) && (WriteCounters == other.WriteCounters) && (OtherCounters == other.OtherCounters);
+        }
+
+        /// <nodoc/>
+        public static bool operator !=(IOCounters t1, IOCounters t2)
+        {
+            return !t1.Equals(t2);
+        }
+
+        /// <nodoc/>
+        public static bool operator ==(IOCounters t1, IOCounters t2)
+        {
+            return t1.Equals(t2);
+        }
+
+        /// <inheritdoc/>
+        public override bool Equals(object obj)
+        {
+            return (obj is IOCounters) ? Equals((IOCounters)obj) : false;
+        }
+
+        /// <inheritdoc/>
+        public override int GetHashCode()
+        {
+            return HashCodeHelper.Combine(ReadCounters.GetHashCode(), WriteCounters.GetHashCode(), OtherCounters.GetHashCode());
+        }
+
+        /// <nodoc />
+        public IOCounters(IOTypeCounters readCounters, IOTypeCounters writeCounters, IOTypeCounters otherCounters)
+        {
+            ReadCounters = readCounters;
+            WriteCounters = writeCounters;
+            OtherCounters = otherCounters;
+        }
+
+        /// <summary>
+        /// Computes the aggregate I/O performed (sum of the read, write, and other counters).
+        /// </summary>
+        [System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Design", "CA1024:UsePropertiesWhereAppropriate")]
+        [Pure]
+        public IOTypeCounters GetAggregateIO()
+        {
+            return new IOTypeCounters(
+                operationCount: ReadCounters.OperationCount + WriteCounters.OperationCount + OtherCounters.OperationCount,
+                transferCount: ReadCounters.TransferCount + WriteCounters.TransferCount + OtherCounters.TransferCount);
+        }
+
+        /// <nodoc />
+        public void Serialize(BinaryWriter writer)
+        {
+            ReadCounters.Serialize(writer);
+            WriteCounters.Serialize(writer);
+            OtherCounters.Serialize(writer);
+        }
+
+        /// <nodoc />
+        public static IOCounters Deserialize(BinaryReader reader) 
+            => new IOCounters(
+                readCounters:  IOTypeCounters.Deserialize(reader), 
+                writeCounters: IOTypeCounters.Deserialize(reader), 
+                otherCounters: IOTypeCounters.Deserialize(reader));
+    }
+}

Public/Src/Utilities/Utilities/BuildXLReader.cs

@@ -457,6 +457,21 @@ namespace BuildXL.Utilities
             return value;
         }
 
+        /// <summary>
+        /// Reads an <see cref="Encoding"/>.
+        /// </summary>
+        public Encoding ReadEncoding()
+        {
+            Start<Encoding>();
+            int codePage = ReadInt32();
+
+#if DISABLE_FEATURE_EXTENDED_ENCODING
+            return new UTF8Encoding(encoderShouldEmitUTF8Identifier: false);
+#else
+            return Encoding.GetEncoding(codePage);
+#endif
+        }
+
         /// <summary>
         /// Reads a Nullable struct
         /// </summary>

Public/Src/Utilities/Utilities/BuildXLReaderExtensions.cs

@@ -0,0 +1,19 @@
+// Copyright (c) Microsoft. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+namespace BuildXL.Utilities
+{
+    /// <summary>
+    /// Extensions for <see cref="BuildXLReader"/>.
+    /// </summary>
+    public static class BuildXLReaderExtensions
+    {
+        /// <summary>
+        /// Reads a string whose result can be null.
+        /// </summary>
+        public static string ReadNullableString(this BuildXLReader reader)
+        {
+            return reader.ReadNullable(r => r.ReadString());
+        }
+    }
+}

Public/Src/Utilities/Utilities/BuildXLWriter.cs

@@ -393,7 +393,18 @@ namespace BuildXL.Utilities
         public void Write(DateTime value)
         {
             Start<DateTime>();
-            this.Write(value.ToBinary());
+            Write(value.ToBinary());
+            End();
+        }
+
+        /// <summary>
+        /// Writes an encoding.
+        /// </summary>
+        public void Write(Encoding value)
+        {
+            Contract.Requires(value != null);
+            Start<Encoding>();
+            Write(value.CodePage);
             End();
         }
 

Public/Src/Utilities/Utilities/BuildXLWriterExtensions.cs

@@ -0,0 +1,19 @@
+// Copyright (c) Microsoft. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+namespace BuildXL.Utilities
+{
+    /// <summary>
+    /// Extensions for <see cref="BuildXLWriter"/>
+    /// </summary>
+    public static class BuildXLWriterExtensions
+    {
+        /// <summary>
+        /// Writes a string whose value can be <code>null</code>.
+        /// </summary>
+        public static void WriteNullableString(this BuildXLWriter writer, string canBeNullString)
+        {
+            writer.Write(canBeNullString, (w, v) => w.Write(v));
+        }
+    }
+}

Public/Src/Utilities/Utilities/Tracing/EventId.cs

@@ -97,7 +97,9 @@ namespace BuildXL.Utilities.Tracing
         PipTableStats = 75,
         PipWriterStats = 76,
         PipIpcFailedDueToInvalidInput = 77,
-        // Free = 78,
+        
+        PipProcessStartExternalTool = 78,
+        PipProcessFinishedExternalTool = 79,
         // Free = 79,
         // Free = 80,
         // Free = 81,

Public/Src/Utilities/Utilities/Tracing/Events.cs

@@ -258,7 +258,7 @@ namespace BuildXL.Utilities.Tracing
 
             public const EventTask UnitTest = (EventTask)4;
 
-            // FREE: 5;
+            public const EventTask SandboxedProcessExecutor = (EventTask)5;
 
             public const EventTask Engine = (EventTask)6;
 

Public/Src/Utilities/Utilities/Warning.cs

@@ -0,0 +1,39 @@
+// Copyright (c) Microsoft. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+namespace BuildXL.Utilities
+{
+    /// <summary>
+    /// Utilities for warnings.
+    /// </summary>
+    public static class Warning
+    {
+        /// <summary>
+        /// Adapted from Microsoft.BUild.Utilities.Core / CanonicalError.cs
+        /// </summary>
+        public const string DefaultWarningPattern =
+
+            // Beginning of line and any amount of whitespace.
+            @"^\s*"
+
+            // Match a [optional project number prefix 'ddd>'], single letter + colon + remaining filename, or
+            // string with no colon followed by a colon.
+            + @"((((((\d+>)?[a-zA-Z]?:[^:]*)|([^:]*))):)"
+
+            // Origin may also be empty. In this case there's no trailing colon.
+            + "|())"
+
+            // Match the empty string or a string without a colon that ends with a space
+            + "(()|([^:]*? ))"
+
+            // Match 'warning'.
+            + @"warning"
+
+            // Match anything starting with a space that's not a colon/space, followed by a colon.
+            // Error code is optional in which case "warning" can be followed immediately by a colon.
+            + @"( \s*([^: ]*))?\s*:"
+
+            // Whatever's left on this line, including colons.
+            + ".*$";
+    }
+}

Shared/Scripts/bxl.ps1

@@ -119,6 +119,9 @@ param(
 
     [string]$CacheNamespace = "BuildXLSelfhost",
 
+    [Parameter(Mandatory=$false)]
+	[switch]$Vs = $false,
+
     [Parameter(ValueFromRemainingArguments=$true)]
     [string[]]$DominoArguments
 )
@@ -235,6 +238,10 @@ if ($DeployStandaloneTest) {
     $AdditionalBuildXLArguments += "/p:[Sdk.BuildXL]DeployStandaloneTest=true";
 }
 
+if ($Vs) {
+    $AdditionalBuildXLArguments += "/p:[Sdk.BuildXL]GenerateVSSolution=true /vs";
+}
+
 # WARNING: CloudBuild selfhost builds do NOT use this script file. When adding a new argument below, we should add the argument to selfhost queues in CloudBuild. Please contact bxl team. 
 $AdditionalBuildXLArguments += @("/remotetelemetry", "/reuseOutputsOnDisk+", "/scrubDirectory:${enlistmentRoot}\out\objects", "/storeFingerprints", "/cacheMiss");
 $AdditionalBuildXLArguments += @("/p:[Sdk.BuildXL]useQTest=true");