diff --git a/Public/Src/Cache/DistributedCache.Host/Service/Deployment/DeploymentLauncher.cs b/Public/Src/Cache/DistributedCache.Host/Service/Deployment/DeploymentLauncher.cs index db81ac22c..fde1e3c9a 100644 --- a/Public/Src/Cache/DistributedCache.Host/Service/Deployment/DeploymentLauncher.cs +++ b/Public/Src/Cache/DistributedCache.Host/Service/Deployment/DeploymentLauncher.cs @@ -163,7 +163,7 @@ namespace BuildXL.Cache.Host.Service }, caller: "ClearDeploymentDirectory").IgnoreFailure(); - if (Settings.CreateJobObject && JobObject.OSSupportsNestedJobs) + if (Settings.CreateJobObject) { JobObject.SetTerminateOnCloseOnCurrentProcessJob(); } diff --git a/Public/Src/Engine/Dll/Engine.cs b/Public/Src/Engine/Dll/Engine.cs index 85fe8097f..bc0e3a54f 100644 --- a/Public/Src/Engine/Dll/Engine.cs +++ b/Public/Src/Engine/Dll/Engine.cs @@ -2566,16 +2566,9 @@ namespace BuildXL.Engine if (Configuration.Schedule.LowPriority) { - if (JobObject.OSSupportsNestedJobs) + if (!JobObject.SetLimitInformationOnCurrentProcessJob(priorityClass: ProcessPriorityClass.BelowNormal)) { - if (!JobObject.SetLimitInformationOnCurrentProcessJob(priorityClass: ProcessPriorityClass.BelowNormal)) - { - Logger.Log.AssignProcessToJobObjectFailed(loggingContext, NativeWin32Exception.GetFormattedMessageForNativeErrorCode(System.Runtime.InteropServices.Marshal.GetLastWin32Error())); - } - } - else - { - Logger.Log.CannotEnforceLowPriority(loggingContext); + Logger.Log.AssignProcessToJobObjectFailed(loggingContext, NativeWin32Exception.GetFormattedMessageForNativeErrorCode(System.Runtime.InteropServices.Marshal.GetLastWin32Error())); } } diff --git a/Public/Src/Engine/Dll/Tracing/Log.cs b/Public/Src/Engine/Dll/Tracing/Log.cs index 3040393ac..78b3f8d74 100644 --- a/Public/Src/Engine/Dll/Tracing/Log.cs +++ b/Public/Src/Engine/Dll/Tracing/Log.cs @@ -1284,15 +1284,6 @@ namespace BuildXL.Engine.Tracing Message = "AssignProcessToJobObject failed: Native Win32 Error: {0}")] public abstract void AssignProcessToJobObjectFailed(LoggingContext context, string nativeError); - [GeneratedEvent( - (ushort)LogEventId.CannotHonorLowPriority, - EventGenerators = EventGenerators.LocalOnly, - EventLevel = Level.Warning, - Keywords = (int)Keywords.UserMessage, - EventTask = (int)Tasks.Engine, - Message = "Unable to honor the /lowPriority option. This requires Win8/Server2012 or newer.")] - public abstract void CannotEnforceLowPriority(LoggingContext context); - [GeneratedEvent( (ushort)LogEventId.ConfigUsingExperimentalOptions, EventGenerators = EventGenerators.LocalAndTelemetry, diff --git a/Public/Src/Engine/Dll/Tracing/LogEventId.cs b/Public/Src/Engine/Dll/Tracing/LogEventId.cs index 37aeb327d..3e4deeaa3 100644 --- a/Public/Src/Engine/Dll/Tracing/LogEventId.cs +++ b/Public/Src/Engine/Dll/Tracing/LogEventId.cs @@ -30,7 +30,6 @@ namespace BuildXL.Engine.Tracing PipWriterStats = 76, PipTableDeserializationContext = 84, EnvironmentValueForTempDisallowed = 94, - CannotHonorLowPriority = 95, ErrorRelatedLocation = 110, BusyOrUnavailableOutputDirectoriesRetry = 214, FileAccessAllowlistCouldNotCreateIdentifier = 270, diff --git a/Public/Src/Engine/Processes/Internal/DetouredProcess.cs b/Public/Src/Engine/Processes/Internal/DetouredProcess.cs index 0bfb09996..948e395e7 100644 --- a/Public/Src/Engine/Processes/Internal/DetouredProcess.cs +++ b/Public/Src/Engine/Processes/Internal/DetouredProcess.cs @@ -533,8 +533,8 @@ namespace BuildXL.Processes.Internal hStdError = new SafeFileHandle(new IntPtr(-1), true); } - // We want a per-process job primarily. If nested job support is not available, then we make sure to not have a BuildXL-level job. - if (JobObject.OSSupportsNestedJobs && m_createJobObjectForCurrentProcess) + // Include the pip process in the job object for the build to ensure it is automatically closed. + if (m_createJobObjectForCurrentProcess) { JobObject.SetTerminateOnCloseOnCurrentProcessJob(); } diff --git a/Public/Src/Engine/UnitTests/Processes/SandboxedProcessTest.cs b/Public/Src/Engine/UnitTests/Processes/SandboxedProcessTest.cs index 83783c0d5..52a525b7d 100644 --- a/Public/Src/Engine/UnitTests/Processes/SandboxedProcessTest.cs +++ b/Public/Src/Engine/UnitTests/Processes/SandboxedProcessTest.cs @@ -225,7 +225,7 @@ namespace Test.BuildXL.Processes [Fact] public async Task StartTimeout() { - if (!JobObject.OSSupportsNestedJobs) + if (!OperatingSystemHelper.IsWindowsOS) { return; } @@ -243,7 +243,7 @@ namespace Test.BuildXL.Processes [Fact] public async Task SuspensionExtendsTimeout() { - if (!JobObject.OSSupportsNestedJobs) + if (!OperatingSystemHelper.IsWindowsOS) { return; } @@ -270,7 +270,7 @@ namespace Test.BuildXL.Processes [Fact] public async Task SuspendResumeTimeout() { - if (!JobObject.OSSupportsNestedJobs) + if (!OperatingSystemHelper.IsWindowsOS) { return; } @@ -321,7 +321,7 @@ namespace Test.BuildXL.Processes [Fact(Skip = "Test is flakey TFS 495531")] public async Task JobCounters() { - if (!JobObject.OSSupportsNestedJobs) + if (!OperatingSystemHelper.IsWindowsOS) { return; } @@ -382,7 +382,7 @@ namespace Test.BuildXL.Processes [InlineData(true)] public async Task Survivors(bool includeAllowedSurvivingChildren) { - if (!JobObject.OSSupportsNestedJobs) + if (!OperatingSystemHelper.IsWindowsOS) { return; } @@ -473,7 +473,7 @@ namespace Test.BuildXL.Processes [MemberData(nameof(CmdExeLocationsData))] public async Task SurvivorsHaveCommandLines(string cmdExeLocation) { - if (!JobObject.OSSupportsNestedJobs) + if (!OperatingSystemHelper.IsWindowsOS) { return; } @@ -511,7 +511,7 @@ namespace Test.BuildXL.Processes [InlineData(false)] public async Task QuickSurvivors(bool waitToFinish) { - if (!JobObject.OSSupportsNestedJobs) + if (!OperatingSystemHelper.IsWindowsOS) { return; } diff --git a/Public/Src/Tools/DistributedBuildRunner/Program.cs b/Public/Src/Tools/DistributedBuildRunner/Program.cs index f82d0ba49..71a02b25a 100644 --- a/Public/Src/Tools/DistributedBuildRunner/Program.cs +++ b/Public/Src/Tools/DistributedBuildRunner/Program.cs @@ -45,11 +45,8 @@ namespace Tool.DistributedBuildRunner } // Ensure that any launched processes will close when this process closes. - if (JobObject.OSSupportsNestedJobs) - { - JobObject.SetTerminateOnCloseOnCurrentProcessJob(); - } - + JobObject.SetTerminateOnCloseOnCurrentProcessJob(); + if (args.Length < 1 || args.Length > 2) { Console.WriteLine(Resources.HelpText); diff --git a/Public/Src/Utilities/Native/IO/Windows/FileSystem.Win.cs b/Public/Src/Utilities/Native/IO/Windows/FileSystem.Win.cs index 1ce921293..6b5e5bfb3 100644 --- a/Public/Src/Utilities/Native/IO/Windows/FileSystem.Win.cs +++ b/Public/Src/Utilities/Native/IO/Windows/FileSystem.Win.cs @@ -196,16 +196,7 @@ namespace BuildXL.Native.IO.Windows public FileIdDescriptor(FileId fileId) { - if (IsExtendedFileIdSupported()) - { - Type = FileIdDescriptorType.ExtendedFileId; - } - else - { - Contract.Assume(fileId.High == 0, "File ID should not have high bytes when extended IDs are not supported on the underlying OS"); - Type = FileIdDescriptorType.FileId; - } - + Type = FileIdDescriptorType.ExtendedFileId; Size = s_size; ExtendedFileId = fileId; } @@ -1028,27 +1019,10 @@ namespace BuildXL.Native.IO.Windows } /// - public static readonly Version MinWindowsVersionThatSupportsLongPaths = new Version(major: 6, minor: 2); - - /// - public static readonly Version MinWindowsVersionThatSupportsNestedJobs = new Version(major: 6, minor: 2); - - /// - public static readonly Version MinWindowsVersionThatSupportsWow64Processes = new Version(major: 5, minor: 1); - - /// - public static readonly int MaxDirectoryPathOld = 130; - - /// - public static readonly int MaxDirectoryPathNew = 260; + public static readonly int MaxDirectoryPath = 260; /// - public int MaxDirectoryPathLength() - { - return StaticIsOSVersionGreaterOrEqual(MinWindowsVersionThatSupportsLongPaths) - ? MaxDirectoryPathNew - : MaxDirectoryPathOld; - } + public int MaxDirectoryPathLength() => MaxDirectoryPath; private readonly Lazy m_supportUnprivilegedCreateSymbolicLinkFlag = default; @@ -1487,21 +1461,6 @@ namespace BuildXL.Native.IO.Windows return new Usn(writtenUsn); } - /// - /// Indicates if the running OS is at least Windows 8.0 / Server 2012 - /// (which is the first version to support nested jobs, hence ) - /// - private static readonly bool s_runningWindows8OrAbove = StaticIsOSVersionGreaterOrEqual(FileSystemWin.MinWindowsVersionThatSupportsNestedJobs); - - /// - /// Indicates if the extended (128-bit) file ID type is supported on this running OS. - /// http://msdn.microsoft.com/en-us/library/windows/desktop/aa364227(v=vs.85).aspx - /// - private static bool IsExtendedFileIdSupported() - { - return s_runningWindows8OrAbove; - } - /// public unsafe FileIdAndVolumeId? TryGetFileIdAndVolumeIdByHandle(SafeFileHandle fileHandle) { diff --git a/Public/Src/Utilities/Native/Processes/IProcessUtilities.cs b/Public/Src/Utilities/Native/Processes/IProcessUtilities.cs index 8d5bff6dd..b9709fa31 100644 --- a/Public/Src/Utilities/Native/Processes/IProcessUtilities.cs +++ b/Public/Src/Utilities/Native/Processes/IProcessUtilities.cs @@ -105,9 +105,6 @@ namespace BuildXL.Native.Processes /// bool TerminateJobObject(IntPtr hJob, int exitCode); - /// - bool OSSupportsNestedJobs(); - /// void AttachContainerToJobObject( IntPtr hJob, diff --git a/Public/Src/Utilities/Native/Processes/JobObject.cs b/Public/Src/Utilities/Native/Processes/JobObject.cs index 2173bac16..0ee435046 100644 --- a/Public/Src/Utilities/Native/Processes/JobObject.cs +++ b/Public/Src/Utilities/Native/Processes/JobObject.cs @@ -43,12 +43,6 @@ namespace BuildXL.Processes /// public const int InitialProcessIdListLength = 2048; // the number needed to make the bufferSizeForProcessIdList 8KB. - /// - /// Nested jobs are only supported on Win8/Server2012 or higher. - /// http://msdn.microsoft.com/en-us/library/windows/desktop/hh448388(v=vs.85).aspx - /// - public static readonly bool OSSupportsNestedJobs = Native.Processes.ProcessUtilities.OSSupportsNestedJobs(); - private static readonly object s_syncRoot = new object(); private static readonly Lazy s_completionPortDrainer = Lazy.Create(() => new CompletionPortDrainer()); private static bool s_terminateOnCloseOnCurrentProcessJob; @@ -491,7 +485,10 @@ namespace BuildXL.Processes /// public static bool SetLimitInformationOnCurrentProcessJob(bool? terminateOnClose = null, ProcessPriorityClass? priorityClass = null) { - Contract.Requires(OSSupportsNestedJobs); + if (!OperatingSystemHelper.IsWindowsOS) + { + return true; + } bool ret = false; using (Process currentProcess = Process.GetCurrentProcess()) @@ -525,7 +522,6 @@ namespace BuildXL.Processes /// public static void SetTerminateOnCloseOnCurrentProcessJob() { - Contract.Requires(OSSupportsNestedJobs); if (!s_terminateOnCloseOnCurrentProcessJob) { lock (s_syncRoot) diff --git a/Public/Src/Utilities/Native/Processes/ProcessUtilities.cs b/Public/Src/Utilities/Native/Processes/ProcessUtilities.cs index deb044de0..378c507b7 100644 --- a/Public/Src/Utilities/Native/Processes/ProcessUtilities.cs +++ b/Public/Src/Utilities/Native/Processes/ProcessUtilities.cs @@ -273,12 +273,6 @@ namespace BuildXL.Native.Processes public static bool TerminateJobObject(IntPtr hJob, int exitCode) => s_nativeMethods.TerminateJobObject(hJob, exitCode); - /// - /// Returns whether the operating system supports nested jobs. - /// - public static bool OSSupportsNestedJobs() - => s_nativeMethods.OSSupportsNestedJobs(); - #region Helium containers /// diff --git a/Public/Src/Utilities/Native/Processes/Unix/ProcessUtilities.Unix.cs b/Public/Src/Utilities/Native/Processes/Unix/ProcessUtilities.Unix.cs index f58c629c5..f199d3f9d 100644 --- a/Public/Src/Utilities/Native/Processes/Unix/ProcessUtilities.Unix.cs +++ b/Public/Src/Utilities/Native/Processes/Unix/ProcessUtilities.Unix.cs @@ -159,9 +159,6 @@ namespace BuildXL.Native.Processes.Unix public bool TerminateJobObject(IntPtr hJob, int exitCode) => throw new NotImplementedException(); - /// - public bool OSSupportsNestedJobs() => false; - /// public void AttachContainerToJobObject( IntPtr hJob, diff --git a/Public/Src/Utilities/Native/Processes/Windows/ProcessUtilities.Win.cs b/Public/Src/Utilities/Native/Processes/Windows/ProcessUtilities.Win.cs index 43490c842..acceaae9f 100644 --- a/Public/Src/Utilities/Native/Processes/Windows/ProcessUtilities.Win.cs +++ b/Public/Src/Utilities/Native/Processes/Windows/ProcessUtilities.Win.cs @@ -97,13 +97,7 @@ namespace BuildXL.Native.Processes.Windows public bool IsWow64Process(SafeProcessHandle? process) { IntPtr handle = process == null ? GetCurrentProcess() : process.DangerousGetHandle(); - - if (IO.Windows.FileSystemWin.StaticIsOSVersionGreaterOrEqual(IO.Windows.FileSystemWin.MinWindowsVersionThatSupportsWow64Processes)) - { - return ExternIsWow64Process(handle, out bool result) && result; - } - - return false; + return ExternIsWow64Process(handle, out bool result) && result; } /// @@ -267,10 +261,6 @@ namespace BuildXL.Native.Processes.Windows public bool TerminateJobObject(IntPtr hJob, int exitCode) => ExternTerminateJobObject(hJob, exitCode); - /// - public bool OSSupportsNestedJobs() - => IO.Windows.FileSystemWin.StaticIsOSVersionGreaterOrEqual(IO.Windows.FileSystemWin.MinWindowsVersionThatSupportsNestedJobs); - internal static void Assert64Process() { Contract.Assert(IntPtr.Size == 8, "BuildXL is 64 bit process only."); diff --git a/README.md b/README.md index 47add6c75..f67acd803 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ ## Introduction -BuildXL (Microsoft Build Accelerator), is a build engine originally developed for large internal teams at Microsoft. Internally at Microsoft, BuildXL runs 150,000+ builds per day on [monorepo](https://en.wikipedia.org/wiki/Monorepo) codebases up to a half-terabyte in size with a half-million process executions per build. It leverages distribution to thousands of data center machines and petabytes of source code, package, and build output caching. Thousands of developers use BuildXL on their desktops for faster builds. +BuildXL (Microsoft Build Accelerator) is a build engine originally developed for large internal teams at Microsoft. Internally at Microsoft, BuildXL runs 150,000+ builds per day on [monorepo](https://en.wikipedia.org/wiki/Monorepo) codebases up to a half-terabyte in size with a half-million process executions per build. It leverages distribution to thousands of data center machines and petabytes of source code, package, and build output caching. Thousands of developers use BuildXL on their desktops for faster builds. BuildXL accelerates multiple build languages, including: @@ -15,6 +15,10 @@ BuildXL accelerates multiple build languages, including: BuildXL has a command-line interface. There are currently no plans to integrate it into Visual Studio. The project is open source in the spirit of transparency of our engineering system. You may find our technology useful if you face similar issues of scale. Note that BuildXL is not intended as a replacement for MSBuild or to indicate any future direction of build languages from Microsoft. +OS compatibility: +* Windows 10+ +* Linux + ## Examples See the `Examples/` folder for basic project examples. @@ -23,4 +27,3 @@ The BuildXL documentation landing page is [here](Documentation/INDEX.md) and loo ## Build Status - Azure DevOps Pipelines [![Build status](https://dev.azure.com/mseng/Domino/_apis/build/status/8196?branchName=master)](https://dev.azure.com/mseng/Domino/_build/latest?definitionId=8196) -