Add Microsoft.IO.Redist for directory enumeration. (#6771)

Fixes #6075

### Context
Microsoft.IO.Redist brings some of the new .NET Core System.IO functionality to .NET Framework. In particular, enumeration in Microsoft.IO was optimized comparing to System.IO: new enumeration API was added and old one was improved. We consider it is beneficial to switch default file system enumeration to the old API from Microsoft.IO.Redist.

### Changes Made

- Added Microsoft.IO.Redist
- Default file system enumeration uses Microsoft.IO instead of System.IO

### Testing
Unit tests & DDRITs

### Notes
We should be wary of possible regressions. There were differences in behavior, see `.NET Framework only` notes in [doc](https://docs.microsoft.com/en-us/dotnet/api/system.io.directory.enumeratefiles?view=net-5.0). The change therefore is under change wave 17_0.

eng/Packages.props

@@ -13,6 +13,7 @@
     <PackageReference Update="Microsoft.CodeQuality.Analyzers" Version="3.3.0" PrivateAssets="all" />
     <PackageReference Update="Microsoft.DotNet.GenAPI" Version="2.1.0-prerelease-02404-02" />
     <PackageReference Update="Microsoft.DotNet.XUnitExtensions" Version="$(MicrosoftDotNetXUnitExtensionsVersion)" />
+    <PackageReference Update="Microsoft.IO.Redist" Version="4.7.1" />
     <PackageReference Update="Microsoft.Net.Compilers.Toolset" Version="$(MicrosoftNetCompilersToolsetVersion)" />
     <PackageReference Update="Microsoft.VisualStudio.SDK.EmbedInteropTypes" Version="15.0.15" />
     <PackageReference Update="Microsoft.VisualStudio.Setup.Configuration.Interop" Version="1.16.30" />

scripts/Deploy-MSBuild.ps1

@@ -93,6 +93,7 @@ if ($runtime -eq "Desktop") {
 
         FileToCopy "$bootstrapBinDirectory\Microsoft.Bcl.AsyncInterfaces.dll"
         FileToCopy "$bootstrapBinDirectory\Microsoft.Data.Entity.targets"
+        FileToCopy "$bootstrapBinDirectory\Microsoft.IO.Redist.dll"
         FileToCopy "$bootstrapBinDirectory\Microsoft.ServiceModel.targets"
         FileToCopy "$bootstrapBinDirectory\Microsoft.WinFx.targets"
         FileToCopy "$bootstrapBinDirectory\Microsoft.WorkflowBuildExtensions.targets"

src/Build.OM.UnitTests/Microsoft.Build.Engine.OM.UnitTests.csproj

@@ -20,6 +20,8 @@
 </PropertyGroup>
 
   <ItemGroup>
+    <PackageReference Include="Microsoft.IO.Redist" Condition="'$(FeatureMSIORedist)' == 'true'" />
+
     <PackageReference Include="Shouldly" />
     <PackageReference Include="System.Net.Http" />
     <PackageDownload Include="NuGet.CommandLine" Version="[$(NuGetCommandLinePackageVersion)]" />

src/Build/Microsoft.Build.csproj

@@ -36,6 +36,8 @@
     <PackageReference Include="System.Text.Json" />
 
     <PackageReference Include="System.Reflection.Metadata" Condition="'$(MonoBuild)' == 'true'" />
+
+    <PackageReference Include="Microsoft.IO.Redist" Condition="'$(FeatureMSIORedist)' == 'true'" />
   </ItemGroup>
 
   <ItemGroup Condition="'$(TargetFrameworkIdentifier)' == '.NETFramework'">

src/Directory.BeforeCommon.targets

@@ -142,4 +142,9 @@
     <DocumentationFile Condition=" '$(GenerateDocumentationFile)' == 'true' ">$(IntermediateOutputPath)\$(AssemblyName).xml</DocumentationFile>
   </PropertyGroup>
 
+  <PropertyGroup Condition="'$(MonoBuild)' != 'true' and $([MSBuild]::GetTargetFrameworkIdentifier('$(TargetFramework)')) == '.NETFramework'">
+    <DefineConstants>$(DefineConstants);FEATURE_MSIOREDIST</DefineConstants>
+    <FeatureMSIORedist>true</FeatureMSIORedist>
+  </PropertyGroup>
+
 </Project>

src/Framework.UnitTests/Microsoft.Build.Framework.UnitTests.csproj

@@ -11,6 +11,7 @@
   </PropertyGroup>
 
   <ItemGroup>
+    <PackageReference Include="Microsoft.IO.Redist" Condition="'$(FeatureMSIORedist)' == 'true'" />
     <PackageReference Include="Microsoft.VisualStudio.Setup.Configuration.Interop" Condition="'$(TargetFrameworkIdentifier)' == '.NETFramework'" />
     <PackageReference Include="Shouldly" />
     <PackageReference Include="System.Net.Http" />

src/MSBuild/MSBuild.csproj

@@ -229,6 +229,7 @@
     </None>
   </ItemGroup>
   <ItemGroup>
+    <PackageReference Include="Microsoft.IO.Redist" Condition="'$(FeatureMSIORedist)' == 'true'" />
     <PackageReference Include="System.Configuration.ConfigurationManager" />
   </ItemGroup>
   <ItemGroup Condition="'$(TargetFrameworkIdentifier)' == '.NETFramework'">

src/Package/MSBuild.Engine.Corext/MsBuild.Engine.Corext.nuspec

@@ -31,6 +31,7 @@
     <file src="$X86BinPath$/Microsoft.Build.Framework.dll" target="v15.0/bin" />
     <file src="$X86BinPath$/Microsoft.Build.Tasks.Core.dll" target="v15.0/bin" />
     <file src="$X86BinPath$/Microsoft.Build.Utilities.Core.dll" target="v15.0/bin" />
+    <file src="$X86BinPath$/Microsoft.IO.Redist.dll" target="v15.0/bin" />
     <file src="$X86BinPath$/System.Collections.Immutable.dll" target="v15.0/bin" />
     <file src="$X86BinPath$/System.Memory.dll" target="v15.0/bin" />
     <file src="$X86BinPath$/System.Text.Json.dll" target="v15.0/bin" />
@@ -87,6 +88,7 @@
     <file src="$X86BinPath$/Microsoft.Build.Framework.dll" target="v15.0/bin/amd64" />
     <file src="$X86BinPath$/Microsoft.Build.Tasks.Core.dll" target="v15.0/bin/amd64" />
     <file src="$X86BinPath$/Microsoft.Build.Utilities.Core.dll" target="v15.0/bin/amd64" />
+    <file src="$X86BinPath$/Microsoft.IO.Redist.dll" target="v15.0/bin/amd64" />
     <file src="$X86BinPath$/System.Collections.Immutable.dll" target="v15.0/bin/amd64" />
     <file src="$X86BinPath$/System.Memory.dll" target="v15.0/bin/amd64" />
     <file src="$X86BinPath$/System.Text.Json.dll" target="v15.0/bin/amd64" />

src/Package/MSBuild.VSSetup/files.swr

@@ -32,6 +32,7 @@ folder InstallDir:\MSBuild\Current\Bin
   file source=$(X86BinPath)Microsoft.Build.Framework.tlb
   file source=$(X86BinPath)Microsoft.Build.Tasks.Core.dll vs.file.ngenApplications="[installDir]\Common7\IDE\vsn.exe" vs.file.ngenApplications="[installDir]\MSBuild\Current\Bin\MSBuild.exe" vs.file.ngenArchitecture=all vs.file.ngenPriority=1
   file source=$(X86BinPath)Microsoft.Build.Utilities.Core.dll vs.file.ngenApplications="[installDir]\Common7\IDE\vsn.exe" vs.file.ngenApplications="[installDir]\MSBuild\Current\Bin\MSBuild.exe" vs.file.ngenArchitecture=all vs.file.ngenPriority=1
+  file source=$(X86BinPath)Microsoft.IO.Redist.dll vs.file.ngenApplications="[installDir]\Common7\IDE\vsn.exe" vs.file.ngenApplications="[installDir]\MSBuild\Current\Bin\MSBuild.exe" vs.file.ngenArchitecture=all vs.file.ngenPriority=1
   file source=$(X86BinPath)MSBuild.exe vs.file.ngenArchitecture=x86 vs.file.ngenPriority=1
   file source=$(X86BinPath)MSBuild.exe.config
   file source=$(TaskHostBinPath)MSBuildTaskHost.exe vs.file.ngenArchitecture=x86
@@ -184,6 +185,7 @@ folder InstallDir:\MSBuild\Current\Bin\amd64
   file source=$(X86BinPath)System.Memory.dll vs.file.ngenArchitecture=all
   file source=$(X86BinPath)System.Text.Json.dll vs.file.ngenArchitecture=all
   file source=$(X86BinPath)Microsoft.Bcl.AsyncInterfaces.dll vs.file.ngenArchitecture=all
+  file source=$(X86BinPath)Microsoft.IO.Redist.dll vs.file.ngenArchitecture=all
   file source=$(X86BinPath)System.Text.Encodings.Web.dll vs.file.ngenArchitecture=all
   file source=$(X86BinPath)System.Threading.Tasks.Extensions.dll vs.file.ngenArchitecture=all
   file source=$(X86BinPath)System.Numerics.Vectors.dll vs.file.ngenArchitecture=all

src/Shared/FileSystem/ManagedFileSystem.cs

@@ -1,6 +1,7 @@
 // Copyright (c) Microsoft. All rights reserved.
 // Licensed under the MIT license. See LICENSE file in the project root for full license information.
 
+using Microsoft.Build.Utilities;
 using System;
 using System.Collections.Generic;
 using System.IO;
@@ -16,6 +17,20 @@ namespace Microsoft.Build.Shared.FileSystem
 
         public static ManagedFileSystem Singleton() => ManagedFileSystem.Instance;
 
+        private static bool ShouldUseMicrosoftIO
+        {
+            get
+            {
+#if !MICROSOFT_BUILD_ENGINE_OM_UNITTESTS
+                return ChangeWaves.AreFeaturesEnabled(ChangeWaves.Wave17_0);
+#else
+                // We need to mock usage of ChangeWaves class,
+                // because Microsoft.Build.Engine.OM.UnitTests should not have access to internals of Microsoft.Build.Framework.
+                return true;
+#endif
+            }
+        }
+
         protected ManagedFileSystem() { }
 
         public TextReader ReadFile(string path)
@@ -38,19 +53,78 @@ namespace Microsoft.Build.Shared.FileSystem
             return File.ReadAllBytes(path);
         }
 
+#if FEATURE_MSIOREDIST
+        private static IEnumerable<string> HandleFileLoadException(
+            Func<string, string, Microsoft.IO.SearchOption, IEnumerable<string>> enumerateFunctionDelegate,
+            string path,
+            string searchPattern,
+            Microsoft.IO.SearchOption searchOption
+        )
+        {
+            try
+            {
+                return enumerateFunctionDelegate(path, searchPattern, searchOption);
+            }
+            // Microsoft.IO.Redist has a dependency on System.Buffers and if System.Buffers assembly is not found the line above throws an exception.
+            // However, FileMatcher class (that in most cases calls the enumeration) does not allow to fail on a IO-related exception. Such behavior hides the actual exception and makes it obscure.
+            // We rethrow it to make it fail with a proper error message and call stack.
+            catch (FileLoadException ex)
+            {
+                throw new InvalidOperationException(ex.Message, ex);
+            }
+            // Sometimes FileNotFoundException is thrown when there is an assembly load failure. In this case it should have FusionLog.
+            catch (FileNotFoundException ex) when (ex.FusionLog != null)
+            {
+                throw new InvalidOperationException(ex.Message, ex);
+            }
+        }
+#endif
+
         public virtual IEnumerable<string> EnumerateFiles(string path, string searchPattern, SearchOption searchOption)
         {
+#if FEATURE_MSIOREDIST
+            return ShouldUseMicrosoftIO
+                ? HandleFileLoadException(
+                    (path, searchPattern, searchOption) => Microsoft.IO.Directory.EnumerateFiles(path, searchPattern, searchOption),
+                    path,
+                    searchPattern,
+                    (Microsoft.IO.SearchOption)searchOption
+                )
+                : Directory.EnumerateFiles(path, searchPattern, searchOption);
+#else
             return Directory.EnumerateFiles(path, searchPattern, searchOption);
+#endif
         }
 
         public virtual IEnumerable<string> EnumerateDirectories(string path, string searchPattern, SearchOption searchOption)
         {
+#if FEATURE_MSIOREDIST
+            return ShouldUseMicrosoftIO
+                ? HandleFileLoadException(
+                    (path, searchPattern, searchOption) => Microsoft.IO.Directory.EnumerateDirectories(path, searchPattern, searchOption),
+                    path,
+                    searchPattern,
+                    (Microsoft.IO.SearchOption)searchOption
+                )
+                : Directory.EnumerateDirectories(path, searchPattern, searchOption);
+#else
             return Directory.EnumerateDirectories(path, searchPattern, searchOption);
+#endif
         }
 
         public virtual IEnumerable<string> EnumerateFileSystemEntries(string path, string searchPattern, SearchOption searchOption)
         {
+#if FEATURE_MSIOREDIST
+            return ShouldUseMicrosoftIO
+                ? HandleFileLoadException(
+                    (path, searchPattern, searchOption) => Microsoft.IO.Directory.EnumerateFileSystemEntries(path, searchPattern, searchOption),
+                    path,
+                    searchPattern, (Microsoft.IO.SearchOption)searchOption
+                )
+                : Directory.EnumerateFileSystemEntries(path, searchPattern, searchOption);
+#else
             return Directory.EnumerateFileSystemEntries(path, searchPattern, searchOption);
+#endif
         }
 
         public FileAttributes GetAttributes(string path)

src/Shared/UnitTests/FileMatcher_Tests.cs

@@ -12,6 +12,7 @@ using System.Text.RegularExpressions;
 using Microsoft.Build.Shared.FileSystem;
 using Xunit;
 using Xunit.Abstractions;
+using Microsoft.Build.Utilities;
 
 namespace Microsoft.Build.UnitTests
 {
@@ -1244,13 +1245,20 @@ namespace Microsoft.Build.UnitTests
         [Fact]
         [PlatformSpecific(TestPlatforms.Windows)] // Nothing's too long for Unix
         [SkipOnTargetFramework(TargetFrameworkMonikers.Netcoreapp)]
-        public void IllegalTooLongPath()
+        public void IllegalTooLongPathOptOutWave17_0()
         {
-            string longString = new string('X', 500) + "*"; // need a wildcard to do anything
-            string[] result = FileMatcher.Default.GetFiles(@"c:\", longString);
+            using (var env = TestEnvironment.Create())
+            {
+                ChangeWaves.ResetStateForTests();
+                env.SetEnvironmentVariable("MSBUILDDISABLEFEATURESFROMVERSION", ChangeWaves.Wave17_0.ToString());
+                BuildEnvironmentHelper.ResetInstance_ForUnitTestsOnly();
 
-            Assert.Equal(longString, result[0]); // Does not throw
+                string longString = new string('X', 500) + "*"; // need a wildcard to do anything
+                string[] result = FileMatcher.Default.GetFiles(@"c:\", longString);
 
+                Assert.Equal(longString, result[0]); // Does not throw
+                ChangeWaves.ResetStateForTests();
+            }
             // Not checking that GetFileSpecMatchInfo returns the illegal-path flag,
             // not certain that won't break something; this fix is merely to avoid a crash.
         }

src/Tasks/Microsoft.Build.Tasks.csproj

@@ -989,6 +989,8 @@
     <ProjectReference Include="..\StringTools\StringTools.csproj" />
   </ItemGroup>
   <ItemGroup>
+    <PackageReference Include="Microsoft.IO.Redist" Condition="'$(FeatureMSIORedist)' == 'true'" />
+
     <PackageReference Include="System.Collections.Immutable" />
     <PackageReference Include="System.Resources.Extensions" />
   </ItemGroup>

src/Utilities/Microsoft.Build.Utilities.csproj

@@ -21,6 +21,7 @@
     <ProjectReference Include="..\Framework\Microsoft.Build.Framework.csproj" />
     <ProjectReference Include="..\StringTools\StringTools.csproj" />
 
+    <PackageReference Include="Microsoft.IO.Redist" Condition="'$(FeatureMSIORedist)' == 'true'" />
     <PackageReference Include="System.Collections.Immutable" />
     <PackageReference Include="System.Configuration.ConfigurationManager" />
   </ItemGroup>