Handle IOException when reading sideband files (#1178)

Public/Src/Engine/Dll/EngineSchedule.cs

@@ -30,6 +30,7 @@ using BuildXL.Scheduler.Tracing;
 using BuildXL.Storage;
 using BuildXL.Tracing;
 using BuildXL.Utilities;
+using BuildXL.Utilities.Collections;
 using BuildXL.Utilities.Configuration;
 using BuildXL.Utilities.Instrumentation.Common;
 using BuildXL.Utilities.Qualifier;
@@ -785,7 +786,7 @@ namespace BuildXL.Engine
                 .AsParallel()
                 .WithDegreeOfParallelism(Environment.ProcessorCount)
                 .WithCancellation(scheduler.Context.CancellationToken)
-                .SelectMany(ReadSidebandFile)
+                .SelectMany(fileName => ReadSidebandFile(loggingContext, fileName))
                 .ToArray();
 
             if (distinctRecordedWrites.Any())
@@ -838,13 +839,21 @@ namespace BuildXL.Engine
             }
         }
 
-        private static string[] ReadSidebandFile(string sidebandFile)
+        private static string[] ReadSidebandFile(LoggingContext loggingContext, string sidebandFile)
         {
             using (var sidebandReader = new SidebandReader(sidebandFile))
             {
-                sidebandReader.ReadHeader(ignoreChecksum: true);
-                sidebandReader.ReadMetadata();
-                return sidebandReader.ReadRecordedPaths().ToArray();
+                try
+                {
+                    sidebandReader.ReadHeader(ignoreChecksum: true);
+                    sidebandReader.ReadMetadata();
+                    return sidebandReader.ReadRecordedPaths().ToArray();
+                }
+                catch (IOException e)
+                {
+                    Logger.Log.CannotReadSidebandFile(loggingContext, sidebandFile, e.Message);
+                    return CollectionUtilities.EmptyArray<string>();
+                }
             }
         }
 

Public/Src/Engine/Dll/Tracing/Log.cs

@@ -1868,6 +1868,15 @@ If you can't update and need this feature after July 2018 please reach out to th
             Message = "/unsafe_DisableSharedOpaqueEmptyDirectoryScrubbing: removal of empty directories within shared opaques has been disabled. This is an unsafe configuration since it may work in detriment of build correctness.")]
         public abstract void ConfigUnsafeDisableSharedOpaqueEmptyDirectoryScrubbing(LoggingContext context);
 
+        [GeneratedEvent(
+            (ushort)EventId.CannotReadSidebandFile,
+            EventGenerators = EventGenerators.LocalOnly,
+            EventLevel = Level.Warning,
+            Keywords = (int)Keywords.UserMessage,
+            EventTask = (int)Tasks.Engine,
+            Message = "Cannot read sideband file '{fileName}': {error}")]
+        public abstract void CannotReadSidebandFile(LoggingContext context, string fileName, string error);
+
         [GeneratedEvent(
             (int)EventId.DeletingOutputsFromSharedOpaqueSidebandFilesStarted,
             EventGenerators = EventGenerators.LocalOnly,

Public/Src/Utilities/Utilities/Tracing/EventId.cs

@@ -540,6 +540,7 @@ namespace BuildXL.Utilities.Tracing
         DeletingOutputsFromSharedOpaqueSidebandFilesStarted = 867,
         DeletingSharedOpaqueSidebandFilesStarted = 868,
         ScrubbingProgress = 869,
+        CannotReadSidebandFile = 870,
 
         // Config
         ConfigUnsafeDisabledFileAccessMonitoring = 900,