Merged PR 800682: Check is scheduler been canceled when ensure historic metadata cache loaded

Previous change addressed the crash when loading cache after bxl cancelled. Crash still happen if load the cache after scheduler been cancelled.
To avoid crash, add access to scheduler cancellation token in PipTwoPhaseCache and check whether cancellation been requested before load cache.

Related work items: #2199303

Public/Src/Engine/Scheduler/Caching/PipTwoPhaseCache.cs

@@ -20,10 +20,10 @@ using BuildXL.Utilities.Collections;
 using BuildXL.Utilities.Configuration;
 using BuildXL.Utilities.Instrumentation.Common;
 using BuildXL.Utilities.Core.Tasks;
-using BuildXL.Utilities.Tracing;
 using static BuildXL.Utilities.Core.FormattableStringEx;
 using static BuildXL.Tracing.Diagnostics;
 using BuildXL.Cache.ContentStore.Interfaces.Sessions;
+using System.Threading;
 
 namespace BuildXL.Scheduler.Cache
 {
@@ -66,6 +66,12 @@ namespace BuildXL.Scheduler.Cache
 
         private readonly PathExpander m_pathExpander;
 
+        /// <summary>
+        /// Cancellation token that used to shutdown the scheduler.
+        /// Different from <see cref="PipExecutionContext.CancellationToken"/> which is used for terminating the entire application like ctrl-c.
+        /// </summary>
+        public CancellationToken SchedulerCancellationToken { get; set; }
+
         /// <nodoc />
         public PipTwoPhaseCache(LoggingContext loggingContext, EngineCache cache, PipExecutionContext context, PathExpander pathExpander)
         {

Public/Src/Engine/Scheduler/Caching/PipTwoPhaseCacheWithHashLookup.cs

@@ -204,7 +204,7 @@ namespace BuildXL.Scheduler.Cache
         /// <exception cref="BuildXLException"></exception>
         protected async Task<bool> EnsureLoadedAsync()
         {
-            if (Context.CancellationToken.IsCancellationRequested)
+            if (Context.CancellationToken.IsCancellationRequested || SchedulerCancellationToken.IsCancellationRequested)
             {
                 return false;
             }

Public/Src/Engine/Scheduler/Scheduler.cs

@@ -1370,6 +1370,7 @@ namespace BuildXL.Scheduler
                 configuration.Logging.GetTimerUpdatePeriodInMs();
             m_previousStatusLogTimeUtc = DateTime.UtcNow.AddMilliseconds(-1 * m_loggingIntervalPeriodMs); // Reducing by loggingIntervalPeriodMs to enable logging in the first call to UpdateStatus
             m_pipTwoPhaseCache = pipTwoPhaseCache ?? new PipTwoPhaseCache(loggingContext, cache, context, m_semanticPathExpander);
+            m_pipTwoPhaseCache.SchedulerCancellationToken = m_schedulerCancellationTokenSource.Token;
             m_runnablePipPerformance = new ConcurrentDictionary<PipId, RunnablePipPerformanceInfo>();
 
             m_fileChangeTrackerFile = m_configuration.Layout.SchedulerFileChangeTrackerFile;