Merged PR 580761: Support untracked scopes/paths in output directories

All changes in sandbox area are cosmetics. Please ignore.

Related work items: #1782665

Private/CloudTest/Gvfs/JournalHelper.cs

@@ -56,8 +56,7 @@ namespace BuildXL.CloudTest.Gvfs
             m_changeTracker = FileChangeTrackingSet.CreateForAllCapableVolumes(
                 loggingContext,
                 volumeMap,
-                m_journal,
-                Utilities.Configuration.FileChangeTrackerSupersedeMode.All);
+                m_journal);
         }
 
         public virtual string GetPath(string path)
@@ -84,7 +83,7 @@ namespace BuildXL.CloudTest.Gvfs
                 StartTracking();
             }
 
-            var result = m_changeTracker.TryEnumerateDirectoryAndTrackMembership(dir, (_, __) => { });
+            var result = m_changeTracker.TryEnumerateDirectoryAndTrackMembership(dir, (_, _) => { });
             XAssert.PossiblySucceeded(result);
             return result.Result;
         }

Public/Src/App/Bxl/Args.cs

@@ -474,9 +474,6 @@ namespace BuildXL
                         OptionHandlerFactory.CreateOption(
                             "fileChangeTrackerInitializationMode",
                             opt => engineConfiguration.FileChangeTrackerInitializationMode = CommandLineUtilities.ParseEnumOption<FileChangeTrackerInitializationMode>(opt)),
-                        OptionHandlerFactory.CreateOption(
-                            "fileChangeTrackerSupersedeMode",
-                            opt => engineConfiguration.FileChangeTrackerSupersedeMode = CommandLineUtilities.ParseEnumOption<FileChangeTrackerSupersedeMode>(opt)),
                         OptionHandlerFactory.CreateOption(
                             "fileChangeTrackingExclusionRoot",
                             opt => cacheConfiguration.FileChangeTrackingExclusionRoots.Add(CommandLineUtilities.ParsePathOption(opt, pathTable))),

Public/Src/App/Bxl/HelpText.cs

@@ -775,11 +775,6 @@ namespace BuildXL
                 Strings.HelpText_DisplayHelp_FileChangeTrackerInitializationMode,
                 HelpLevel.Verbose);
 
-            hw.WriteOption(
-                "/fileChangeTrackerSupersedeMode:<mode>",
-                Strings.HelpText_DisplayHelp_FileChangeTrackerSupersedeMode,
-                HelpLevel.Verbose);
-
             hw.WriteOption(
                 "/fileChangeTrackingExclusionRoot:<path>",
                 Strings.HelpText_DisplayHelp_FileChangeTrackingExclusionRoot,

Public/Src/App/Bxl/Strings.resx

@@ -1051,9 +1051,6 @@ Example: ad2d42d2ec5d2ca0c0b7ad65402d07c7ef40b91e</value>
   <data name="HelpText_DisplayHelp_ManageMemoryMode" xml:space="preserve">
     <value>Specifies the mode to manage memory under pressure. Defaults to CancellationRam where {ShortProductName} attemps to cancel processes. EmptyWorkingSet mode will empty working set of processes instead of cancellation. Suspend mode will suspend processes to free memory.</value>
   </data>
-  <data name="HelpText_DisplayHelp_FileChangeTrackerSupersedeMode" xml:space="preserve">
-    <value>Supersede mode to be applied to file tracking. Allowed values are All and FileOnly. Defaults to All</value>
-  </data>
   <data name="HelpText_DisplayHelp_FileContentTableEntryTimeToLive" xml:space="preserve">
     <value>Time-to-live for file content table entries. Min value is 1, max value is 65535. Defaults to 255.</value>
   </data>

Public/Src/Engine/Cache/Artifacts/ILocalDiskFileSystemView.cs

@@ -17,7 +17,10 @@ namespace BuildXL.Engine.Cache.Artifacts
         /// <summary>
         /// Tries to enumerate a directory and track membership.
         /// </summary>
-        Possible<PathExistence> TryEnumerateDirectoryAndTrackMembership(AbsolutePath path, Action<string, FileAttributes> handleEntry);
+        Possible<PathExistence> TryEnumerateDirectoryAndTrackMembership(
+            AbsolutePath path,
+            Action<string, FileAttributes> handleEntry,
+            Func<string, FileAttributes, bool> shouldIncludeEntry);
 
         /// <summary>
         /// Tracks a non-existent relative path chain from a tracked parent root.

Public/Src/Engine/Cache/Artifacts/LocalDiskContentStore.cs

@@ -345,11 +345,21 @@ namespace BuildXL.Engine.Cache.Artifacts
         /// </summary>
         public Possible<PathExistence> TryEnumerateDirectoryAndTrackMembership(
             AbsolutePath path,
-            Action<string, FileAttributes> handleEntry)
+            Action<string, FileAttributes> handleEntry,
+            Func<string, FileAttributes, bool> shouldIncludeEntry) => TryEnumerateDirectoryAndTrackMembership(path, handleEntry, shouldIncludeEntry, false);
+
+        /// <summary>
+        /// Tries to enumerate a directory and track membership.
+        /// </summary>
+        public Possible<PathExistence> TryEnumerateDirectoryAndTrackMembership(
+            AbsolutePath path,
+            Action<string, FileAttributes> handleEntry,
+            Func<string, FileAttributes, bool> shouldIncludeEntry,
+            bool supersedeWithLastEntryUsn)
         {
             return m_fileChangeTrackerSelector
                 .GetTracker(path)
-                .TryEnumerateDirectoryAndTrackMembership(path.ToString(m_pathTable), handleEntry)
+                .TryEnumerateDirectoryAndTrackMembership(path.ToString(m_pathTable), handleEntry, shouldIncludeEntry, supersedeWithLastEntryUsn)
                 .Then(enumerationResult => enumerationResult.Existence);
         }
 

Public/Src/Engine/Dll/Engine.cs

@@ -2843,7 +2843,6 @@ namespace BuildXL.Engine
                                 loggingContext,
                                 FileContentTable,
                                 journalState,
-                                Configuration.Engine.FileChangeTrackerSupersedeMode,
                                 graphFingerprint.ExactFingerprint);
                         }
                     }

Public/Src/Engine/Dll/InputTracker.cs

@@ -174,7 +174,6 @@ namespace BuildXL.Engine
             LoggingContext loggingContext,
             FileContentTable fileContentTable,
             JournalState journalState,
-            FileChangeTrackerSupersedeMode supersedeMode,
             CompositeGraphFingerprint graphFingerprint)
         {
             Contract.Requires(loggingContext != null);
@@ -185,7 +184,7 @@ namespace BuildXL.Engine
                 fileContentTable,
                 graphFingerprint,
                 journalState.IsEnabled
-                    ? FileChangeTracker.StartTrackingChanges(loggingContext, journalState.VolumeMap, journalState.Journal, supersedeMode, graphFingerprint.BuildEngineHash.ToString())
+                    ? FileChangeTracker.StartTrackingChanges(loggingContext, journalState.VolumeMap, journalState.Journal, graphFingerprint.BuildEngineHash.ToString())
                     : FileChangeTracker.CreateDisabledTracker(loggingContext),
                 true);
         }
@@ -327,7 +326,10 @@ namespace BuildXL.Engine
 
             if (members != null)
             {
-                possibleEnumResult = FileChangeTracker.TryTrackDirectoryMembership(directoryPath, members);
+                possibleEnumResult = FileChangeTracker.TryTrackDirectoryMembership(
+                    directoryPath,
+                    members,
+                    supersedeWithStrongIdentity: false /* enumeration only happens in sources, no need to supersede */);
             }
             else
             {
@@ -335,7 +337,9 @@ namespace BuildXL.Engine
                 // In future, this method will be responsible for enumeration as well.
                 possibleEnumResult = FileChangeTracker.TryEnumerateDirectoryAndTrackMembership(
                     directoryPath,
-                    (entryName, entryAttributes) => { });
+                    (entryName, entryAttributes) => { },
+                    shouldIncludeEntry: null /* include all entries */,
+                    supersedeWithStrongIdentity: false /* enumeration only happens in sources, no need to supersede */);
             }
 
             Possible<DirectoryMembershipTrackingFingerprint> possiblyDirectoryFingerprint;
@@ -377,7 +381,9 @@ namespace BuildXL.Engine
 
             var possibleEnumResult = tracker.TryEnumerateDirectoryAndTrackMembership(
                     directoryPath,
-                    (entryName, entryAttributes) => { });
+                    (entryName, entryAttributes) => { },
+                    shouldIncludeEntry: null /* include all entries */,
+                    supersedeWithStrongIdentity: false);
 
             if (possibleEnumResult.Succeeded)
             {
@@ -995,7 +1001,6 @@ namespace BuildXL.Engine
                     loggingContext, 
                     journalState.VolumeMap, 
                     journalState.Journal,
-                    configuration.Engine.FileChangeTrackerSupersedeMode,
                     graphFingerprint.ExactFingerprint.BuildEngineHash.ToString(),
                     atomicSaveToken); // Passing the save token ensure that the file change tracker is owned by (or correlated to) the input tracker.
             }
@@ -1005,7 +1010,6 @@ namespace BuildXL.Engine
                     loggingContext,
                     journalState.VolumeMap,
                     journalState.Journal,
-                    configuration.Engine.FileChangeTrackerSupersedeMode,
                     changeTrackingStatePath,
                     graphFingerprint.ExactFingerprint.BuildEngineHash.ToString(),
                     out fileChangeTracker);
@@ -1030,7 +1034,6 @@ namespace BuildXL.Engine
                         loggingContext,
                         journalState.VolumeMap,
                         journalState.Journal,
-                        configuration.Engine.FileChangeTrackerSupersedeMode,
                         graphFingerprint.ExactFingerprint.BuildEngineHash.ToString(),
                         atomicSaveToken); // Passing the save token ensure that the file change tracker is owned by (or correlated to) the input tracker.
 
@@ -1071,7 +1074,6 @@ namespace BuildXL.Engine
                     loggingContext,
                     journalState.VolumeMap,
                     journalState.Journal,
-                    configuration.Engine.FileChangeTrackerSupersedeMode,
                     graphFingerprint.ExactFingerprint.BuildEngineHash.ToString(),
                     atomicSaveToken); // Passing the save token ensure that the file change tracker is owned by (or correlated to) the input tracker.
             }

Public/Src/Engine/Processes/SandboxedProcessPipExecutor.cs

@@ -2309,8 +2309,8 @@ namespace BuildXL.Processes
                     {
                         // Compute whether the output directory is under an exclusion. In that case we want to block writes, but configure the rest of the policy in the regular way so tools
                         // can operate normally as long as they don't produce any outputs under it
-                        bool isUnderAnExclusion = (pip.OutputDirectoryExclusions.Any(exclusion => directory.Path.IsWithin(m_pathTable, exclusion)));
-
+                        bool isUnderAnExclusion = pip.OutputDirectoryExclusions.Any(exclusion => directory.Path.IsWithin(m_pathTable, exclusion));
+                        
                         // We need to allow the real timestamp to be seen under a directory output (since these are outputs). If this directory output happens to share the root with
                         // a directory dependency (shared opaque case), this is overridden for specific input files when processing directory dependencies below
                         var values =
@@ -2347,12 +2347,12 @@ namespace BuildXL.Processes
                     {
                         if (directory.IsSharedOpaque)
                         {
-                            // All members of the shared opaque need to be added to the manifest explicitly so timestamp faking happens for them
+                            // All members of the shared opaque need to be added to the manifest explicitly so timestamp faking happens for them.
                             AddSharedOpaqueInputContentToManifest(directory, allInputPathsUnderSharedOpaques);
                         }
 
-                        // If this directory dependency is also a directory output, then we don't set any additional policy: we don't want to restrict
-                        // writes
+                        // If this directory dependency is also a directory output, then we don't set any additional policy, i.e.,
+                        // we don't want to restrict writes.
                         if (!directoryOutputIds.Contains(directory.Path))
                         {
                             // Directories here represent inputs, we want to apply the timestamp faking logic
@@ -3615,15 +3615,17 @@ namespace BuildXL.Processes
 
                 foreach (ReportedFileAccess reported in result.ExplicitlyReportedFileAccesses)
                 {
-                    Contract.Assume(reported.Status == FileAccessStatus.Allowed ||
-                        reported.Method == FileAccessStatusMethod.FileExistenceBased, "Explicitly reported accesses are defined to be successful or denied only based on file existence");
+                    Contract.Assert(
+                        reported.Status == FileAccessStatus.Allowed || reported.Method == FileAccessStatusMethod.FileExistenceBased,
+                        "Explicitly reported accesses are defined to be successful or denied only based on file existence");
 
                     // Enumeration probes have a corresponding Enumeration access (also explicitly reported).
                     // Presently we are interested in capturing the existence of enumerations themselves rather than what was seen
                     // (and for NtQueryDirectoryFile, we can't always report the individual probes anyway).
                     if (reported.RequestedAccess == RequestedAccess.EnumerationProbe)
                     {
-                        // If it is an incremental tool and the pip allows preserving outputs, do not ignore.
+                        // If it is an incremental tool and the pip allows preserving outputs, then do not ignore because
+                        // the tool may depend on the directory membership.
                         if (!IsIncrementalToolAccess(reported))
                         {
                             continue;
@@ -3643,7 +3645,8 @@ namespace BuildXL.Processes
                     // Let's resolve all intermediate symlink dirs if configured.
                     // TODO: this logic will be eventually replaced by doing the right thing on detours side.
                     // This option is Windows-specific
-                    ReportedFileAccess finalReported;
+                    ReportedFileAccess finalReported = reported;
+
                     if (m_sandboxConfig.UnsafeSandboxConfiguration.ProcessSymlinkedAccesses())
                     {
                         Contract.Assume(m_symlinkedAccessResolver != null);
@@ -3663,11 +3666,7 @@ namespace BuildXL.Processes
                             parsedPath = finalPath;
                         }
                     }
-                    else
-                    {
-                        finalReported = reported;
-                    }
-
+                    
                     bool shouldExclude = false;
 
                     // Remove special accesses see Bug: #121875.
@@ -3690,10 +3689,7 @@ namespace BuildXL.Processes
                     }
 
                     accessesByPath.TryGetValue(parsedPath, out CompactSet<ReportedFileAccess> existingAccessesToPath);
-                    accessesByPath[parsedPath] =
-                        !shouldExclude
-                        ? existingAccessesToPath.Add(finalReported)
-                        : existingAccessesToPath;
+                    accessesByPath[parsedPath] = !shouldExclude ? existingAccessesToPath.Add(finalReported) : existingAccessesToPath;
                 }
 
                 foreach (var output in m_pip.FileOutputs)
@@ -3702,7 +3698,7 @@ namespace BuildXL.Processes
                     {
                         if (RequireOutputObservation(output))
                         {
-                            unobservedOutputs = unobservedOutputs ?? new List<AbsolutePath>();
+                            unobservedOutputs ??= new List<AbsolutePath>();
                             unobservedOutputs.Add(output.Path);
                         }
                     }
@@ -3712,10 +3708,8 @@ namespace BuildXL.Processes
                     }
                 }
 
-                using (PooledObjectWrapper<Dictionary<AbsolutePath, HashSet<AbsolutePath>>> dynamicWriteAccessWrapper =
-                    ProcessPools.DynamicWriteAccesses.GetInstance())
-                using (PooledObjectWrapper<List<ObservedFileAccess>> accessesUnsortedWrapper =
-                    ProcessPools.AccessUnsorted.GetInstance())
+                using (PooledObjectWrapper<Dictionary<AbsolutePath, HashSet<AbsolutePath>>> dynamicWriteAccessWrapper = ProcessPools.DynamicWriteAccesses.GetInstance())
+                using (PooledObjectWrapper<List<ObservedFileAccess>> accessesUnsortedWrapper = ProcessPools.AccessUnsorted.GetInstance())
                 using (var excludedPathsWrapper = Pools.GetAbsolutePathSet())
                 using (var fileExistenceDenialsWrapper = Pools.GetAbsolutePathSet())
                 using (var createdDirectoriesMutableWrapper = Pools.GetAbsolutePathSet())
@@ -3748,8 +3742,7 @@ namespace BuildXL.Processes
                         // Discard entries that have a single MacLookup report on a path that contains an intermediate directory symlink.
                         // Reason: observed accesses computed here should only contain fully expanded paths to avoid ambiguity;
                         //         on Mac, all access reports except for MacLookup report fully expanded paths, so only MacLookup paths need to be curated
-                        if (
-                            entry.Value.Count == 1 &&
+                        if (entry.Value.Count == 1 &&
                             firstAccess.Operation == ReportedFileOperation.MacLookup &&
                             firstAccess.ManifestPath.IsValid &&
                             CheckIfPathContainsSymlinks(firstAccess.ManifestPath.GetParent(m_context.PathTable)))
@@ -3762,28 +3755,21 @@ namespace BuildXL.Processes
                         foreach (var access in entry.Value)
                         {
                             // Detours reports a directory probe with a trailing backslash.
-                            if (// If the path is available and ends with a trailing backlash, we know that represents
+                            isDirectoryLocation =
+                                // If the path is available and ends with a trailing backlash, we know that represents
                                 // a directory
-                                ((isDirectoryLocation == null || isDirectoryLocation.Value) &&
+                                ((isDirectoryLocation == null || !isDirectoryLocation.Value) &&
                                  access.Path != null && access.Path.EndsWith("\\", StringComparison.OrdinalIgnoreCase))
                                 ||
                                 // If FILE_ATTRIBUTE_DIRECTORY flag is present, that means detours understood the operation
                                 // as happening on a directory.
                                 // TODO: this flag is not properly propagated for all detours operations.
-                                access.FlagsAndAttributes.HasFlag(FlagsAndAttributes.FILE_ATTRIBUTE_DIRECTORY))
-                            {
-                                isDirectoryLocation = true;
-                            }
-                            else
-                            {
-                                isDirectoryLocation = false;
-                            }
-
+                                access.FlagsAndAttributes.HasFlag(FlagsAndAttributes.FILE_ATTRIBUTE_DIRECTORY);
+                            
                             // To treat the paths as file probes, all accesses to the path must be the probe access.
                             isProbe &= access.RequestedAccess == RequestedAccess.Probe;
 
-                            if (access.RequestedAccess == RequestedAccess.Probe
-                                && IsIncrementalToolAccess(access))
+                            if (access.RequestedAccess == RequestedAccess.Probe && IsIncrementalToolAccess(access))
                             {
                                 isProbe = false;
                             }
@@ -3819,7 +3805,6 @@ namespace BuildXL.Processes
                         // in the cone of a shared opaque, then it is a dynamic write access
                         bool? isAccessUnderASharedOpaque = null;
                         if (isPathCandidateToBeOwnedByASharedOpaque && IsAccessUnderASharedOpaque(
-                                entry.Key,
                                 firstAccess,
                                 dynamicWriteAccesses,
                                 out AbsolutePath sharedDynamicDirectoryRoot))
@@ -3844,11 +3829,12 @@ namespace BuildXL.Processes
                             // then we just skip reporting the access. Together with the above step, this means that no accesses under shared opaques that represent outputs are actually
                             // reported as observed accesses. This matches the same behavior that occurs on static outputs.
                             if (!allInputPathsUnderSharedOpaques.Contains(entry.Key) &&
-                                (isAccessUnderASharedOpaque == true || IsAccessUnderASharedOpaque(entry.Key, firstAccess, dynamicWriteAccesses, out _)))
+                                (isAccessUnderASharedOpaque == true || IsAccessUnderASharedOpaque(firstAccess, dynamicWriteAccesses, out _)))
                             {
                                 continue;
                             }
                         }
+
                         ObservationFlags observationFlags = ObservationFlags.None;
 
                         if (isProbe)
@@ -3981,9 +3967,8 @@ namespace BuildXL.Processes
         }
 
         private bool IsAccessUnderASharedOpaque(
-            AbsolutePath accessPath,
-            ReportedFileAccess access,
-            Dictionary<AbsolutePath, HashSet<AbsolutePath>> dynamicWriteAccesses,
+            ReportedFileAccess access, 
+            Dictionary<AbsolutePath, HashSet<AbsolutePath>> dynamicWriteAccesses, 
             out AbsolutePath sharedDynamicDirectoryRoot)
         {
             sharedDynamicDirectoryRoot = AbsolutePath.Invalid;
@@ -4005,10 +3990,6 @@ namespace BuildXL.Processes
 
             var initialNode = access.ManifestPath.Value;
 
-            using var outputDirectoryExclusionSetWrapper = Pools.AbsolutePathSetPool.GetInstance();
-            var outputDirectoryExclusionSet = outputDirectoryExclusionSetWrapper.Instance;
-            outputDirectoryExclusionSet.AddRange(m_pip.OutputDirectoryExclusions);
-
             // TODO: consider adding a cache from manifest paths to containing shared opaques. It is likely
             // that many writes for a given pip happens under the same cones.
 
@@ -4031,7 +4012,7 @@ namespace BuildXL.Processes
         /// </summary>
         private static bool PathStartsWith(string path, string prefix, StringComparison? comparison = default)
         {
-            comparison = comparison ?? OperatingSystemHelper.PathComparison;
+            comparison ??= OperatingSystemHelper.PathComparison;
 
             return path.StartsWith(prefix, comparison.Value)
                    || path.StartsWith(@"\??\" + prefix, comparison.Value)

Public/Src/Engine/Scheduler/Artifacts/FileContentManager.cs

@@ -189,6 +189,11 @@ namespace BuildXL.Scheduler.Artifacts
 
         private readonly ConcurrentBigSet<AbsolutePath> m_pathsWithoutFileArtifact = new ConcurrentBigSet<AbsolutePath>();
 
+        private readonly ObjectPool<OutputDirectoryEnumerationData> m_outputEnumerationDataPool =
+            new ObjectPool<OutputDirectoryEnumerationData>(
+                () => new OutputDirectoryEnumerationData(),
+                data => { data.Clear(); return data; });
+
         #endregion
 
         #region External State (i.e. passed into constructor)
@@ -390,7 +395,7 @@ namespace BuildXL.Scheduler.Artifacts
                     }
                 }
 
-                return await TryHashArtifactsAsync(operationContext, state, pip.ProcessAllowsUndeclaredSourceReads);
+                return await TryHashArtifactsAsync(operationContext, state, pip.ProcessAllowsUndeclaredSourceReads, artifactsProducer: null);
             }
         }
 
@@ -410,7 +415,7 @@ namespace BuildXL.Scheduler.Artifacts
                 // by consumers of the seal directory
                 MarkDirectoryMaterializations(state);
 
-                return await TryHashArtifactsAsync(operationContext, state, pip.ProcessAllowsUndeclaredSourceReads);
+                return await TryHashArtifactsAsync(operationContext, state, pip.ProcessAllowsUndeclaredSourceReads, artifactsProducer: pip);
             }
         }
 
@@ -422,9 +427,13 @@ namespace BuildXL.Scheduler.Artifacts
             return m_allowedFileRewriteOutputs.Contains(path);
         }
 
-        private async Task<Possible<Unit>> TryHashArtifactsAsync(OperationContext operationContext, PipArtifactsState state, bool allowUndeclaredSourceReads)
+        private async Task<Possible<Unit>> TryHashArtifactsAsync(
+            OperationContext operationContext,
+            PipArtifactsState state,
+            bool allowUndeclaredSourceReads,
+            Pip artifactsProducer)
         {
-            var maybeReported = EnumerateAndReportDynamicOutputDirectories(state.PipArtifacts);
+            var maybeReported = EnumerateOutputDirectories(state, artifactsProducer, shouldReport: true);
 
             if (!maybeReported.Succeeded)
             {
@@ -569,6 +578,12 @@ namespace BuildXL.Scheduler.Artifacts
                     return state.GetFailure();
                 }
 
+                var enumerateResult = EnumerateOutputDirectories(state, pip, shouldReport: false);
+                if (!enumerateResult.Succeeded)
+                {
+                    return enumerateResult.Failure;
+                }
+
                 if (hasExcludedOutput)
                 {
                     return PipOutputOrigin.NotMaterialized;
@@ -671,41 +686,83 @@ namespace BuildXL.Scheduler.Artifacts
         /// <summary>
         /// Enumerates dynamic output directory.
         /// </summary>
-        public Possible<Unit> EnumerateDynamicOutputDirectory(
+        public Possible<Unit> EnumerateOutputDirectory(
             DirectoryArtifact directoryArtifact,
+            OutputDirectoryEnumerationData enumerationData,
             Action<FileArtifact> handleFile,
             Action<AbsolutePath> handleDirectory)
         {
             Contract.Requires(directoryArtifact.IsValid);
-
+            Contract.Requires(enumerationData != null);
+            
             var pathTable = Context.PathTable;
             var directoryPath = directoryArtifact.Path;
-            var queue = new Queue<AbsolutePath>();
-            queue.Enqueue(directoryPath);
+            var queue = new Queue<(AbsolutePath path, bool shouldTrack)>();
+            queue.Enqueue((directoryPath, true));
 
             while (queue.Count > 0)
             {
-                var currentDirectoryPath = queue.Dequeue();
-                var result = m_host.LocalDiskContentStore.TryEnumerateDirectoryAndTrackMembership(
-                    currentDirectoryPath,
-                    handleEntry: (entry, attributes) =>
-                    {
-                        var path = currentDirectoryPath.Combine(pathTable, entry);
-                        // must treat directory reparse points as files: recursing on directory reparse points can lead to infinite loops
-                        if (!FileUtilities.IsDirectoryNoFollow(attributes))
-                        {
-                            var fileArtifact = FileArtifact.CreateOutputFile(path);
-                            handleFile?.Invoke(fileArtifact);
-                        }
-                        else
-                        {
-                            handleDirectory?.Invoke(path);
-                            queue.Enqueue(path);
-                        }
-                    });
-                if (!result.Succeeded)
+                (AbsolutePath currentDirectoryPath, bool shouldTrack) = queue.Dequeue();
+
+                if (shouldTrack)
                 {
-                    return result.Failure;
+                    var result = m_host.LocalDiskContentStore.TryEnumerateDirectoryAndTrackMembership(
+                        currentDirectoryPath,
+                        handleEntry: (entry, attributes) =>
+                        {
+                            var path = currentDirectoryPath.Combine(pathTable, entry);
+                            
+                            if (!FileUtilities.IsDirectoryNoFollow(attributes))
+                            {
+                                handleFile?.Invoke(FileArtifact.CreateOutputFile(path));
+                            }
+                            else
+                            {
+                                handleDirectory?.Invoke(path);
+                            }
+                        },
+                        shouldIncludeEntry: (entry, attributes) =>
+                        {
+                            var path = currentDirectoryPath.Combine(pathTable, entry);
+                            bool shouldIncludeEntry = !enumerationData.UntrackedPaths.Contains(path) && !enumerationData.UntrackedScopes.Contains(path);
+
+                            // Treat directory reparse points as files: recursing on directory reparse points can lead to infinite loops
+                            if (FileUtilities.IsDirectoryNoFollow(attributes) && !enumerationData.OutputDirectoryExclusions.Contains(path))
+                            {
+                                queue.Enqueue((path, shouldIncludeEntry));
+                            }
+
+                            return shouldIncludeEntry;
+                        },
+                        supersedeWithLastEntryUsn: true);
+
+                    if (!result.Succeeded)
+                    {
+                        return result.Failure;
+                    }
+                }
+                else
+                {
+                    FileUtilities.EnumerateDirectoryEntries(
+                        currentDirectoryPath.ToString(pathTable),
+                        handleEntry: (entry, attributes) =>
+                        {
+                            var path = currentDirectoryPath.Combine(pathTable, entry);
+
+                            // Treat directory reparse points as files: recursing on directory reparse points can lead to infinite loops
+                            if (FileUtilities.IsDirectoryNoFollow(attributes))
+                            {
+                                // Once the directory is untracked, its descendent directories will always be untracked.
+                                queue.Enqueue((path, false));
+                            }
+                            else
+                            {
+                                if (enumerationData.OutputFilePaths.Contains(path))
+                                {
+                                    handleFile?.Invoke(FileArtifact.CreateOutputFile(path));
+                                }
+                            }
+                        });
                 }
             }
 
@@ -1160,32 +1217,57 @@ namespace BuildXL.Scheduler.Artifacts
 
         // TODO: Consider calling this from TryHashDependencies. That would allow us to remove logic which requires
         // that direct seal directory dependencies are scheduled
-        private Possible<Unit> EnumerateAndReportDynamicOutputDirectories(HashSet<FileOrDirectoryArtifact> artifacts)
+        private Possible<Unit> EnumerateOutputDirectories(PipArtifactsState state, Pip artifactsProducer, bool shouldReport)
         {
-            foreach (var artifact in artifacts)
+            using (var outputDirectoryEnumerationDataWrapper = m_outputEnumerationDataPool.GetInstance())
             {
-                if (artifact.IsDirectory)
+                OutputDirectoryEnumerationData outputDirectoryEnumerationData = null;
+                Process process = artifactsProducer as Process;
+
+                if (process != null)
                 {
-                    var directory = artifact.DirectoryArtifact;
-                    SealDirectoryKind sealDirectoryKind = m_host.GetSealDirectoryKind(directory);
+                    outputDirectoryEnumerationData = outputDirectoryEnumerationDataWrapper.Instance;
+                    outputDirectoryEnumerationData.Process = process;
+                }
 
-                    if (sealDirectoryKind == SealDirectoryKind.Opaque)
+                foreach (var artifact in state.PipArtifacts)
+                {
+                    if (artifact.IsDirectory)
                     {
-                        if (m_sealContents.ContainsKey(directory))
-                        {
-                            // Only enumerate and report if the directory has not already been reported
-                            continue;
-                        }
+                        var directory = artifact.DirectoryArtifact;
+                        SealDirectoryKind sealDirectoryKind = m_host.GetSealDirectoryKind(directory);
 
-                        if (Context.CancellationToken.IsCancellationRequested)
+                        if (sealDirectoryKind == SealDirectoryKind.Opaque)
                         {
-                            return Context.CancellationToken.CreateFailure();
-                        }
+                            if (m_sealContents.ContainsKey(directory))
+                            {
+                                // Only enumerate and report if the directory has not already been reported
+                                continue;
+                            }
 
-                        var result = EnumerateAndReportDynamicOutputDirectory(directory);
-                        if (!result.Succeeded)
-                        {
-                            return result;
+                            if (Context.CancellationToken.IsCancellationRequested)
+                            {
+                                return Context.CancellationToken.CreateFailure();
+                            }
+
+                            using var outputDirectoryEnumerationDataInnerWrapper = m_outputEnumerationDataPool.GetInstance();
+                            
+                            OutputDirectoryEnumerationData outputDirectoryEnumerationDataInner = outputDirectoryEnumerationData;
+                            
+                            if (outputDirectoryEnumerationDataInner == null)
+                            {
+                                process = m_host.GetProducer(directory) as Process;
+                                Contract.Assert(process != null, "Opaque directory artifact must be produced by a process");
+
+                                outputDirectoryEnumerationDataInner = outputDirectoryEnumerationDataInnerWrapper.Instance;
+                                outputDirectoryEnumerationDataInner.Process = process;
+                            }
+
+                            var result = EnumerateOutputDirectory(directory, outputDirectoryEnumerationDataInner, shouldReport);
+                            if (!result.Succeeded)
+                            {
+                                return result;
+                            }
                         }
                     }
                 }
@@ -1194,24 +1276,39 @@ namespace BuildXL.Scheduler.Artifacts
             return Unit.Void;
         }
 
-        private Possible<Unit> EnumerateAndReportDynamicOutputDirectory(DirectoryArtifact directory)
+        private Possible<Unit> EnumerateOutputDirectory(
+            DirectoryArtifact directoryArtifact,
+            OutputDirectoryEnumerationData outputDirectoryEnumerationData,
+            bool shouldReport)
         {
+            Contract.Requires(directoryArtifact.IsValid);
+            Contract.Requires(outputDirectoryEnumerationData != null);
+
             using (var poolFileList = Pools.GetFileArtifactList())
             {
                 var fileList = poolFileList.Instance;
                 fileList.Clear();
 
-                var result = EnumerateDynamicOutputDirectory(directory, handleFile: file => fileList.Add(file), handleDirectory: null);
+                var result = EnumerateOutputDirectory(
+                    directoryArtifact,
+                    outputDirectoryEnumerationData,
+                    handleFile: file => fileList.Add(file),
+                    handleDirectory: null);
 
                 if (!result.Succeeded)
                 {
                     return result.Failure;
                 }
 
-                // When enumerating the local disk in order to get an output directory contents, the result is always required artifacts with no rewritten sources
-                // The source of rewritten sources are always reported from the pip executor
-                ReportDynamicDirectoryContents(directory, fileList.Select(fa => 
-                    FileArtifactWithAttributes.Create(fa, FileExistence.Required, isUndeclaredFileRewrite: false)), PipOutputOrigin.UpToDate);
+                if (shouldReport)
+                {
+                    // When enumerating the local disk in order to get an output directory contents, the result is always required artifacts with no rewritten sources
+                    // The source of rewritten sources are always reported from the pip executor
+                    ReportDynamicDirectoryContents(
+                        directoryArtifact,
+                        fileList.Select(fa => FileArtifactWithAttributes.Create(fa, FileExistence.Required, isUndeclaredFileRewrite: false)),
+                        PipOutputOrigin.UpToDate);
+                }
             }
 
             return Unit.Void;
@@ -1355,9 +1452,8 @@ namespace BuildXL.Scheduler.Artifacts
                 {
                     // Directory artifact contents are not hashed since they will be hashed dynamically
                     // if the pip accesses them, so the file is the declared artifact
-                    FileMaterializationInfo fileContentInfo;
                     FileArtifact file = artifact.FileArtifact;
-                    if (!m_fileArtifactContentHashes.TryGetValue(file, out fileContentInfo))
+                    if (!m_fileArtifactContentHashes.TryGetValue(file, out _))
                     {
                         // Directory artifact contents are not hashed since they will be hashed dynamically
                         // if the pip accesses them, so the file is the declared artifact

Public/Src/Engine/Scheduler/FileSystem/FileSystemView.cs

@@ -225,7 +225,12 @@ namespace BuildXL.Scheduler.FileSystem
                     using (Counters.StartStopwatch(FileSystemViewCounters.RealFileSystemEnumerationsDuration))
                     {
 
-                        possibleExistence = LocalDiskFileSystem.TryEnumerateDirectoryAndTrackMembership(path, handleDirectoryEntry);
+                        possibleExistence = LocalDiskFileSystem.TryEnumerateDirectoryAndTrackMembership(
+                            path,
+                            handleDirectoryEntry,
+                            // This method is called during observed input processing. Currently, we simply include all entries.
+                            // TODO: In the future, we may want to restrict it based on pip's untracked scopes/paths.
+                            shouldIncludeEntry: null /* include all entries */);
                     }
 
                     if (possibleExistence.Succeeded)

Public/Src/Engine/Scheduler/IncrementalScheduling/GraphAgnosticIncrementalSchedulingState.cs

@@ -872,7 +872,8 @@ namespace BuildXL.Scheduler.IncrementalScheduling
             IEnumerable<string> dynamicallyObservedFilePaths,
             IEnumerable<string> dynamicallyProbedFilePaths,
             IEnumerable<string> dynamicallyObservedEnumerationPaths,
-            IEnumerable<(string directory, IEnumerable<string> fileArtifactsCollection)> dynamicDirectoryContents)
+            IEnumerable<(string directory, IEnumerable<string> fileArtifactsCollection)> outputDirectoriesContents,
+            IEnumerable<string> untrackedScopes)
         {
             if (!PipGraph.TryGetPipFingerprint(nodeId.ToPipId(), out ContentFingerprint fingerprint))
             {
@@ -905,13 +906,19 @@ namespace BuildXL.Scheduler.IncrementalScheduling
                 m_dynamicallyObservedEnumerations.AddEntry(pipStableId, dynamicallyObservedEnumeration);
             }
 
-            foreach (var dynamicDirectoryContent in dynamicDirectoryContents)
+            foreach (var dynamicDirectoryContent in outputDirectoriesContents)
             {
                 var directoryPath = AbsolutePath.Create(m_internalPathTable, dynamicDirectoryContent.directory);
 
+                using (var untrackedScopesPool = Pools.GetAbsolutePathSet())
                 using (var pools = Pools.GetAbsolutePathSet())
                 {
                     var pathSet = pools.Instance;
+                    var untrackedScopeSet = untrackedScopesPool.Instance;
+                    untrackedScopeSet.UnionWith(
+                        untrackedScopes
+                            .Select(p => AbsolutePath.Create(m_internalPathTable, p))
+                            .Where(p => p.IsWithin(m_internalPathTable, directoryPath)));
 
                     foreach (var member in dynamicDirectoryContent.fileArtifactsCollection)
                     {
@@ -925,7 +932,9 @@ namespace BuildXL.Scheduler.IncrementalScheduling
                             {
                                 m_dynamicallyObservedFiles.AddEntry(pipStableId, currentPath);
                             }
-                            else
+                            else if (untrackedScopeSet.Count == 0 // Typical case.
+                                || (!untrackedScopeSet.Contains(currentPath)
+                                    && !untrackedScopeSet.Any(u => currentPath.IsWithin(m_internalPathTable, u))))
                             {
                                 m_dynamicallyObservedEnumerations.AddEntry(pipStableId, currentPath);
                             }

Public/Src/Engine/Scheduler/IncrementalScheduling/IIncrementalSchedulingState.cs

@@ -53,13 +53,29 @@ namespace BuildXL.Scheduler.IncrementalScheduling
         /// <param name="dynamicallyObservedFilePaths">Dynamically observed read files.</param>
         /// <param name="dynamicallyProbedFilePaths">Dynamically observed probed files.</param>
         /// <param name="dynamicallyObservedEnumerationPaths">Dynamically observed enumerations.</param>
-        /// <param name="dynamicDirectoryContents">Dynamic directory contents.</param>
+        /// <param name="outputDirectoriesContents">Contents of output directories.</param>
+        /// <param name="untrackedScopes">Untracked scopes.</param>
+        /// <remarks>
+        /// <paramref name="untrackedScopes"/> is used to filter which directory in <paramref name="outputDirectoriesContents"/>
+        /// needs to be associated with directory enumeration. For example, suppose that we have an output directory D with
+        /// the following structure
+        /// D\E\
+        ///     1
+        ///     2
+        ///     3   -- untrack path
+        ///   F\    -- untracked scope
+        ///     4   -- statically declared output file
+        /// In this case, D\E\3 will not appear in the content of D. However, for tracking enumeration, we have to include D and D\E
+        /// as enumerated, but not F. At the same time, we need to include D\F\4 as dynamically observed. We use <paramref name="untrackedScopes"/>
+        /// to not associate D\F with enumeration.
+        /// </remarks>
         void RecordDynamicObservations(
             NodeId nodeId,
             IEnumerable<string> dynamicallyObservedFilePaths,
             IEnumerable<string> dynamicallyProbedFilePaths,
             IEnumerable<string> dynamicallyObservedEnumerationPaths,
-            IEnumerable<(string directory, IEnumerable<string> fileArtifactsCollection)> dynamicDirectoryContents);
+            IEnumerable<(string directory, IEnumerable<string> fileArtifactsCollection)> outputDirectoriesContents,
+            IEnumerable<string> untrackedScopes);
 
         /// <summary>
         /// Writes textual format of the instance of <see cref="IIncrementalSchedulingState"/>.

Public/Src/Engine/Scheduler/OutputDirectoryEnumerationData.cs

@@ -0,0 +1,70 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+using System;
+using System.Collections.Generic;
+using System.Diagnostics.ContractsLight;
+using System.Linq;
+using BuildXL.Pips.Operations;
+using BuildXL.Utilities;
+
+namespace BuildXL.Scheduler
+{
+    /// <summary>
+    /// Data needed when enumerating process output directories.
+    /// </summary>
+    public sealed class OutputDirectoryEnumerationData
+    {
+        /// <summary>
+        /// Underlying process.
+        /// </summary>
+        public Process Process 
+        {
+            get => m_process;
+            set
+            {
+                Contract.Requires(value != null);
+                Clear();
+                m_process = value;
+                UntrackedPaths.UnionWith(Process.UntrackedPaths);
+                UntrackedScopes.UnionWith(Process.UntrackedScopes);
+                OutputDirectoryExclusions.UnionWith(Process.OutputDirectoryExclusions);
+                OutputFilePaths.UnionWith(Process.FileOutputs.Select(f => f.Path));
+            }
+        }
+
+        /// <summary>
+        /// Untracked paths.
+        /// </summary>
+        public readonly HashSet<AbsolutePath> UntrackedPaths = new HashSet<AbsolutePath>();
+
+        /// <summary>
+        /// Untracked scopes.
+        /// </summary>
+        public readonly HashSet<AbsolutePath> UntrackedScopes = new HashSet<AbsolutePath>();
+
+        /// <summary>
+        /// Excluded directories.
+        /// </summary>
+        public readonly HashSet<AbsolutePath> OutputDirectoryExclusions = new HashSet<AbsolutePath>();
+
+        /// <summary>
+        /// Statically declared output files.
+        /// </summary>
+        public readonly HashSet<AbsolutePath> OutputFilePaths = new HashSet<AbsolutePath>();
+
+        private Process m_process;
+
+        /// <summary>
+        /// Clears data.
+        /// </summary>
+        public void Clear()
+        {
+            m_process = null;
+            UntrackedPaths.Clear();
+            UntrackedScopes.Clear();
+            OutputDirectoryExclusions.Clear();
+            OutputFilePaths.Clear();
+        }
+    }
+}

Public/Src/Engine/Scheduler/PipExecutor.cs

@@ -84,6 +84,11 @@ namespace BuildXL.Scheduler
                 () => new Dictionary<FileArtifact, Task<Possible<FileMaterializationInfo>>>(),
                 map => { map.Clear(); return map; });
 
+        private static readonly ObjectPool<OutputDirectoryEnumerationData> s_outputEnumerationDataPool =
+            new ObjectPool<OutputDirectoryEnumerationData>(
+                () => new OutputDirectoryEnumerationData(),
+                data => { data.Clear(); return data; });
+
         /// <summary>
         /// Materializes pip's inputs.
         /// </summary>
@@ -4315,6 +4320,10 @@ namespace BuildXL.Scheduler
                     allOutputs.Add(output);
                 }
 
+                using var outputDirectoryDataWrapper = s_outputEnumerationDataPool.GetInstance();
+                var outputDirectoryData = outputDirectoryDataWrapper.Instance;
+                outputDirectoryData.Process = process;
+
                 // We need to discover dynamic outputs in the given opaque directories.
                 var fileList = poolFileList.Instance;
 
@@ -4329,8 +4338,9 @@ namespace BuildXL.Scheduler
                     // For the case of an opaque directory, the content is determined by scanning the file system
                     if (!directoryArtifact.IsSharedOpaque)
                     {
-                        var enumerationResult = environment.State.FileContentManager.EnumerateDynamicOutputDirectory(
+                        var enumerationResult = environment.State.FileContentManager.EnumerateOutputDirectory(
                             directoryArtifact,
+                            outputDirectoryData,
                             handleFile: fileArtifact =>
                             {
                                 if (!CheckForAllowedReparsePointProduction(fileArtifact.Path, operationContext, description, pathTable, processExecutionResult, environment.Configuration))

Public/Src/Engine/Scheduler/Scheduler.cs

@@ -3065,6 +3065,11 @@ namespace BuildXL.Scheduler
                                     {
                                         using (runnablePip.OperationContext.StartOperation(PipExecutorCounter.RecordDynamicObservationsDuration))
                                         {
+                                            // Note that we don't include untracked paths when recoring dynamic observation for output
+                                            // directories. This is because m_fileContentManager.ListSealedDirectoryContents(d) contains
+                                            // only tracked paths. See m_fileContentManager.EnumerateOutputDirectory method for details.
+                                            // Also see the documentation of RecordDynamicObservations for details why untracked scopes
+                                            // are still needed for recording dynamic observation into incremental scheduling state.
                                             IncrementalSchedulingState.RecordDynamicObservations(
                                                 nodeId,
                                                 result.DynamicallyObservedFiles.Select(path => path.ToString(Context.PathTable)),
@@ -3075,7 +3080,8 @@ namespace BuildXL.Scheduler
                                                         (
                                                             d.Path.ToString(Context.PathTable),
                                                             m_fileContentManager.ListSealedDirectoryContents(d)
-                                                                .Select(f => f.Path.ToString(Context.PathTable)))));
+                                                                .Select(f => f.Path.ToString(Context.PathTable)))),
+                                                processPip.UntrackedScopes.Select(p => p.ToString(Context.PathTable)));
                                         }
                                     }
                                 }
@@ -6037,7 +6043,6 @@ namespace BuildXL.Scheduler
                         loggingContext,
                         m_journalState.VolumeMap,
                         m_journalState.Journal,
-                        m_configuration.Engine.FileChangeTrackerSupersedeMode,
                         m_buildEngineFingerprint);
                     loadingResult = null;
                 }
@@ -6047,7 +6052,6 @@ namespace BuildXL.Scheduler
                         loggingContext,
                         m_journalState.VolumeMap,
                         m_journalState.Journal,
-                        m_configuration.Engine.FileChangeTrackerSupersedeMode,
                         m_fileChangeTrackerFile.ToString(Context.PathTable),
                         m_buildEngineFingerprint,
                         out m_fileChangeTracker);

Public/Src/Engine/UnitTests/Cache/LocalDiskContentStoreTests.cs

@@ -581,7 +581,10 @@ namespace Test.BuildXL.Engine.Cache
 
                 // Not try directory operations against the parent path. The assumption is that the directory is at or under the inclusion/exclusion root as well
                 var directoryPath = filePath.GetParent(pathTable);
-                var enumerationResult = harness.Store.TryEnumerateDirectoryAndTrackMembership(directoryPath, (name, attr) => { });
+                var enumerationResult = harness.Store.TryEnumerateDirectoryAndTrackMembership(
+                    directoryPath,
+                    (name, attr) => { },
+                    shouldIncludeEntry: null /* include all entries */);
                 XAssert.IsTrue(enumerationResult.Succeeded);
                 XAssert.AreEqual(PathExistence.ExistsAsDirectory, enumerationResult.Result);
 

Public/Src/Engine/UnitTests/Scheduler.IntegrationTest/IncrementalSchedulingTests/IncrementalSchedulingTests.cs

@@ -619,15 +619,8 @@ namespace IntegrationTest.BuildXL.Scheduler.IncrementalSchedulingTests
         }
 
         [Theory]
-        [InlineData(true, FileChangeTrackerSupersedeMode.FileOnly)]
-        [InlineData(true, FileChangeTrackerSupersedeMode.All)]
-        [InlineData(true, FileChangeTrackerSupersedeMode.FileAndParents)]
-        [InlineData(false, FileChangeTrackerSupersedeMode.FileOnly)]
-        [InlineData(false, FileChangeTrackerSupersedeMode.All)]
-        [InlineData(false, FileChangeTrackerSupersedeMode.FileAndParents)]
-        public void ProducingNestedOutputDirectoryShouldNotInvalidateIncrementalScheduling(
-            bool runPosixDeleteFirst,
-            FileChangeTrackerSupersedeMode supersedeMode)
+        [MemberData(nameof(TruthTable.GetTable), 1, MemberType = typeof(TruthTable))]
+        public void ProducingNestedOutputDirectoryShouldNotInvalidateIncrementalScheduling(bool runPosixDeleteFirst)
         {
             var outputDirectory = CreateOutputDirectoryArtifact();
             var outputFileInNestedOutputDirectory = CreateOutputFileArtifact(outputDirectory.Path.Combine(Context.PathTable, "nested"));
@@ -645,8 +638,6 @@ namespace IntegrationTest.BuildXL.Scheduler.IncrementalSchedulingTests
             var originalPosixDeleteMode = FileUtilities.PosixDeleteMode;
             FileUtilities.PosixDeleteMode = runPosixDeleteFirst ? PosixDeleteMode.RunFirst : PosixDeleteMode.NoRun;
 
-            Configuration.Engine.FileChangeTrackerSupersedeMode = supersedeMode;
-
             try
             {
                 // 1st Run:
@@ -664,8 +655,8 @@ namespace IntegrationTest.BuildXL.Scheduler.IncrementalSchedulingTests
                 RunScheduler(runNameOrDescription: "2nd Run").AssertCacheMiss(process.PipId);
 
                 var runResult = RunScheduler(runNameOrDescription: "3rd Run");
-
-                if (supersedeMode != FileChangeTrackerSupersedeMode.FileOnly && runPosixDeleteFirst)
+                
+                if (runPosixDeleteFirst)
                 {
                     runResult.AssertNotScheduled(process.PipId);
                 }

Public/Src/Engine/UnitTests/Scheduler.IntegrationTest/OpaqueDirectoryTests.cs

@@ -9,7 +9,6 @@ using BuildXL.Pips.Builders;
 using BuildXL.Pips.Operations;
 using BuildXL.Scheduler.Tracing;
 using BuildXL.Utilities;
-using BuildXL.Utilities.Tracing;
 using BuildXL.Utilities.Configuration;
 using BuildXL.Utilities.Configuration.Mutable;
 using Test.BuildXL.Executables.TestProcess;
@@ -412,5 +411,180 @@ namespace IntegrationTest.BuildXL.Scheduler
 
             RunScheduler().AssertSuccess();
         }
+
+        [Fact]
+        public void OutputDirectoryWithUntrackedPaths()
+        {
+            var outputDirectory = CreateUniqueObjPath("outputDir");
+            var outputFileA = CreateOutputFileArtifact(outputDirectory, "fileA");
+            var outputFileB = CreateOutputFileArtifact(outputDirectory, "fileB");
+            var outputFileC = CreateOutputFileArtifact(outputDirectory, "fileC");
+            var sourceFile = CreateSourceFile();
+
+            var builder = CreatePipBuilder(
+                new[]
+                {
+                    Operation.ReadFile(sourceFile),
+                    Operation.WriteFile(outputFileA, doNotInfer: true),
+                    Operation.WriteFile(outputFileB, doNotInfer: true),
+                    Operation.WriteFile(outputFileC, doNotInfer: true),
+                });
+            builder.AddOutputDirectory(outputDirectory);
+            builder.AddUntrackedFile(outputFileC);
+            var process = SchedulePipBuilder(builder);
+
+            RunScheduler().AssertSuccess();
+
+            XAssert.IsTrue(File.Exists(ArtifactToString(outputFileC)));
+
+            XAssert.IsTrue(File.Exists(ArtifactToString(outputFileA)));
+            XAssert.IsTrue(File.Exists(ArtifactToString(outputFileB)));
+
+            File.WriteAllText(ArtifactToString(outputFileC), "Modified-C");
+
+            var result = RunScheduler();
+
+            if (Configuration.Schedule.IncrementalScheduling)
+            {
+                result.AssertNotScheduled(process.Process.PipId);
+
+                // Pip is not scheduled, output fileC is still there.
+                XAssert.IsTrue(File.Exists(ArtifactToString(outputFileC)));
+            }
+            else
+            {
+                result.AssertCacheHit(process.Process.PipId);
+
+                // Since output fileC is untracked, it should no longer exist when pip replayed outputs from cache.
+                XAssert.IsFalse(File.Exists(ArtifactToString(outputFileC)));
+            }
+
+            XAssert.IsTrue(File.Exists(ArtifactToString(outputFileA)));
+            XAssert.IsTrue(File.Exists(ArtifactToString(outputFileB)));
+        }
+
+        [Fact]
+        public void OutputDirectoryWithUntrackedScopes()
+        {
+            var outputDirectory = CreateUniqueObjPath("outputDir");
+            var nestedOutputDirectory = outputDirectory.Combine(Context.PathTable, "nested");
+            var outputFileA = CreateOutputFileArtifact(nestedOutputDirectory, "fileA");
+            var outputFileB = CreateOutputFileArtifact(nestedOutputDirectory, "fileB");
+            var outputFileC = CreateOutputFileArtifact(nestedOutputDirectory, "fileC");
+            var outputFileX = CreateOutputFileArtifact(outputDirectory, "fileX");
+            var outputFileY = CreateOutputFileArtifact(outputDirectory, "fileY");
+            var sourceFile = CreateSourceFile();
+
+            var builder = CreatePipBuilder(
+                new[]
+                {
+                    Operation.ReadFile(sourceFile),
+                    Operation.WriteFile(outputFileA, doNotInfer: true),
+                    Operation.WriteFile(outputFileB, doNotInfer: true),
+                    Operation.WriteFile(outputFileC),
+                    Operation.WriteFile(outputFileX, doNotInfer: true),
+                    Operation.WriteFile(outputFileY, doNotInfer: true),
+                });
+            builder.AddOutputDirectory(outputDirectory);
+            builder.AddUntrackedDirectoryScope(nestedOutputDirectory);
+            var process = SchedulePipBuilder(builder);
+
+            RunScheduler().AssertSuccess();
+
+            XAssert.IsTrue(File.Exists(ArtifactToString(outputFileA)));
+            XAssert.IsTrue(File.Exists(ArtifactToString(outputFileB)));
+            XAssert.IsTrue(File.Exists(ArtifactToString(outputFileC)));
+            XAssert.IsTrue(File.Exists(ArtifactToString(outputFileX)));
+            XAssert.IsTrue(File.Exists(ArtifactToString(outputFileY)));
+
+            var result = RunScheduler();
+
+            if (Configuration.Schedule.IncrementalScheduling)
+            {
+                result.AssertNotScheduled(process.Process.PipId);
+
+                XAssert.IsTrue(File.Exists(ArtifactToString(outputFileA)));
+                XAssert.IsTrue(File.Exists(ArtifactToString(outputFileB)));
+            }
+            else
+            {
+                result.AssertCacheHit(process.Process.PipId);
+
+                // Output fileA and fileB should no longer exist after cache replay, because nested directory is untracked.
+                XAssert.IsFalse(File.Exists(ArtifactToString(outputFileA)));
+                XAssert.IsFalse(File.Exists(ArtifactToString(outputFileB)));
+            }
+
+            // Although fileC is inside untracked nested directory, but it is specified as an output file, and so
+            // it has to exist after cache replay.
+            XAssert.IsTrue(File.Exists(ArtifactToString(outputFileC)));
+
+            XAssert.IsTrue(File.Exists(ArtifactToString(outputFileX)));
+            XAssert.IsTrue(File.Exists(ArtifactToString(outputFileY)));
+
+            File.WriteAllText(ArtifactToString(outputFileC), "Modified-C");
+
+            result = RunScheduler();
+            result.AssertScheduled(process.Process.PipId).AssertCacheHit(process.Process.PipId);
+
+            // Output fileA and fileB should no longer exist after cache replay, because nested directory is untracked.
+            XAssert.IsFalse(File.Exists(ArtifactToString(outputFileA)));
+            XAssert.IsFalse(File.Exists(ArtifactToString(outputFileB)));
+
+            XAssert.IsTrue(File.Exists(ArtifactToString(outputFileC)));
+
+            XAssert.IsTrue(File.Exists(ArtifactToString(outputFileX)));
+            XAssert.IsTrue(File.Exists(ArtifactToString(outputFileY)));
+        }
+
+        [Fact]
+        public void EnumerateOutputDirectoryWithUntrackedPaths()
+        {
+            var outputDirectory = CreateUniqueObjPath("outputDir");
+            var outputFileA = CreateOutputFileArtifact(outputDirectory, "fileA");
+            var outputFileB = CreateOutputFileArtifact(outputDirectory, "fileB");
+            var outputFileC = CreateOutputFileArtifact(outputDirectory, "fileC");
+            var sourceFile = CreateSourceFile();
+
+            var builderA = CreatePipBuilder(
+                new[]
+                {
+                    Operation.ReadFile(sourceFile),
+                    Operation.WriteFile(outputFileA, doNotInfer: true),
+                    Operation.WriteFile(outputFileB, doNotInfer: true),
+                    Operation.WriteFile(outputFileC, doNotInfer: true),
+                });
+            builderA.AddOutputDirectory(outputDirectory);
+            builderA.AddUntrackedFile(outputFileC);
+            var processA = SchedulePipBuilder(builderA);
+
+            var builderB = CreatePipBuilder(
+                new[]
+                {
+                    Operation.ReadFile(sourceFile),
+                    Operation.EnumerateDir(processA.ProcessOutputs.GetOpaqueDirectory(outputDirectory)),
+                    Operation.WriteFile(CreateOutputFileArtifact()),
+                });
+            builderB.AddInputDirectory(processA.ProcessOutputs.GetOpaqueDirectory(outputDirectory));
+            var processB = SchedulePipBuilder(builderB);
+
+            // In this 1st build, process B will see the output directory outputDir consisting of
+            // outputDir\A, outputDir\B, and outputDirC.
+            RunScheduler().AssertSuccess();
+
+            var result = RunScheduler();
+
+            if (Configuration.Schedule.IncrementalScheduling)
+            {
+                result.AssertNotScheduled(processA.Process.PipId, processB.Process.PipId);
+            }
+            else
+            {
+                // In the 2nd build, since process A comes from the cache, outputDir will only consist of
+                // outputDir\A and outputDir\B. Thus, in principle process B should re-run. However, due to
+                // graph file system enumeration, process B gets a cache hit.
+                RunScheduler().AssertCacheHit(processA.Process.PipId, processB.Process.PipId);
+            }
+        }
     }
 }

Public/Src/Engine/UnitTests/Scheduler/FileSystemViewTests.cs

@@ -272,7 +272,10 @@ namespace Test.BuildXL.Scheduler
                 }
             }
 
-            public Possible<PathExistence> TryEnumerateDirectoryAndTrackMembership(AbsolutePath path, Action<string, FileAttributes> handleEntry)
+            public Possible<PathExistence> TryEnumerateDirectoryAndTrackMembership(
+                AbsolutePath path,
+                Action<string, FileAttributes> handleEntry,
+                Func<string, FileAttributes, bool> shouldIncludeEntry)
             {
                 AssertAdd(EnumeratedDirectories, path);
 

Public/Src/Engine/UnitTests/Scheduler/IncrementalSchedulingTests.cs

@@ -928,7 +928,6 @@ ENDLOCAL && EXIT /b 1
                 LoggingContext,
                 m_volumeMap,
                 m_journal,
-                m_configuration.Engine.FileChangeTrackerSupersedeMode,
                 fileChangeTrackerPath,
                 engineFingerprint,
                 out fileChangeTracker);
@@ -1108,7 +1107,6 @@ ENDLOCAL && EXIT /b 1
                 LoggingContext,
                 m_volumeMap,
                 m_journal,
-                m_configuration.Engine.FileChangeTrackerSupersedeMode,
                 null);
             IIncrementalSchedulingState iss = CreateNewState(tracker.FileEnvelopeId, pipGraph);
 

Public/Src/Sandbox/Windows/DetoursServices/DataTypes.h

@@ -158,7 +158,7 @@ enum FileAccessStatus
 };
 
 // Keep this in sync with the C# version declared in ReportType.cs
-enum ProcessDetouringStatus
+enum class ProcessDetouringStatus
 {
     ProcessDetouringStatus_None = 0,
     ProcessDetouringStatus_Starting = 1,
@@ -172,7 +172,7 @@ enum ProcessDetouringStatus
 };
 
 // Keep this in sync with the C# version declared in ReportType.cs
-enum ReportType
+enum class ReportType
 {
     ReportType_None = 0,
     ReportType_FileAccess = 1,
@@ -694,7 +694,7 @@ typedef const ManifestRecord * PCManifestRecord; // duplicated for use in scopes
 // Characterization of the currently running process
 // These are special processes for which we remove some artificial file accesses from reporting.
 // We should not detour anything if the process is WinDbg.
-enum SpecialProcessKind {
+enum class SpecialProcessKind {
     NotSpecial,
     WinDbg,
     RC,

Public/Src/Sandbox/Windows/DetoursServices/DebuggingHelpers.cpp

@@ -136,7 +136,7 @@ void Dbg(PCWSTR format, ...)
         return;
     }
 
-    std::wstring report = DebugStringFormat(L"%d,", ReportType_DebugMessage);
+    std::wstring report = DebugStringFormat(L"%d,", ReportType::ReportType_DebugMessage);
     report.append(resultArgs);
     report.append(L"\r\n");
 

Public/Src/Sandbox/Windows/DetoursServices/DetouredFunctions.cpp

@@ -2723,7 +2723,7 @@ HANDLE WINAPI Detoured_CreateFileW(
 
                 ReportFileAccess(
                     operationContext,
-                    FileAccessStatus_Allowed,
+                    FileAccessStatus::FileAccessStatus_Allowed,
                     policyResult,
                     AccessCheckResult(RequestedAccess::None, ResultAction::Deny, ReportLevel::Report),
                     0,
@@ -2817,7 +2817,7 @@ HANDLE WINAPI Detoured_CreateFileW(
 
             ReportFileAccess(
                 opContext,
-                FileAccessStatus_CannotDeterminePolicy,
+                FileAccessStatus::FileAccessStatus_CannotDeterminePolicy,
                 policyResult,
                 AccessCheckResult(RequestedAccess::None, ResultAction::Deny, ReportLevel::Report),
                 getUsnError,
@@ -5989,7 +5989,7 @@ NTSTATUS NTAPI Detoured_ZwCreateFile(
 
                 ReportFileAccess(
                     operationContext,
-                    FileAccessStatus_Allowed,
+                    FileAccessStatus::FileAccessStatus_Allowed,
                     policyResult,
                     AccessCheckResult(RequestedAccess::None, ResultAction::Deny, ReportLevel::Report),
                     0,
@@ -6285,7 +6285,7 @@ NTSTATUS NTAPI Detoured_NtCreateFile(
 
                 ReportFileAccess(
                     operationContext,
-                    FileAccessStatus_Allowed,
+                    FileAccessStatus::FileAccessStatus_Allowed,
                     policyResult,
                     AccessCheckResult(RequestedAccess::None, ResultAction::Deny, ReportLevel::Report),
                     0,
@@ -6563,7 +6563,7 @@ NTSTATUS NTAPI Detoured_ZwOpenFile(
 
                 ReportFileAccess(
                     operationContext,
-                    FileAccessStatus_Allowed,
+                    FileAccessStatus::FileAccessStatus_Allowed,
                     policyResult,
                     AccessCheckResult(RequestedAccess::None, ResultAction::Deny, ReportLevel::Report),
                     0,

Public/Src/Sandbox/Windows/DetoursServices/DetoursHelpers.cpp

@@ -991,7 +991,7 @@ bool ParseFileAccessManifest(
 
         ReportFileAccess(
             fileOperationContextWithoutModuleName,
-            FileAccessStatus_CannotDeterminePolicy,
+            FileAccessStatus::FileAccessStatus_CannotDeterminePolicy,
             PolicyResult(), // Indeterminate
             AccessCheckResult(RequestedAccess::None, ResultAction::Deny, ReportLevel::Report),
             GetLastError(),

Public/Src/Sandbox/Windows/DetoursServices/DetoursServices.cpp

@@ -515,7 +515,7 @@ InternalCreateDetouredProcess(
     if (LogProcessDetouringStatus())
     {
         ReportProcessDetouringStatus(
-            ProcessDetouringStatus_Starting,
+            ProcessDetouringStatus::ProcessDetouringStatus_Starting,
             lpApplicationName,
             lpCommandLine,
             needsInjection,
@@ -640,7 +640,7 @@ InternalCreateDetouredProcess(
     if (LogProcessDetouringStatus())
     {
         ReportProcessDetouringStatus(
-            ProcessDetouringStatus_Done,
+            ProcessDetouringStatus::ProcessDetouringStatus_Done,
             lpApplicationName,
             lpCommandLine,
             needsInjection,
@@ -808,7 +808,7 @@ static CreateDetouredProcessStatus CreateProcessAttributes(
         if (LogProcessDetouringStatus())
         {
             ReportProcessDetouringStatus(
-                ProcessDetouringStatus_Done,
+                ProcessDetouringStatus::ProcessDetouringStatus_Done,
                 L"",
                 (LPWSTR)lpcwCommandLine,
                 0,
@@ -839,7 +839,7 @@ static CreateDetouredProcessStatus CreateProcessAttributes(
         if (LogProcessDetouringStatus())
         {
             ReportProcessDetouringStatus(
-                ProcessDetouringStatus_Done,
+                ProcessDetouringStatus::ProcessDetouringStatus_Done,
                 L"",
                 (LPWSTR)lpcwCommandLine,
                 0,
@@ -869,7 +869,7 @@ static CreateDetouredProcessStatus CreateProcessAttributes(
             if (LogProcessDetouringStatus())
             {
                 ReportProcessDetouringStatus(
-                    ProcessDetouringStatus_Done,
+                    ProcessDetouringStatus::ProcessDetouringStatus_Done,
                     L"",
                     (LPWSTR)lpcwCommandLine,
                     0,

Public/Src/Sandbox/Windows/DetoursServices/FileAccessHelpers.h

@@ -90,7 +90,7 @@ private:
     static unsigned long GetNextId();
 };
 
-enum FileExistence {
+enum class FileExistence {
     Existent,
     Nonexistent,
     InvalidPath,
@@ -113,19 +113,19 @@ public:
     void InferExistenceFromNtStatus(NTSTATUS status);
 };
 
-enum ReportLevel {
+enum class ReportLevel {
     Ignore,
     Report,
     ReportExplicit
 };
 
-enum ResultAction {
+enum class ResultAction {
     Allow,
     Deny,
     Warn
 };
 
-enum PathValidity {
+enum class PathValidity {
     Valid,
     // We observed ERROR_PATH_NOT_FOUND (not ERROR_FILE_NOT_FOUND); unfortunately this is possible
     // with C:\foo\"bar" where C:\foo doesn't exist; if it did, we'd get ERROR_INVALID_NAME for "bar".
@@ -189,7 +189,7 @@ public:
     
     // Returns a corresponding report line status. Note that warning-level access failures (allowed to proceed) map to FileAccessStatus_Denied.
     FileAccessStatus GetFileAccessStatus() const {
-        return (Result != ResultAction::Allow) ? FileAccessStatus_Denied : FileAccessStatus_Allowed;
+        return Result != ResultAction::Allow ? FileAccessStatus::FileAccessStatus_Denied : FileAccessStatus::FileAccessStatus_Allowed;
     }
 
     // Indicates if access to a file should be denied entirely (i.e., return an invalid handle and some error such as ERROR_ACCESS_DENIED).

Public/Src/Sandbox/Windows/DetoursServices/PolicyResult.cpp

@@ -97,7 +97,7 @@ void PolicyResult::ReportIndeterminatePolicyAndSetLastError(FileOperationContext
 
     ReportFileAccess(
         fileOperationContext,
-        FileAccessStatus_CannotDeterminePolicy,
+        FileAccessStatus::FileAccessStatus_CannotDeterminePolicy,
         *this,
         fakeAccessCheck,
         ERROR_SUCCESS,
@@ -135,17 +135,14 @@ bool PolicyResult::AllowWrite() const {
             // Observe this implies that in this case we never block accesses on detours based on file existence, but generate a DFA on managed code
             bool fileExists = ExistsAsFile(m_canonicalizedPath.GetPathString());
 
-            AccessCheckResult accessCheck = AccessCheckResult(RequestedAccess::Read, ResultAction::Allow, ReportLevel::Ignore);
-            FileOperationContext operationContext =
+            FileOperationContext operationContext = 
                 FileOperationContext::CreateForRead(L"FirstAllowWriteCheckInProcess", this->GetCanonicalizedPath().GetPathString());
 
             ReportFileAccess(
                 operationContext,
-                fileExists?
-                    FileAccessStatus_Denied :
-                    FileAccessStatus_Allowed,
+                fileExists ? FileAccessStatus::FileAccessStatus_Denied : FileAccessStatus::FileAccessStatus_Allowed,
                 *this,
-                AccessCheckResult(RequestedAccess::None, ResultAction::Deny, ReportLevel::Report),
+                AccessCheckResult(RequestedAccess::Write, fileExists ? ResultAction::Deny : ResultAction::Allow, ReportLevel::Report),
                 0,
                 -1);
 

Public/Src/Sandbox/Windows/DetoursServices/PolicyResult_common.cpp

@@ -99,8 +99,8 @@ AccessCheckResult PolicyResult::CheckReadAccess(RequestedReadAccess readAccessRe
         : (FailUnexpectedFileAccesses() ? ResultAction::Deny : ResultAction::Warn);
 
     bool explicitReport = !context.OpenedDirectory &&
-        ((exists && ((m_policy & FileAccessPolicy_ReportAccessIfExistent) != 0)) ||
-         (!exists && ((m_policy & FileAccessPolicy_ReportAccessIfNonExistent) != 0)));
+        ((exists && ((m_policy & FileAccessPolicy::FileAccessPolicy_ReportAccessIfExistent) != 0)) ||
+         (!exists && ((m_policy & FileAccessPolicy::FileAccessPolicy_ReportAccessIfNonExistent) != 0)));
 
     ReportLevel reportLevel = explicitReport 
         ? ReportLevel::ReportExplicit 
@@ -151,7 +151,7 @@ AccessCheckResult PolicyResult::CreateAccessCheckResult(bool isAllowed) const
         ? ResultAction::Allow
         : (FailUnexpectedFileAccesses() ? ResultAction::Deny : ResultAction::Warn);
 
-    ReportLevel reportLevel = ((m_policy & FileAccessPolicy_ReportAccess) != 0)
+    ReportLevel reportLevel = ((m_policy & FileAccessPolicy::FileAccessPolicy_ReportAccess) != 0)
         ? ReportLevel::ReportExplicit
         : (ReportAnyAccess(result != ResultAction::Allow) ? ReportLevel::Report : ReportLevel::Ignore);
 

Public/Src/Sandbox/Windows/DetoursServices/SendReport.cpp

@@ -147,7 +147,7 @@ void ReportFileAccess(
         std::replace(commandLine.begin(), commandLine.end(), L'\n', L' ');
 
         constructReportResult = swprintf_s(report.get(), reportBufferSize, L"%d,%s:%lx|%lx|%lx|%x|%x|%x|%lx|%llx|%lx|%lx|%lx|%lx|%lx|%s|%s|%s\r\n",
-            ReportType_FileAccess,
+            ReportType::ReportType_FileAccess,
             fileOperationContext.Operation,
             g_currentProcessId,
             fileOperationContext.Id,
@@ -169,7 +169,7 @@ void ReportFileAccess(
     else
     {
         constructReportResult = swprintf_s(report.get(), reportBufferSize, L"%d,%s:%lx|%lx|%lx|%x|%x|%x|%lx|%llx|%lx|%lx|%lx|%lx|%lx|%s|%s\r\n",
-            ReportType_FileAccess,
+            ReportType::ReportType_FileAccess,
             fileOperationContext.Operation,
             g_currentProcessId,
             fileOperationContext.Id,
@@ -270,7 +270,7 @@ void ReportProcessDetouringStatus(
 
 #pragma warning(suppress: 4826)
     int const constructReportResult = swprintf_s(report.get(), reportBufferSize, L"%u,%lu|%u|%s|%s|%u|%u|%u|%u|%u|%llu|%u|%u|%u|%u|%u|%s\r\n",
-        ReportType_ProcessDetouringStatus,
+        ReportType::ReportType_ProcessDetouringStatus,
         GetCurrentProcessId(),
         status,
         processName.get() != nullptr ? processName.get() : nullStringPtr,
@@ -351,7 +351,7 @@ void ReportProcessData(
     }
 
     int const constructReportResult = swprintf_s(report.get(), reportBufferSize, L"%u,%lu|%I64u|%I64u|%I64u|%I64u|%I64u|%I64u|%lu|%lu|%lu|%lu|%lu|%lu|%lu|%lu|%s|%lu|%lu|%I64u|%lu|%I64u|%lu|%I64u|%I64u\r\n",
-        ReportType_ProcessData,
+        ReportType::ReportType_ProcessData,
         GetCurrentProcessId(),
         ioCounters.ReadOperationCount,
         ioCounters.WriteOperationCount,

Public/Src/Tools/Execution.Analyzer/Analyzers.Partner/FileChangeTrackerAnalyzer.cs

@@ -69,7 +69,6 @@ namespace BuildXL.Execution.Analyzer
                 loggingContext,
                 null,
                 null,
-                default(Utilities.Configuration.FileChangeTrackerSupersedeMode?),
                 m_inputFile,
                 // if we do not pass the fingerprint, FileChangeTracker will not check for fingerprint match
                 null, 

Public/Src/Tools/Execution.Analyzer/Analyzers.Partner/IncrementalSchedulingStateAnalyzer.cs

@@ -69,7 +69,6 @@ namespace BuildXL.Execution.Analyzer
                 loggingContext,
                 null,
                 null,
-                default(Utilities.Configuration.FileChangeTrackerSupersedeMode?),
                 trackerFile,
                 null, 
                 out fileChangeTracker,

Public/Src/Utilities/Configuration/FileChangeTrackerSupersedeMode.cs

@@ -1,26 +0,0 @@
-// Copyright (c) Microsoft Corporation.
-// Licensed under the MIT License.
-
-namespace BuildXL.Utilities.Configuration
-{
-    /// <summary>
-    /// Supersede mode for file change tracker.
-    /// </summary>
-    public enum FileChangeTrackerSupersedeMode : byte
-    {
-        /// <summary>
-        /// Supersede on all paths including files and directories (or container paths).
-        /// </summary>
-        All = 0,
-
-        /// <summary>
-        /// Legacy mode where superseding is only applied to file paths.
-        /// </summary>
-        FileOnly = 1,
-
-        /// <summary>
-        /// In addition to superseding a file as in <see cref="FileOnly"/>, the file's parents are superseded with the file's USN.
-        /// </summary>
-        FileAndParents = 2,
-    }
-}

Public/Src/Utilities/Configuration/IEngineConfiguration.cs

@@ -151,11 +151,6 @@ namespace BuildXL.Utilities.Configuration
         /// </summary>
         FileChangeTrackerInitializationMode FileChangeTrackerInitializationMode { get; }
 
-        /// <summary>
-        /// Supersede mode for file change tracker.
-        /// </summary>
-        FileChangeTrackerSupersedeMode FileChangeTrackerSupersedeMode { get; }
-
         /// <summary>
         /// Whether to track the builds in a textfile in the user folder.
         /// </summary>

Public/Src/Utilities/Configuration/Mutable/EngineConfiguration.cs

@@ -30,7 +30,6 @@ namespace BuildXL.Utilities.Configuration.Mutable
             ScrubDirectories = new List<AbsolutePath>();
             CompressGraphFiles = false;
             FileChangeTrackerInitializationMode = FileChangeTrackerInitializationMode.ResumeExisting;
-            FileChangeTrackerSupersedeMode = FileChangeTrackerSupersedeMode.FileAndParents;
             LogStatistics = true;
             TrackBuildsInUserFolder = true;
             TrackGvfsProjections = false;
@@ -70,7 +69,6 @@ namespace BuildXL.Utilities.Configuration.Mutable
             ScrubDirectories = pathRemapper.Remap(template.ScrubDirectories);
             CompressGraphFiles = template.CompressGraphFiles;
             FileChangeTrackerInitializationMode = template.FileChangeTrackerInitializationMode;
-            FileChangeTrackerSupersedeMode = template.FileChangeTrackerSupersedeMode;
             LogStatistics = template.LogStatistics;
             TrackBuildsInUserFolder = template.TrackBuildsInUserFolder;
             TrackGvfsProjections = template.TrackGvfsProjections;
@@ -150,9 +148,6 @@ namespace BuildXL.Utilities.Configuration.Mutable
         /// <inheritdoc />
         public FileChangeTrackerInitializationMode FileChangeTrackerInitializationMode { get; set; }
 
-        /// <inheritdoc />
-        public FileChangeTrackerSupersedeMode FileChangeTrackerSupersedeMode { get; set; }
-
         /// <inheritdoc />
         public bool LogStatistics { get; set; }
 

Public/Src/Utilities/Storage/ChangeTracking/DirectoryMembershipTrackingFingerprinter.cs

@@ -56,7 +56,8 @@ namespace BuildXL.Storage.ChangeTracking
         /// </summary>
         public static Possible<DirectoryMembershipFingerprintResult> ComputeFingerprint(
             string path,
-            Action<string, FileAttributes> handleEntry = null)
+            Action<string, FileAttributes> handleEntry = null,
+            Func<string, FileAttributes, bool> shouldIncludeEntry = null)
         {
             Contract.Requires(!string.IsNullOrWhiteSpace(path));
 
@@ -67,9 +68,12 @@ namespace BuildXL.Storage.ChangeTracking
                 path,
                 (entryName, entryAttributes) =>
                 {
-                    calculator = calculator.Accumulate(entryName, entryAttributes);
-                    memberCount++;
-                    handleEntry?.Invoke(entryName, entryAttributes);
+                    if (shouldIncludeEntry?.Invoke(entryName, entryAttributes) ?? true)
+                    {
+                        calculator = calculator.Accumulate(entryName, entryAttributes);
+                        memberCount++;
+                        handleEntry?.Invoke(entryName, entryAttributes);
+                    }
                 });
 
             DirectoryMembershipTrackingFingerprint fingerprint = calculator.GetFingerprint();
@@ -151,13 +155,16 @@ namespace BuildXL.Storage.ChangeTracking
         /// <summary>
         /// Compute directory membership fingerprint given its members.
         /// </summary>
-        public static DirectoryMembershipTrackingFingerprint ComputeFingerprint(IReadOnlyList<(string, FileAttributes)> members)
+        public static DirectoryMembershipTrackingFingerprint ComputeFingerprint(
+            IReadOnlyList<(string, FileAttributes)> members,
+            Action<string, FileAttributes> handle = null)
         {
             var calculator = DirectoryMembershipTrackingFingerprint.CreateCalculator();
 
-            foreach (var valueTuple in members.AsStructEnumerable())
+            foreach (var (entry, attributes) in members.AsStructEnumerable())
             {
-                calculator = calculator.Accumulate(valueTuple.Item1, valueTuple.Item2);
+                calculator = calculator.Accumulate(entry, attributes);
+                handle?.Invoke(entry, attributes);
             }
 
             return calculator.GetFingerprint();

Public/Src/Utilities/Storage/ChangeTracking/FileChangeTracker.cs

@@ -13,7 +13,6 @@ using BuildXL.Native.IO;
 using BuildXL.Storage.ChangeJournalService;
 using BuildXL.Storage.Tracing;
 using BuildXL.Utilities;
-using BuildXL.Utilities.Configuration;
 using BuildXL.Utilities.Instrumentation.Common;
 using BuildXL.Utilities.Tracing;
 using Microsoft.Win32.SafeHandles;
@@ -62,8 +61,10 @@ namespace BuildXL.Storage.ChangeTracking
         /// <remarks>
         /// 10: Added All supersede mode.
         /// 11: Added FileAndParents supersede mode.
+        /// 12: Supersede directory tracking.
+        /// 13: Remove kinds of supersede mode.
         /// </remarks>
-        public static readonly FileEnvelope FileEnvelope = new FileEnvelope(name: nameof(FileChangeTracker), version: 11);
+        public static readonly FileEnvelope FileEnvelope = new FileEnvelope(name: nameof(FileChangeTracker), version: 13);
 
         private int m_trackingStateValue;
 
@@ -169,7 +170,6 @@ namespace BuildXL.Storage.ChangeTracking
             LoggingContext loggingContext,
             VolumeMap volumeMap,
             IChangeJournalAccessor journal,
-            FileChangeTrackerSupersedeMode supersedeMode,
             string buildEngineFingerprint,
             FileEnvelopeId? correlatedId = default)
         {
@@ -184,7 +184,7 @@ namespace BuildXL.Storage.ChangeTracking
                 FileChangeTrackingState.BuildingInitialChangeTrackingSet,
                 volumeMap,
                 journal,
-                FileChangeTrackingSet.CreateForAllCapableVolumes(loggingContext, volumeMap, journal, supersedeMode),
+                FileChangeTrackingSet.CreateForAllCapableVolumes(loggingContext, volumeMap, journal),
                 buildEngineFingerprint);
 
             foreach (var gvfsProjectionFile in volumeMap.GvfsProjections)
@@ -240,7 +240,6 @@ namespace BuildXL.Storage.ChangeTracking
             LoggingContext loggingContext,
             VolumeMap volumeMap,
             IChangeJournalAccessor journal,
-            FileChangeTrackerSupersedeMode supersedeMode,
             string path,
             string buildEngineFingerprint,
             out FileChangeTracker tracker)
@@ -262,7 +261,7 @@ namespace BuildXL.Storage.ChangeTracking
             {
 
                 // Note that TryLoad may throw in the event of spooky I/O errors.
-                var loadingTrackerResult = TryLoad(pm.LoggingContext, path, volumeMap, journal, supersedeMode, buildEngineFingerprint);
+                var loadingTrackerResult = TryLoad(pm.LoggingContext, path, volumeMap, journal, buildEngineFingerprint);
 
                 if (loadingTrackerResult.Succeeded)
                 {
@@ -285,7 +284,7 @@ namespace BuildXL.Storage.ChangeTracking
                     // Or, we might be unable to re-use the persisted state. In that case we start over. Note that there's nothing to do with the correlating save token here;
                     // on save, a new or existing token will be provided as appropriate.
                     // The reason of the failure is already logged in the TryLoad() method above.
-                    tracker = StartTrackingChanges(pm.LoggingContext, volumeMap, journal, supersedeMode, buildEngineFingerprint);
+                    tracker = StartTrackingChanges(pm.LoggingContext, volumeMap, journal, buildEngineFingerprint);
                 }
 
                 Logger.Log.LoadingChangeTracker(
@@ -309,7 +308,6 @@ namespace BuildXL.Storage.ChangeTracking
             LoggingContext loggingContext,
             VolumeMap volumeMap,
             IChangeJournalAccessor journal,
-            FileChangeTrackerSupersedeMode? supersedeMode,
             string path,
             string buildEngineFingerprint,
             out FileChangeTracker tracker,
@@ -332,7 +330,6 @@ namespace BuildXL.Storage.ChangeTracking
                     path,
                     volumeMap,
                     journal,
-                    supersedeMode,
                     buildEngineFingerprint,
                     loadForAllCapableVolumes: loadForAllCapableVolumes);
 
@@ -383,7 +380,6 @@ namespace BuildXL.Storage.ChangeTracking
             string path,
             VolumeMap volumeMap,
             IChangeJournalAccessor journal,
-            FileChangeTrackerSupersedeMode? supersedeMode,
             string buildEngineFingerprint,
             bool loadForAllCapableVolumes = true)
         {
@@ -452,7 +448,6 @@ namespace BuildXL.Storage.ChangeTracking
                                             reader,
                                             volumeMap,
                                             journal,
-                                            supersedeMode,
                                             stopwatch,
                                             loadForAllCapableVolumes);
                                     }
@@ -638,14 +633,18 @@ namespace BuildXL.Storage.ChangeTracking
         /// The membership of the directory will be invalidated if a name is added or removed directly inside the directory (i.e., when <c>FindFirstFile</c>
         /// and <c>FindNextFile</c> would see a different set of names).
         /// </summary>
-        public Possible<FileChangeTrackingSet.EnumerationResult> TryEnumerateDirectoryAndTrackMembership(string path, Action<string, FileAttributes> handleEntry)
+        public Possible<FileChangeTrackingSet.EnumerationResult> TryEnumerateDirectoryAndTrackMembership(
+            string path,
+            Action<string, FileAttributes> handleEntry,
+            Func<string, FileAttributes, bool> shouldIncludeEntry,
+            bool supersedeWithStrongIdentity)
         {
             Contract.Requires(!string.IsNullOrWhiteSpace(path));
             Contract.Requires(handleEntry != null);
 
             if (IsDisabledOrNullTrackingSet)
             {
-                var possibleFingerprintResult = DirectoryMembershipTrackingFingerprinter.ComputeFingerprint(path, handleEntry);
+                var possibleFingerprintResult = DirectoryMembershipTrackingFingerprinter.ComputeFingerprint(path, handleEntry, shouldIncludeEntry);
                 if (!possibleFingerprintResult.Succeeded)
                 {
                     return possibleFingerprintResult.Failure;
@@ -661,7 +660,10 @@ namespace BuildXL.Storage.ChangeTracking
             // (and if additional tracking succeeds while disabled, there's no harm).
             Possible<FileChangeTrackingSet.EnumerationResult> possibleEnumerationResult = m_changeTrackingSet.TryEnumerateDirectoryAndTrackMembership(
                 path,
-                handleEntry);
+                handleEntry,
+                shouldIncludeEntry,
+                fingerprintFilter: null,
+                supersedeWithStrongIdentity: supersedeWithStrongIdentity);
 
             if (!possibleEnumerationResult.Succeeded)
             {
@@ -677,7 +679,8 @@ namespace BuildXL.Storage.ChangeTracking
         /// </summary>
         public Possible<FileChangeTrackingSet.EnumerationResult> TryTrackDirectoryMembership(
             string path,
-            IReadOnlyList<(string, FileAttributes)> members)
+            IReadOnlyList<(string, FileAttributes)> members,
+            bool supersedeWithStrongIdentity)
         {
             Contract.Requires(!string.IsNullOrWhiteSpace(path));
 
@@ -690,7 +693,10 @@ namespace BuildXL.Storage.ChangeTracking
                     new Failure<string>("Tracking set is disabled"));
             }
 
-            Possible<FileChangeTrackingSet.EnumerationResult> possibleEnumerationResult = m_changeTrackingSet.TryTrackDirectoryMembership(path, members);
+            Possible<FileChangeTrackingSet.EnumerationResult> possibleEnumerationResult = m_changeTrackingSet.TryTrackDirectoryMembership(
+                path,
+                members,
+                supersedeWithStrongIdentity);
 
             if (!possibleEnumerationResult.Succeeded)
             {
@@ -855,8 +861,7 @@ namespace BuildXL.Storage.ChangeTracking
                     m_changeTrackingSet = FileChangeTrackingSet.CreateForAllCapableVolumes(
                         m_loggingContext,
                         m_volumeMap,
-                        m_journal,
-                        m_changeTrackingSet.SupersedeMode);
+                        m_journal);
                 }
 
                 ReportProcessChangesCompletion(scanningJournalResult);

Public/Src/Utilities/Storage/ChangeTracking/FileChangeTrackingSet.cs

@@ -17,7 +17,6 @@ using BuildXL.Storage.ChangeJournalService;
 using BuildXL.Storage.ChangeJournalService.Protocol;
 using BuildXL.Storage.Tracing;
 using BuildXL.Utilities;
-using BuildXL.Utilities.Collections;
 using BuildXL.Utilities.Configuration;
 using BuildXL.Utilities.Instrumentation.Common;
 using BuildXL.Utilities.Tasks;
@@ -170,19 +169,13 @@ namespace BuildXL.Storage.ChangeTracking
         /// </remarks>
         private static readonly int s_maxConcurrencyForRecordProcessing = Math.Max(16, 256 / Environment.ProcessorCount);
 
-        /// <summary>
-        /// Supersede mode.
-        /// </summary>
-        public FileChangeTrackerSupersedeMode SupersedeMode { get; private set; }
-
         private FileChangeTrackingSet(
             LoggingContext loggingContext,
             PathTable internalPathTable,
             VolumeMap volumeMap,
             Dictionary<ulong, SingleVolumeFileChangeTrackingSet> perVolumeChangeTrackingSets,
             ConcurrentDictionary<AbsolutePath, DirectoryMembershipTrackingFingerprint> membershipFingerprints,
-            bool hasNewFileOrCheckpointData,
-            FileChangeTrackerSupersedeMode supersedeMode)
+            bool hasNewFileOrCheckpointData)
         {
             Contract.Requires(loggingContext != null);
             Contract.Requires(volumeMap != null);
@@ -201,8 +194,6 @@ namespace BuildXL.Storage.ChangeTracking
             {
                 trackingSet.OwningFileChangeTrackingSet = this;
             }
-
-            SupersedeMode = supersedeMode;
         }
 
         /// <summary>
@@ -230,8 +221,7 @@ namespace BuildXL.Storage.ChangeTracking
         public static FileChangeTrackingSet CreateForAllCapableVolumes(
             LoggingContext loggingContext,
             VolumeMap volumeMap,
-            IChangeJournalAccessor journalAccessor,
-            FileChangeTrackerSupersedeMode supersedeMode)
+            IChangeJournalAccessor journalAccessor)
         {
             Contract.Requires(loggingContext != null);
             Contract.Requires(volumeMap != null);
@@ -248,7 +238,6 @@ namespace BuildXL.Storage.ChangeTracking
                 membershipFingerprints: new ConcurrentDictionary<AbsolutePath, DirectoryMembershipTrackingFingerprint>(),
                 knownCheckpoints: null,
                 hasNewFileOrCheckpointData: true,
-                supersedeMode: supersedeMode,
                 trackedJournalsSizeBytes: out _,
                 nextUsns: out _);
         }
@@ -265,7 +254,6 @@ namespace BuildXL.Storage.ChangeTracking
             VolumeMap volumeMap,
             IChangeJournalAccessor journalAccessor,
             bool hasNewFileOrCheckpointData,
-            FileChangeTrackerSupersedeMode supersedeMode,
             PathTable internalPathTable,
             ConcurrentDictionary<AbsolutePath, DirectoryMembershipTrackingFingerprint> membershipFingerprints,
             Dictionary<ulong, Usn> knownCheckpoints,
@@ -296,8 +284,8 @@ namespace BuildXL.Storage.ChangeTracking
                 }
 
                 QueryUsnJournalResult journalStatus = response.Response;
-                Usn? maybeNextUsn = default(Usn?);
-                Usn? maybeCheckpoint = default(Usn?);
+                Usn? maybeNextUsn = default;
+                Usn? maybeCheckpoint = default;
 
                 if (journalStatus.Status == QueryUsnJournalStatus.Success)
                 {
@@ -344,15 +332,13 @@ namespace BuildXL.Storage.ChangeTracking
                 volumeMap: volumeMap,
                 perVolumeChangeTrackingSets: perVolumeChangeTrackingSets,
                 membershipFingerprints: membershipFingerprints,
-                hasNewFileOrCheckpointData: hasNewFileOrCheckpointData,
-                supersedeMode: supersedeMode);
+                hasNewFileOrCheckpointData: hasNewFileOrCheckpointData);
         }
 
         private static FileChangeTrackingSet CreateForKnownVolumesFromPriorCheckpoint(
             LoggingContext loggingContext,
             VolumeMap volumeMap,
             bool hasNewFileOrCheckpointData,
-            FileChangeTrackerSupersedeMode supersedeMode,
             PathTable internalPathTable,
             ConcurrentDictionary<AbsolutePath, DirectoryMembershipTrackingFingerprint> membershipFingerprints,
             Dictionary<ulong, ulong> knownJournalIds,
@@ -405,8 +391,7 @@ namespace BuildXL.Storage.ChangeTracking
                 volumeMap: volumeMap,
                 perVolumeChangeTrackingSets: perVolumeChangeTrackingSets,
                 membershipFingerprints: membershipFingerprints,
-                hasNewFileOrCheckpointData: hasNewFileOrCheckpointData,
-                supersedeMode: supersedeMode);
+                hasNewFileOrCheckpointData: hasNewFileOrCheckpointData);
         }
 
         /// <summary>
@@ -898,6 +883,20 @@ namespace BuildXL.Storage.ChangeTracking
                                 TryEnumerateDirectoryAndTrackMembership(
                                     expandedPath,
                                     (name, attributes) => { },
+                                    // Passing null here by default makes the enumeration include all directory entries into the membership. 
+                                    // Output directories of pips can have untracked scopes/paths. Since file change tracking set does not
+                                    // have any knowledge about pips, this can cause unnecassary invalidation of incremental scheduling.
+                                    //
+                                    // For example, suppose that a pip has an output directory whose content is D\E\1.txt and D\E\2.txt, and
+                                    // the latter is specified to be untracked by the pip. Pip executor and file content manager has knowledge
+                                    // about pip, and thus it can exclude D\E\2.txt from the membership.
+                                    // Now, if there is a change that potentially updates the membership of D\E, e.g., adding D\E\3.txt and deleting
+                                    // it again, then, although in principle the membership of D\E does not change in the eye of the pip,
+                                    // but since the file change tracking set does not have any knowledge about the pip, it will signal that the
+                                    // membership of D\E has changed because it includes D\E\2.txt in the computed membership.
+                                    //
+                                    // We consider such an above change rare.
+                                    null,
                                     fingerprintFilter: directoryAndFingerprint.Value);
 
                             bool shouldEmitChange = true;
@@ -1381,7 +1380,9 @@ namespace BuildXL.Storage.ChangeTracking
         public Possible<EnumerationResult> TryEnumerateDirectoryAndTrackMembership(
             string path,
             Action<string, FileAttributes> handleEntry,
-            DirectoryMembershipTrackingFingerprint? fingerprintFilter = null)
+            Func<string, FileAttributes, bool> shouldIncludeEntry = null,
+            DirectoryMembershipTrackingFingerprint? fingerprintFilter = null,
+            bool supersedeWithStrongIdentity = false)
         {
             using (Counters.StartStopwatch(FileChangeTrackingCounter.TryEnumerateDirectoryAndTrackMembershipTime))
             {
@@ -1395,7 +1396,11 @@ namespace BuildXL.Storage.ChangeTracking
                 Func<SafeFileHandle, bool> enumerateAndCheckFingerprint =
                     handle =>
                     {
-                        var possibleFingerprintResult = DirectoryMembershipTrackingFingerprinter.ComputeFingerprint(path, handleEntry);
+                        var possibleFingerprintResult = DirectoryMembershipTrackingFingerprinter.ComputeFingerprint(
+                            path,
+                            handleEntry,
+                            shouldIncludeEntry);
+
                         if (!possibleFingerprintResult.Succeeded)
                         {
                             possibleEnumeration = possibleFingerprintResult.Failure;
@@ -1425,14 +1430,15 @@ namespace BuildXL.Storage.ChangeTracking
                         ExistenceTrackingFilter.TrackAlways,
                         existentFileFilter: enumerateAndCheckFingerprint);
 
-                return TrackDirectoryMembership(possibleTrackAndOpenResult, possibleEnumeration, internalPath);
+                return TrackDirectoryMembership(possibleTrackAndOpenResult, possibleEnumeration, internalPath, supersedeWithStrongIdentity);
             }
         }
 
         private Possible<EnumerationResult> TrackDirectoryMembership(
             Possible<TrackAndOpenResult> possibleTrackAndOpenResult,
             Possible<(DirectoryMembershipTrackingFingerprint, PathExistence)>? possibleEnumeration,
-            AbsolutePath internalPath)
+            AbsolutePath internalPath,
+            bool supersedeWithStrongIdentity)
         {
             return possibleTrackAndOpenResult.Then(
                 trackAndOpenResult =>
@@ -1454,6 +1460,14 @@ namespace BuildXL.Storage.ChangeTracking
                                         // TODO: trackAndOpenResult should have a subscription on it.
                                         //       We have an existent path and tracking succeeded, so we can safely invent one here.
                                         TrackDirectoryMembership(new FileChangeTrackingSubscription(internalPath), enumeration.Item1);
+
+                                        if (supersedeWithStrongIdentity)
+                                        {
+                                            Analysis.IgnoreResult(TryTrackChangesToFileInternal(
+                                                trackAndOpenResult.Handle,
+                                                internalPath,
+                                                updateMode: TrackingUpdateMode.Supersede));
+                                        }
                                     }
 
                                     // possibleEnumeration contains a (fingerprint, file-or-directory) pair.
@@ -1479,9 +1493,11 @@ namespace BuildXL.Storage.ChangeTracking
         /// </summary>
         public Possible<EnumerationResult> TryTrackDirectoryMembership(
             string path,
-            IReadOnlyList<(string, FileAttributes)> members)
+            IReadOnlyList<(string, FileAttributes)> members,
+            bool supersedeWithStrongIdentity = false)
         {
             AbsolutePath internalPath = AbsolutePath.Create(m_internalPathTable, path);
+
             DirectoryMembershipTrackingFingerprint fingerprint = DirectoryMembershipTrackingFingerprinter.ComputeFingerprint(members);
 
             Possible<TrackAndOpenResult> possibleTrackAndOpenResult = TryOpenAndTrackPathInternal(
@@ -1499,7 +1515,7 @@ namespace BuildXL.Storage.ChangeTracking
                     fingerprint,
                     possibleTrackAndOpenResult.Result.Existent ? PathExistence.ExistsAsDirectory : PathExistence.Nonexistent);
 
-            return TrackDirectoryMembership(possibleTrackAndOpenResult, possibleEnumeration, internalPath);
+            return TrackDirectoryMembership(possibleTrackAndOpenResult, possibleEnumeration, internalPath, supersedeWithStrongIdentity);
         }
 
         #endregion
@@ -2121,8 +2137,6 @@ namespace BuildXL.Storage.ChangeTracking
         {
             Contract.Requires(writer != null);
 
-            writer.Write((byte)SupersedeMode);
-
             m_internalPathTable.StringTable.Serialize(writer);
             m_internalPathTable.Serialize(writer);
             
@@ -2184,7 +2198,6 @@ namespace BuildXL.Storage.ChangeTracking
             BuildXLReader reader,
             VolumeMap volumeMap,
             IChangeJournalAccessor journal,
-            FileChangeTrackerSupersedeMode? supersedeMode,
             Stopwatch stopwatch,
             bool createForAllCapableVolumes = true)
         {
@@ -2193,16 +2206,6 @@ namespace BuildXL.Storage.ChangeTracking
             Contract.Requires(!createForAllCapableVolumes || volumeMap != null);
             Contract.Requires(!createForAllCapableVolumes || journal != null);
 
-            byte sm = reader.ReadByte();
-            Contract.Assert(Enum.IsDefined(typeof(FileChangeTrackerSupersedeMode), sm));
-
-            var readSupersedeMode = (FileChangeTrackerSupersedeMode)sm;
-
-            if (supersedeMode.HasValue && supersedeMode.Value != readSupersedeMode)
-            {
-                return LoadingTrackerResult.FailMismatchedSupersedeModes(supersedeMode.Value, readSupersedeMode);
-            }
-
             StringTable internalStringTable = StringTable.DeserializeAsync(reader).GetAwaiter().GetResult();
             PathTable internalPathTable = PathTable.DeserializeAsync(reader, Task.FromResult(internalStringTable)).GetAwaiter().GetResult();
 
@@ -2260,7 +2263,6 @@ namespace BuildXL.Storage.ChangeTracking
                     internalPathTable: internalPathTable,
                     membershipFingerprints: membershipFingerprints,
                     hasNewFileOrCheckpointData: true,
-                    supersedeMode: readSupersedeMode,
                     knownCheckpoints: knownCheckpoints,
                     trackedJournalsSizeBytes: out trackedJournalsSizeBytes,
                     nextUsns: out nextUsns);
@@ -2275,8 +2277,7 @@ namespace BuildXL.Storage.ChangeTracking
                     internalPathTable: internalPathTable,
                     membershipFingerprints: membershipFingerprints,
                     knownJournalIds: knownJournalIds,
-                    knownCheckpoints: knownCheckpoints,
-                    supersedeMode: readSupersedeMode);
+                    knownCheckpoints: knownCheckpoints);
             }
 
             foreach (var knownVolumeSerial in trackedVolumeSerials)
@@ -2329,8 +2330,6 @@ namespace BuildXL.Storage.ChangeTracking
         {
             Contract.Requires(writer != null);
 
-            writer.WriteLine(I($"Supersede mode: {SupersedeMode}"));
-
             foreach (var singleVolumeFileChangeTrackingSet in m_perVolumeChangeTrackingSets)
             {
                 singleVolumeFileChangeTrackingSet.Value.WriteText(writer);
@@ -2484,8 +2483,6 @@ namespace BuildXL.Storage.ChangeTracking
             /// </summary>
             public FileChangeTrackingSet OwningFileChangeTrackingSet { get; set; }
 
-            private FileChangeTrackerSupersedeMode SupersedeMode => OwningFileChangeTrackingSet.SupersedeMode;
-
             /// <summary>
             /// Creates an instance of <see cref="SingleVolumeFileChangeTrackingSet"/>.
             /// </summary>
@@ -2591,8 +2588,7 @@ namespace BuildXL.Storage.ChangeTracking
                     bool isPrimaryPath = currentPathHierarchicalNameId == path.Value;
                     isPathContainer = !isPrimaryPath || isPathContainer;
 
-                    if ((containerOfCurrentPathAndFlagsOfCurrentPath.flags & Tracked) != 0
-                        && (SupersedeMode == FileChangeTrackerSupersedeMode.FileOnly || SupersedeMode == FileChangeTrackerSupersedeMode.FileAndParents))
+                    if ((containerOfCurrentPathAndFlagsOfCurrentPath.flags & Tracked) != 0)
                     {
                         bool superseding = !isPathContainer && updateMode == TrackingUpdateMode.Supersede;
 
@@ -2611,11 +2607,6 @@ namespace BuildXL.Storage.ChangeTracking
                                 path,
                                 addValue: identity.Usn,
                                 updateValueFactory: (p, existingSupersessionUsn) => new Usn(Math.Max(identity.Usn.Value, existingSupersessionUsn.Value)));
-
-                            if (SupersedeMode == FileChangeTrackerSupersedeMode.FileAndParents)
-                            {
-                                SupersedeParentPaths(path, identity);
-                            }
                         }
 
                         // This path (and therefore all parent paths, by construction) have already been tracked.
@@ -2673,7 +2664,7 @@ namespace BuildXL.Storage.ChangeTracking
                                 Contract.Assert(otherVolumeTrackingSet != null);
 
                                 Possible<FileChangeTrackingSubscription> maybetrackingParent = otherVolumeTrackingSet.TryTrackChangesToParentPath(
-                                    SupersedeMode == FileChangeTrackerSupersedeMode.All ? updateMode : TrackingUpdateMode.Preserve,
+                                    TrackingUpdateMode.Preserve,
                                     currentPathHandle,
                                     currentPath,
                                     currentPathIdentity);
@@ -2689,19 +2680,7 @@ namespace BuildXL.Storage.ChangeTracking
                     }
 
                     AddRecordForFile(currentPathIdentity.FileId, currentPathIdentity.Usn, currentPath);
-
-                    if ((containerOfCurrentPathAndFlagsOfCurrentPath.flags & Tracked) != 0 && SupersedeMode == FileChangeTrackerSupersedeMode.All)
-                    {
-                        // Note that the intent of superseding is only relevant for path is already tracked (or else there is nothing to supersede).
-                        if (updateMode == TrackingUpdateMode.Supersede)
-                        {
-                            m_pathSupersessionLimits.AddOrUpdate(
-                                currentPath,
-                                addValue: currentPathIdentity.Usn,
-                                updateValueFactory: (p, existingSupersessionUsn) => new Usn(Math.Max(currentPathIdentity.Usn.Value, existingSupersessionUsn.Value)));
-                        }
-                    }
-
+                    
                     added = !isPathContainer ? true : added;
 
                     // We've succeeded in adding a file record sufficient to invalidate this path later. Marking the path notes this, so that we don't
@@ -2724,30 +2703,6 @@ namespace BuildXL.Storage.ChangeTracking
                 return new FileChangeTrackingSubscription(path);
             }
 
-            private void SupersedeParentPaths(AbsolutePath path, VersionedFileIdentity identity)
-            {
-                // Superseding parent paths does not work if parents belong to a different volume. However, it is fine to add
-                // an entry about that parent in the current volume's supersession limits because that supersession limit entry
-                // will never be considered during the journal scanning.
-                AbsolutePath parentPath = path.GetParent(m_internalPathTable);
-                HierarchicalNameId parentNameId = parentPath.Value;
-
-                foreach (HierarchicalNameId currentPathNameId in m_internalPathTable.EnumerateHierarchyBottomUp(parentNameId))
-                {
-                    (HierarchicalNameId nameId, HierarchicalNameTable.NameFlags flags) containerOfCurrentPathAndFlagsOfCurrentPath =
-                        m_internalPathTable.GetContainerAndFlags(currentPathNameId);
-
-                    if ((containerOfCurrentPathAndFlagsOfCurrentPath.flags & Tracked) != 0)
-                    {
-                        m_pathSupersessionLimits.AddOrUpdate(
-                            new AbsolutePath(currentPathNameId),
-                            addValue: identity.Usn,
-                            updateValueFactory: (p, existingSupersessionUsn) => new Usn(Math.Max(identity.Usn.Value, existingSupersessionUsn.Value)));
-                    }
-
-                }
-            }
-
             private void AddRecordForFile(FileId fileId, Usn usn, AbsolutePath path)
             {
                 while (true)
@@ -3229,9 +3184,9 @@ namespace BuildXL.Storage.ChangeTracking
                                         {
                                             // If childPath is non-existent, then it will be re-tracked by CheckAndMaybeInvalidateAntiDependencies.
                                             handleRelevantRecord(
-                                                    PathChanges.NewlyPresent,
-                                                    childPath,
-                                                    usnRecord);
+                                                PathChanges.NewlyPresent,
+                                                childPath,
+                                                usnRecord);
 
                                             if (FileUtilities.DirectoryExistsNoFollow(childPath.ToString(m_internalPathTable)))
                                             {
@@ -3253,14 +3208,40 @@ namespace BuildXL.Storage.ChangeTracking
                     }
 
                     // Deletion or Creation+Deletion may invalidate enumerated directories (the container itself).
-                    if (m_internalPathTable.SetFlags(containerPath.Value, Enumerated, clear: true))
+                    if (EmitMembershipChange(handleRelevantRecord, containerPath, ref usnRecord))
                     {
                         numberOfExistentialChanges++;
-                        handleRelevantRecord(PathChanges.MembershipChanged, containerPath, usnRecord);
                     }
                 }
             }
 
+            private bool EmitMembershipChange(
+                Action<PathChanges, AbsolutePath, UsnRecord> handle,
+                AbsolutePath containerPath,
+                ref UsnRecord record)
+            {
+                HierarchicalNameTable.NameFlags currentFlags = m_internalPathTable.GetContainerAndFlags(containerPath.Value).flags;
+
+                if ((Enumerated & currentFlags) == 0)
+                {
+                    return false;
+                }
+
+                Usn supersessionLowerBound;
+
+                if (m_pathSupersessionLimits.TryGetValue(containerPath, out supersessionLowerBound) &&
+                    record.Usn < supersessionLowerBound)
+                {
+                    return false;
+                }
+
+                m_internalPathTable.SetFlags(containerPath.Value, Enumerated, clear: true);
+                handle(PathChanges.MembershipChanged, containerPath, record);
+
+                return true;
+            }
+
+
             private bool EmitAllChangesIfTracked(
                 Action<PathChanges, AbsolutePath, UsnRecord> handle,
                 AbsolutePath path,
@@ -3309,15 +3290,13 @@ namespace BuildXL.Storage.ChangeTracking
                 }
 
                 // ifAnySet filter passed; for non-Container paths, we may additionally filter
-                // out changes preceding some supercede-level tracking. When path is a container and the superseding is applied
-                // to all subpaths (supersede mode == All), we distinguish whether the change is rename or delete. If it's a rename,
-                // then we should not supersede because the tracked children can be left orphan (see remarks on Container). If it's a delete,
-                // all the children will be deleted as well, and so they will be untracked automatically.
+                // out changes preceding some supercede-level tracking. When path is a container, we distinguish whether the change is rename or delete.
+                // If it's a rename, then we should not supersede because the tracked children can be left orphan (see remarks on Container).
+                // If it's a delete, all the children will be deleted as well, and so they will be untracked automatically.
                 bool shouldSupersede =
                     respectSupersession                                                 // Supersession limit should be respected,
                     && (((currentFlags & Container) == 0)                               // and, path is a file
-                        || (SupersedeMode != FileChangeTrackerSupersedeMode.FileOnly    //      or superseding is applied not only to file paths,
-                            && (record.Reason & UsnChangeReasons.RenameOldName) == 0)); //         but change is not rename/move.
+                        || (record.Reason & UsnChangeReasons.RenameOldName) == 0);      //      or change is not rename/move.
 
                 if (shouldSupersede)
                 {
@@ -3683,8 +3662,6 @@ namespace BuildXL.Storage.ChangeTracking
 
         private Usn m_checkPointUsn;
 
-        private (FileChangeTrackerSupersedeMode expected, FileChangeTrackerSupersedeMode loaded) m_supersedeModes;
-
         /// <summary>
         /// Return a description for each status
         /// </summary>
@@ -3713,8 +3690,6 @@ namespace BuildXL.Storage.ChangeTracking
                         return I($"Change tracking set could not be loaded due to build engine fingerprint mismatch.");
                     case LoadingTrackerStatus.JournalGoesBackInTime:
                         return I($"Journal goes back in time because the next Usn '{m_nextUsn}' is smaller than the checkpoint Usn '{m_checkPointUsn}'");
-                    case LoadingTrackerStatus.MismatchedSupersedeMode:
-                        return I($"Change tracking set has mismatched supersede modes, expected: '{m_supersedeModes.expected}, loaded: '{m_supersedeModes.loaded}'");
                     default:
                         throw Contract.AssertFailure(I($"Unrecognized {nameof(LoadingTrackerStatus)}"));
                 }
@@ -3832,18 +3807,6 @@ namespace BuildXL.Storage.ChangeTracking
         {
             return new LoadingTrackerResult(FileEnvelopeId.Invalid,  LoadingTrackerStatus.BuildEngineFingerprintMismatch, null);
         }
-
-        /// <summary>
-        /// Creates an instance of <see cref="LoadingTrackerResult"/> for failure due to mismatched supersede modes.
-        /// </summary>
-        public static LoadingTrackerResult FailMismatchedSupersedeModes(FileChangeTrackerSupersedeMode expected, FileChangeTrackerSupersedeMode loaded)
-        {
-            return new LoadingTrackerResult(FileEnvelopeId.Invalid, LoadingTrackerStatus.MismatchedSupersedeMode, null)
-            {
-                m_supersedeModes = (expected, loaded)
-            };
-        }
-
     }
 
     /// <summary>
@@ -3897,12 +3860,7 @@ namespace BuildXL.Storage.ChangeTracking
         /// <summary>
         /// Journal goes back in time because the next USN is smaller than the checkpoint.
         /// </summary>
-        JournalGoesBackInTime,
-
-        /// <summary>
-        /// Mismatched supersede mode.
-        /// </summary>
-        MismatchedSupersedeMode
+        JournalGoesBackInTime
     }
 
     /// <summary>

Public/Src/Utilities/Storage/ChangeTracking/IFileChangeTrackingSubscriptionSource.cs

@@ -40,9 +40,36 @@ namespace BuildXL.Storage.ChangeTracking
         /// The membership of the directory will be invalidated if a name is added or removed directly inside the directory (i.e., when <c>FindFirstFile</c>
         /// and <c>FindNextFile</c> would see a different set of names).
         /// </summary>
+        /// <remarks>
+        /// <paramref name="handleEntry"/> will only be called on an entry if <paramref name="shouldIncludeEntry"/> on that entry returns true.
+        /// <paramref name="supersedeWithStrongIdentity"/> is used to supersede the tracked USN of <paramref name="path"/> with the max USN of
+        /// its entries that are included by <paramref name="shouldIncludeEntry"/>. This is essential for tracking output directory.
+        /// 
+        /// Suppose that we have the following output directory:
+        /// D\
+        ///   1.txt
+        ///   2.txt
+        ///   3.txt
+        /// When we produce D, we track D and record its membership fingerprint. By tracking, D is recorded with some USN-D.
+        /// The issue is D's members are typically created after D. Thus, each of them has higher USN, and that USN contains CREATED change reason.
+        /// When we scan our journal later we find this higher USN containing CREATED, and journal processing thinks that D's membership
+        /// has probably changed. Our journal scanning then needs to work extra to prove that the membership has not changed by enumerating D again
+        /// and compare its recorded fingerprint with the current one. See CheckAndMaybeInvalidateEnumerationDependencies in FileChangeTrackingSet.cs
+        /// for details.
+        /// 
+        /// When <paramref name="supersedeWithStrongIdentity"/> is true, after the enumeration, we retrack again D by establishing a strong
+        /// identity for it, i.e., we create a dummy CLOSE record so that D will have higher USN than all of its members. Then, we use that higher
+        /// USN as D's supersession limit. Thus, any membership change below that higher USN will be ignored during the journal scanning.
+        /// 
+        /// One can think what about if D is superseded by max USN among its members. First, one needs to query the USNs of D's members, and it
+        /// affects performance of enumeration. Second, the enumeratioon only care about the member included by <paramref name="shouldIncludeEntry"/>.
+        /// The excluded member may have higher USN then any included members'.
+        /// </remarks>
         Possible<FileChangeTrackingSet.EnumerationResult> TryEnumerateDirectoryAndTrackMembership(
             [NotNull]string path,
-            [NotNull]Action<string, FileAttributes> handleEntry);
+            [NotNull]Action<string, FileAttributes> handleEntry,
+            Func<string, FileAttributes, bool> shouldIncludeEntry,
+            bool supersedeWithStrongIdentity);
 
         /// <summary>
         /// Probes for the existence of a path, while also tracking the result (e.g. if a file does not exist and is later created, that change will be detected).

Public/Src/Utilities/Storage/FileContentTable.cs

@@ -13,7 +13,6 @@ using System.Threading;
 using System.Threading.Tasks;
 using BuildXL.Cache.ContentStore.Hashing;
 using BuildXL.Native.IO;
-using BuildXL.Native.IO.Windows;
 using BuildXL.Storage.ChangeTracking;
 using BuildXL.Storage.FileContentTableAccessor;
 using BuildXL.Tracing;

Public/Src/Utilities/Storage/VersionedFileIdentity.cs

@@ -102,6 +102,11 @@ namespace BuildXL.Storage
             Kind = kind;
         }
 
+        /// <summary>
+        /// Creates a new <see cref="VersionedFileIdentity"/> with a specified <see cref="Usn"/>.
+        /// </summary>
+        public VersionedFileIdentity WithUsn(Usn usn) => new VersionedFileIdentity(m_volumeSerialNumber, m_fileId, usn, Kind);
+
         /// <summary>
         /// Failure reason for <see cref="VersionedFileIdentity.TryQuery" /> or <see cref="VersionedFileIdentity.TryEstablishStrong"/>
         /// </summary>

Public/Src/Utilities/UnitTests/Storage.Untracked/FileChangeDetectionTests.cs

@@ -1549,6 +1549,37 @@ namespace Test.BuildXL.Storage.Admin
             changedPaths.AssertNoChangesDetected();
         }
 
+        [FactIfSupported(requiresJournalScan: true)]
+        public void TrackDirectoryWithFilter()
+        {
+            const string DirectoryName = "D";
+            const string FileA = @"D\A";
+            const string FileB = @"D\B";
+            const string FileC = @"D\C";
+
+            CreateDirectory(DirectoryName);
+            WriteFile(FileA, "A");
+            WriteFile(FileB, "B");
+
+            ChangeDetectionSupport support = InitializeChangeDetectionSupport();
+            support.TrackDirectoryMembership(
+                DirectoryName,
+                (relativeName, _) => string.Equals(FileA, relativeName, OperatingSystemHelper.PathComparison),
+                true,
+                FileA);
+
+            WriteFile(FileB, "B-modified");
+            DetectedChanges changedPaths = support.ProcessChanges();
+            changedPaths.AssertNoChangesDetected();
+
+            WriteFile(FileC, "C");
+            changedPaths = support.ProcessChanges();
+
+            // Although D is tracked with a filter, but during the journal scanning, the tracker
+            // does not have the filter, and so it detects some membership change.
+            changedPaths.AssertChangedExactly(MembershipChanged(DirectoryName));
+        }
+
         #endregion
 
         #region Superseding updates (last tracker wins)
@@ -1699,21 +1730,20 @@ namespace Test.BuildXL.Storage.Admin
             changedPaths.AssertChangedExactly(Removed(Directory1), MembershipChanged(Directory1));
         }
 
-        [TheoryIfSupported(requiresJournalScan: true)]
-        [InlineData(FileChangeTrackerSupersedeMode.All)]
-        [InlineData(FileChangeTrackerSupersedeMode.FileOnly)]
-        [InlineData(FileChangeTrackerSupersedeMode.FileAndParents)]
-        public void SupersedeRetrackingEffectiveOnDirectoryDeletionInAllSupersede(FileChangeTrackerSupersedeMode supersedeMode)
+        [FactIfSupported(requiresJournalScan: true)]
+        public void SupersedeTrackingWithLatestUsnEffectiveOnDirectoryDeletion()
         {
             const string Directory = @"D";
             const string File1 = @"D\F1";
             const string File2 = @"D\F2";
 
-            ChangeDetectionSupport support = InitializeChangeDetectionSupport(supersedeMode);
+            ChangeDetectionSupport support = InitializeChangeDetectionSupport();
 
             CreateDirectory(Directory);
             WriteFile(File1, "Stuff1");
             WriteFile(File2, "Stuff2");
+            
+            support.TrackDirectoryMembership(Directory, File1, File2);
             support.Track(File1);
             support.Track(File2);
 
@@ -1722,20 +1752,38 @@ namespace Test.BuildXL.Storage.Admin
             // Now recreate Directory and File1, then retrack with 'supersede'.
             CreateDirectory(Directory);
             WriteFile(File1, "Re-stuff1");
+            support.TrackDirectoryMembership(Directory, null, true, File1);
             support.Track(File1, TrackingUpdateMode.Supersede);
 
             DetectedChanges changedPaths = support.ProcessChanges();
 
-            if (supersedeMode == FileChangeTrackerSupersedeMode.All
-                || supersedeMode == FileChangeTrackerSupersedeMode.FileAndParents)
-            {
-                // Removal of Directory and File1 has been superseded. 
-                changedPaths.AssertChangedExactly(Removed(File2));
-            }
-            else if (supersedeMode == FileChangeTrackerSupersedeMode.FileOnly)
-            {
-                changedPaths.AssertChangedExactly(Removed(File1), Removed(File2), Removed(Directory));
-            }
+            // Removal of Directory and File1 has been superseded. 
+            changedPaths.AssertChangedExactly(Removed(File2));
+        }
+
+        [FactIfSupported(requiresJournalScan: true)]
+        public void SupersedeTrackingWithLatestUsnEffectiveOnDirectoryEnumeration()
+        {
+            const string Directory = @"D";
+            const string File1 = @"D\F1";
+            const string File2 = @"D\F2";
+
+            ChangeDetectionSupport support = InitializeChangeDetectionSupport();
+
+            CreateDirectory(Directory);
+            WriteFile(File1, "Stuff1");
+            WriteFile(File2, "Stuff2");
+
+            // System.Diagnostics.Debugger.Launch();
+            support.TrackDirectoryMembership(
+                Directory,
+                (relativePath, _) => string.Equals(relativePath, File1, OperatingSystemHelper.PathComparison),
+                true,
+                File1);
+            support.Track(File1, TrackingUpdateMode.Supersede);
+
+            DetectedChanges changedPaths = support.ProcessChanges();
+            changedPaths.AssertNoChangesDetected();
         }
 
         #endregion
@@ -2087,7 +2135,14 @@ namespace Test.BuildXL.Storage.Admin
                 }
             }
 
-            public void TrackDirectoryMembership(string relative, params string[] expectedMembers)
+            public void TrackDirectoryMembership(string relative, params string[] expectedMembers) =>
+                TrackDirectoryMembership(relative, null, false, expectedMembers);
+
+            public void TrackDirectoryMembership(
+                string relative,
+                Func<string, FileAttributes, bool> shouldIncludeRelativeEntry,
+                bool supersedeWithLastestEntryUsn,
+                params string[] expectedMembers)
             {
                 string path = GetFullPath(relative);
 
@@ -2100,7 +2155,9 @@ namespace Test.BuildXL.Storage.Admin
                     {
                         diff.Add(Path.Combine(path, entry));
                         calculator = calculator.Accumulate(entry, attributes);
-                    });
+                    },
+                    shouldIncludeEntry: (entry, attributes) => shouldIncludeRelativeEntry?.Invoke(Path.Combine(relative, entry), attributes) ?? true,
+                    supersedeWithStrongIdentity: supersedeWithLastestEntryUsn);
 
                 var fingerprint = calculator.GetFingerprint();
 
@@ -2273,7 +2330,7 @@ namespace Test.BuildXL.Storage.Admin
             }
         }
 
-        private ChangeDetectionSupport InitializeChangeDetectionSupport(FileChangeTrackerSupersedeMode supersedeMode = FileChangeTrackerSupersedeMode.FileAndParents)
+        private ChangeDetectionSupport InitializeChangeDetectionSupport()
         {
             var loggingContext = new LoggingContext("Dummy", "Dummy");
             VolumeMap volumeMap = JournalUtils.TryCreateMapOfAllLocalVolumes(loggingContext);
@@ -2282,7 +2339,7 @@ namespace Test.BuildXL.Storage.Admin
             var maybeJournal = JournalUtils.TryGetJournalAccessorForTest(volumeMap);
             XAssert.IsTrue(maybeJournal.Succeeded, "Could not connect to journal");
 
-            FileChangeTrackingSet trackingSet = FileChangeTrackingSet.CreateForAllCapableVolumes(loggingContext, volumeMap, maybeJournal.Result, supersedeMode);
+            FileChangeTrackingSet trackingSet = FileChangeTrackingSet.CreateForAllCapableVolumes(loggingContext, volumeMap, maybeJournal.Result);
 
             return new ChangeDetectionSupport(TemporaryDirectory, JournalState.CreateEnabledJournal(volumeMap, maybeJournal.Result), trackingSet);
         }

Public/Src/Utilities/UnitTests/Storage.Untracked/FileContentTableWithTrackerTests.cs

@@ -191,7 +191,6 @@ namespace Test.BuildXL.Storage.Admin
                     loggingContext,
                     volumeMap,
                     maybeJournal.Result,
-                    global::BuildXL.Utilities.Configuration.FileChangeTrackerSupersedeMode.FileAndParents,
                     null);
 
                 return new ChangeTrackerSupport(
@@ -218,7 +217,6 @@ namespace Test.BuildXL.Storage.Admin
                     m_loggingContext,
                     m_volumeMap,
                     m_journal,
-                    global::BuildXL.Utilities.Configuration.FileChangeTrackerSupersedeMode.FileAndParents,
                     m_fileChangeTrackerPath.ToString(m_pathTable),
                     m_buildEngineFingerprint,
                     out m_fileChangeTracker);

Public/Src/Utilities/UnitTests/StorageTestUtilities/FileChangeTrackingRecorder.cs

@@ -83,12 +83,14 @@ namespace Test.BuildXL.StorageTestUtilities
 
         public Possible<FileChangeTrackingSet.EnumerationResult> TryEnumerateDirectoryAndTrackMembership(
             string path,
-            Action<string, FileAttributes> handleEntry)
+            Action<string, FileAttributes> handleEntry,
+            Func<string, FileAttributes, bool> shouldIncludeEntry,
+            bool supersedeWithStrongIdentity)
         {
             Contract.Requires(path != null);
             Contract.Requires(handleEntry != null);
 
-            var possibleFingerprintResult = DirectoryMembershipTrackingFingerprinter.ComputeFingerprint(path, handleEntry);
+            var possibleFingerprintResult = DirectoryMembershipTrackingFingerprinter.ComputeFingerprint(path, handleEntry, shouldIncludeEntry);
 
             if (!possibleFingerprintResult.Succeeded)
             {