Fix to remove default off detectors from experiments (#739)

Fix to remove default off detectors from experiments

src/Microsoft.ComponentDetection.Orchestrator/DetectorRestrictions.cs

@@ -1,6 +1,7 @@
 namespace Microsoft.ComponentDetection.Orchestrator;
 
 using System.Collections.Generic;
+using Microsoft.ComponentDetection.Contracts;
 
 public class DetectorRestrictions
 {
@@ -9,4 +10,6 @@ public class DetectorRestrictions
     public IEnumerable<string> ExplicitlyEnabledDetectorIds { get; set; }
 
     public IEnumerable<string> AllowedDetectorCategories { get; set; }
+
+    public IEnumerable<IComponentDetector> DisabledDetectors { get; set; }
 }

src/Microsoft.ComponentDetection.Orchestrator/Experiments/ExperimentService.cs

@@ -108,6 +108,18 @@ public class ExperimentService : IExperimentService
         }
     }
 
+    public void RemoveUnwantedExperimentsbyDetectors(IEnumerable<IComponentDetector> detectors)
+    {
+        var experimentsToRemove = this.experiments
+            .Where(x => detectors.Any(detector => x.Key.IsInControlGroup(detector) || x.Key.IsInExperimentGroup(detector)))
+            .Select(x => x.Key).ToList();
+
+        foreach (var config in experimentsToRemove.Where(config => this.experiments.TryRemove(config, out _)))
+        {
+            this.logger.LogDebug("Removing {Experiment} from active experiments", config.Name);
+        }
+    }
+
     /// <inheritdoc />
     public async Task FinishAsync()
     {
@@ -125,7 +137,6 @@ public class ExperimentService : IExperimentService
         {
             var controlComponents = experiment.ControlGroupComponents;
             var experimentComponents = experiment.ExperimentGroupComponents;
-
             this.logger.LogInformation(
                 "Experiment {Experiment} finished with {ControlCount} components in the control group and {ExperimentCount} components in the experiment group",
                 config.Name,

src/Microsoft.ComponentDetection.Orchestrator/Experiments/IExperimentService.cs

@@ -1,5 +1,6 @@
 namespace Microsoft.ComponentDetection.Orchestrator.Experiments;
 
+using System.Collections.Generic;
 using System.Threading.Tasks;
 using Microsoft.ComponentDetection.Common.DependencyGraph;
 using Microsoft.ComponentDetection.Contracts;
@@ -23,4 +24,10 @@ public interface IExperimentService
     /// </summary>
     /// <returns>A <see cref="Task"/> representing the asynchronous operation.</returns>
     Task FinishAsync();
+
+    /// <summary>
+    /// Removes any experimentsthat contains a detector that is not needed.
+    /// </summary>
+    /// <param name="detectors"> List of all detectors. </param>
+    void RemoveUnwantedExperimentsbyDetectors(IEnumerable<IComponentDetector> detectors);
 }

src/Microsoft.ComponentDetection.Orchestrator/Services/BcdeScanExecutionService.cs

@@ -1,4 +1,4 @@
-namespace Microsoft.ComponentDetection.Orchestrator.Services;
+namespace Microsoft.ComponentDetection.Orchestrator.Services;
 
 using System;
 using System.Collections.Generic;
@@ -38,14 +38,15 @@ public class BcdeScanExecutionService : IBcdeScanExecutionService
         using var scope = this.logger.BeginScope("Executing BCDE scan");
 
         var detectorRestrictions = this.GetDetectorRestrictions(detectionArguments);
-        var detectors = this.detectorRestrictionService.ApplyRestrictions(detectorRestrictions, this.detectors).ToImmutableList();
+        var restrictedDetectors = this.detectorRestrictionService.ApplyRestrictions(detectorRestrictions, this.detectors).ToImmutableList();
+        detectorRestrictions.DisabledDetectors = this.detectors.Except(restrictedDetectors).ToList();
 
         this.logger.LogDebug("Finished applying restrictions to detectors.");
 
-        var processingResult = await this.detectorProcessingService.ProcessDetectorsAsync(detectionArguments, detectors, detectorRestrictions);
+        var processingResult = await this.detectorProcessingService.ProcessDetectorsAsync(detectionArguments, restrictedDetectors, detectorRestrictions);
         var scanResult = this.graphTranslationService.GenerateScanResultFromProcessingResult(processingResult, detectionArguments);
 
-        scanResult.DetectorsInScan = detectors.Select(x => ConvertToContract(x)).ToList();
+        scanResult.DetectorsInScan = restrictedDetectors.Select(x => ConvertToContract(x)).ToList();
         scanResult.ResultCode = processingResult.ResultCode;
 
         return scanResult;

src/Microsoft.ComponentDetection.Orchestrator/Services/DetectorProcessingService.cs

@@ -52,6 +52,8 @@ public class DetectorProcessingService : IDetectorProcessingService
             ? this.GenerateDirectoryExclusionPredicate(detectionArguments.SourceDirectory.ToString(), detectionArguments.DirectoryExclusionList, detectionArguments.DirectoryExclusionListObsolete, allowWindowsPaths: false, ignoreCase: false)
             : this.GenerateDirectoryExclusionPredicate(detectionArguments.SourceDirectory.ToString(), detectionArguments.DirectoryExclusionList, detectionArguments.DirectoryExclusionListObsolete, allowWindowsPaths: true, ignoreCase: true);
 
+        this.experimentService.RemoveUnwantedExperimentsbyDetectors(detectorRestrictions.DisabledDetectors);
+
         IEnumerable<Task<(IndividualDetectorScanResult, ComponentRecorder, IComponentDetector)>> scanTasks = detectors
             .Select(async detector =>
             {

src/Microsoft.ComponentDetection.Orchestrator/Services/IDetectorProcessingService.cs

@@ -1,4 +1,4 @@
-namespace Microsoft.ComponentDetection.Orchestrator.Services;
+namespace Microsoft.ComponentDetection.Orchestrator.Services;
 
 using System.Collections.Generic;
 using System.Threading.Tasks;

test/Microsoft.ComponentDetection.Orchestrator.Tests/Experiments/ExperimentServiceTests.cs

@@ -8,6 +8,7 @@ using FluentAssertions;
 using Microsoft.ComponentDetection.Common.DependencyGraph;
 using Microsoft.ComponentDetection.Contracts;
 using Microsoft.ComponentDetection.Contracts.BcdeModels;
+using Microsoft.ComponentDetection.Detectors.NuGet;
 using Microsoft.ComponentDetection.Orchestrator.ArgumentSets;
 using Microsoft.ComponentDetection.Orchestrator.Experiments;
 using Microsoft.ComponentDetection.Orchestrator.Experiments.Configs;
@@ -262,4 +263,66 @@ public class ExperimentServiceTests
             x => x.ProcessExperimentAsync(this.experimentConfigMock.Object, It.IsAny<ExperimentDiff>()),
             Times.Never());
     }
+
+    [TestMethod]
+    public async Task RecordDetectorRun_CheckUnwantedDetectors_RemoveExperimentAsync()
+    {
+        var components = ExperimentTestUtils.CreateRandomComponents();
+
+        var service = new ExperimentService(
+            new[] { this.experimentConfigMock.Object },
+            new[] { this.experimentProcessorMock.Object },
+            this.graphTranslationServiceMock.Object,
+            this.loggerMock.Object);
+        this.SetupGraphMock(components);
+
+        var detectorList = new List<IComponentDetector>
+        {
+            new NuGetComponentDetector(
+            new Mock<IComponentStreamEnumerableFactory>().Object,
+            new Mock<IObservableDirectoryWalkerFactory>().Object,
+            new Mock<ILogger<NuGetComponentDetector>>().Object), this.detectorMock.Object,
+        };
+
+        service.RemoveUnwantedExperimentsbyDetectors(detectorList);
+
+        service.RecordDetectorRun(this.detectorMock.Object, this.componentRecorder, this.detectionArgsMock.Object);
+
+        await service.FinishAsync();
+
+        this.experimentProcessorMock.Verify(
+            x => x.ProcessExperimentAsync(this.experimentConfigMock.Object, It.IsAny<ExperimentDiff>()),
+            Times.Never());
+    }
+
+    [TestMethod]
+    public async Task RecordDetectorRun_CheckUnwantedDetectors_KeepExperimentAsync()
+    {
+        var components = ExperimentTestUtils.CreateRandomComponents();
+
+        var service = new ExperimentService(
+            new[] { this.experimentConfigMock.Object },
+            new[] { this.experimentProcessorMock.Object },
+            this.graphTranslationServiceMock.Object,
+            this.loggerMock.Object);
+        this.SetupGraphMock(components);
+
+        var detectorList = new List<IComponentDetector>
+        {
+            new NuGetComponentDetector(
+            new Mock<IComponentStreamEnumerableFactory>().Object,
+            new Mock<IObservableDirectoryWalkerFactory>().Object,
+            new Mock<ILogger<NuGetComponentDetector>>().Object),
+        };
+
+        service.RemoveUnwantedExperimentsbyDetectors(detectorList);
+
+        service.RecordDetectorRun(this.detectorMock.Object, this.componentRecorder, this.detectionArgsMock.Object);
+
+        await service.FinishAsync();
+
+        this.experimentProcessorMock.Verify(
+            x => x.ProcessExperimentAsync(this.experimentConfigMock.Object, It.IsAny<ExperimentDiff>()),
+            Times.Once());
+    }
 }