feat: disable detector experiments by default (#688)

docs/environment-variables.md

@@ -12,4 +12,10 @@ Otherwise, the Go detector uses go-cli command: `go list -m all` to discover Go
 The environment variable `PyPiMaxCacheEntries` is used to control the size of the in-memory LRU cache that caches responses from PyPi.
 The default value is 4096.
 
+## `CD_DETECTOR_EXPERIMENTS`
+
+When set to any value, enables detector experiments, a feature to compare the results of different detectors for the
+same ecosystem. The available experiments are found in the [`Experiments\Config`](../src/Microsoft.ComponentDetection.Orchestrator/Experiments/Configs)
+folder.
+
 [1]: https://go.dev/ref/mod#go-mod-graph

src/Microsoft.ComponentDetection.Orchestrator/Experiments/DetectorExperiments.cs

@@ -0,0 +1,19 @@
+namespace Microsoft.ComponentDetection.Orchestrator.Experiments;
+
+using System;
+
+/// <summary>
+/// Enables or disables detector experiments in Component Detection.
+/// </summary>
+public static class DetectorExperiments
+{
+    /// <summary>
+    /// Manually enables detector experiments.
+    /// </summary>
+    public static bool Enable { get; set; }
+
+    private static bool EnvironmentEnabled =>
+        !string.IsNullOrEmpty(Environment.GetEnvironmentVariable("CD_DETECTOR_EXPERIMENTS"));
+
+    internal static bool AreExperimentsEnabled => Enable || EnvironmentEnabled;
+}

src/Microsoft.ComponentDetection.Orchestrator/Experiments/ExperimentService.cs

@@ -46,6 +46,11 @@ public class ExperimentService : IExperimentService
     /// <inheritdoc />
     public void RecordDetectorRun(IComponentDetector detector, ComponentRecorder componentRecorder, IDetectionArguments detectionArguments)
     {
+        if (!DetectorExperiments.AreExperimentsEnabled)
+        {
+            return;
+        }
+
         try
         {
             var scanResult = this.graphTranslationService.GenerateScanResultFromProcessingResult(
@@ -106,6 +111,11 @@ public class ExperimentService : IExperimentService
     /// <inheritdoc />
     public async Task FinishAsync()
     {
+        if (!DetectorExperiments.AreExperimentsEnabled)
+        {
+            return;
+        }
+
         foreach (var (config, experiment) in this.experiments)
         {
             var controlComponents = experiment.ControlGroupComponents;

test/Microsoft.ComponentDetection.Orchestrator.Tests/Experiments/ExperimentServiceTests.cs

@@ -54,6 +54,9 @@ public class ExperimentServiceTests
             .Returns(new ScanResult() { ComponentsFound = components });
     }
 
+    [TestInitialize]
+    public void EnableDetectorExperiments() => DetectorExperiments.Enable = true;
+
     [TestMethod]
     public void RecordDetectorRun_AddsComponentsToControlAndExperimentGroup()
     {
@@ -109,6 +112,32 @@ public class ExperimentServiceTests
             Times.Once());
     }
 
+    [TestMethod]
+    public async Task RecordDetectorRun_Respects_DetectorExperiments_EnableAsync()
+    {
+        DetectorExperiments.Enable = false;
+        var filterConfigMock = new Mock<IExperimentConfiguration>();
+
+        var components = ExperimentTestUtils.CreateRandomComponents();
+
+        var service = new ExperimentService(
+            new[] { this.experimentConfigMock.Object, filterConfigMock.Object },
+            new[] { this.experimentProcessorMock.Object },
+            this.graphTranslationServiceMock.Object,
+            this.loggerMock.Object);
+
+        service.RecordDetectorRun(this.detectorMock.Object, this.componentRecorder, this.detectionArgsMock.Object);
+        await service.FinishAsync();
+
+        filterConfigMock.Verify(x => x.ShouldRecord(this.detectorMock.Object, components.Count), Times.Never());
+        this.experimentProcessorMock.Verify(
+            x => x.ProcessExperimentAsync(filterConfigMock.Object, It.IsAny<ExperimentDiff>()),
+            Times.Never());
+        this.experimentProcessorMock.Verify(
+            x => x.ProcessExperimentAsync(this.experimentConfigMock.Object, It.IsAny<ExperimentDiff>()),
+            Times.Never());
+    }
+
     [TestMethod]
     public async Task FinishAsync_ProcessesExperimentsAsync()
     {
@@ -165,4 +194,26 @@ public class ExperimentServiceTests
             x => x.ProcessExperimentAsync(It.IsAny<IExperimentConfiguration>(), It.IsAny<ExperimentDiff>()),
             Times.Never());
     }
+
+    [TestMethod]
+    public async Task FinishAsync_Respects_DetectorExperiments_EnableAsync()
+    {
+        DetectorExperiments.Enable = false;
+
+        var components = ExperimentTestUtils.CreateRandomComponents();
+        this.SetupGraphMock(components);
+
+        var service = new ExperimentService(
+            new[] { this.experimentConfigMock.Object },
+            new[] { this.experimentProcessorMock.Object },
+            this.graphTranslationServiceMock.Object,
+            this.loggerMock.Object);
+        service.RecordDetectorRun(this.detectorMock.Object, this.componentRecorder, this.detectionArgsMock.Object);
+
+        await service.FinishAsync();
+
+        this.experimentProcessorMock.Verify(
+            x => x.ProcessExperimentAsync(this.experimentConfigMock.Object, It.IsAny<ExperimentDiff>()),
+            Times.Never());
+    }
 }