diff --git a/src/Microsoft.ComponentDetection.Detectors/go/GoComponentDetector.cs b/src/Microsoft.ComponentDetection.Detectors/go/GoComponentDetector.cs index c780dec0..f1d4a100 100644 --- a/src/Microsoft.ComponentDetection.Detectors/go/GoComponentDetector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/go/GoComponentDetector.cs @@ -5,8 +5,10 @@ using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; using System.IO; using System.Linq; +using System.Reactive.Linq; using System.Text.RegularExpressions; using System.Threading.Tasks; +using Microsoft.ComponentDetection.Common; using Microsoft.ComponentDetection.Common.Telemetry.Records; using Microsoft.ComponentDetection.Contracts; using Microsoft.ComponentDetection.Contracts.Internal; @@ -16,11 +18,11 @@ using Newtonsoft.Json; public class GoComponentDetector : FileComponentDetector { - private static readonly Regex GoSumRegex = new Regex( + private static readonly Regex GoSumRegex = new( @"(?.*)\s+(?.*?)(/go\.mod)?\s+(?.*)", RegexOptions.Compiled | RegexOptions.ExplicitCapture | RegexOptions.IgnoreCase); - private readonly HashSet projectRoots = new HashSet(); + private readonly HashSet projectRoots = new(); private readonly ICommandLineInvocationService commandLineInvocationService; private readonly IEnvironmentVariableService envVarService; @@ -39,7 +41,7 @@ public class GoComponentDetector : FileComponentDetector this.Logger = logger; } - public override string Id { get; } = "Go"; + public override string Id => "Go"; public override IEnumerable Categories => new[] { Enum.GetName(typeof(DetectorClass), DetectorClass.GoMod) }; @@ -47,7 +49,101 @@ public class GoComponentDetector : FileComponentDetector public override IEnumerable SupportedComponentTypes { get; } = new[] { ComponentType.Go }; - public override int Version => 6; + public override int Version => 7; + + protected override Task> OnPrepareDetectionAsync( + IObservable processRequests, IDictionary detectorArgs) + { + // Filter out any go.sum process requests if the adjacent go.mod file is present and has a go version >= 1.17 + var goModProcessRequests = processRequests.Where(processRequest => + { + if (Path.GetFileName(processRequest.ComponentStream.Location) != "go.sum") + { + return true; + } + + var goModFile = this.FindAdjacentGoModComponentStreams(processRequest).FirstOrDefault(); + + if (goModFile == null) + { + this.Logger.LogDebug( + "go.sum file found without an adjacent go.mod file. Location: {Location}", + processRequest.ComponentStream.Location); + + return true; + } + + // parse the go.mod file to get the go version + using var reader = new StreamReader(goModFile.Stream); + var goModFileContents = reader.ReadToEnd(); + goModFile.Stream.Dispose(); + + return this.CheckGoModVersion(goModFileContents, processRequest, goModFile); + }); + + return Task.FromResult(goModProcessRequests); + } + + private IEnumerable FindAdjacentGoModComponentStreams(ProcessRequest processRequest) => + this.ComponentStreamEnumerableFactory.GetComponentStreams( + new FileInfo(processRequest.ComponentStream.Location).Directory, + new[] { "go.mod" }, + (_, _) => false, + false) + .Select(x => + { + // The stream will be disposed at the end of this method, so we need to copy it to a new stream. + var memoryStream = new MemoryStream(); + + x.Stream.CopyTo(memoryStream); + memoryStream.Position = 0; + + return new ComponentStream + { + Stream = memoryStream, + Location = x.Location, + Pattern = x.Pattern, + }; + }); + + private bool CheckGoModVersion(string goModFileContents, ProcessRequest processRequest, ComponentStream goModFile) + { + var goVersionMatch = Regex.Match(goModFileContents, @"go\s(?\d+\.\d+)"); + + if (!goVersionMatch.Success) + { + this.Logger.LogDebug( + "go.sum file found with an adjacent go.mod file that does not contain a go version. Location: {Location}", + processRequest.ComponentStream.Location); + return true; + } + + var goVersion = goVersionMatch.Groups["version"].Value; + if (System.Version.TryParse(goVersion, out var version)) + { + if (version < new Version(1, 17)) + { + this.Logger.LogWarning( + "go.mod file at {GoModLocation} does not have a go version >= 1.17. Scanning this go.sum file: {GoSumLocation} which may lead to over reporting components", + goModFile.Location, + processRequest.ComponentStream.Location); + + return true; + } + + this.Logger.LogInformation( + "go.sum file found with an adjacent go.mod file that has a go version >= 1.17. Will not scan this go.sum file. Location: {Location}", + processRequest.ComponentStream.Location); + + return false; + } + + this.Logger.LogWarning( + "go.sum file found with an adjacent go.mod file that has an invalid go version. Scanning both for components. Location: {Location}", + processRequest.ComponentStream.Location); + + return true; + } protected override async Task OnFileFoundAsync(ProcessRequest processRequest, IDictionary detectorArgs) { @@ -75,7 +171,7 @@ public class GoComponentDetector : FileComponentDetector { record.WasGoCliDisabled = true; this.Logger.LogInformation("Go cli scan was manually disabled, fallback strategy performed." + - " More info: https://github.com/microsoft/component-detection/blob/main/docs/detectors/go.md#fallback-detection-strategy"); + " More info: https://github.com/microsoft/component-detection/blob/main/docs/detectors/go.md#fallback-detection-strategy"); } } catch (Exception ex) @@ -97,7 +193,7 @@ public class GoComponentDetector : FileComponentDetector case ".MOD": { this.Logger.LogDebug("Found Go.mod: {Location}", file.Location); - this.ParseGoModFile(singleFileComponentRecorder, file, record); + await this.ParseGoModFileAsync(singleFileComponentRecorder, file, record); break; } @@ -135,8 +231,8 @@ public class GoComponentDetector : FileComponentDetector } this.Logger.LogInformation("Go CLI was found in system and will be used to generate dependency graph. " + - "Detection time may be improved by activating fallback strategy (https://github.com/microsoft/component-detection/blob/main/docs/detectors/go.md#fallback-detection-strategy). " + - "But, it will introduce noise into the detected components."); + "Detection time may be improved by activating fallback strategy (https://github.com/microsoft/component-detection/blob/main/docs/detectors/go.md#fallback-detection-strategy). " + + "But, it will introduce noise into the detected components."); var goDependenciesProcess = await this.commandLineInvocationService.ExecuteCommandAsync("go", null, workingDirectory: projectRootDirectory, new[] { "list", "-mod=readonly", "-m", "-json", "all" }); if (goDependenciesProcess.ExitCode != 0) { @@ -159,36 +255,53 @@ public class GoComponentDetector : FileComponentDetector return true; } - private void ParseGoModFile( + private void TryRegisterDependencyFromModLine(string line, ISingleFileComponentRecorder singleFileComponentRecorder) + { + if (this.TryToCreateGoComponentFromModLine(line, out var goComponent)) + { + singleFileComponentRecorder.RegisterUsage(new DetectedComponent(goComponent)); + } + else + { + var lineTrim = line.Trim(); + this.Logger.LogWarning("Line could not be parsed for component [{LineTrim}]", lineTrim); + singleFileComponentRecorder.RegisterPackageParseFailure(lineTrim); + } + } + + private async Task ParseGoModFileAsync( ISingleFileComponentRecorder singleFileComponentRecorder, IComponentStream file, GoGraphTelemetryRecord goGraphTelemetryRecord) { using var reader = new StreamReader(file.Stream); - var line = reader.ReadLine(); - while (line != null && !line.StartsWith("require (")) + // There can be multiple require( ) sections in go 1.17+. loop over all of them. + while (!reader.EndOfStream) { - if (line.StartsWith("go ")) + var line = await reader.ReadLineAsync(); + + while (line != null && !line.StartsWith("require (")) { - goGraphTelemetryRecord.GoModVersion = line[3..].Trim(); + if (line.StartsWith("go ")) + { + goGraphTelemetryRecord.GoModVersion = line[3..].Trim(); + } + + // In go >= 1.17, direct dependencies are listed as "require x/y v1.2.3", and transitive dependencies + // are listed in the require () section + if (line.StartsWith("require ")) + { + this.TryRegisterDependencyFromModLine(line[8..], singleFileComponentRecorder); + } + + line = await reader.ReadLineAsync(); } - line = reader.ReadLine(); - } - - // Stopping at the first ) restrict the detection to only the require section. - while ((line = reader.ReadLine()) != null && !line.EndsWith(")")) - { - if (this.TryToCreateGoComponentFromModLine(line, out var goComponent)) + // Stopping at the first ) restrict the detection to only the require section. + while ((line = await reader.ReadLineAsync()) != null && !line.EndsWith(")")) { - singleFileComponentRecorder.RegisterUsage(new DetectedComponent(goComponent)); - } - else - { - var lineTrim = line.Trim(); - this.Logger.LogWarning("Line could not be parsed for component [{LineTrim}]", lineTrim); - singleFileComponentRecorder.RegisterPackageParseFailure(lineTrim); + this.TryRegisterDependencyFromModLine(line, singleFileComponentRecorder); } } } diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/GoComponentDetectorTests.cs b/test/Microsoft.ComponentDetection.Detectors.Tests/GoComponentDetectorTests.cs index bb68b171..8fc5f3ad 100644 --- a/test/Microsoft.ComponentDetection.Detectors.Tests/GoComponentDetectorTests.cs +++ b/test/Microsoft.ComponentDetection.Detectors.Tests/GoComponentDetectorTests.cs @@ -183,6 +183,67 @@ $#26^#25%4"; Assert.AreEqual(0, componentRecorder.GetDetectedComponents().Count()); } + [TestMethod] + public async Task TestGoModDetector_SkipsGoSumFilesAsync() + { + var goMod = + @"module contoso.com/greetings +go 1.18 + +require github.com/go-sql-driver/mysql v1.7.1 // indirect"; + + var goSum = + @"github.com/go-sql-driver/mysql v1.7.1 h1:lUIinVbN1DY0xBg0eMOzmmtGoHwWBbvnWubQUrtU8EI= +github.com/go-sql-driver/mysql v1.7.1/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U="; + + var (scanResult, componentRecorder) = await this.DetectorTestUtility + .WithFile("go.mod", goMod) + .WithFile("go.mod", goMod, new[] { "go.mod" }) + .WithFile("go.sum", goSum) + .ExecuteDetectorAsync(); + + scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); + componentRecorder.GetDetectedComponents().Should().ContainSingle(); + + var component = componentRecorder.GetDetectedComponents().First(); + component.Component.Id.Should().Be("github.com/go-sql-driver/mysql v1.7.1 - Go"); + } + + [TestMethod] + public async Task TestGoModDetector_HandlesTwoRequiresSectionsAsync() + { + var goMod = + @"module microsoft/component-detection + +go 1.18 + +require ( + github.com/go-sql-driver/mysql v1.7.1 + rsc.io/quote v1.5.2 +) + +require ( + golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c // indirect + rsc.io/sampler v1.3.0 // indirect +)"; + + var (scanResult, componentRecorder) = await this.DetectorTestUtility + .WithFile("go.mod", goMod) + .ExecuteDetectorAsync(); + + scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); + componentRecorder.GetDetectedComponents().Should().HaveCount(4); + + var expectedComponentIds = new[] + { + "github.com/go-sql-driver/mysql v1.7.1 - Go", "rsc.io/quote v1.5.2 - Go", + "golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c - Go", "rsc.io/sampler v1.3.0 - Go", + }; + + componentRecorder.GetDetectedComponents().Select(c => c.Component.Id).Should().BeEquivalentTo(expectedComponentIds); + } + [TestMethod] public async Task TestGoSumDetection_TwoEntriesForTheSameComponent_ReturnsSuccessfullyAsync() {