From 541b6468a2001c7e7f9b76ca471a0a1d7e57faea Mon Sep 17 00:00:00 2001 From: Paul Dorsch Date: Fri, 13 Sep 2024 09:45:43 -0400 Subject: [PATCH] add additional removal indicator --- .../FileUtilityService.cs | 5 +++-- .../IFileUtilityService.cs | 4 ++-- .../pip/PipCommandService.cs | 4 ++-- .../FileUtilityServiceTests.cs | 9 +++++++-- .../Microsoft.ComponentDetection.Common.Tests.csproj | 6 +++--- .../Resources/test-file-duplicate.txt | 1 + .../PipCommandServiceTests.cs | 6 +++--- 7 files changed, 21 insertions(+), 14 deletions(-) diff --git a/src/Microsoft.ComponentDetection.Common/FileUtilityService.cs b/src/Microsoft.ComponentDetection.Common/FileUtilityService.cs index 5faf0d06..714aebf7 100644 --- a/src/Microsoft.ComponentDetection.Common/FileUtilityService.cs +++ b/src/Microsoft.ComponentDetection.Common/FileUtilityService.cs @@ -2,6 +2,7 @@ namespace Microsoft.ComponentDetection.Common; using System.Collections.Generic; using System.IO; +using System.Linq; using System.Threading.Tasks; using Microsoft.ComponentDetection.Contracts; @@ -39,14 +40,14 @@ public class FileUtilityService : IFileUtilityService } /// - public (string DuplicateFilePath, bool CreatedDuplicate) DuplicateFileWithoutLines(string fileName, string removalIndicator) + public (string DuplicateFilePath, bool CreatedDuplicate) DuplicateFileWithoutLines(string fileName, params string[] removalIndicators) { // Read all lines from the file and filter out the lines that start with the removal indicator. var removedAnyLines = false; var linesToKeep = new List(); foreach (var line in File.ReadLines(fileName)) { - if (line == null || line.Trim().StartsWith(removalIndicator, System.StringComparison.OrdinalIgnoreCase)) + if (line == null || removalIndicators.Any(removalIndicator => line.Trim().StartsWith(removalIndicator, System.StringComparison.OrdinalIgnoreCase))) { removedAnyLines = true; } diff --git a/src/Microsoft.ComponentDetection.Contracts/IFileUtilityService.cs b/src/Microsoft.ComponentDetection.Contracts/IFileUtilityService.cs index 88c1675c..eaea4c61 100644 --- a/src/Microsoft.ComponentDetection.Contracts/IFileUtilityService.cs +++ b/src/Microsoft.ComponentDetection.Contracts/IFileUtilityService.cs @@ -47,7 +47,7 @@ public interface IFileUtilityService /// Duplicates a file, removing any lines that starts with the given string. /// /// Path to the file. - /// The string that indicates a line should be removed. + /// The strings that indicates a line should be removed. /// Returns the path of the new file, and whether or not one was created. - (string DuplicateFilePath, bool CreatedDuplicate) DuplicateFileWithoutLines(string fileName, string removalIndicator); + (string DuplicateFilePath, bool CreatedDuplicate) DuplicateFileWithoutLines(string fileName, params string[] removalIndicators); } diff --git a/src/Microsoft.ComponentDetection.Detectors/pip/PipCommandService.cs b/src/Microsoft.ComponentDetection.Detectors/pip/PipCommandService.cs index 8e8bd518..b2adf11c 100644 --- a/src/Microsoft.ComponentDetection.Detectors/pip/PipCommandService.cs +++ b/src/Microsoft.ComponentDetection.Detectors/pip/PipCommandService.cs @@ -149,7 +149,7 @@ public class PipCommandService : IPipCommandService if (this.environmentService.IsEnvironmentVariableValueTrue(PipReportIgnoreFileLevelIndexUrlEnvVar)) { // check for --index-url in requirements.txt and remove it from the file, since we want to use PIP_INDEX_URL from the environment. - var (duplicateFilePath, createdDuplicate) = this.fileUtilityService.DuplicateFileWithoutLines(formattedPath, "--index-url"); + var (duplicateFilePath, createdDuplicate) = this.fileUtilityService.DuplicateFileWithoutLines(formattedPath, "--index-url", "-i"); if (createdDuplicate) { var duplicateFileName = Path.GetFileName(duplicateFilePath); @@ -169,7 +169,7 @@ public class PipCommandService : IPipCommandService // When PIP_INDEX_URL is set, we need to pass it as a parameter to pip install command. // This should be done before running detection by the build system, otherwise the detection // will default to the public PyPI index if not configured in pip defaults. Note this index URL may have credentials, we need to remove it when logging. - pipReportCommand += $" --dry-run --ignore-installed --quiet --no-input --report {reportName}"; + pipReportCommand += $" --dry-run --ignore-installed --quiet --no-cache-dir --no-input --report {reportName}"; if (this.environmentService.DoesEnvironmentVariableExist("PIP_INDEX_URL")) { pipReportCommand += $" --index-url {this.environmentService.GetEnvironmentVariable("PIP_INDEX_URL")}"; diff --git a/test/Microsoft.ComponentDetection.Common.Tests/FileUtilityServiceTests.cs b/test/Microsoft.ComponentDetection.Common.Tests/FileUtilityServiceTests.cs index ca44ae84..fc02cea7 100644 --- a/test/Microsoft.ComponentDetection.Common.Tests/FileUtilityServiceTests.cs +++ b/test/Microsoft.ComponentDetection.Common.Tests/FileUtilityServiceTests.cs @@ -23,16 +23,21 @@ public class FileUtilityServiceTests // Arrange var fileName = $"{directory}/Resources/test-file-duplicate.txt"; - var removalIndicator = "//REMOVE"; var expectedDuplicateFilePath = Path.Combine(directory, "Resources", "temp.test-file-duplicate.txt"); // Act - var (duplicateFilePath, createdDuplicate) = this.fileUtilityService.DuplicateFileWithoutLines(fileName, removalIndicator); + var (duplicateFilePath, createdDuplicate) = this.fileUtilityService.DuplicateFileWithoutLines(fileName, "//REMOVE", "//ME"); // Assert createdDuplicate.Should().BeTrue(); duplicateFilePath.Should().Be(expectedDuplicateFilePath); File.Exists(expectedDuplicateFilePath).Should().BeTrue(); + + var contents = File.ReadAllText(expectedDuplicateFilePath); + contents.Should().NotContain("//REMOVE"); + contents.Should().NotContain("//ME"); + contents.Should().Contain("hello"); + contents.Should().Contain("world"); } [TestMethod] diff --git a/test/Microsoft.ComponentDetection.Common.Tests/Microsoft.ComponentDetection.Common.Tests.csproj b/test/Microsoft.ComponentDetection.Common.Tests/Microsoft.ComponentDetection.Common.Tests.csproj index b4be8372..76651e7e 100644 --- a/test/Microsoft.ComponentDetection.Common.Tests/Microsoft.ComponentDetection.Common.Tests.csproj +++ b/test/Microsoft.ComponentDetection.Common.Tests/Microsoft.ComponentDetection.Common.Tests.csproj @@ -1,4 +1,4 @@ - + @@ -9,8 +9,8 @@ - - + + diff --git a/test/Microsoft.ComponentDetection.Common.Tests/Resources/test-file-duplicate.txt b/test/Microsoft.ComponentDetection.Common.Tests/Resources/test-file-duplicate.txt index 9e8b2a86..4eb15c7a 100644 --- a/test/Microsoft.ComponentDetection.Common.Tests/Resources/test-file-duplicate.txt +++ b/test/Microsoft.ComponentDetection.Common.Tests/Resources/test-file-duplicate.txt @@ -1,3 +1,4 @@ hello //REMOVE world +//ME diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/PipCommandServiceTests.cs b/test/Microsoft.ComponentDetection.Detectors.Tests/PipCommandServiceTests.cs index 2ecf7611..cfbc4878 100644 --- a/test/Microsoft.ComponentDetection.Detectors.Tests/PipCommandServiceTests.cs +++ b/test/Microsoft.ComponentDetection.Detectors.Tests/PipCommandServiceTests.cs @@ -500,7 +500,7 @@ public class PipCommandServiceTests this.envVarService.Setup(x => x.IsEnvironmentVariableValueTrue("PipReportIgnoreFileLevelIndexUrl")).Returns(true); this.commandLineInvokationService.Setup(x => x.CanCommandBeLocatedAsync("pip", It.IsAny>(), "--version")).ReturnsAsync(true); - this.fileUtilityService.Setup(x => x.DuplicateFileWithoutLines(It.IsAny(), "--index-url")) + this.fileUtilityService.Setup(x => x.DuplicateFileWithoutLines(It.IsAny(), "--index-url", "-i")) .Returns(("C:/asdf/temp.requirements.txt", true)) .Verifiable(); this.fileUtilityService.Setup(x => x.ReadAllTextAsync(It.IsAny())) @@ -535,7 +535,7 @@ public class PipCommandServiceTests this.envVarService.Setup(x => x.IsEnvironmentVariableValueTrue("PipReportIgnoreFileLevelIndexUrl")).Returns(true); this.commandLineInvokationService.Setup(x => x.CanCommandBeLocatedAsync("pip", It.IsAny>(), "--version")).ReturnsAsync(true); - this.fileUtilityService.Setup(x => x.DuplicateFileWithoutLines(It.IsAny(), "--index-url")) + this.fileUtilityService.Setup(x => x.DuplicateFileWithoutLines(It.IsAny(), "--index-url", "-i")) .Returns((null, false)) .Verifiable(); this.fileUtilityService.Setup(x => x.ReadAllTextAsync(It.IsAny())) @@ -590,7 +590,7 @@ public class PipCommandServiceTests this.logger.Object); var (report, reportFile) = await service.GenerateInstallationReportAsync(testPath); - this.fileUtilityService.Verify(x => x.DuplicateFileWithoutLines(It.IsAny(), "--index-url"), Times.Never); + this.fileUtilityService.Verify(x => x.DuplicateFileWithoutLines(It.IsAny(), "--index-url", "-i"), Times.Never); this.commandLineInvokationService.Verify(); }