microsoft
/
component-detection
зеркало из
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

WIP update to setuptools

This commit is contained in:
Coby Allred 2024-01-25 10:08:44 -08:00
Родитель b65fa8a7d1
Коммит d06e0a8110
5 изменённых файлов: 127 добавлений и 34 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

5
src/Microsoft.ComponentDetection.Common/PathUtilityService.cs
Просмотреть файл

@ -1,4 +1,4 @@
namespace Microsoft.ComponentDetection.Common;
namespace Microsoft.ComponentDetection.Common;
using System;
using System.IO;
@ -74,5 +74,8 @@ public class PathUtilityService : IPathUtilityService
return fileInfo.Exists ? this.ResolvePathFromInfo(fileInfo) : null;
}
public string NormalizePath(string path) =>
path?.Replace('\\', Path.DirectorySeparatorChar).Replace('/', Path.DirectorySeparatorChar);
private string ResolvePathFromInfo(FileSystemInfo info) => info.LinkTarget ?? info.FullName;
}

9
src/Microsoft.ComponentDetection.Contracts/IPathUtilityService.cs
Просмотреть файл

@ -1,4 +1,4 @@
namespace Microsoft.ComponentDetection.Contracts;
namespace Microsoft.ComponentDetection.Contracts;
/// <summary>
/// Wraps some common folder operations, shared across command line app and service.
@ -34,4 +34,11 @@ public interface IPathUtilityService
/// <param name="fileName">File name without directory.</param>
/// <returns>Returns true if file name matches a pattern, otherwise false.</returns>
bool MatchesPattern(string searchPattern, string fileName);
/// <summary>
/// Normalizes the path to the system based separator.
/// </summary>
/// <param name="path">the path.</param>
/// <returns>normalized path.</returns>
string NormalizePath(string path);
}

42
src/Microsoft.ComponentDetection.Detectors/pip/PythonCommandService.cs
Просмотреть файл

@ -8,17 +8,29 @@ using System.Text.RegularExpressions;
using System.Threading.Tasks;
using Microsoft.ComponentDetection.Contracts;
using Microsoft.ComponentDetection.Contracts.TypedComponent;
using Microsoft.Extensions.Logging;
public class PythonCommandService : IPythonCommandService
{
private const string ModuleImportError = "ModuleNotFoundError";
private readonly ICommandLineInvocationService commandLineInvocationService;
private readonly IPathUtilityService pathUtilityService;
private readonly ILogger<PythonCommandService> logger;
public PythonCommandService()
{
}
public PythonCommandService(ICommandLineInvocationService commandLineInvocationService) =>
public PythonCommandService(
ICommandLineInvocationService commandLineInvocationService,
IPathUtilityService pathUtilityService,
ILogger<PythonCommandService> logger)
{
this.commandLineInvocationService = commandLineInvocationService;
this.pathUtilityService = pathUtilityService;
this.logger = logger;
}
public async Task<bool> PythonExistsAsync(string pythonPath = null)
{
@ -57,12 +69,34 @@ public class PythonCommandService : IPythonCommandService
throw new PythonNotFoundException();
}
// This calls out to python and prints out an array like: [ packageA, packageB, packageC ]
// We need to have python interpret this file because install_requires can be composed at runtime
var command = await this.commandLineInvocationService.ExecuteCommandAsync(pythonExecutable, null, $"-c \"import distutils.core; setup=distutils.core.run_setup('{filePath.Replace('\\', '/')}'); print(setup.install_requires)\"");
// File paths will need to be normalized before passing to the python cmdline
var formattedFilePath = this.pathUtilityService.NormalizePath(filePath);
var workingDirectory = new DirectoryInfo(this.pathUtilityService.GetParentDirectory(formattedFilePath));
// This calls out to python and prints out an array like: [ packageA, packageB, packageC ].
// We need to have python interpret this file because install_requires can be composed at runtime.
// Attempt to use setuptools which is the replacement for deprecated distutils.
var command = await this.commandLineInvocationService.ExecuteCommandAsync(
pythonExecutable,
null,
workingDirectory,
$"-c \"from setuptools import setup; result=setup(); print(result.install_requires)\"");
// If importing setuptools fails, try and fallback to distutils.
if (command.ExitCode != 0 && command.StdErr.Contains(ModuleImportError, StringComparison.OrdinalIgnoreCase))
{
this.logger.LogInformation("Python: setuptools is not available to the Python runtime. Attempting to fall back to distutils...");
command = await this.commandLineInvocationService.ExecuteCommandAsync(
pythonExecutable,
null,
workingDirectory,
$"-c \"import distutils.core; setup=distutils.core.run_setup('{formattedFilePath}'); print(setup.install_requires)\"");
}
// If both invocations fail, log a warning and exit without declaring any packages.
if (command.ExitCode != 0)
{
this.logger.LogWarning("Python: Failure while attempting to parse install_requires from {FilePath}. Message: {StdErrOutput}", filePath, command.StdErr);
return new List<string>();
}

29
test/Microsoft.ComponentDetection.Common.Tests/PathUtilityServiceTests.cs Normal file
Просмотреть файл

@ -0,0 +1,29 @@
namespace Microsoft.ComponentDetection.Common.Tests;
using System.IO;
using FluentAssertions;
using Microsoft.ComponentDetection.Common;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.VisualStudio.TestTools.UnitTesting;
[TestClass]
[TestCategory("Governance/All")]
[TestCategory("Governance/ComponentDetection")]
public class PathUtilityServiceTests
{
[TestMethod]
public void PathShouldBeNormalized()
{
var service = new PathUtilityService(new NullLogger<PathUtilityService>());
var path = "Users\\SomeUser\\someDir\\someFile";
var normalizedPath = service.NormalizePath(path);
if (Path.DirectorySeparatorChar == '\\')
{
normalizedPath.Should().Be(path);
}
else
{
normalizedPath.Should().Be(string.Join(Path.DirectorySeparatorChar, path.Split('\\')));
}
}
}

76
test/Microsoft.ComponentDetection.Detectors.Tests/PythonCommandServiceTests.cs
Просмотреть файл

@ -9,6 +9,7 @@ using FluentAssertions;
using Microsoft.ComponentDetection.Contracts;
using Microsoft.ComponentDetection.Contracts.TypedComponent;
using Microsoft.ComponentDetection.Detectors.Pip;
using Microsoft.Extensions.Logging;
using Microsoft.VisualStudio.TestTools.UnitTesting;
using Moq;
@ -47,16 +48,23 @@ something=1.3
git+git://github.com/path/to/package-two@41b95ec#egg=package-two
other=2.1";
private readonly Mock<ICommandLineInvocationService> commandLineInvokationService;
private readonly Mock<ICommandLineInvocationService> mockCliService;
private readonly Mock<IPathUtilityService> mockPathUtilityService;
private readonly Mock<ILogger<PythonCommandService>> mockLogger;
public PythonCommandServiceTests() => this.commandLineInvokationService = new Mock<ICommandLineInvocationService>();
public PythonCommandServiceTests()
{
this.mockCliService = new();
this.mockPathUtilityService = new();
this.mockLogger = new();
}
[TestMethod]
public async Task PythonCommandService_ReturnsTrueWhenPythonExistsAsync()
{
this.commandLineInvokationService.Setup(x => x.CanCommandBeLocatedAsync("python", It.IsAny<IEnumerable<string>>(), "--version")).ReturnsAsync(true);
this.mockCliService.Setup(x => x.CanCommandBeLocatedAsync("python", It.IsAny<IEnumerable<string>>(), "--version")).ReturnsAsync(true);
var service = new PythonCommandService(this.commandLineInvokationService.Object);
var service = new PythonCommandService(this.mockCliService.Object, this.mockPathUtilityService.Object, this.mockLogger.Object);
(await service.PythonExistsAsync()).Should().BeTrue();
}
@ -64,9 +72,9 @@ other=2.1";
[TestMethod]
public async Task PythonCommandService_ReturnsFalseWhenPythonExistsAsync()
{
this.commandLineInvokationService.Setup(x => x.CanCommandBeLocatedAsync("python", It.IsAny<IEnumerable<string>>(), "--version")).ReturnsAsync(false);
this.mockCliService.Setup(x => x.CanCommandBeLocatedAsync("python", It.IsAny<IEnumerable<string>>(), "--version")).ReturnsAsync(false);
var service = new PythonCommandService(this.commandLineInvokationService.Object);
var service = new PythonCommandService(this.mockCliService.Object, this.mockPathUtilityService.Object, this.mockLogger.Object);
(await service.PythonExistsAsync()).Should().BeFalse();
}
@ -74,9 +82,9 @@ other=2.1";
[TestMethod]
public async Task PythonCommandService_ReturnsTrueWhenPythonExistsForAPathAsync()
{
this.commandLineInvokationService.Setup(x => x.CanCommandBeLocatedAsync("test", It.IsAny<IEnumerable<string>>(), "--version")).ReturnsAsync(true);
this.mockCliService.Setup(x => x.CanCommandBeLocatedAsync("test", It.IsAny<IEnumerable<string>>(), "--version")).ReturnsAsync(true);
var service = new PythonCommandService(this.commandLineInvokationService.Object);
var service = new PythonCommandService(this.mockCliService.Object, this.mockPathUtilityService.Object, this.mockLogger.Object);
(await service.PythonExistsAsync("test")).Should().BeTrue();
}
@ -84,9 +92,9 @@ other=2.1";
[TestMethod]
public async Task PythonCommandService_ReturnsFalseWhenPythonExistsForAPathAsync()
{
this.commandLineInvokationService.Setup(x => x.CanCommandBeLocatedAsync("test", It.IsAny<IEnumerable<string>>(), "--version")).ReturnsAsync(false);
this.mockCliService.Setup(x => x.CanCommandBeLocatedAsync("test", It.IsAny<IEnumerable<string>>(), "--version")).ReturnsAsync(false);
var service = new PythonCommandService(this.commandLineInvokationService.Object);
var service = new PythonCommandService(this.mockCliService.Object, this.mockPathUtilityService.Object, this.mockLogger.Object);
(await service.PythonExistsAsync("test")).Should().BeFalse();
}
@ -97,11 +105,14 @@ other=2.1";
var fakePath = @"c:\the\fake\path.py";
var fakePathAsPassedToPython = fakePath.Replace("\\", "/");
this.commandLineInvokationService.Setup(x => x.CanCommandBeLocatedAsync("python", It.IsAny<IEnumerable<string>>(), "--version")).ReturnsAsync(true);
this.commandLineInvokationService.Setup(x => x.ExecuteCommandAsync("python", It.IsAny<IEnumerable<string>>(), It.Is<string>(c => c.Contains(fakePathAsPassedToPython))))
this.mockPathUtilityService.Setup(x => x.NormalizePath(fakePath)).Returns(fakePathAsPassedToPython);
this.mockPathUtilityService.Setup(x => x.GetParentDirectory(fakePathAsPassedToPython)).Returns(Path.GetDirectoryName(fakePathAsPassedToPython));
this.mockCliService.Setup(x => x.CanCommandBeLocatedAsync("python", It.IsAny<IEnumerable<string>>(), "--version")).ReturnsAsync(true);
this.mockCliService.Setup(x => x.ExecuteCommandAsync("python", It.IsAny<IEnumerable<string>>(), It.IsAny<DirectoryInfo>(), It.Is<string>(c => c.Contains(fakePathAsPassedToPython))))
.ReturnsAsync(new CommandLineExecutionResult { ExitCode = 0, StdOut = "[]", StdErr = string.Empty });
var service = new PythonCommandService(this.commandLineInvokationService.Object);
var service = new PythonCommandService(this.mockCliService.Object, this.mockPathUtilityService.Object, this.mockLogger.Object);
var result = await service.ParseFileAsync(fakePath);
@ -114,11 +125,14 @@ other=2.1";
var fakePath = @"c:\the\fake\path.py";
var fakePathAsPassedToPython = fakePath.Replace("\\", "/");
this.commandLineInvokationService.Setup(x => x.CanCommandBeLocatedAsync("python", It.IsAny<IEnumerable<string>>(), "--version")).ReturnsAsync(true);
this.commandLineInvokationService.Setup(x => x.ExecuteCommandAsync("python", It.IsAny<IEnumerable<string>>(), It.Is<string>(c => c.Contains(fakePathAsPassedToPython))))
this.mockPathUtilityService.Setup(x => x.NormalizePath(fakePath)).Returns(fakePathAsPassedToPython);
this.mockPathUtilityService.Setup(x => x.GetParentDirectory(fakePathAsPassedToPython)).Returns(Path.GetDirectoryName(fakePathAsPassedToPython));
this.mockCliService.Setup(x => x.CanCommandBeLocatedAsync("python", It.IsAny<IEnumerable<string>>(), "--version")).ReturnsAsync(true);
this.mockCliService.Setup(x => x.ExecuteCommandAsync("python", It.IsAny<IEnumerable<string>>(), It.IsAny<DirectoryInfo>(), It.Is<string>(c => c.Contains(fakePathAsPassedToPython))))
.ReturnsAsync(new CommandLineExecutionResult { ExitCode = 0, StdOut = "None", StdErr = string.Empty });
var service = new PythonCommandService(this.commandLineInvokationService.Object);
var service = new PythonCommandService(this.mockCliService.Object, this.mockPathUtilityService.Object, this.mockLogger.Object);
var result = await service.ParseFileAsync(fakePath);
@ -131,11 +145,14 @@ other=2.1";
var fakePath = @"c:\the\fake\path.py";
var fakePathAsPassedToPython = fakePath.Replace("\\", "/");
this.commandLineInvokationService.Setup(x => x.CanCommandBeLocatedAsync("python", It.IsAny<IEnumerable<string>>(), "--version")).ReturnsAsync(true);
this.commandLineInvokationService.Setup(x => x.ExecuteCommandAsync("python", It.IsAny<IEnumerable<string>>(), It.Is<string>(c => c.Contains(fakePathAsPassedToPython))))
this.mockPathUtilityService.Setup(x => x.NormalizePath(fakePath)).Returns(fakePathAsPassedToPython);
this.mockPathUtilityService.Setup(x => x.GetParentDirectory(fakePathAsPassedToPython)).Returns(Path.GetDirectoryName(fakePathAsPassedToPython));
this.mockCliService.Setup(x => x.CanCommandBeLocatedAsync("python", It.IsAny<IEnumerable<string>>(), "--version")).ReturnsAsync(true);
this.mockCliService.Setup(x => x.ExecuteCommandAsync("python", It.IsAny<IEnumerable<string>>(), It.IsAny<DirectoryInfo>(), It.Is<string>(c => c.Contains(fakePathAsPassedToPython))))
.ReturnsAsync(new CommandLineExecutionResult { ExitCode = 0, StdOut = "['None']", StdErr = string.Empty });
var service = new PythonCommandService(this.commandLineInvokationService.Object);
var service = new PythonCommandService(this.mockCliService.Object, this.mockPathUtilityService.Object, this.mockLogger.Object);
var result = await service.ParseFileAsync(fakePath);
@ -149,11 +166,14 @@ other=2.1";
var fakePath = @"c:\the\fake\path.py";
var fakePathAsPassedToPython = fakePath.Replace("\\", "/");
this.commandLineInvokationService.Setup(x => x.CanCommandBeLocatedAsync("python", It.IsAny<IEnumerable<string>>(), "--version")).ReturnsAsync(true);
this.commandLineInvokationService.Setup(x => x.ExecuteCommandAsync("python", It.IsAny<IEnumerable<string>>(), It.Is<string>(c => c.Contains(fakePathAsPassedToPython))))
this.mockPathUtilityService.Setup(x => x.NormalizePath(fakePath)).Returns(fakePathAsPassedToPython);
this.mockPathUtilityService.Setup(x => x.GetParentDirectory(fakePathAsPassedToPython)).Returns(Path.GetDirectoryName(fakePathAsPassedToPython));
this.mockCliService.Setup(x => x.CanCommandBeLocatedAsync("python", It.IsAny<IEnumerable<string>>(), "--version")).ReturnsAsync(true);
this.mockCliService.Setup(x => x.ExecuteCommandAsync("python", It.IsAny<IEnumerable<string>>(), It.IsAny<DirectoryInfo>(), It.Is<string>(c => c.Contains(fakePathAsPassedToPython))))
.ReturnsAsync(new CommandLineExecutionResult { ExitCode = 0, StdOut = "['knack==0.4.1', 'setuptools>=1.0,!=1.1', 'vsts-cli-common==0.1.3', 'vsts-cli-admin==0.1.3', 'vsts-cli-build==0.1.3', 'vsts-cli-code==0.1.3', 'vsts-cli-team==0.1.3', 'vsts-cli-package==0.1.3', 'vsts-cli-work==0.1.3']", StdErr = string.Empty });
var service = new PythonCommandService(this.commandLineInvokationService.Object);
var service = new PythonCommandService(this.mockCliService.Object, this.mockPathUtilityService.Object, this.mockLogger.Object);
var result = await service.ParseFileAsync(fakePath);
var expected = new string[] { "knack==0.4.1", "setuptools>=1.0,!=1.1", "vsts-cli-common==0.1.3", "vsts-cli-admin==0.1.3", "vsts-cli-build==0.1.3", "vsts-cli-code==0.1.3", "vsts-cli-team==0.1.3", "vsts-cli-package==0.1.3", "vsts-cli-work==0.1.3" }.Select<string, (string, GitComponent)>(dep => (dep, null)).ToArray();
@ -171,8 +191,8 @@ other=2.1";
{
var testPath = Path.Join(Directory.GetCurrentDirectory(), string.Join(Guid.NewGuid().ToString(), ".txt"));
this.commandLineInvokationService.Setup(x => x.CanCommandBeLocatedAsync("python", It.IsAny<IEnumerable<string>>(), "--version")).ReturnsAsync(true);
var service = new PythonCommandService(this.commandLineInvokationService.Object);
this.mockCliService.Setup(x => x.CanCommandBeLocatedAsync("python", It.IsAny<IEnumerable<string>>(), "--version")).ReturnsAsync(true);
var service = new PythonCommandService(this.mockCliService.Object, this.mockPathUtilityService.Object, this.mockLogger.Object);
try
{
@ -208,8 +228,8 @@ other=2.1";
{
var testPath = Path.Join(Directory.GetCurrentDirectory(), string.Join(Guid.NewGuid().ToString(), ".txt"));
this.commandLineInvokationService.Setup(x => x.CanCommandBeLocatedAsync("python", It.IsAny<IEnumerable<string>>(), "--version")).ReturnsAsync(true);
var service = new PythonCommandService(this.commandLineInvokationService.Object);
this.mockCliService.Setup(x => x.CanCommandBeLocatedAsync("python", It.IsAny<IEnumerable<string>>(), "--version")).ReturnsAsync(true);
var service = new PythonCommandService(this.mockCliService.Object, this.mockPathUtilityService.Object, this.mockLogger.Object);
try
{
@ -390,8 +410,8 @@ other=2.1";
{
var testPath = Path.Join(Directory.GetCurrentDirectory(), string.Join(Guid.NewGuid().ToString(), ".txt"));
this.commandLineInvokationService.Setup(x => x.CanCommandBeLocatedAsync("python", It.IsAny<IEnumerable<string>>(), "--version")).ReturnsAsync(true);
var service = new PythonCommandService(this.commandLineInvokationService.Object);
this.mockCliService.Setup(x => x.CanCommandBeLocatedAsync("python", It.IsAny<IEnumerable<string>>(), "--version")).ReturnsAsync(true);
var service = new PythonCommandService(this.mockCliService.Object, this.mockPathUtilityService.Object, this.mockLogger.Object);
using (var writer = File.CreateText(testPath))
{