Add poetry detector (#23)

Co-authored-by: Jamie Magee <jamagee@microsoft.com>

README.md

@@ -31,6 +31,7 @@ ComponentDetection is a package scanning tool intended to be used at build time.
 | NPM (including Yarn, Pnpm) | ✔ | ✔ |
 | NuGet | ✔ | ✔ |
 | Pip (Python) | ✔ | ✔ |
+| Poetry (Python, lockfiles only) | ✔ | ❌ |
 | Ruby | ✔ | ✔ |
 | Rust | ✔ | ✔ |
 

docs/creating-a-new-detector.md

@@ -129,7 +129,7 @@ From the example above you can see each test is initialized with a new `Detector
 ## How to run/debug your detector
 
 ```
-dotnet run -p "[YOUR REPO PATH]\src\Microsoft.ComponentDetection\Microsoft.ComponentDetection.csproj" scan
+dotnet run --project "[YOUR REPO PATH]\src\Microsoft.ComponentDetection\Microsoft.ComponentDetection.csproj" scan
 --Verbosity Verbose
 --SourceDirectory [PATH TO THE REPO TO SCAN]
 --DetectorArgs [YOUR DETECTOR ID]=EnableIfDefaultOff

docs/detectors/README.md

@@ -8,5 +8,6 @@
 - NPM
 - NuGet
 - [Pip](pip.md)
+- [Poetry](poetry.md)
 - Ruby
 - Rust

docs/detectors/poetry.md

@@ -0,0 +1,11 @@
+# Poetry Detection
+## Requirements
+Poetry detection relies on a poetry.lock file being present.
+
+## Detection strategy
+Poetry detection is performed by parsing a <em>poetry.lock</em> found under the scan directory.
+
+## Known limitations
+Poetry detection will not work if lock files are not being used.
+
+Full dependency graph generation is not supported.

docs/feature-overview.md

@@ -13,6 +13,7 @@
 | Pnpm | <ul><li>shrinkwrap.yaml</li><li>pnpm-lock.yaml</li></ul> | - | ✔ (packages/{package}/dev flag) | ✔ |
 | NuGet | <ul><li>project.assets.json</li><li>*.nupkg</li><li>*.nuspec</li><li>nuget.config</li></ul> | - | - | ✔ (required project.assets.json) |
 | Pip (Python) | <ul><li>setup.py</li><li>requirements.txt</li><li>*setup=distutils.core.run_setup({setup.py}); setup.install_requires*</li><li>dist package METADATA file</li></ul> | <ul><li>Python 2 or Python 3</li><li>Internet connection</li></ul> | ❌ | ✔ |
+| Poetry (Python) | <ul><li>poetry.lock</li><ul> | - | ✔ | ❌ |
 | Ruby | <ul><li>gemfile.lock</li></ul> | - | ❌ | ✔ |
 | Cargo | <ul><li>cargo.lock (v1, v2)</li><li>cargo.toml</li></ul> | - | ✔ (dev-dependencies in cargo.toml) | ✔ | 
 

src/Microsoft.ComponentDetection.Detectors/poetry/Contracts/PoetryLock.cs

@@ -0,0 +1,12 @@
+using System.Diagnostics.CodeAnalysis;
+using Nett;
+
+namespace Microsoft.ComponentDetection.Detectors.Poetry.Contracts
+{
+    // Represents Poetry.Lock file structure.
+    public class PoetryLock
+    {
+        [SuppressMessage("StyleCop.CSharp.NamingRules", "SA1300:ElementMustBeginWithUpperCaseLetter", Justification = "Deserialization contract. Casing cannot be overwritten.")]
+        public PoetryPackage[] package { get; set; }
+    }
+}

src/Microsoft.ComponentDetection.Detectors/poetry/Contracts/PoetryPackage.cs

@@ -0,0 +1,21 @@
+using System;
+using System.Collections.Generic;
+using System.Diagnostics.CodeAnalysis;
+
+namespace Microsoft.ComponentDetection.Detectors.Poetry.Contracts
+{
+    public class PoetryPackage
+    {
+        [SuppressMessage("StyleCop.CSharp.NamingRules", "SA1300:ElementMustBeginWithUpperCaseLetter", Justification = "Deserialization contract. Casing cannot be overwritten.")]
+        public string category { get; set; }
+
+        [SuppressMessage("StyleCop.CSharp.NamingRules", "SA1300:ElementMustBeginWithUpperCaseLetter", Justification = "Deserialization contract. Casing cannot be overwritten.")]
+        public string name { get; set; }
+
+        [SuppressMessage("StyleCop.CSharp.NamingRules", "SA1300:ElementMustBeginWithUpperCaseLetter", Justification = "Deserialization contract. Casing cannot be overwritten.")]
+        public string version { get; set; }
+
+        [SuppressMessage("StyleCop.CSharp.NamingRules", "SA1300:ElementMustBeginWithUpperCaseLetter", Justification = "Deserialization contract. Casing cannot be overwritten.")]
+        public PoetrySource source { get; set; }
+    }
+}

src/Microsoft.ComponentDetection.Detectors/poetry/Contracts/PoetrySource.cs

@@ -0,0 +1,21 @@
+using System;
+using System.Collections.Generic;
+using System.Diagnostics.CodeAnalysis;
+
+namespace Microsoft.ComponentDetection.Detectors.Poetry.Contracts
+{
+    public class PoetrySource
+    {
+        [SuppressMessage("StyleCop.CSharp.NamingRules", "SA1300:ElementMustBeginWithUpperCaseLetter", Justification = "Deserialization contract. Casing cannot be overwritten.")]
+        public string type { get; set; }
+
+        [SuppressMessage("StyleCop.CSharp.NamingRules", "SA1300:ElementMustBeginWithUpperCaseLetter", Justification = "Deserialization contract. Casing cannot be overwritten.")]
+        public string url { get; set; }
+
+        [SuppressMessage("StyleCop.CSharp.NamingRules", "SA1300:ElementMustBeginWithUpperCaseLetter", Justification = "Deserialization contract. Casing cannot be overwritten.")]
+        public string reference { get; set; }
+
+        [SuppressMessage("StyleCop.CSharp.NamingRules", "SA1300:ElementMustBeginWithUpperCaseLetter", Justification = "Deserialization contract. Casing cannot be overwritten.")]
+        public string resolved_reference { get; set; }
+    }
+}

src/Microsoft.ComponentDetection.Detectors/poetry/PoetryComponentDetector.cs

@@ -0,0 +1,54 @@
+using System;
+using System.Collections.Generic;
+using System.Composition;
+using System.IO;
+using System.Linq;
+using System.Threading.Tasks;
+using Microsoft.ComponentDetection.Contracts;
+using Microsoft.ComponentDetection.Contracts.Internal;
+using Microsoft.ComponentDetection.Contracts.TypedComponent;
+using Microsoft.ComponentDetection.Detectors.Poetry.Contracts;
+using Nett;
+
+namespace Microsoft.ComponentDetection.Detectors.Poetry
+{
+    [Export(typeof(IComponentDetector))]
+    public class PoetryComponentDetector : FileComponentDetector, IDefaultOffComponentDetector
+    {
+        public override string Id => "Poetry";
+
+        public override IList<string> SearchPatterns { get; } = new List<string> { "poetry.lock" };
+
+        public override IEnumerable<ComponentType> SupportedComponentTypes => new[] { ComponentType.Pip };
+
+        public override int Version { get; } = 1;
+
+        public override IEnumerable<string> Categories => new List<string> { "Python" };
+
+        protected override Task OnFileFound(ProcessRequest processRequest, IDictionary<string, string> detectorArgs)
+        {
+            var singleFileComponentRecorder = processRequest.SingleFileComponentRecorder;
+            var poetryLockFile = processRequest.ComponentStream;
+            Logger.LogVerbose("Found Poetry lockfile: " + poetryLockFile);
+
+            var poetryLock = StreamTomlSerializer.Deserialize(poetryLockFile.Stream, TomlSettings.Create()).Get<PoetryLock>();
+            poetryLock.package.ToList().ForEach(package =>
+            {
+                var isDevelopmentDependency = package.category != "main";
+
+                if (package.source != null && package.source.type == "git")
+                {
+                    var component = new DetectedComponent(new GitComponent(new Uri(package.source.url), package.source.resolved_reference));
+                    singleFileComponentRecorder.RegisterUsage(component, isDevelopmentDependency: isDevelopmentDependency);
+                }
+                else
+                {
+                    var component = new DetectedComponent(new PipComponent(package.name, package.version));
+                    singleFileComponentRecorder.RegisterUsage(component, isDevelopmentDependency: isDevelopmentDependency);
+                }
+            });
+
+            return Task.CompletedTask;
+        }
+    }
+}

test/Microsoft.ComponentDetection.Detectors.Tests/PoetryComponentDetectorTests.cs

@@ -0,0 +1,149 @@
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading.Tasks;
+using Microsoft.ComponentDetection.Contracts;
+using Microsoft.ComponentDetection.Contracts.TypedComponent;
+using Microsoft.ComponentDetection.Detectors.Poetry;
+using Microsoft.ComponentDetection.Detectors.Tests.Utilities;
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+using Microsoft.ComponentDetection.TestsUtilities;
+
+namespace Microsoft.ComponentDetection.Detectors.Tests
+{
+    [TestClass]
+    [TestCategory("Governance/All")]
+    [TestCategory("Governance/ComponentDetection")]
+    public class PoetryComponentDetectorTests
+    {
+        private DetectorTestUtility<PoetryComponentDetector> detectorTestUtility;
+
+        [TestInitialize]
+        public void TestInitialize()
+        {
+            detectorTestUtility = DetectorTestUtilityCreator.Create<PoetryComponentDetector>();
+        }
+
+        [TestMethod]
+        public async Task TestPoetryDetector_TestCustomSource()
+        {
+            var poetryLockContent = @"[[package]]
+name = ""certifi""
+version = ""2021.10.8""
+description = ""Python package for providing Mozilla's CA Bundle.""
+category = ""main""
+optional = false
+python-versions = ""*""
+
+[package.source]
+type = ""legacy""
+url = ""https://pypi.custom.com//simple""
+reference = ""custom""
+";
+
+            var (scanResult, componentRecorder) = await detectorTestUtility
+                                                    .WithFile("poetry.lock", poetryLockContent)
+                                                    .ExecuteDetector();
+
+            Assert.AreEqual(ProcessingResultCode.Success, scanResult.ResultCode);
+
+            var detectedComponents = componentRecorder.GetDetectedComponents();
+            Assert.AreEqual(1, detectedComponents.Count());
+
+            AssertPipComponentNameAndVersion(detectedComponents, "certifi", "2021.10.8");
+            var queryString = detectedComponents.Single(component => ((PipComponent)component.Component).Name.Contains("certifi"));
+            Assert.IsFalse(componentRecorder.GetEffectiveDevDependencyValue(queryString.Component.Id).GetValueOrDefault(false));
+        }
+
+        [TestMethod]
+        public async Task TestPoetryDetector_TestDevDependency()
+        {
+            var poetryLockContent = @"[[package]]
+name = ""certifi""
+version = ""2021.10.8""
+description = ""Python package for providing Mozilla's CA Bundle.""
+category = ""dev""
+optional = false
+python-versions = ""*""
+";
+
+            var (scanResult, componentRecorder) = await detectorTestUtility
+                                                    .WithFile("poetry.lock", poetryLockContent)
+                                                    .ExecuteDetector();
+
+            Assert.AreEqual(ProcessingResultCode.Success, scanResult.ResultCode);
+
+            var detectedComponents = componentRecorder.GetDetectedComponents();
+            Assert.AreEqual(1, detectedComponents.Count());
+
+            AssertPipComponentNameAndVersion(detectedComponents, "certifi", "2021.10.8");
+
+            var queryString = detectedComponents.Single(component => ((PipComponent)component.Component).Name.Contains("certifi"));
+            Assert.IsTrue(componentRecorder.GetEffectiveDevDependencyValue(queryString.Component.Id).GetValueOrDefault(false));
+        }
+
+        [TestMethod]
+        public async Task TestPoetryDetector_TestGitDependency()
+        {
+            var poetryLockContent = @"[[package]]
+name = ""certifi""
+version = ""2021.10.8""
+description = ""Python package for providing Mozilla's CA Bundle.""
+category = ""dev""
+optional = false
+python-versions = ""*""
+
+[[package]]
+name = ""requests""
+version = ""2.26.0""
+description = ""Python HTTP for Humans.""
+category = ""main""
+optional = false
+python-versions = "">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*""
+develop = false
+
+[package.dependencies]
+certifi = "">=2017.4.17""
+charset-normalizer = {version = "">=2.0.0,<2.1.0"", markers = ""python_version >= \""3\""""}
+idna = {version = "">=2.5,<4"", markers = ""python_version >= \""3\""""}
+urllib3 = "">=1.21.1,<1.27""
+
+[package.extras]
+socks = [""PySocks (>=1.5.6,!=1.5.7)"", ""win-inet-pton""]
+use_chardet_on_py3 = [""chardet (>=3.0.2,<5)""]
+
+[package.source]
+type = ""git""
+url = ""https://github.com/requests/requests.git""
+reference = ""master""
+resolved_reference = ""232a5596424c98d11c3cf2e29b2f6a6c591c2ff3""";
+
+            var (scanResult, componentRecorder) = await detectorTestUtility
+                                                    .WithFile("poetry.lock", poetryLockContent)
+                                                    .ExecuteDetector();
+
+            Assert.AreEqual(ProcessingResultCode.Success, scanResult.ResultCode);
+
+            var detectedComponents = componentRecorder.GetDetectedComponents();
+            Assert.AreEqual(2, detectedComponents.Count());
+
+            AssertGitComponentHashAndUrl(detectedComponents, "232a5596424c98d11c3cf2e29b2f6a6c591c2ff3", "https://github.com/requests/requests.git");
+        }
+
+        private void AssertPipComponentNameAndVersion(IEnumerable<DetectedComponent> detectedComponents, string name, string version)
+        {
+            Assert.IsNotNull(
+                detectedComponents.SingleOrDefault(c =>
+            c.Component is PipComponent component &&
+            component.Name.Equals(name) &&
+            component.Version.Equals(version)), $"Component with name {name} and version {version} was not found");
+        }
+
+        private void AssertGitComponentHashAndUrl(IEnumerable<DetectedComponent> detectedComponents, string commitHash, string repositoryUrl)
+        {
+            Assert.IsNotNull(detectedComponents.SingleOrDefault(c =>
+            c.Component is GitComponent component &&
+            component.CommitHash.Equals(commitHash) &&
+            component.RepositoryUrl.Equals(repositoryUrl)));
+        }
+    }
+}