New PypiClient using the new Simple Api (#672)

* Added a new client for SimplePypi and fixed disposal in the original pypi client

Directory.Packages.props

@@ -15,6 +15,7 @@
     <PackageVersion Include="Microsoft.Extensions.Caching.Memory" Version="7.0.0" />
     <PackageVersion Include="Microsoft.Extensions.DependencyInjection" Version="7.0.0" />
     <PackageVersion Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="7.0.0" />
+    <PackageVersion Include="Microsoft.Extensions.Http" Version="7.0.0" />
     <PackageVersion Include="Microsoft.Extensions.Logging" Version="7.0.0" />
     <PackageVersion Include="Microsoft.NET.Test.Sdk" Version="17.6.3" />
     <PackageVersion Include="Microsoft.SourceLink.GitHub" Version="1.1.1" />

src/Microsoft.ComponentDetection.Common/Telemetry/Records/SimplePypiCacheTelemetryRecord.cs

@@ -0,0 +1,26 @@
+namespace Microsoft.ComponentDetection.Common.Telemetry.Records;
+
+public class SimplePypiCacheTelemetryRecord : BaseDetectionTelemetryRecord
+{
+    public override string RecordName => "SimplePyPiCache";
+
+    /// <summary>
+    /// Gets or sets total number of PyPi project requests that hit the cache instead of SimplePyPi API.
+    /// </summary>
+    public int NumSimpleProjectCacheHits { get; set; }
+
+    /// <summary>
+    /// Gets or sets total number of project wheel file requests that hit the cache instead of API.
+    /// </summary>
+    public int NumProjectFileCacheHits { get; set; }
+
+    /// <summary>
+    /// Gets or sets the size of the Simple Project cache at class destruction.
+    /// </summary>
+    public int FinalSimpleProjectCacheSize { get; set; }
+
+    /// <summary>
+    /// Gets or sets the size of the Project Wheel File cache at class destruction.
+    /// </summary>
+    public int FinalProjectFileCacheSize { get; set; }
+}

src/Microsoft.ComponentDetection.Detectors/Microsoft.ComponentDetection.Detectors.csproj

@@ -1,7 +1,8 @@
-<Project Sdk="Microsoft.NET.Sdk">
+<Project Sdk="Microsoft.NET.Sdk">
 
     <ItemGroup>
         <PackageReference Include="DotNet.Glob" />
+        <PackageReference Include="Microsoft.Extensions.Http" />
         <PackageReference Include="Microsoft.Extensions.Logging" />
         <PackageReference Include="morelinq" />
         <PackageReference Include="NuGet.ProjectModel" />

src/Microsoft.ComponentDetection.Detectors/pip/IPyPiClient.cs

@@ -1,4 +1,4 @@
-using System;
+using System;
 using System.Collections.Generic;
 using System.IO;
 using System.IO.Compression;
@@ -27,7 +27,7 @@ public interface IPyPiClient
     Task<SortedDictionary<string, IList<PythonProjectRelease>>> GetReleasesAsync(PipDependencySpecification spec);
 }
 
-public class PyPiClient : IPyPiClient
+public sealed class PyPiClient : IPyPiClient, IDisposable
 {
     // Values used for cache creation
     private const long CACHEINTERVALSECONDS = 60;
@@ -82,12 +82,6 @@ public class PyPiClient : IPyPiClient
         this.logger = logger;
     }
 
-    ~PyPiClient()
-    {
-        this.cacheTelemetry.FinalCacheSize = this.cachedResponses.Count;
-        this.cacheTelemetry.Dispose();
-    }
-
     public static HttpClient HttpClient { get; internal set; } = new HttpClient(HttpClientHandler);
 
     public async Task<IList<PipDependencySpecification>> FetchPackageDependenciesAsync(string name, string version, PythonProjectRelease release)
@@ -282,4 +276,11 @@ public class PyPiClient : IPyPiClient
 
         this.checkedMaxEntriesVariable = true;
     }
+
+    public void Dispose()
+    {
+        this.cacheTelemetry.FinalCacheSize = this.cachedResponses.Count;
+        this.cacheTelemetry.Dispose();
+        this.cachedResponses.Dispose();
+    }
 }

src/Microsoft.ComponentDetection.Detectors/pip/ISimplePyPiClient.cs

@@ -0,0 +1,22 @@
+namespace Microsoft.ComponentDetection.Detectors.Pip;
+
+using System;
+using System.IO;
+using System.Threading.Tasks;
+
+public interface ISimplePyPiClient
+{
+    /// <summary>
+    /// Uses the release url to retrieve the project file.
+    /// </summary>
+    /// <param name="releaseUrl">The url to fetch dependencies from. </param>
+    /// <returns>Returns a project from the simplepypi api. </returns>
+    Task<Stream> FetchPackageFileStreamAsync(Uri releaseUrl);
+
+    /// <summary>
+    /// Calls simplepypi and retrieves the project specified with the spec name.
+    /// </summary>
+    /// <param name="spec">The PipDependencySpecification for the project. </param>
+    /// <returns>Returns a project from the simplepypi api. </returns>
+    Task<SimplePypiProject> GetSimplePypiProjectAsync(PipDependencySpecification spec);
+}

src/Microsoft.ComponentDetection.Detectors/pip/SimplePypiClient.cs

@@ -0,0 +1,285 @@
+namespace Microsoft.ComponentDetection.Detectors.Pip;
+
+using System;
+using System.IO;
+using System.Linq;
+using System.Net.Http;
+using System.Net.Http.Headers;
+using System.Reflection;
+using System.Text.Json;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.ComponentDetection.Common.Telemetry.Records;
+using Microsoft.ComponentDetection.Contracts;
+using Microsoft.Extensions.Caching.Memory;
+using Microsoft.Extensions.Logging;
+using Polly;
+
+public sealed class SimplePyPiClient : ISimplePyPiClient, IDisposable
+{
+    // Values used for cache creation
+    private const long CACHEINTERVALSECONDS = 60;
+    private const long DEFAULTCACHEENTRIES = 4096;
+
+    // max number of retries allowed, to cap the total delay period
+    public const long MAXRETRIES = 15;
+
+    private static readonly ProductInfoHeaderValue ProductValue = new(
+    "ComponentDetection",
+    Assembly.GetEntryAssembly().GetCustomAttribute<AssemblyInformationalVersionAttribute>()?.InformationalVersion);
+
+    private static readonly ProductInfoHeaderValue CommentValue = new("(+https://github.com/microsoft/component-detection)");
+
+    // time to wait before retrying a failed call to pypi.org
+    private static readonly TimeSpan RETRYDELAY = TimeSpan.FromSeconds(1);
+
+    private readonly IEnvironmentVariableService environmentVariableService;
+    private readonly ILogger<SimplePyPiClient> logger;
+
+    // Keep telemetry on how the cache is being used for future refinements
+    private readonly SimplePypiCacheTelemetryRecord cacheTelemetry = new SimplePypiCacheTelemetryRecord();
+
+    private readonly HttpClient httpClient;
+
+    /// <summary>
+    /// A thread safe cache implementation which contains a mapping of URI -> SimpleProject for simplepypi api projects
+    /// and has a limited number of entries which will expire after the cache fills or a specified interval.
+    /// </summary>
+    private MemoryCache cachedSimplePyPiProjects = new MemoryCache(new MemoryCacheOptions { SizeLimit = DEFAULTCACHEENTRIES });
+
+    /// <summary>
+    /// A thread safe cache implementation which contains a mapping of URI -> Stream for project wheel files
+    /// and has a limited number of entries which will expire after the cache fills or a specified interval.
+    /// </summary>
+    private MemoryCache cachedProjectWheelFiles = new MemoryCache(new MemoryCacheOptions { SizeLimit = DEFAULTCACHEENTRIES });
+
+    private bool checkedMaxEntriesVariable;
+
+    // retries used so far for calls to pypi.org
+    private long retries;
+
+    public SimplePyPiClient(IEnvironmentVariableService environmentVariableService, IHttpClientFactory httpClientFactory, ILogger<SimplePyPiClient> logger)
+    {
+        this.environmentVariableService = environmentVariableService;
+        this.logger = logger;
+        this.httpClient = httpClientFactory.CreateClient();
+    }
+
+    /// <inheritdoc />
+    public async Task<SimplePypiProject> GetSimplePypiProjectAsync(PipDependencySpecification spec)
+    {
+        var requestUri = new Uri($"https://pypi.org/simple/{spec.Name}");
+
+        var project = await this.GetAndCacheSimpleProjectAsync(requestUri, spec);
+
+        return project;
+    }
+
+    /// <inheritdoc />
+    public async Task<Stream> FetchPackageFileStreamAsync(Uri releaseUrl)
+    {
+        var projectStream = await this.GetAndCacheProjectFileAsync(releaseUrl);
+
+        return projectStream;
+    }
+
+    /// <summary>
+    /// Returns a cached response if it exists, otherwise returns the response from PyPi REST call.
+    /// The response from PyPi is automatically added to the cache.
+    /// </summary>
+    /// <param name="uri">The REST Uri to call.</param>
+    /// <returns>The cached project file or a new result from Simple PyPi.</returns>
+    private async Task<Stream> GetAndCacheProjectFileAsync(Uri uri)
+    {
+        if (!this.checkedMaxEntriesVariable)
+        {
+            this.cachedProjectWheelFiles = this.InitializeNonDefaultMemoryCache(this.cachedProjectWheelFiles);
+        }
+
+        if (this.cachedProjectWheelFiles.TryGetValue(uri, out Stream result))
+        {
+            this.cacheTelemetry.NumProjectFileCacheHits++;
+            this.logger.LogDebug("Retrieved cached Python data from {Uri}", uri);
+            return result;
+        }
+
+        var response = await this.GetPypiResponseAsync(uri);
+
+        if (!response.IsSuccessStatusCode)
+        {
+            this.logger.LogWarning("Http GET at {ReleaseUrl} failed with status code {ResponseStatusCode}", uri, response.StatusCode);
+            return new MemoryStream();
+        }
+
+        var responseContent = await response.Content.ReadAsStreamAsync();
+
+        // The `first - wins` response accepted into the cache. This might be different from the input if another caller wins the race.
+        return await this.cachedProjectWheelFiles.GetOrCreateAsync(uri, cacheEntry =>
+        {
+            cacheEntry.SlidingExpiration = TimeSpan.FromSeconds(CACHEINTERVALSECONDS); // This entry will expire after CACHEINTERVALSECONDS seconds from last use
+            cacheEntry.Size = 1; // Specify a size of 1 so a set number of entries can always be in the cache
+            return Task.FromResult(responseContent);
+        });
+    }
+
+    /// <summary>
+    /// Returns a cached response if it exists, otherwise returns the response from PyPi REST call.
+    /// The response from PyPi is automatically added to the cache.
+    /// </summary>
+    /// <param name="uri">The REST Uri to call.</param>
+    /// <param name="spec">The PipDependencySpecification for the project. </param>
+    /// <returns>The cached deserialized json object or a new result from Simple PyPi.</returns>
+    private async Task<SimplePypiProject> GetAndCacheSimpleProjectAsync(Uri uri, PipDependencySpecification spec)
+    {
+        var pythonProject = new SimplePypiProject();
+        if (!this.checkedMaxEntriesVariable)
+        {
+            this.cachedSimplePyPiProjects = this.InitializeNonDefaultMemoryCache(this.cachedSimplePyPiProjects);
+        }
+
+        if (this.cachedSimplePyPiProjects.TryGetValue(uri, out SimplePypiProject result))
+        {
+            this.cacheTelemetry.NumSimpleProjectCacheHits++;
+            this.logger.LogDebug("Retrieved cached Python data from {Uri}", uri);
+            return result;
+        }
+
+        var response = await this.RetryPypiRequestAsync(uri, spec);
+
+        var responseContent = await response.Content.ReadAsStringAsync();
+        if (string.IsNullOrEmpty(responseContent))
+        {
+            return pythonProject;
+        }
+
+        try
+        {
+            pythonProject = JsonSerializer.Deserialize<SimplePypiProject>(responseContent);
+        }
+        catch (JsonException e)
+        {
+            this.logger.LogError(
+                    e,
+                    "Unable to deserialize simple pypi project. This is possibly because the server responded with an unexpected content type. Spec Name = {SpecName}",
+                    spec.Name);
+            return new SimplePypiProject();
+        }
+
+        // The `first - wins` response accepted into the cache. This might be different from the input if another caller wins the race.
+        return await this.cachedSimplePyPiProjects.GetOrCreateAsync(uri, cacheEntry =>
+        {
+            cacheEntry.SlidingExpiration = TimeSpan.FromSeconds(CACHEINTERVALSECONDS); // This entry will expire after CACHEINTERVALSECONDS seconds from last use
+            cacheEntry.Size = 1; // Specify a size of 1 so a set number of entries can always be in the cache
+            return Task.FromResult(pythonProject);
+        });
+    }
+
+    /// <summary>
+    /// On the initial caching attempt, see if the user specified an override for
+    /// PyPiMaxCacheEntries and recreate the cache if needed.
+    /// </summary>
+    private MemoryCache InitializeNonDefaultMemoryCache(MemoryCache cache)
+    {
+        var maxEntriesVariable = this.environmentVariableService.GetEnvironmentVariable("PyPiMaxCacheEntries");
+        if (!string.IsNullOrEmpty(maxEntriesVariable) && long.TryParse(maxEntriesVariable, out var maxEntries))
+        {
+            this.logger.LogInformation("Setting ISimplePyPiClient max cache entries to {MaxEntries}", maxEntries);
+            cache = new MemoryCache(new MemoryCacheOptions { SizeLimit = maxEntries });
+        }
+
+        this.checkedMaxEntriesVariable = true;
+        return cache;
+    }
+
+    /// <summary>
+    /// Retries the request to PyPi if the response is not successful.
+    /// </summary>
+    /// <param name="uri"> uri of the request.</param>
+    /// <param name="spec"> The pip dependency specification. </param>
+    /// <returns> Returns the HttpResponseMessage. </returns>
+    private async Task<HttpResponseMessage> RetryPypiRequestAsync(Uri uri, PipDependencySpecification spec)
+    {
+        var request = await Policy
+           .HandleResult<HttpResponseMessage>(message =>
+           {
+               // stop retrying if MAXRETRIES was hit!
+               if (message == null)
+               {
+                   return false;
+               }
+
+               var statusCode = (int)message.StatusCode;
+
+               // only retry if server doesn't classify the call as a client error!
+               var isRetryable = statusCode != 300 && statusCode != 406 && (statusCode < 400 || statusCode > 499);
+               return !message.IsSuccessStatusCode && isRetryable;
+           })
+           .WaitAndRetryAsync((int)MAXRETRIES - 1, i => RETRYDELAY, (result, timeSpan, retryCount, context) =>
+           {
+               using var r = new PypiRetryTelemetryRecord { Name = spec.Name, DependencySpecifiers = spec.DependencySpecifiers?.ToArray(), StatusCode = result.Result.StatusCode };
+               this.logger.LogWarning(
+                   "Received {StatusCode} {ReasonPhrase} from {RequestUri}. Waiting {TimeSpan} before retry attempt {RetryCount}",
+                   result.Result.StatusCode,
+                   result.Result.ReasonPhrase,
+                   uri,
+                   timeSpan,
+                   retryCount);
+
+               Interlocked.Increment(ref this.retries);
+           })
+           .ExecuteAsync(() =>
+           {
+               if (Interlocked.Read(ref this.retries) >= MAXRETRIES)
+               {
+                   return Task.FromResult<HttpResponseMessage>(null);
+               }
+
+               return this.GetPypiResponseAsync(uri);
+           });
+        if (request == null)
+        {
+            using var r = new PypiMaxRetriesReachedTelemetryRecord { Name = spec.Name, DependencySpecifiers = spec.DependencySpecifiers?.ToArray() };
+
+            this.logger.LogWarning($"Call to simple pypi api failed, but no more retries allowed!");
+
+            return new HttpResponseMessage();
+        }
+
+        if (!request.IsSuccessStatusCode)
+        {
+            using var r = new PypiFailureTelemetryRecord { Name = spec.Name, DependencySpecifiers = spec.DependencySpecifiers?.ToArray(), StatusCode = request.StatusCode };
+
+            this.logger.LogWarning("Received {StatusCode} {ReasonPhrase} from {RequestUri}", request.StatusCode, request.ReasonPhrase, uri);
+
+            return new HttpResponseMessage();
+        }
+
+        return request;
+    }
+
+    /// <summary>
+    /// Sends a request to pypi.
+    /// </summary>
+    /// <param name="uri">The uri of the request. </param>
+    /// <returns> Returns the httpresponsemessage. </returns>
+    private async Task<HttpResponseMessage> GetPypiResponseAsync(Uri uri)
+    {
+        this.logger.LogInformation("Getting Python data from {Uri}", uri);
+        using var request = new HttpRequestMessage(HttpMethod.Get, uri);
+        request.Headers.UserAgent.Add(ProductValue);
+        request.Headers.UserAgent.Add(CommentValue);
+        request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/vnd.pypi.simple.v1+json"));
+        var response = await this.httpClient.SendAsync(request);
+        return response;
+    }
+
+    public void Dispose()
+    {
+        this.cacheTelemetry.FinalSimpleProjectCacheSize = this.cachedSimplePyPiProjects.Count;
+        this.cacheTelemetry.FinalProjectFileCacheSize = this.cachedProjectWheelFiles.Count;
+        this.cacheTelemetry.Dispose();
+        this.cachedProjectWheelFiles.Dispose();
+        this.cachedSimplePyPiProjects.Dispose();
+        this.httpClient.Dispose();
+    }
+}

src/Microsoft.ComponentDetection.Detectors/pip/SimplePypiProject.cs

@@ -0,0 +1,13 @@
+namespace Microsoft.ComponentDetection.Detectors.Pip;
+
+using System.Collections.Generic;
+using System.Text.Json.Serialization;
+
+/// <summary>
+/// A project from the new simple pypi api.
+/// </summary>
+public sealed record SimplePypiProject
+{
+    [JsonPropertyName("files")]
+    public IList<SimplePypiProjectRelease> Files { get; init; }
+}

src/Microsoft.ComponentDetection.Detectors/pip/SimplePypiProjectRelease.cs

@@ -0,0 +1,19 @@
+namespace Microsoft.ComponentDetection.Detectors.Pip;
+
+using System;
+using System.Text.Json.Serialization;
+
+/// <summary>
+/// A specific release of a project from the new simple pypi api.
+/// </summary>
+public sealed record SimplePypiProjectRelease
+{
+    [JsonPropertyName("filename")]
+    public string FileName { get; init; }
+
+    [JsonPropertyName("size")]
+    public double Size { get; init; }
+
+    [JsonPropertyName("url")]
+    public Uri Url { get; init; }
+}

src/Microsoft.ComponentDetection.Orchestrator/Extensions/ServiceCollectionExtensions.cs

@@ -113,6 +113,7 @@ public static class ServiceCollectionExtensions
 
         // PIP
         services.AddSingleton<IPyPiClient, PyPiClient>();
+        services.AddSingleton<ISimplePyPiClient, SimplePyPiClient>();
         services.AddSingleton<IPythonCommandService, PythonCommandService>();
         services.AddSingleton<IPythonResolver, PythonResolver>();
         services.AddSingleton<IComponentDetector, PipComponentDetector>();
@@ -140,6 +141,9 @@ public static class ServiceCollectionExtensions
         services.AddSingleton<IYarnLockFileFactory, YarnLockFileFactory>();
         services.AddSingleton<IComponentDetector, YarnLockComponentDetector>();
 
+        // HttpClient
+        services.AddHttpClient();
+
         return services;
     }
 

src/Microsoft.ComponentDetection.Orchestrator/Microsoft.ComponentDetection.Orchestrator.csproj

@@ -4,6 +4,7 @@
         <PackageReference Include="CommandLineParser" />
         <PackageReference Include="DotNet.Glob" />
         <PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" />
+        <PackageReference Include="Microsoft.Extensions.Http" />
         <PackageReference Include="Microsoft.Extensions.Logging" />
         <PackageReference Include="Newtonsoft.Json" />
         <PackageReference Include="Polly" />

src/Microsoft.ComponentDetection/Microsoft.ComponentDetection.csproj

@@ -8,6 +8,7 @@
 
     <ItemGroup>
       <PackageReference Include="Microsoft.Extensions.DependencyInjection" />
+      <PackageReference Include="Microsoft.Extensions.Http" />
       <PackageReference Include="Microsoft.Extensions.Logging" />
       <PackageReference Include="Serilog" />
       <PackageReference Include="Serilog.Extensions.Hosting" />

src/Microsoft.ComponentDetection/Program.cs

@@ -1,4 +1,4 @@
-using System;
+using System;
 using System.Diagnostics;
 using System.Linq;
 using System.Threading.Tasks;

test/Microsoft.ComponentDetection.Detectors.Tests/Microsoft.ComponentDetection.Detectors.Tests.csproj

@@ -5,6 +5,7 @@
     </ItemGroup>
 
     <ItemGroup Label="Package References">
+        <PackageReference Include="Microsoft.Extensions.Http" />
         <PackageReference Include="Microsoft.Extensions.Logging" />
         <PackageReference Include="Microsoft.NET.Test.Sdk" />
         <PackageReference Include="NuGet.Versioning" />

test/Microsoft.ComponentDetection.Detectors.Tests/SimplePypiClientTests.cs

@@ -0,0 +1,361 @@
+namespace Microsoft.ComponentDetection.Detectors.Tests;
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Net;
+using System.Net.Http;
+using System.Threading;
+using System.Threading.Tasks;
+using FluentAssertions;
+using Microsoft.ComponentDetection.Common;
+using Microsoft.ComponentDetection.Contracts;
+using Microsoft.ComponentDetection.Detectors.Pip;
+using Microsoft.Extensions.Logging;
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+using Moq;
+using Moq.Protected;
+using Newtonsoft.Json;
+
+[TestClass]
+public class SimplePyPiClientTests
+{
+    private readonly Mock<IHttpClientFactory> mockHttpClientFactory = new Mock<IHttpClientFactory>();
+
+    private Mock<HttpMessageHandler> MockHttpMessageHandler(string content, HttpStatusCode statusCode)
+    {
+        var handlerMock = new Mock<HttpMessageHandler>();
+        handlerMock.Protected()
+            .Setup<Task<HttpResponseMessage>>(
+                "SendAsync",
+                ItExpr.IsAny<HttpRequestMessage>(),
+                ItExpr.IsAny<CancellationToken>())
+            .ReturnsAsync(new HttpResponseMessage()
+            {
+                StatusCode = statusCode,
+                Content = new StringContent(content),
+            });
+
+        return handlerMock;
+    }
+
+    private ISimplePyPiClient CreateSimplePypiClient(HttpMessageHandler messageHandler, IEnvironmentVariableService evs, ILogger<SimplePyPiClient> logger)
+    {
+        var httpClient = new HttpClient(messageHandler);
+        this.mockHttpClientFactory.Setup(x => x.CreateClient(It.IsAny<string>())).Returns(httpClient);
+        return new SimplePyPiClient(evs, this.mockHttpClientFactory.Object, logger);
+    }
+
+    [TestMethod]
+    public async Task GetSimplePypiProject_DuplicateEntries_CallsGetAsync_OnceAsync()
+    {
+        var pythonSpecs = new PipDependencySpecification { DependencySpecifiers = new List<string> { "==1.0.0" }, Name = "boto3" };
+        var pythonProject = this.SampleValidApiJsonResponse("boto3", "0.0.1");
+        var mockHandler = this.MockHttpMessageHandler(pythonProject, HttpStatusCode.OK);
+
+        var simplePypiClient = this.CreateSimplePypiClient(mockHandler.Object, new Mock<EnvironmentVariableService>().Object, new Mock<ILogger<SimplePyPiClient>>().Object);
+        var action = async () => await simplePypiClient.GetSimplePypiProjectAsync(pythonSpecs);
+
+        await action.Should().NotThrowAsync();
+        await action.Should().NotThrowAsync();
+
+        // Verify the API call was performed only once
+        mockHandler.Protected().Verify(
+            "SendAsync",
+            Times.Once(),
+            ItExpr.IsAny<HttpRequestMessage>(),
+            ItExpr.IsAny<CancellationToken>());
+    }
+
+    [TestMethod]
+    public async Task GetSimplePypiProject_DifferentEntries_CallsGetAsync_TwiceAsync()
+    {
+        var pythonSpecs = new PipDependencySpecification { DependencySpecifiers = new List<string> { "==1.0.0" } };
+        var pythonProject = new SimplePypiProject()
+        {
+            Files = new List<SimplePypiProjectRelease> { new SimplePypiProjectRelease() },
+        };
+
+        var mockHandler = this.MockHttpMessageHandler(JsonConvert.SerializeObject(pythonProject), HttpStatusCode.OK);
+        var simplePypiClient = this.CreateSimplePypiClient(mockHandler.Object, new Mock<EnvironmentVariableService>().Object, new Mock<ILogger<SimplePyPiClient>>().Object);
+
+        var action = async () =>
+        {
+            pythonSpecs.Name = Guid.NewGuid().ToString();
+            await simplePypiClient.GetSimplePypiProjectAsync(pythonSpecs);
+        };
+
+        await action.Should().NotThrowAsync();
+        await action.Should().NotThrowAsync();
+
+        // Verify the API call was performed twice
+        mockHandler.Protected().Verify(
+            "SendAsync",
+            Times.Exactly(2),
+            ItExpr.IsAny<HttpRequestMessage>(),
+            ItExpr.IsAny<CancellationToken>());
+    }
+
+    [TestMethod]
+    public async Task GetSimplePypiProject_ReturnsValidSimplePypiProjectAsync()
+    {
+        var pythonSpecs = new PipDependencySpecification { DependencySpecifiers = new List<string> { "==1.0.0" }, Name = "boto3" };
+        var sampleApiResponse = this.SampleValidApiJsonResponse("boto3", "0.0.1");
+        var expectedResult = new SimplePypiProject()
+        {
+            Files = new List<SimplePypiProjectRelease>
+            {
+                new SimplePypiProjectRelease() { FileName = "boto3-0.0.1-py2.py3-none-any.whl", Url = new Uri("https://files.pythonhosted.org/packages/3f/95/a24847c245befa8c50a9516cbdca309880bd21b5879e7c895e953217e947/boto3-0.0.1-py2.py3-none-any.whl"), Size = 45469 },
+                new SimplePypiProjectRelease() { FileName = "boto3-0.0.1.tar.gz", Url = new Uri("https://files.pythonhosted.org/packages/df/18/4e36b93f6afb79b5f67b38f7d235773a21831b193602848c590f8a008608/boto3-0.0.1.tar.gz"), Size = 33415 },
+            },
+        };
+
+        var mockHandler = this.MockHttpMessageHandler(sampleApiResponse, HttpStatusCode.OK);
+        var simplePypiClient = this.CreateSimplePypiClient(mockHandler.Object, new Mock<EnvironmentVariableService>().Object, new Mock<ILogger<SimplePyPiClient>>().Object);
+
+        var actualResult = await simplePypiClient.GetSimplePypiProjectAsync(pythonSpecs);
+        actualResult.Should().BeEquivalentTo(expectedResult);
+    }
+
+    [TestMethod]
+    public async Task GetSimplePypiProject_InvalidSpec_NotThrowAsync()
+    {
+        var pythonSpecs = new PipDependencySpecification { DependencySpecifiers = new List<string> { "==1.0.0" }, Name = "randomName" };
+
+        var mockHandler = this.MockHttpMessageHandler("404 Not Found", HttpStatusCode.NotFound);
+        var simplePypiClient = this.CreateSimplePypiClient(mockHandler.Object, new Mock<EnvironmentVariableService>().Object, new Mock<ILogger<SimplePyPiClient>>().Object);
+
+        var action = async () => await simplePypiClient.GetSimplePypiProjectAsync(pythonSpecs);
+
+        await action.Should().NotThrowAsync();
+    }
+
+    [TestMethod]
+    public async Task GetSimplePypiProject_UnexpectedContentTypeReturnedByApi_NotThrowAsync()
+    {
+        var pythonSpecs = new PipDependencySpecification { DependencySpecifiers = new List<string> { "==1.0.0" } };
+
+        var content = "<!DOCTYPE html><body>\r\n\t<h1>Links for boto3</h1>\r\n\t<a\r\n\t\thref=\"some link\">boto3-0.0.1-py2.py3-none-any.whl</a><br /></html>";
+        var mockHandler = this.MockHttpMessageHandler(content, HttpStatusCode.OK);
+        var simplePypiClient = this.CreateSimplePypiClient(mockHandler.Object, new Mock<EnvironmentVariableService>().Object, new Mock<ILogger<SimplePyPiClient>>().Object);
+
+        var action = async () => await simplePypiClient.GetSimplePypiProjectAsync(pythonSpecs);
+
+        await action.Should().NotThrowAsync();
+    }
+
+    [TestMethod]
+    public async Task GetSimplePypiProject_ShouldRetryAsync()
+    {
+        var pythonSpecs = new PipDependencySpecification { DependencySpecifiers = new List<string> { "==1.0.0" } };
+
+        var mockHandler = this.MockHttpMessageHandler(string.Empty, HttpStatusCode.InternalServerError);
+        var simplePypiClient = this.CreateSimplePypiClient(mockHandler.Object, new Mock<EnvironmentVariableService>().Object, new Mock<ILogger<SimplePyPiClient>>().Object);
+
+        var action = async () => await simplePypiClient.GetSimplePypiProjectAsync(pythonSpecs);
+
+        await action.Should().NotThrowAsync();
+
+        // Verify the API call was retried max retry times
+        mockHandler.Protected().Verify(
+            "SendAsync",
+            Times.Exactly((int)SimplePyPiClient.MAXRETRIES),
+            ItExpr.IsAny<HttpRequestMessage>(),
+            ItExpr.IsAny<CancellationToken>());
+    }
+
+    [TestMethod]
+    public async Task GetSimplePypiProject_ShouldNotRetryAsync()
+    {
+        var pythonSpecs = new PipDependencySpecification { DependencySpecifiers = new List<string> { "==1.0.0" } };
+        var mockHandler = this.MockHttpMessageHandler("some content", HttpStatusCode.MultipleChoices);
+        var simplePypiClient = this.CreateSimplePypiClient(mockHandler.Object, new Mock<EnvironmentVariableService>().Object, new Mock<ILogger<SimplePyPiClient>>().Object);
+
+        var action = async () => await simplePypiClient.GetSimplePypiProjectAsync(pythonSpecs);
+        await action.Should().NotThrowAsync();
+
+        // Verify the API call was called only once
+        mockHandler.Protected().Verify(
+            "SendAsync",
+            Times.Once(),
+            ItExpr.IsAny<HttpRequestMessage>(),
+            ItExpr.IsAny<CancellationToken>());
+    }
+
+    [TestMethod]
+    public async Task GetSimplePypiProject_AddsCorrectHeadersAsync()
+    {
+        var pythonSpecs = new PipDependencySpecification { DependencySpecifiers = new List<string> { "==1.0.0" } };
+        var pythonProject = this.SampleValidApiJsonResponse("boto3", "0.0.1");
+
+        var mockHandler = this.MockHttpMessageHandler(pythonProject, HttpStatusCode.OK);
+        var simplePypiClient = this.CreateSimplePypiClient(mockHandler.Object, new Mock<EnvironmentVariableService>().Object, new Mock<ILogger<SimplePyPiClient>>().Object);
+
+        var action = async () => await simplePypiClient.GetSimplePypiProjectAsync(pythonSpecs);
+
+        await action.Should().NotThrowAsync();
+
+        mockHandler.Protected().Verify(
+            "SendAsync",
+            Times.Once(),
+            ItExpr.Is<HttpRequestMessage>(
+                req => req.Headers.UserAgent.Count == 2 &&
+                       req.Headers.UserAgent.First().Product.Name == "ComponentDetection"
+                       && req.Headers.UserAgent.Last().Comment == "(+https://github.com/microsoft/component-detection)"
+                       && req.Headers.Accept.First().ToString() == "application/vnd.pypi.simple.v1+json"),
+            ItExpr.IsAny<CancellationToken>());
+    }
+
+    [TestMethod]
+    public async Task GetSimplePypiProject_MaxEntriesVariable_CreatesNewCacheAsync()
+    {
+        var pythonSpecs = new PipDependencySpecification { DependencySpecifiers = new List<string> { "==1.0.0" } };
+        var pythonProject = this.SampleValidApiJsonResponse("boto3", "0.0.1");
+        var mockHandler = this.MockHttpMessageHandler(pythonProject, HttpStatusCode.OK);
+
+        var mockLogger = new Mock<ILogger<SimplePyPiClient>>();
+        var mockEvs = new Mock<IEnvironmentVariableService>();
+        mockEvs.Setup(x => x.GetEnvironmentVariable(It.Is<string>(s => s.Equals("PyPiMaxCacheEntries")))).Returns("32");
+
+        var simplePyPiClient = this.CreateSimplePypiClient(mockHandler.Object, mockEvs.Object, mockLogger.Object);
+
+        var action = async () => await simplePyPiClient.GetSimplePypiProjectAsync(pythonSpecs);
+
+        await action.Should().NotThrowAsync();
+        await action.Should().NotThrowAsync();
+
+        // Verify the cache setup call was performed only once
+        mockEvs.Verify(x => x.GetEnvironmentVariable(It.IsAny<string>()), Times.Once());
+        mockLogger.Verify(
+            x => x.Log(
+            It.IsAny<LogLevel>(),
+            It.IsAny<EventId>(),
+            It.IsAny<It.IsAnyType>(),
+            It.IsAny<Exception>(),
+            (Func<It.IsAnyType, Exception, string>)It.IsAny<object>()),
+            Times.Exactly(3));
+    }
+
+    [TestMethod]
+    public async Task FetchPackageFileStream_MaxEntriesVariable_CreatesNewCacheAsync()
+    {
+        var mockHandler = this.MockHttpMessageHandler(string.Empty, HttpStatusCode.OK);
+
+        var mockLogger = new Mock<ILogger<SimplePyPiClient>>();
+        var mockEvs = new Mock<IEnvironmentVariableService>();
+        mockEvs.Setup(x => x.GetEnvironmentVariable(It.Is<string>(s => s.Equals("PyPiMaxCacheEntries")))).Returns("32");
+
+        var mockedPyPi = this.CreateSimplePypiClient(mockHandler.Object, mockEvs.Object, mockLogger.Object);
+
+        var action = async () => await mockedPyPi.FetchPackageFileStreamAsync(new Uri($"https://testurl"));
+
+        await action.Should().NotThrowAsync();
+        await action.Should().NotThrowAsync();
+
+        // Verify the cache setup call was performed only once
+        mockEvs.Verify(x => x.GetEnvironmentVariable(It.IsAny<string>()), Times.Once());
+        mockLogger.Verify(
+            x => x.Log(
+            It.IsAny<LogLevel>(),
+            It.IsAny<EventId>(),
+            It.IsAny<It.IsAnyType>(),
+            It.IsAny<Exception>(),
+            (Func<It.IsAnyType, Exception, string>)It.IsAny<object>()),
+            Times.Exactly(3));
+    }
+
+    [TestMethod]
+    public async Task FetchPackageFileStream_DuplicateEntries_CallsGetAsync_OnceAsync()
+    {
+        var mockHandler = this.MockHttpMessageHandler(string.Empty, HttpStatusCode.OK);
+        var simplePypiClient = this.CreateSimplePypiClient(mockHandler.Object, new Mock<EnvironmentVariableService>().Object, new Mock<ILogger<SimplePyPiClient>>().Object);
+
+        var action = async () => await simplePypiClient.FetchPackageFileStreamAsync(new Uri($"https://testurl"));
+
+        await action.Should().NotThrowAsync();
+        await action.Should().NotThrowAsync();
+
+        // Verify the API call was performed only once
+        mockHandler.Protected().Verify(
+            "SendAsync",
+            Times.Once(),
+            ItExpr.IsAny<HttpRequestMessage>(),
+            ItExpr.IsAny<CancellationToken>());
+    }
+
+    [TestMethod]
+    public async Task FetchPackageFileStream_DifferentEntries_CallsGetAsync_TwiceAsync()
+    {
+        var mockHandler = this.MockHttpMessageHandler(string.Empty, HttpStatusCode.OK);
+        var simplePypiClient = this.CreateSimplePypiClient(mockHandler.Object, new Mock<EnvironmentVariableService>().Object, new Mock<ILogger<SimplePyPiClient>>().Object);
+
+        var action = async () => await simplePypiClient.FetchPackageFileStreamAsync(new Uri($"https://{Guid.NewGuid()}"));
+
+        await action.Should().NotThrowAsync();
+        await action.Should().NotThrowAsync();
+
+        // Verify the API call was performed twice
+        mockHandler.Protected().Verify(
+            "SendAsync",
+            Times.Exactly(2),
+            ItExpr.IsAny<HttpRequestMessage>(),
+            ItExpr.IsAny<CancellationToken>());
+    }
+
+    [TestMethod]
+    public async Task FetchPackageFileStream_UnableToRetrievePackageAsync()
+    {
+        var mockHandler = this.MockHttpMessageHandler(string.Empty, HttpStatusCode.InternalServerError);
+        var simplePypiClient = this.CreateSimplePypiClient(mockHandler.Object, new Mock<EnvironmentVariableService>().Object, new Mock<ILogger<SimplePyPiClient>>().Object);
+
+        var action = async () => { return await simplePypiClient.FetchPackageFileStreamAsync(new Uri($"https://{Guid.NewGuid()}")); };
+
+        await action.Should().NotThrowAsync();
+    }
+
+    public string SampleValidApiJsonResponse(string packageName, string version)
+    {
+        var packageJson = @"{{
+                ""files"": [
+        {{
+            ""core-metadata"": false,
+            ""data-dist-info-metadata"": false,
+            ""filename"": ""{0}-{1}-py2.py3-none-any.whl"",
+            ""hashes"": {{
+                ""sha256"": ""bc9b3ce78d3863e45b43a33d076c7b0561f6590205c94f0f8a23a4738e79a13f""
+            }},
+            ""requires-python"": null,
+            ""size"": 45469,
+            ""upload-time"": ""2014-11-11T20:30:49.562183Z"",
+            ""url"": ""https://files.pythonhosted.org/packages/3f/95/a24847c245befa8c50a9516cbdca309880bd21b5879e7c895e953217e947/{0}-{1}-py2.py3-none-any.whl"",
+            ""yanked"": false
+        }},
+        {{
+            ""core-metadata"": false,
+            ""data-dist-info-metadata"": false,
+            ""filename"": ""{0}-{1}.tar.gz"",
+            ""hashes"": {{
+                ""sha256"": ""bc018a3aedc5cf7329dcdeb435ece8a296b605c19fb09842c1821935f1b14cfd""
+            }},
+            ""requires-python"": null,
+            ""size"": 33415,
+            ""upload-time"": ""2014-11-11T20:30:40.057636Z"",
+            ""url"": ""https://files.pythonhosted.org/packages/df/18/4e36b93f6afb79b5f67b38f7d235773a21831b193602848c590f8a008608/{0}-{1}.tar.gz"",
+            ""yanked"": false
+        }}
+    ],
+    ""meta"": {{
+        ""_last-serial"": 18925095,
+        ""api-version"": ""1.1""
+    }},
+    ""name"": ""{0}"",
+    ""versions"": [
+        ""{1}""
+    ]
+            }}";
+
+        var packageJsonTemplate = string.Format(packageJson, packageName, version);
+
+        return packageJsonTemplate;
+    }
+}