* First pass of Check Enforcer
This commit is contained in:
Mitch Denny 2019-09-19 10:25:10 +10:00 коммит произвёл GitHub
Родитель 0da3c2416e
Коммит 090ec0a4bf
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: 4AEE18F83AFDEB23
14 изменённых файлов: 676 добавлений и 0 удалений

3
.gitignore поставляемый
Просмотреть файл

@ -464,3 +464,6 @@ src/dotnet/Mgmt.CI.BuildTools/NugetToolsPackage/CI.Tools.Package/build/tasks/net
# Visual Studio launch settings (frequently user-specific) # Visual Studio launch settings (frequently user-specific)
launchSettings.json launchSettings.json
# Azure Functions
local.settings.json

Просмотреть файл

@ -0,0 +1,24 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>netcoreapp2.1</TargetFramework>
<AzureFunctionsVersion>v2</AzureFunctionsVersion>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Azure.Identity" Version="1.0.0-preview.4" />
<PackageReference Include="Azure.Security.KeyVault.Keys" Version="4.0.0-preview.4" />
<PackageReference Include="Azure.Security.KeyVault.Secrets" Version="4.0.0-preview.4" />
<PackageReference Include="Microsoft.NET.Sdk.Functions" Version="1.0.29" />
<PackageReference Include="Octokit" Version="0.32.0" />
<PackageReference Include="System.IdentityModel.Tokens.Jwt" Version="5.5.0" />
<PackageReference Include="YamlDotNet" Version="6.1.2" />
</ItemGroup>
<ItemGroup>
<None Update="host.json">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</None>
<None Update="local.settings.json">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
<CopyToPublishDirectory>Never</CopyToPublishDirectory>
</None>
</ItemGroup>
</Project>

Просмотреть файл

@ -0,0 +1,17 @@
using System;
using System.Collections.Generic;
using System.Text;
namespace Azure.Sdk.Tools.CheckEnforcer
{
[Serializable]
public class CheckEnforcerConfigurationException : Exception
{
public CheckEnforcerConfigurationException(string message) : base(message) { }
public CheckEnforcerConfigurationException(string message, Exception inner) : base(message, inner) { }
protected CheckEnforcerConfigurationException(
System.Runtime.Serialization.SerializationInfo info,
System.Runtime.Serialization.StreamingContext context) : base(info, context) { }
}
}

Просмотреть файл

@ -0,0 +1,17 @@
using System;
using System.Collections.Generic;
using System.Text;
namespace Azure.Sdk.Tools.CheckEnforcer
{
[Serializable]
public class CheckEnforcerException : Exception
{
public CheckEnforcerException(string message) : base(message) { }
public CheckEnforcerException(string message, Exception inner) : base(message, inner) { }
protected CheckEnforcerException(
System.Runtime.Serialization.SerializationInfo info,
System.Runtime.Serialization.StreamingContext context) : base(info, context) { }
}
}

Просмотреть файл

@ -0,0 +1,17 @@
using System;
using System.Collections.Generic;
using System.Text;
namespace Azure.Sdk.Tools.CheckEnforcer
{
[Serializable]
public class CheckEnforcerUnsupportedEventException : CheckEnforcerException
{
public CheckEnforcerUnsupportedEventException(string eventName) : base($"The GitHub event '{eventName}' cannot be processed.") { }
public CheckEnforcerUnsupportedEventException(string eventName, Exception inner) : base($"The GitHub event '{eventName}' does not have a handler.", inner) { }
protected CheckEnforcerUnsupportedEventException(
System.Runtime.Serialization.SerializationInfo info,
System.Runtime.Serialization.StreamingContext context) : base(info, context) { }
}
}

Просмотреть файл

@ -0,0 +1,74 @@
using Microsoft.Build.Utilities;
using Microsoft.Extensions.Options;
using Microsoft.Extensions.Primitives;
using Octokit;
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.IdentityModel.Tokens.Jwt;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using YamlDotNet.RepresentationModel;
using YamlDotNet.Serialization;
using YamlDotNet.Serialization.NodeDeserializers;
namespace Azure.Sdk.Tools.CheckEnforcer
{
public class ConfigurationStore
{
public ConfigurationStore(GitHubClientFactory clientFactory)
{
this.clientFactory = clientFactory;
}
private GitHubClientFactory clientFactory;
private ConcurrentDictionary<string, IRepositoryConfiguration> cachedConfigurations = new ConcurrentDictionary<string, IRepositoryConfiguration>();
public async Task<IRepositoryConfiguration> GetRepositoryConfigurationAsync(long installationId, long repositoryId, string pullRequestSha, CancellationToken cancellationToken)
{
try
{
var client = await clientFactory.GetInstallationClientAsync(installationId, cancellationToken);
var searchResults = await client.Repository.Content.GetAllContents(repositoryId, "eng/CHECKENFORCER");
var configurationFile = searchResults.Single();
ThrowIfInvalidFormat(configurationFile);
var builder = new DeserializerBuilder().Build();
var configuration = builder.Deserialize<RepositoryConfiguration>(configurationFile.Content);
return configuration;
}
catch (NotFoundException) // OK, we just disable if it isn't configured.
{
return new RepositoryConfiguration()
{
IsEnabled = false
};
}
}
private static void ThrowIfInvalidFormat(RepositoryContent configurationFile)
{
// TODO: This is gross. I want to look more closely at the YamlDotNet API
// to see if there is a way I can parse this file once and then do
// deserialization of a document. At the moment I'm parsing the string
// twice. I suspect that I'm just not grokking the API properly yet.
var stream = new StringReader(configurationFile.Content);
var yaml = new YamlStream();
yaml.Load(stream);
var mapping = (YamlMappingNode)yaml.Documents[0].RootNode;
var formatScalar = (YamlScalarNode)mapping.Children[new YamlScalarNode("format")];
if (formatScalar.Value != "v0.1-alpha")
{
throw new CheckEnforcerConfigurationException("The value for the 'format' was not valid. Try v0.1-alpha.");
}
}
}
}

Просмотреть файл

@ -0,0 +1,13 @@
using System;
using System.Collections.Generic;
using System.Text;
namespace Azure.Sdk.Tools.CheckEnforcer
{
internal static class Constants
{
public const int ApplicationID = 41380;
public const string ApplicationName = "check-enforcer";
public const int ApplicationTokenLifetimeInMinutes = 10;
}
}

Просмотреть файл

@ -0,0 +1,196 @@
using Azure.Core;
using Azure.Identity;
using Azure.Security.KeyVault.Keys;
using Azure.Security.KeyVault.Keys.Cryptography;
using Microsoft.IdentityModel.Tokens;
using Octokit;
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.IdentityModel.Tokens.Jwt;
using System.Security.Cryptography;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
namespace Azure.Sdk.Tools.CheckEnforcer
{
public class GitHubClientFactory
{
public GitHubClientFactory()
{
}
private Tuple<DateTimeOffset, string> cachedApplicationToken;
private async Task<string> GetTokenAsync(CancellationToken cancellationToken)
{
if (cachedApplicationToken == null || cachedApplicationToken.Item1 < DateTimeOffset.UtcNow)
{
// NOTE: There is potential for a cache stampeed issue here, but it may
// not be enough of an issue to worry about it. Will need to evaluate
// once we get some realistic load numbers. If necessary we can switch
// to a sync programming model and use locking.
//
// Cache stampeed will be visible in the KeyVault diagostics because
// we'll see a spike in get and sign requests. Stampeed duration will
// be limited in duration.
var headerAndPayloadString = GenerateJwtTokenHeaderAndPayload();
var digest = ComputeHeaderAndPayloadDigest(headerAndPayloadString);
var encodedSignature = await SignHeaderAndPayloadDigestWithGitHubApplicationKey(digest, cancellationToken);
var token = AppendSignatureToHeaderAndPayload(headerAndPayloadString, encodedSignature);
// Let's get a token a full minute before it times out.
cachedApplicationToken = new Tuple<DateTimeOffset, string>(
DateTimeOffset.UtcNow.AddMinutes(Constants.ApplicationTokenLifetimeInMinutes - 1),
token
);
}
return cachedApplicationToken.Item2;
}
private string AppendSignatureToHeaderAndPayload(string headerAndPayloadString, string encodedSignature)
{
return $"{headerAndPayloadString}.{encodedSignature}";
}
private async Task<string> SignHeaderAndPayloadDigestWithGitHubApplicationKey(byte[] digest, CancellationToken cancellationToken)
{
var cryptographyClient = await GetCryptographyClient(cancellationToken);
var signResult = await cryptographyClient.SignAsync(
SignatureAlgorithm.RS256,
digest,
cancellationToken
);
var encodedSignature = Base64UrlEncoder.Encode(signResult.Signature);
return encodedSignature;
}
private byte[] ComputeHeaderAndPayloadDigest(string headerAndPayloadString)
{
var headerAndPayloadBytes = Encoding.UTF8.GetBytes(headerAndPayloadString);
var sha256 = new SHA256CryptoServiceProvider();
var digest = sha256.ComputeHash(headerAndPayloadBytes);
return digest;
}
private string GenerateJwtTokenHeaderAndPayload()
{
var jwtHeader = new JwtHeader();
jwtHeader["alg"] = "RS256";
var jwtPayload = new JwtPayload(
issuer: Constants.ApplicationID.ToString(),
audience: null,
claims: null,
notBefore: DateTime.UtcNow,
expires: DateTime.UtcNow.AddMinutes(Constants.ApplicationTokenLifetimeInMinutes),
issuedAt: DateTime.UtcNow
);
var jwtToken = new JwtSecurityToken(jwtHeader, jwtPayload);
var headerAndPayloadString = $"{jwtToken.EncodedHeader}.{jwtToken.EncodedPayload}";
return headerAndPayloadString;
}
private async Task<CryptographyClient> GetCryptographyClient(CancellationToken cancellationToken)
{
// Using DefaultAzureCredential to support local development. If developing
// locally you'll need to register an AAD application and set the following
// variables:
//
// AZURE_TENANT_ID (the ID of the AAD tenant)
// AZURE_CLIENT_ID (the iD of the AAD application you registered)
// AZURE_CLIENT_SECRET (the secret for the AAD application you registered)
//
// You can get these values when you configure the application. Set them in
// the Debug section of the project properties. Once this is done you will need
// to create a KeyVault instance and then register a GitHub application and upload
// the private key into the vault. The AAD application that you just created needs
// to have Get and Sign rights - so set an access policy up which grants the app
// those rights.
//
var credential = new DefaultAzureCredential();
var keyClient = GetKeyClient(credential);
var key = await GetKey(keyClient, cancellationToken);
var cryptographyClient = new CryptographyClient(key.Id, credential);
return cryptographyClient;
}
private async Task<Key> GetKey(KeyClient keyClient, CancellationToken cancellationToken)
{
var keyVaultGitHubKeyName = Environment.GetEnvironmentVariable("KEYVAULT_GITHUBAPP_KEY_NAME");
var keyResponse = await keyClient.GetKeyAsync(
keyVaultGitHubKeyName,
cancellationToken: cancellationToken
);
var key = keyResponse.Value;
return key;
}
private KeyClient GetKeyClient(TokenCredential credential)
{
// We need a variable that tells Check Enforcer which KeyVault to talk to,
// this is currently done via an environment variable.
//
// KEYVAULT_URI
//
var keyVaultUriEnvironmentVariable = Environment.GetEnvironmentVariable("KEYVAULT_URI");
var keyVaultUri = new Uri(keyVaultUriEnvironmentVariable);
var keyClient = new KeyClient(keyVaultUri, credential);
return keyClient;
}
public async Task<GitHubClient> GetApplicationClientAsync(CancellationToken cancellationToken)
{
var token = await GetTokenAsync(cancellationToken);
var appClient = new GitHubClient(new ProductHeaderValue(Constants.ApplicationName))
{
Credentials = new Credentials(token, AuthenticationType.Bearer)
};
return appClient;
}
private ConcurrentDictionary<long, Octokit.AccessToken> cachedInstallationTokens = new ConcurrentDictionary<long, Octokit.AccessToken>();
private async Task<string> GetInstallationTokenAsync(long installationId, CancellationToken cancellationToken)
{
cachedInstallationTokens.TryGetValue(installationId, out Octokit.AccessToken accessToken);
if (accessToken == null || accessToken.ExpiresAt < DateTimeOffset.UtcNow)
{
// NOTE: There is a possible cache stampeed here as well. We'll see in time
// if we need to do anything about it. If there is a problem it will
// manifest as exceptions being thrown out of this method.
var appClient = await GetApplicationClientAsync(cancellationToken);
accessToken = await appClient.GitHubApps.CreateInstallationToken(installationId);
cachedInstallationTokens[installationId] = accessToken;
}
return accessToken.Token;
}
public async Task<GitHubClient> GetInstallationClientAsync(long installationId, CancellationToken cancellationToken)
{
var installationToken = await GetInstallationTokenAsync(installationId, cancellationToken);
var installationClient = new GitHubClient(new ProductHeaderValue($"{Constants.ApplicationName}-{installationId}"))
{
Credentials = new Credentials(installationToken)
};
return installationClient;
}
}
}

Просмотреть файл

@ -0,0 +1,59 @@
using System;
using System.Linq;
using System.IO;
using System.Threading.Tasks;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Azure.WebJobs;
using Microsoft.Azure.WebJobs.Extensions.Http;
using Microsoft.AspNetCore.Http;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Primitives;
using Octokit;
using Octokit.Internal;
using System.Collections;
using System.Collections.Generic;
using Azure.Security.KeyVault.Keys;
using Azure.Identity;
using System.Threading;
using Azure.Security.KeyVault.Keys.Cryptography;
using System.Security.Cryptography.X509Certificates;
using System.IdentityModel.Tokens.Jwt;
using Microsoft.IdentityModel.Tokens;
using System.Text;
using System.Security.Cryptography;
using Azure.Core;
using System.Web.Http;
using System.Runtime.CompilerServices;
namespace Azure.Sdk.Tools.CheckEnforcer
{
public static class GitHubWebhookFunction
{
private static GitHubClientFactory clientFactory = new GitHubClientFactory();
private static ConfigurationStore configurationStore = new ConfigurationStore(clientFactory);
[FunctionName("webhook")]
public static async Task<IActionResult> Run(
[HttpTrigger(AuthorizationLevel.Anonymous, "get", "post", Route = null)] HttpRequest req,
ILogger log, CancellationToken cancellationToken)
{
try
{
var processor = new GitHubWebhookProcessor(log, clientFactory, configurationStore);
await processor.ProcessWebhookAsync(req, cancellationToken);
return new OkResult();
}
catch (CheckEnforcerUnsupportedEventException ex)
{
log.LogWarning(ex, "An error occured because the event is not supported.");
return new BadRequestResult();
}
catch (Exception ex)
{
log.LogError(ex, "An error occured processing the webhook.");
return new InternalServerErrorResult();
}
}
}
}

Просмотреть файл

@ -0,0 +1,183 @@
using Azure.Core;
using Azure.Identity;
using Azure.Security.KeyVault.Keys;
using Azure.Security.KeyVault.Keys.Cryptography;
using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Primitives;
using Microsoft.IdentityModel.Tokens;
using Octokit;
using Octokit.Internal;
using System;
using System.Collections;
using System.Collections.Generic;
using System.IdentityModel.Tokens.Jwt;
using System.IO;
using System.Linq;
using System.Security.Cryptography;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
namespace Azure.Sdk.Tools.CheckEnforcer
{
public class GitHubWebhookProcessor
{
public GitHubWebhookProcessor(ILogger log, GitHubClientFactory gitHubClientFactory, ConfigurationStore configurationStore)
{
this.Log = log;
this.ClientFactory = gitHubClientFactory;
this.ConfigurationStore = configurationStore;
}
public ILogger Log { get; private set; }
public GitHubClientFactory ClientFactory { get; private set; }
public ConfigurationStore ConfigurationStore { get; private set; }
private const string GitHubEventHeader = "X-GitHub-Event";
private async Task<TEvent> DeserializePayloadAsync<TEvent>(Stream stream)
{
using (var reader = new StreamReader(stream))
{
var rawPayload = await reader.ReadToEndAsync();
Log.LogInformation("Received payload from GitHub: {rawPayload}", rawPayload);
var serializer = new SimpleJsonSerializer();
var payload = serializer.Deserialize<TEvent>(rawPayload);
return payload;
}
}
private async Task<CheckRun> EnsureCheckEnforcerRunAsync(GitHubClient client, long repositoryId, string headSha, IReadOnlyList<CheckRun> runs, bool recreate)
{
var checkRun = runs.SingleOrDefault(r => r.Name == Constants.ApplicationName);
if (checkRun == null || recreate)
{
Log.LogDebug("Creating Check Enforcer run.");
checkRun = await client.Check.Run.Create(
repositoryId,
new NewCheckRun(Constants.ApplicationName, headSha)
);
}
return checkRun;
}
private async Task EvaluateAndUpdateCheckEnforcerRunStatusAsync(IRepositoryConfiguration configuration, long installationId, long repositoryId, string pullRequestSha, CancellationToken cancellationToken)
{
Log.LogDebug("Fetching installation client.");
var client = await this.ClientFactory.GetInstallationClientAsync(installationId, cancellationToken);
Log.LogDebug("Fetching check runs.");
var runsResponse = await client.Check.Run.GetAllForReference(repositoryId, pullRequestSha);
var runs = runsResponse.CheckRuns;
var checkEnforcerRun = await EnsureCheckEnforcerRunAsync(client, repositoryId, pullRequestSha, runs, false);
var otherRuns = from run in runs
where run.Name != Constants.ApplicationName
select run;
var totalOtherRuns = otherRuns.Count();
var outstandingOtherRuns = from run in otherRuns
where run.Conclusion != new StringEnum<CheckConclusion>(CheckConclusion.Success)
select run;
var totalOutstandingOtherRuns = outstandingOtherRuns.Count();
Log.LogDebug("{totalOutstandingOtherRuns}/{totalOtherRuns} other runs outstanding.", totalOutstandingOtherRuns, totalOtherRuns);
if (totalOtherRuns >= configuration.MinimumCheckRuns && totalOutstandingOtherRuns == 0)
{
Log.LogDebug("Check Enforcer criteria met, marking check successful.");
await client.Check.Run.Update(repositoryId, checkEnforcerRun.Id, new CheckRunUpdate()
{
Conclusion = new StringEnum<CheckConclusion>(CheckConclusion.Success)
});
}
else
{
if (checkEnforcerRun.Conclusion == new StringEnum<CheckConclusion>(CheckConclusion.Success) && totalOutstandingOtherRuns > 0)
{
Log.LogDebug("Check Enforcer run was previously marked successful, but there are now outstanding runs. Recreating check.");
await EnsureCheckEnforcerRunAsync(client, repositoryId, pullRequestSha, runs, true);
}
else
{
if (checkEnforcerRun.Status != new StringEnum<CheckStatus>(CheckStatus.InProgress))
{
Log.LogDebug("Updating Check Enforcer status to in-progress.");
await client.Check.Run.Update(repositoryId, checkEnforcerRun.Id, new CheckRunUpdate()
{
Status = new StringEnum<CheckStatus>(CheckStatus.InProgress)
});
}
}
}
}
public async Task ProcessWebhookAsync(HttpRequest request, CancellationToken cancellationToken)
{
if (request.Headers.TryGetValue(GitHubEventHeader, out StringValues eventName))
{
if (eventName == "check_run")
{
var rawPayload = request.Body;
var payload = await DeserializePayloadAsync<CheckRunEventPayload>(rawPayload);
var installationId = payload.Installation.Id;
var repositoryId = payload.Repository.Id;
var pullRequestSha = payload.CheckRun.CheckSuite.HeadSha;
Log.LogDebug("Fetching repository configuration.");
var configuration = await this.ConfigurationStore.GetRepositoryConfigurationAsync(installationId, repositoryId, pullRequestSha, cancellationToken);
Log.LogDebug("Repository configuration: {configuration}", configuration.ToString());
if (configuration.IsEnabled)
{
Log.LogDebug("Repository was enabled for Check Enforcer.",
installationId,
repositoryId,
pullRequestSha
);
await EvaluateAndUpdateCheckEnforcerRunStatusAsync(configuration, installationId, repositoryId, pullRequestSha, cancellationToken);
}
else
{
Log.LogInformation("Repository was not enabled for Check Enforcer.");
}
}
else if (eventName == "check_suite")
{
// HACK: We swallow check_suite events. Technically we could register
// the check enforcer check run earlier (before the PR is even created)
// but at this point we don't know the target branch for sure so we
// can't potentially cache the configuration entry.
//
// However - we can't opt out of receiving this event even then check suite
// is unchecked in the app's event setup. So rather than returning a 400
// for this event we just swallow it to eliminate the noise.
return;
}
else
{
throw new CheckEnforcerUnsupportedEventException(eventName);
}
}
else
{
throw new CheckEnforcerException($"Could not find header '{GitHubEventHeader}'.");
}
}
}
}

Просмотреть файл

@ -0,0 +1,14 @@
using System;
using System.Collections.Generic;
using System.Text;
namespace Azure.Sdk.Tools.CheckEnforcer
{
public interface IRepositoryConfiguration
{
string Format { get; }
bool IsEnabled { get; }
uint MinimumCheckRuns { get; }
}
}

Просмотреть файл

@ -0,0 +1,31 @@
using Newtonsoft.Json;
using System;
using System.Collections.Generic;
using System.Text;
using YamlDotNet.Serialization;
namespace Azure.Sdk.Tools.CheckEnforcer
{
public class RepositoryConfiguration : IRepositoryConfiguration
{
public RepositoryConfiguration()
{
MinimumCheckRuns = 1;
}
[YamlMember(Alias = "minimumCheckRuns")]
public uint MinimumCheckRuns { get; internal set; }
[YamlMember(Alias = "enabled")]
public bool IsEnabled { get; internal set; }
[YamlMember(Alias = "format")]
public string Format { get; internal set; }
public override string ToString()
{
var json = JsonConvert.SerializeObject(this);
return json;
}
}
}

Просмотреть файл

@ -0,0 +1,3 @@
{
"version": "2.0"
}

Просмотреть файл

@ -0,0 +1,25 @@

Microsoft Visual Studio Solution File, Format Version 12.00
# Visual Studio Version 16
VisualStudioVersion = 16.0.29215.179
MinimumVisualStudioVersion = 10.0.40219.1
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Azure.Sdk.Tools.CheckEnforcer", "Azure.Sdk.Tools.CheckEnforcer\Azure.Sdk.Tools.CheckEnforcer.csproj", "{97850C3F-0A6C-446B-B00B-B1E571426661}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
Release|Any CPU = Release|Any CPU
EndGlobalSection
GlobalSection(ProjectConfigurationPlatforms) = postSolution
{97850C3F-0A6C-446B-B00B-B1E571426661}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{97850C3F-0A6C-446B-B00B-B1E571426661}.Debug|Any CPU.Build.0 = Debug|Any CPU
{97850C3F-0A6C-446B-B00B-B1E571426661}.Release|Any CPU.ActiveCfg = Release|Any CPU
{97850C3F-0A6C-446B-B00B-B1E571426661}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
EndGlobalSection
GlobalSection(ExtensibilityGlobals) = postSolution
SolutionGuid = {14C0E54C-F47D-4D15-AAFC-0279F42D6537}
EndGlobalSection
EndGlobal