Azure
/
azure-sdk-tools
зеркало из https://github.com/Azure/azure-sdk-tools.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

Remove failure analysis and apply code cleanup (#8153)

This commit is contained in:
Patrick Hallisey 2024-04-26 10:49:12 -07:00 коммит произвёл GitHub
Родитель a8b08796fb
Коммит 478ff5c2d2
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: B5690EEEBB952194
45 изменённых файлов: 377 добавлений и 1207 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

9
.editorconfig
Просмотреть файл

@ -81,9 +81,9 @@ dotnet_naming_style.prefix_underscore.capitalization = camel_case
###############################
[*.cs]
# var preferences
csharp_style_var_for_built_in_types = true:silent
csharp_style_var_when_type_is_apparent = true:silent
csharp_style_var_elsewhere = true:silent
csharp_style_var_for_built_in_types = false:silent
csharp_style_var_when_type_is_apparent = false:silent
csharp_style_var_elsewhere = false:silent
# Expression-bodied members
csharp_style_expression_bodied_methods = false:silent
csharp_style_expression_bodied_constructors = false:silent
@ -105,6 +105,8 @@ csharp_style_deconstructed_variable_declaration = true:suggestion
csharp_prefer_simple_default_expression = true:suggestion
csharp_style_pattern_local_over_anonymous_function = true:suggestion
csharp_style_inlined_variable_declaration = true:suggestion
# one class per file
csharp_style_single_file_classes = true:suggestion
###############################
# C# Formatting Rules #
@ -136,3 +138,4 @@ csharp_space_between_method_call_empty_parameter_list_parentheses = false
# Wrapping preferences
csharp_preserve_single_line_statements = true
csharp_preserve_single_line_blocks = true

50
tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness.Tests/BlobUploadProcessorIntegrationTests.cs
Просмотреть файл

@ -17,53 +17,53 @@ namespace Azure.Sdk.Tools.PipelineWitness.Tests
{
public class BlobUploadProcessorIntegrationTests
{
private VssCredentials VisualStudioCredentials;
private VssConnection VisualStudioConnection;
private string TARGET_ACCOUNT_ID = "azure-sdk";
private Guid TARGET_PROJECT_ID = new Guid("29ec6040-b234-4e31-b139-33dc4287b756");
private int TARGET_DEFINITION_ID = 297;
private string DEVOPS_PATH = "https://dev.azure.com/azure-sdk";
private PipelineWitnessSettings TestSettings = new PipelineWitnessSettings()
private const string TARGET_ACCOUNT_ID = "azure-sdk";
private const string TARGET_PROJECT_ID = "29ec6040-b234-4e31-b139-33dc4287b756";
private const int TARGET_DEFINITION_ID = 297;
private const string DEVOPS_PATH = "https://dev.azure.com/azure-sdk";
private readonly VssCredentials visualStudioCredentials;
private readonly VssConnection visualStudioConnection;
private readonly PipelineWitnessSettings testSettings = new()
{
PipelineOwnersDefinitionId = 5112,
PipelineOwnersFilePath = "pipelineOwners/pipelineOwners.json",
PipelineOwnersArtifactName = "pipelineOwners"
};
PipelineOwnersFilePath = "pipelineOwners/pipelineOwners.json",
PipelineOwnersArtifactName = "pipelineOwners"
};
public BlobUploadProcessorIntegrationTests()
{
var pat = Environment.GetEnvironmentVariable("AZURESDK_DEVOPS_TOKEN");
var blobUri = Environment.GetEnvironmentVariable("AZURESDK_BLOB_CS");
string pat = Environment.GetEnvironmentVariable("AZURESDK_DEVOPS_TOKEN");
string blobUri = Environment.GetEnvironmentVariable("AZURESDK_BLOB_CS");
if (!string.IsNullOrWhiteSpace(pat) && !string.IsNullOrWhiteSpace(blobUri) )
if (!string.IsNullOrWhiteSpace(pat) && !string.IsNullOrWhiteSpace(blobUri))
{
VisualStudioCredentials = new VssBasicCredential("nobody", pat);
VisualStudioConnection = new VssConnection(new Uri(DEVOPS_PATH), VisualStudioCredentials);
this.visualStudioCredentials = new VssBasicCredential("nobody", pat);
this.visualStudioConnection = new VssConnection(new Uri(DEVOPS_PATH), this.visualStudioCredentials);
}
}
[EnvironmentConditionalSkipFact]
public async Task BasicBlobProcessInvokesSuccessfully()
{
var buildLogProvider = new BuildLogProvider(logger: new NullLogger<BuildLogProvider>(), VisualStudioConnection);
var blobServiceClient = new BlobServiceClient(Environment.GetEnvironmentVariable("AZURESDK_BLOB_CS"));
var buildHttpClient = VisualStudioConnection.GetClient<BuildHttpClient>();
var testResultsBuiltClient = VisualStudioConnection.GetClient<TestResultsHttpClient>();
BuildLogProvider buildLogProvider = new(logger: new NullLogger<BuildLogProvider>(), this.visualStudioConnection);
BlobServiceClient blobServiceClient = new(Environment.GetEnvironmentVariable("AZURESDK_BLOB_CS"));
BuildHttpClient buildHttpClient = this.visualStudioConnection.GetClient<BuildHttpClient>();
TestResultsHttpClient testResultsBuiltClient = this.visualStudioConnection.GetClient<TestResultsHttpClient>();
List<Build> recentBuilds = await buildHttpClient.GetBuildsAsync(TARGET_PROJECT_ID, definitions: new[] { TARGET_DEFINITION_ID }, resultFilter: BuildResult.Succeeded, statusFilter: BuildStatus.Completed, top: 1, queryOrder: BuildQueryOrder.FinishTimeDescending);
Assert.True(recentBuilds.Count > 0);
var targetBuildId = recentBuilds.First().Id;
int targetBuildId = recentBuilds.First().Id;
BlobUploadProcessor processor = new BlobUploadProcessor(logger: new NullLogger<BlobUploadProcessor>(),
BlobUploadProcessor processor = new(logger: new NullLogger<BlobUploadProcessor>(),
logProvider: buildLogProvider,
blobServiceClient: blobServiceClient,
buildClient: buildHttpClient,
testResultsClient: testResultsBuiltClient,
options: Options.Create<PipelineWitnessSettings>(TestSettings),
failureAnalyzer: new PassThroughFailureAnalyzer());
options: Options.Create<PipelineWitnessSettings>(this.testSettings));
await processor.UploadBuildBlobsAsync(TARGET_ACCOUNT_ID, TARGET_PROJECT_ID, targetBuildId);
await processor.UploadBuildBlobsAsync(TARGET_ACCOUNT_ID, new Guid(TARGET_PROJECT_ID), targetBuildId);
}
[Theory]
@ -73,7 +73,7 @@ namespace Azure.Sdk.Tools.PipelineWitness.Tests
[InlineData(0, 10000, 0)]
public void TestBatching(int startingNumber, int batchSize, int expectedBatchNumber)
{
var numberOfBatches = BlobUploadProcessor.CalculateBatches(startingNumber, batchSize);
int numberOfBatches = BlobUploadProcessor.CalculateBatches(startingNumber, batchSize);
Assert.Equal(expectedBatchNumber, numberOfBatches);
}

4
tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness.Tests/EnvironmentConditionalSkipFact.cs
Просмотреть файл

@ -11,8 +11,8 @@ namespace Azure.Sdk.Tools.PipelineWitness.Tests
{
public EnvironmentConditionalSkipFact()
{
var devopsPat = Environment.GetEnvironmentVariable("AZURESDK_DEVOPS_TOKEN");
var blobToken = Environment.GetEnvironmentVariable("AZURESDK_BLOB_CS");
string devopsPat = Environment.GetEnvironmentVariable("AZURESDK_DEVOPS_TOKEN");
string blobToken = Environment.GetEnvironmentVariable("AZURESDK_BLOB_CS");
// and if we don't, skip this test
if (string.IsNullOrWhiteSpace(devopsPat) || string.IsNullOrWhiteSpace(blobToken))

16
tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness.Tests/PassThroughFailureAnalyzer.cs
Просмотреть файл

@ -1,16 +0,0 @@
using System.Collections.Generic;
using System.Threading.Tasks;
using Azure.Sdk.Tools.PipelineWitness.Entities.AzurePipelines;
using Azure.Sdk.Tools.PipelineWitness.Services.FailureAnalysis;
using Microsoft.TeamFoundation.Build.WebApi;
namespace Azure.Sdk.Tools.PipelineWitness.Tests
{
internal class PassThroughFailureAnalyzer : IFailureAnalyzer
{
public Task<IEnumerable<Failure>> AnalyzeFailureAsync(Build build, Timeline timeline)
{
return Task.FromResult<IEnumerable<Failure>>(new List<Failure>());
}
}
}

25
tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness.Tests/TestLogger.cs
Просмотреть файл

@ -24,29 +24,4 @@ namespace Azure.Sdk.Tools.PipelineWitness.Tests
Logs.Add(state);
}
}
public class TestLoggingFactory : ILoggerFactory
{
private readonly TestLogger _logger;
public TestLoggingFactory(TestLogger logger)
{
_logger = logger;
}
public void Dispose()
{
throw new NotImplementedException();
}
public void AddProvider(ILoggerProvider provider)
{
throw new NotImplementedException();
}
public ILogger CreateLogger(string categoryName)
{
return _logger;
}
}
}

19
tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/ApplicationInsights/ApplicationVersionTelemetryInitializer.cs
Просмотреть файл

@ -1,31 +1,32 @@
using System.Reflection;
using System.Reflection;
using Microsoft.ApplicationInsights.Channel;
using Microsoft.ApplicationInsights.Extensibility;
using Microsoft.ApplicationInsights.Extensibility.Implementation;
namespace Azure.Sdk.Tools.PipelineWitness.ApplicationInsights
{
public class ApplicationVersionTelemetryInitializer : ITelemetryInitializer
{
private static string _version = GetVersion();
private static readonly string version = GetVersion();
public void Initialize(ITelemetry telemetry)
{
if (!string.IsNullOrEmpty(_version))
if (!string.IsNullOrEmpty(version))
{
var component = telemetry.Context?.Component;
ComponentContext component = telemetry.Context?.Component;
if (component != null)
{
component.Version = _version;
component.Version = version;
}
}
}
private static string GetVersion()
{
var assembly = typeof(ApplicationVersionTelemetryInitializer).Assembly;
var version = assembly.GetCustomAttribute<AssemblyInformationalVersionAttribute>()?.InformationalVersion
Assembly assembly = typeof(ApplicationVersionTelemetryInitializer).Assembly;
string version = assembly.GetCustomAttribute<AssemblyInformationalVersionAttribute>()?.InformationalVersion
?? assembly.GetName().Version?.ToString();
return version;

8
tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/ApplicationInsights/BlobNotFoundTelemetryProcessor.cs
Просмотреть файл

@ -13,14 +13,14 @@ namespace Azure.Sdk.Tools.PipelineWitness.ApplicationInsights
{
this.next = next;
}
public void Process(ITelemetry telemetry)
{
if (telemetry is DependencyTelemetry { Success: false, Type: "Azure blob" or "Microsoft.Storage" } blobRequestTelemetry)
{
blobRequestTelemetry.Properties.TryGetValue("Error", out var errorProperty);
var isNotFound = blobRequestTelemetry.ResultCode is "404" or "409"
blobRequestTelemetry.Properties.TryGetValue("Error", out string errorProperty);
bool isNotFound = blobRequestTelemetry.ResultCode is "404" or "409"
|| (blobRequestTelemetry.ResultCode == "" && errorProperty?.Contains("Status: 404") == true);
if (isNotFound)

292
tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/BlobUploadProcessor.cs
Просмотреть файл

@ -1,44 +1,45 @@
using System;
using System.Collections.Generic;
using System.Diagnostics.CodeAnalysis;
using System.IO;
using System.IO.Compression;
using System.Linq;
using System.Net;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using Azure.Sdk.Tools.PipelineWitness.Configuration;
using Azure.Sdk.Tools.PipelineWitness.Services;
using Azure.Storage.Blobs;
using Azure.Storage.Blobs.Models;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using Microsoft.TeamFoundation.Build.WebApi;
using Microsoft.TeamFoundation.TestManagement.WebApi;
using Microsoft.VisualStudio.Services.TestResults.WebApi;
using Microsoft.VisualStudio.Services.WebApi;
using Newtonsoft.Json;
using Newtonsoft.Json.Converters;
using Newtonsoft.Json.Serialization;
namespace Azure.Sdk.Tools.PipelineWitness
{
using System;
using System.Collections.Generic;
using System.IO;
using System.IO.Compression;
using System.Linq;
using System.Net;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using Azure.Sdk.Tools.PipelineWitness.Configuration;
using Azure.Sdk.Tools.PipelineWitness.Services;
using Azure.Sdk.Tools.PipelineWitness.Services.FailureAnalysis;
using Azure.Storage.Blobs;
using Azure.Storage.Blobs.Models;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using Microsoft.TeamFoundation.Build.WebApi;
using Microsoft.TeamFoundation.TestManagement.WebApi;
using Microsoft.VisualStudio.Services.TestResults.WebApi;
using Newtonsoft.Json;
using Newtonsoft.Json.Converters;
using Newtonsoft.Json.Serialization;
[SuppressMessage("Style", "IDE0037:Use inferred member name", Justification = "Explicit member names are added to json export objects for clarity")]
public class BlobUploadProcessor
{
private const string BuildsContainerName = "builds";
private const string BuildLogLinesContainerName = "buildloglines";
private const string BuildTimelineRecordsContainerName = "buildtimelinerecords";
private const string BuildDefinitionsContainerName = "builddefinitions";
private const string BuildFailuresContainerName = "buildfailures";
private const string PipelineOwnersContainerName = "pipelineowners";
private const string TestRunsContainerName = "testruns";
private const string TestResultsContainerName = "testrunresults";
private const int ApiBatchSize = 10000;
private const string TimeFormat = @"yyyy-MM-dd\THH:mm:ss.fffffff\Z";
private static readonly JsonSerializerSettings jsonSettings = new JsonSerializerSettings()
private static readonly JsonSerializerSettings jsonSettings = new()
{
ContractResolver = new CamelCasePropertyNamesContractResolver(),
Converters = { new StringEnumConverter(new CamelCaseNamingStrategy()) },
@ -55,11 +56,9 @@ namespace Azure.Sdk.Tools.PipelineWitness
private readonly BlobContainerClient testRunsContainerClient;
private readonly BlobContainerClient testResultsContainerClient;
private readonly BlobContainerClient buildDefinitionsContainerClient;
private readonly BlobContainerClient buildFailuresContainerClient;
private readonly BlobContainerClient pipelineOwnersContainerClient;
private readonly IOptions<PipelineWitnessSettings> options;
private readonly Dictionary<string, int?> cachedDefinitionRevisions = new();
private readonly IFailureAnalyzer failureAnalyzer;
public BlobUploadProcessor(
ILogger<BlobUploadProcessor> logger,
@ -67,8 +66,7 @@ namespace Azure.Sdk.Tools.PipelineWitness
BlobServiceClient blobServiceClient,
BuildHttpClient buildClient,
TestResultsHttpClient testResultsClient,
IOptions<PipelineWitnessSettings> options,
IFailureAnalyzer failureAnalyzer)
IOptions<PipelineWitnessSettings> options)
{
if (blobServiceClient == null)
{
@ -84,17 +82,15 @@ namespace Azure.Sdk.Tools.PipelineWitness
this.buildTimelineRecordsContainerClient = blobServiceClient.GetBlobContainerClient(BuildTimelineRecordsContainerName);
this.buildLogLinesContainerClient = blobServiceClient.GetBlobContainerClient(BuildLogLinesContainerName);
this.buildDefinitionsContainerClient = blobServiceClient.GetBlobContainerClient(BuildDefinitionsContainerName);
this.buildFailuresContainerClient = blobServiceClient.GetBlobContainerClient(BuildFailuresContainerName);
this.testRunsContainerClient = blobServiceClient.GetBlobContainerClient(TestRunsContainerName);
this.testResultsContainerClient = blobServiceClient.GetBlobContainerClient(TestResultsContainerName);
this.buildDefinitionsContainerClient = blobServiceClient.GetBlobContainerClient(BuildDefinitionsContainerName);
this.pipelineOwnersContainerClient = blobServiceClient.GetBlobContainerClient(PipelineOwnersContainerName);
this.failureAnalyzer = failureAnalyzer;
}
public async Task UploadBuildBlobsAsync(string account, Guid projectId, int buildId)
{
var build = await GetBuildAsync(projectId, buildId);
Build build = await GetBuildAsync(projectId, buildId);
if (build == null)
{
@ -102,7 +98,7 @@ namespace Azure.Sdk.Tools.PipelineWitness
return;
}
var skipBuild = false;
bool skipBuild = false;
// Project name is used in blob paths and cannot be empty
if (build.Project == null)
@ -157,34 +153,33 @@ namespace Azure.Sdk.Tools.PipelineWitness
await UploadTestRunBlobsAsync(account, build);
var timeline = await this.buildClient.GetBuildTimelineAsync(projectId, buildId);
Timeline timeline = await this.buildClient.GetBuildTimelineAsync(projectId, buildId);
if (timeline == null)
{
logger.LogWarning("No timeline available for build {Project}: {BuildId}", build.Project.Name, build.Id);
this.logger.LogWarning("No timeline available for build {Project}: {BuildId}", build.Project.Name, build.Id);
}
else
{
await UploadTimelineBlobAsync(account, build, timeline);
await UploadBuildFailureBlobAsync(account, build, timeline);
}
var logs = await buildClient.GetBuildLogsAsync(build.Project.Id, build.Id);
List<BuildLog> logs = await this.buildClient.GetBuildLogsAsync(build.Project.Id, build.Id);
if (logs == null || logs.Count == 0)
{
logger.LogWarning("No logs available for build {Project}: {BuildId}", build.Project.Name, build.Id);
this.logger.LogWarning("No logs available for build {Project}: {BuildId}", build.Project.Name, build.Id);
return;
}
var buildLogInfos = GetBuildLogInfos(account, build, timeline, logs);
List<BuildLogInfo> buildLogInfos = GetBuildLogInfos(build, timeline, logs);
foreach (var log in buildLogInfos)
foreach (BuildLogInfo log in buildLogInfos)
{
await UploadLogLinesBlobAsync(account, build, log);
}
if (build.Definition.Id == options.Value.PipelineOwnersDefinitionId)
if (build.Definition.Id == this.options.Value.PipelineOwnersDefinitionId)
{
await UploadPipelineOwnersBlobAsync(account, build, timeline);
}
@ -194,8 +189,8 @@ namespace Azure.Sdk.Tools.PipelineWitness
{
try
{
var blobPath = $"{build.Project.Name}/{build.FinishTime:yyyy/MM/dd}/{build.Id}-{timeline.ChangeId}.jsonl";
var blobClient = this.pipelineOwnersContainerClient.GetBlobClient(blobPath);
string blobPath = $"{build.Project.Name}/{build.FinishTime:yyyy/MM/dd}/{build.Id}-{timeline.ChangeId}.jsonl";
BlobClient blobClient = this.pipelineOwnersContainerClient.GetBlobClient(blobPath);
if (await blobClient.ExistsAsync())
{
@ -203,7 +198,7 @@ namespace Azure.Sdk.Tools.PipelineWitness
return;
}
var owners = await GetOwnersFromBuildArtifactAsync(build);
Dictionary<int, string[]> owners = await GetOwnersFromBuildArtifactAsync(build);
if (owners == null)
{
@ -213,11 +208,11 @@ namespace Azure.Sdk.Tools.PipelineWitness
this.logger.LogInformation("Creating owners blob for build {DefinitionId} change {ChangeId}", build.Id, timeline.ChangeId);
var stringBuilder = new StringBuilder();
StringBuilder stringBuilder = new();
foreach (var owner in owners)
foreach (KeyValuePair<int, string[]> owner in owners)
{
var contentLine = JsonConvert.SerializeObject(new
string contentLine = JsonConvert.SerializeObject(new
{
OrganizationName = account,
BuildDefinitionId = owner.Key,
@ -244,15 +239,15 @@ namespace Azure.Sdk.Tools.PipelineWitness
private async Task<Dictionary<int, string[]>> GetOwnersFromBuildArtifactAsync(Build build)
{
var artifactName = this.options.Value.PipelineOwnersArtifactName;
var filePath = this.options.Value.PipelineOwnersFilePath;
string artifactName = this.options.Value.PipelineOwnersArtifactName;
string filePath = this.options.Value.PipelineOwnersFilePath;
try
{
await using var artifactStream = await this.buildClient.GetArtifactContentZipAsync(build.Project.Id, build.Id, artifactName);
using var zip = new ZipArchive(artifactStream);
await using Stream artifactStream = await this.buildClient.GetArtifactContentZipAsync(build.Project.Id, build.Id, artifactName);
using ZipArchive zip = new(artifactStream);
var fileEntry = zip.GetEntry(filePath);
ZipArchiveEntry fileEntry = zip.GetEntry(filePath);
if (fileEntry == null)
{
@ -260,9 +255,9 @@ namespace Azure.Sdk.Tools.PipelineWitness
return null;
}
await using var contentStream = fileEntry.Open();
using var contentReader = new StreamReader(contentStream);
var content = await contentReader.ReadToEndAsync();
await using Stream contentStream = fileEntry.Open();
using StreamReader contentReader = new(contentStream);
string content = await contentReader.ReadToEndAsync();
if (string.IsNullOrEmpty(content))
{
@ -270,7 +265,7 @@ namespace Azure.Sdk.Tools.PipelineWitness
return null;
}
var ownersDictionary = JsonConvert.DeserializeObject<Dictionary<int, string[]>>(content);
Dictionary<int, string[]> ownersDictionary = JsonConvert.DeserializeObject<Dictionary<int, string[]>>(content);
if (ownersDictionary == null)
{
@ -298,72 +293,15 @@ namespace Azure.Sdk.Tools.PipelineWitness
return null;
}
private async Task UploadBuildFailureBlobAsync(string account, Build build, Timeline timeline)
{
try
{
var blobPath = $"{build.Project.Name}/{build.FinishTime:yyyy/MM/dd}/{build.Id}-{timeline.ChangeId}.jsonl";
var blobClient = this.buildFailuresContainerClient.GetBlobClient(blobPath);
if (await blobClient.ExistsAsync())
{
this.logger.LogInformation("Skipping existing build failure blob for build {BuildId}", build.Id);
return;
}
var failures = await this.failureAnalyzer.AnalyzeFailureAsync(build, timeline);
if (!failures.Any())
{
return;
}
this.logger.LogInformation("Creating failure blob for build {DefinitionId} change {ChangeId}", build.Id, timeline.ChangeId);
var stringBuilder = new StringBuilder();
foreach (var failure in failures)
{
var contentLine = JsonConvert.SerializeObject(new
{
OrganizationName = account,
ProjectId = build.Project.Id,
ProjectName = build.Project.Name,
BuildDefinitionId = build.Definition.Id,
BuildDefinitionName = build.Definition.Name,
BuildId = build.Id,
BuildFinishTime = build.FinishTime,
RecordFinishTime = failure.Record.FinishTime,
ChangeId = timeline.ChangeId,
RecordId = failure.Record.Id,
BuildTimelineId = timeline.Id,
ErrorClassification = failure.Classification,
EtlIngestDate = DateTimeOffset.UtcNow
}, jsonSettings);
stringBuilder.AppendLine(contentLine);
}
await blobClient.UploadAsync(new BinaryData(stringBuilder.ToString()));
}
catch (RequestFailedException ex) when (ex.Status == (int)HttpStatusCode.Conflict)
{
this.logger.LogInformation("Ignoring exception from existing failure blob for build {BuildId}", build.Id);
}
catch (Exception ex)
{
this.logger.LogError(ex, "Error processing build failure blob for build {BuildId}", build.Id);
throw;
}
}
public async Task UploadBuildDefinitionBlobsAsync(string account, string projectName)
{
var definitions = await buildClient.GetFullDefinitionsAsync2(project: projectName);
IPagedList<BuildDefinition> definitions = await this.buildClient.GetFullDefinitionsAsync2(project: projectName);
foreach (var definition in definitions)
foreach (BuildDefinition definition in definitions)
{
var cacheKey = $"{definition.Project.Id}:{definition.Id}";
string cacheKey = $"{definition.Project.Id}:{definition.Id}";
if (!this.cachedDefinitionRevisions.TryGetValue(cacheKey, out var cachedRevision) || cachedRevision != definition.Revision)
if (!this.cachedDefinitionRevisions.TryGetValue(cacheKey, out int? cachedRevision) || cachedRevision != definition.Revision)
{
await UploadBuildDefinitionBlobAsync(account, definition);
}
@ -371,14 +309,14 @@ namespace Azure.Sdk.Tools.PipelineWitness
this.cachedDefinitionRevisions[cacheKey] = definition.Revision;
}
}
private async Task UploadBuildDefinitionBlobAsync(string account, BuildDefinition definition)
{
var blobPath = $"{definition.Project.Name}/{definition.Id}-{definition.Revision}.jsonl";
string blobPath = $"{definition.Project.Name}/{definition.Id}-{definition.Revision}.jsonl";
try
{
var blobClient = this.buildDefinitionsContainerClient.GetBlobClient(blobPath);
BlobClient blobClient = this.buildDefinitionsContainerClient.GetBlobClient(blobPath);
if (await blobClient.ExistsAsync())
{
@ -388,7 +326,7 @@ namespace Azure.Sdk.Tools.PipelineWitness
this.logger.LogInformation("Creating blob for build definition {DefinitionId} revision {Revision} project {Project}", definition.Id, definition.Revision, definition.Project.Name);
var content = JsonConvert.SerializeObject(new
string content = JsonConvert.SerializeObject(new
{
OrganizationName = account,
ProjectId = definition.Project.Id,
@ -446,32 +384,32 @@ namespace Azure.Sdk.Tools.PipelineWitness
}
}
private List<BuildLogInfo> GetBuildLogInfos(string account, Build build, Timeline timeline, List<BuildLog> logs)
private List<BuildLogInfo> GetBuildLogInfos(Build build, Timeline timeline, List<BuildLog> logs)
{
var logsById = logs.ToDictionary(l => l.Id);
Dictionary<int, BuildLog> logsById = logs.ToDictionary(l => l.Id);
var buildLogInfos = new List<BuildLogInfo>();
List<BuildLogInfo> buildLogInfos = new();
foreach (var log in logs)
foreach (BuildLog log in logs)
{
var logRecords = timeline.Records.Where(x => x.Log?.Id == log.Id).ToArray();
TimelineRecord[] logRecords = timeline.Records.Where(x => x.Log?.Id == log.Id).ToArray();
if(logRecords.Length > 1)
if (logRecords.Length > 1)
{
this.logger.LogWarning("Found multiple timeline records for build {BuildId}, log {LogId}", build.Id, log.Id);
}
var logRecord = logRecords.FirstOrDefault();
TimelineRecord logRecord = logRecords.FirstOrDefault();
// Job logs are typically just a duplication of their child task logs with the addition of extra start and end lines.
// If we can, we skip the redundant lines.
if (string.Equals(logRecord?.RecordType, "job", StringComparison.OrdinalIgnoreCase))
{
// find all of the child task records
var childRecords = timeline.Records.Where(x => x.ParentId == logRecord.Id);
IEnumerable<TimelineRecord> childRecords = timeline.Records.Where(x => x.ParentId == logRecord.Id);
// sum the line counts for all of the child task records
var childLineCount = childRecords
long childLineCount = childRecords
.Where(x => x.Log != null && logsById.ContainsKey(x.Log.Id))
.Sum(x => logsById[x.Log.Id].LineCount);
@ -502,8 +440,8 @@ namespace Azure.Sdk.Tools.PipelineWitness
try
{
long changeTime = ((DateTimeOffset)build.LastChangedDate).ToUnixTimeSeconds();
var blobPath = $"{build.Project.Name}/{build.FinishTime:yyyy/MM/dd}/{build.Id}-{changeTime}.jsonl";
var blobClient = this.buildsContainerClient.GetBlobClient(blobPath);
string blobPath = $"{build.Project.Name}/{build.FinishTime:yyyy/MM/dd}/{build.Id}-{changeTime}.jsonl";
BlobClient blobClient = this.buildsContainerClient.GetBlobClient(blobPath);
if (await blobClient.ExistsAsync())
{
@ -511,7 +449,7 @@ namespace Azure.Sdk.Tools.PipelineWitness
return;
}
var content = JsonConvert.SerializeObject(new
string content = JsonConvert.SerializeObject(new
{
OrganizationName = account,
ProjectId = build.Project?.Id,
@ -582,8 +520,8 @@ namespace Azure.Sdk.Tools.PipelineWitness
return;
}
var blobPath = $"{build.Project.Name}/{build.FinishTime:yyyy/MM/dd}/{build.Id}-{timeline.ChangeId}.jsonl";
var blobClient = this.buildTimelineRecordsContainerClient.GetBlobClient(blobPath);
string blobPath = $"{build.Project.Name}/{build.FinishTime:yyyy/MM/dd}/{build.Id}-{timeline.ChangeId}.jsonl";
BlobClient blobClient = this.buildTimelineRecordsContainerClient.GetBlobClient(blobPath);
if (await blobClient.ExistsAsync())
{
@ -591,8 +529,8 @@ namespace Azure.Sdk.Tools.PipelineWitness
return;
}
var builder = new StringBuilder();
foreach (var record in timeline.Records)
StringBuilder builder = new();
foreach (TimelineRecord record in timeline.Records)
{
builder.AppendLine(JsonConvert.SerializeObject(
new
@ -656,8 +594,8 @@ namespace Azure.Sdk.Tools.PipelineWitness
{
// we don't use FinishTime in the logs blob path to prevent duplicating logs when processing retries.
// i.e. logs with a given buildid/logid are immutable and retries only add new logs.
var blobPath = $"{build.Project.Name}/{build.QueueTime:yyyy/MM/dd}/{build.Id}-{log.LogId}.jsonl";
var blobClient = this.buildLogLinesContainerClient.GetBlobClient(blobPath);
string blobPath = $"{build.Project.Name}/{build.QueueTime:yyyy/MM/dd}/{build.Id}-{log.LogId}.jsonl";
BlobClient blobClient = this.buildLogLinesContainerClient.GetBlobClient(blobPath);
if (await blobClient.ExistsAsync())
{
@ -667,44 +605,44 @@ namespace Azure.Sdk.Tools.PipelineWitness
this.logger.LogInformation("Processing log for build {BuildId}, record {RecordId}, log {LogId}", build.Id, log.RecordId, log.LogId);
var lineNumber = 0;
var characterCount = 0;
int lineNumber = 0;
int characterCount = 0;
// Over an open read stream and an open write stream, one line at a time, read, process, and write to
// blob storage
using (var logStream = await this.logProvider.GetLogStreamAsync(build.Project.Name, build.Id, log.LogId))
using (var logReader = new StreamReader(logStream))
using (var blobStream = await blobClient.OpenWriteAsync(overwrite: true, new BlobOpenWriteOptions()))
using (var blobWriter = new StreamWriter(blobStream))
using (Stream logStream = await this.logProvider.GetLogStreamAsync(build.Project.Name, build.Id, log.LogId))
using (StreamReader logReader = new(logStream))
using (Stream blobStream = await blobClient.OpenWriteAsync(overwrite: true, new BlobOpenWriteOptions()))
using (StreamWriter blobWriter = new(blobStream))
{
var lastTimeStamp = log.LogCreatedOn;
DateTimeOffset lastTimeStamp = log.LogCreatedOn;
while (true)
{
var line = await logReader.ReadLineAsync();
string line = await logReader.ReadLineAsync();
if (line == null)
{
break;
}
var isLastLine = logReader.EndOfStream;
bool isLastLine = logReader.EndOfStream;
lineNumber += 1;
characterCount += line.Length;
// log lines usually follow the format:
// 2022-03-30T21:38:38.7007903Z Downloading task: AzureKeyVault (1.200.0)
// Sometimes, there's no leading timestamp, so we'll use the last timestamp we saw.
var match = Regex.Match(line, @"^([^Z]{20,28}Z) (.*)$");
Match match = Regex.Match(line, @"^([^Z]{20,28}Z) (.*)$");
var timestamp = match.Success
DateTimeOffset timestamp = match.Success
? DateTime.ParseExact(match.Groups[1].Value, TimeFormat, null,
System.Globalization.DateTimeStyles.AssumeUniversal).ToUniversalTime()
: lastTimeStamp;
lastTimeStamp = timestamp;
var message = match.Success ? match.Groups[2].Value : line;
string message = match.Success ? match.Groups[2].Value : line;
await blobWriter.WriteLineAsync(JsonConvert.SerializeObject(new
{
@ -725,7 +663,7 @@ namespace Azure.Sdk.Tools.PipelineWitness
}
}
logger.LogInformation("Processed {CharacterCount} characters and {LineCount} lines for build {BuildId}, record {RecordId}, log {LogId}", characterCount, lineNumber, build.Id, log.RecordId, log.LogId);
this.logger.LogInformation("Processed {CharacterCount} characters and {LineCount} lines for build {BuildId}, record {RecordId}, log {LogId}", characterCount, lineNumber, build.Id, log.RecordId, log.LogId);
}
catch (RequestFailedException ex) when (ex.Status == (int)HttpStatusCode.Conflict)
{
@ -737,31 +675,31 @@ namespace Azure.Sdk.Tools.PipelineWitness
throw;
}
}
private async Task UploadTestRunBlobsAsync(string account, Build build)
{
try
{
var continuationToken = string.Empty;
var buildIds = new[] { build.Id };
string continuationToken = string.Empty;
int[] buildIds = new[] { build.Id };
var minLastUpdatedDate = build.QueueTime.Value.AddHours(-1);
var maxLastUpdatedDate = build.FinishTime.Value.AddHours(1);
DateTime minLastUpdatedDate = build.QueueTime.Value.AddHours(-1);
DateTime maxLastUpdatedDate = build.FinishTime.Value.AddHours(1);
var rangeStart = minLastUpdatedDate;
DateTime rangeStart = minLastUpdatedDate;
while(rangeStart < maxLastUpdatedDate)
while (rangeStart < maxLastUpdatedDate)
{
// Ado limits test run queries to a 7 day range, so we'll chunk on 6 days.
var rangeEnd = rangeStart.AddDays(6);
if(rangeEnd > maxLastUpdatedDate)
DateTime rangeEnd = rangeStart.AddDays(6);
if (rangeEnd > maxLastUpdatedDate)
{
rangeEnd = maxLastUpdatedDate;
}
do
{
var page = await testResultsClient.QueryTestRunsAsync2(
IPagedList<TestRun> page = await this.testResultsClient.QueryTestRunsAsync2(
build.Project.Id,
rangeStart,
rangeEnd,
@ -769,7 +707,7 @@ namespace Azure.Sdk.Tools.PipelineWitness
buildIds: buildIds
);
foreach (var testRun in page)
foreach (TestRun testRun in page)
{
await UploadTestRunBlobAsync(account, build, testRun);
await UploadTestRunResultBlobAsync(account, build, testRun);
@ -792,8 +730,8 @@ namespace Azure.Sdk.Tools.PipelineWitness
{
try
{
var blobPath = $"{build.Project.Name}/{testRun.CompletedDate:yyyy/MM/dd}/{testRun.Id}.jsonl";
var blobClient = this.testRunsContainerClient.GetBlobClient(blobPath);
string blobPath = $"{build.Project.Name}/{testRun.CompletedDate:yyyy/MM/dd}/{testRun.Id}.jsonl";
BlobClient blobClient = this.testRunsContainerClient.GetBlobClient(blobPath);
if (await blobClient.ExistsAsync())
{
@ -801,9 +739,9 @@ namespace Azure.Sdk.Tools.PipelineWitness
return;
}
var stats = testRun.RunStatistics.ToDictionary(x => x.Outcome, x => x.Count);
Dictionary<string, int> stats = testRun.RunStatistics.ToDictionary(x => x.Outcome, x => x.Count);
var content = JsonConvert.SerializeObject(new
string content = JsonConvert.SerializeObject(new
{
OrganizationName = account,
ProjectId = build.Project?.Id,
@ -821,7 +759,7 @@ namespace Azure.Sdk.Tools.PipelineWitness
BranchName = build.SourceBranch,
HasDetail = default(bool?),
IsAutomated = testRun.IsAutomated,
ResultAbortedCount = stats.TryGetValue("Aborted", out var value) ? value : 0,
ResultAbortedCount = stats.TryGetValue("Aborted", out int value) ? value : 0,
ResultBlockedCount = stats.TryGetValue("Blocked", out value) ? value : 0,
ResultCount = testRun.TotalTests,
ResultErrorCount = stats.TryGetValue("Error", out value) ? value : 0,
@ -875,8 +813,8 @@ namespace Azure.Sdk.Tools.PipelineWitness
{
try
{
var blobPath = $"{build.Project.Name}/{testRun.CompletedDate:yyyy/MM/dd}/{testRun.Id}.jsonl";
var blobClient = this.testResultsContainerClient.GetBlobClient(blobPath);
string blobPath = $"{build.Project.Name}/{testRun.CompletedDate:yyyy/MM/dd}/{testRun.Id}.jsonl";
BlobClient blobClient = this.testResultsContainerClient.GetBlobClient(blobPath);
if (await blobClient.ExistsAsync())
{
@ -884,14 +822,14 @@ namespace Azure.Sdk.Tools.PipelineWitness
return;
}
var builder = new StringBuilder();
StringBuilder builder = new();
int batchCount = BlobUploadProcessor.CalculateBatches(testRun.TotalTests, batchSize: ApiBatchSize);
for(int batchMultiplier = 0; batchMultiplier < batchCount; batchMultiplier++)
for (int batchMultiplier = 0; batchMultiplier < batchCount; batchMultiplier++)
{
var data = await testResultsClient.GetTestResultsAsync(build.Project.Id, testRun.Id, top: ApiBatchSize, skip: batchMultiplier * ApiBatchSize);
List<TestCaseResult> data = await this.testResultsClient.GetTestResultsAsync(build.Project.Id, testRun.Id, top: ApiBatchSize, skip: batchMultiplier * ApiBatchSize);
foreach (var record in data)
foreach (TestCaseResult record in data)
{
builder.AppendLine(JsonConvert.SerializeObject(
new
@ -936,7 +874,7 @@ namespace Azure.Sdk.Tools.PipelineWitness
try
{
build = await buildClient.GetBuildAsync(projectId, buildId);
build = await this.buildClient.GetBuildAsync(projectId, buildId);
}
catch (BuildNotFoundException)
{

10
tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Entities/BuildLogBundle.cs
Просмотреть файл

@ -14,15 +14,15 @@ namespace Azure.Sdk.Tools.PipelineWitness
public int BuildId { get; set; }
public DateTimeOffset StartTime { get; set; }
public DateTimeOffset FinishTime { get; set; }
public DateTimeOffset QueueTime { get; set; }
public int DefinitionId { get; set; }
public string DefinitionPath { get; set; }
public string DefinitionName { get; set; }
public List<BuildLogInfo> TimelineLogs { get; } = new List<BuildLogInfo>();

10
tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Entities/BuildLogInfo.cs
Просмотреть файл

@ -1,7 +1,7 @@
namespace Azure.Sdk.Tools.PipelineWitness
{
using System;
using System;
namespace Azure.Sdk.Tools.PipelineWitness
{
public class BuildLogInfo
{
public int LogId { get; set; }
@ -11,9 +11,9 @@
public DateTimeOffset LogCreatedOn { get; set; }
public string RecordType { get; set; }
public Guid? RecordId { get; set; }
public Guid? ParentRecordId { get; set; }
}
}

6
tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Program.cs
Просмотреть файл

@ -13,8 +13,8 @@ namespace Azure.Sdk.Tools.PipelineWitness
// per queue storage performance docs, set the default connection limit to >= 100
// https://learn.microsoft.com/en-us/azure/storage/queues/storage-performance-checklist#increase-default-connection-limit
ServicePointManager.DefaultConnectionLimit = 100;
var builder = WebApplication.CreateBuilder(args);
WebApplicationBuilder builder = WebApplication.CreateBuilder(args);
// Add services to the container.
builder.Services.AddControllers();
@ -26,7 +26,7 @@ namespace Azure.Sdk.Tools.PipelineWitness
Startup.Configure(builder);
var app = builder.Build();
WebApplication app = builder.Build();
// Configure the HTTP request pipeline.
if (app.Environment.IsDevelopment())

22
tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/AzurePipelinesBuildDefinitionWorker.cs
Просмотреть файл

@ -1,12 +1,12 @@
using System;
using System.Diagnostics;
using System.Threading;
using System.Threading.Tasks;
using Azure.Sdk.Tools.PipelineWitness.Configuration;
using Azure.Sdk.Tools.PipelineWitness.Services.WorkTokens;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using System.Diagnostics;
using Microsoft.Extensions.Options;
using Azure.Sdk.Tools.PipelineWitness.Services.WorkTokens;
using Azure.Sdk.Tools.PipelineWitness.Configuration;
namespace Azure.Sdk.Tools.PipelineWitness.Services
{
@ -15,7 +15,7 @@ namespace Azure.Sdk.Tools.PipelineWitness.Services
private readonly ILogger<AzurePipelinesBuildDefinitionWorker> logger;
private readonly BlobUploadProcessor runProcessor;
private readonly IOptions<PipelineWitnessSettings> options;
private IAsyncLockProvider asyncLockProvider;
private readonly IAsyncLockProvider asyncLockProvider;
public AzurePipelinesBuildDefinitionWorker(
ILogger<AzurePipelinesBuildDefinitionWorker> logger,
@ -31,32 +31,32 @@ namespace Azure.Sdk.Tools.PipelineWitness.Services
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
var processEvery = TimeSpan.FromMinutes(60);
TimeSpan processEvery = TimeSpan.FromMinutes(60);
while (true)
{
var stopWatch = Stopwatch.StartNew();
var settings = this.options.Value;
Stopwatch stopWatch = Stopwatch.StartNew();
PipelineWitnessSettings settings = this.options.Value;
try
{
await using var asyncLock = await this.asyncLockProvider.GetLockAsync("UpdateBuildDefinitions", processEvery, stoppingToken);
await using IAsyncLock asyncLock = await this.asyncLockProvider.GetLockAsync("UpdateBuildDefinitions", processEvery, stoppingToken);
// if there's no asyncLock, this process has alread completed in the last hour
if (asyncLock != null)
{
foreach (var project in settings.Projects)
foreach (string project in settings.Projects)
{
await this.runProcessor.UploadBuildDefinitionBlobsAsync(settings.Account, project);
}
}
}
catch(Exception ex)
catch (Exception ex)
{
this.logger.LogError(ex, "Error processing build definitions");
}
var duration = settings.BuildDefinitionLoopPeriod - stopWatch.Elapsed;
TimeSpan duration = settings.BuildDefinitionLoopPeriod - stopWatch.Elapsed;
if (duration > TimeSpan.Zero)
{
await Task.Delay(duration, stoppingToken);

21
tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/BuildCompleteQueueWorker.cs
Просмотреть файл

@ -1,3 +1,4 @@
using System;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
@ -17,7 +18,6 @@ namespace Azure.Sdk.Tools.PipelineWitness.Services
internal class BuildCompleteQueueWorker : QueueWorkerBackgroundService
{
private readonly ILogger logger;
private readonly TelemetryClient telemetryClient;
private readonly BlobUploadProcessor runProcessor;
public BuildCompleteQueueWorker(
@ -35,16 +35,15 @@ namespace Azure.Sdk.Tools.PipelineWitness.Services
{
this.logger = logger;
this.runProcessor = runProcessor;
this.telemetryClient = telemetryClient;
}
internal override async Task ProcessMessageAsync(QueueMessage message, CancellationToken cancellationToken)
{
this.logger.LogInformation("Processing build.complete event: {MessageText}", message.MessageText);
var devopsEvent = JObject.Parse(message.MessageText);
JObject devopsEvent = JObject.Parse(message.MessageText);
var buildUrl = devopsEvent["resource"]?.Value<string>("url");
string buildUrl = devopsEvent["resource"]?.Value<string>("url");
if (buildUrl == null)
{
@ -52,7 +51,7 @@ namespace Azure.Sdk.Tools.PipelineWitness.Services
return;
}
var match = Regex.Match(buildUrl, @"^https://dev.azure.com/(?<account>[\w-]+)/(?<project>[0-9a-fA-F-]+)/_apis/build/Builds/(?<build>\d+)$");
Match match = Regex.Match(buildUrl, @"^https://dev.azure.com/(?<account>[\w-]+)/(?<project>[0-9a-fA-F-]+)/_apis/build/Builds/(?<build>\d+)$");
if (!match.Success)
{
@ -60,23 +59,23 @@ namespace Azure.Sdk.Tools.PipelineWitness.Services
return;
}
var account = match.Groups["account"].Value;
var projectIdString = match.Groups["project"].Value;
var buildIdString = match.Groups["build"].Value;
string account = match.Groups["account"].Value;
string projectIdString = match.Groups["project"].Value;
string buildIdString = match.Groups["build"].Value;
if (!System.Guid.TryParse(projectIdString, out var projectId))
if (!Guid.TryParse(projectIdString, out Guid projectId))
{
this.logger.LogError("Could not parse project id as a guid '{ProjectId}'", projectIdString);
return;
}
if (!int.TryParse(buildIdString, out var buildId))
if (!int.TryParse(buildIdString, out int buildId))
{
this.logger.LogError("Could not parse build id as a guid '{BuildId}'", buildIdString);
return;
}
await runProcessor.UploadBuildBlobsAsync(account, projectId, buildId);
await this.runProcessor.UploadBuildBlobsAsync(account, projectId, buildId);
}
}
}

14
tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/BuildLogProvider.cs
Просмотреть файл

@ -22,22 +22,22 @@ namespace Azure.Sdk.Tools.PipelineWitness.Services
public virtual async Task<IReadOnlyList<string>> GetLogLinesAsync(Build build, int logId)
{
logger.LogTrace("Getting logs for build {BuildId}, log {LogId} from rest api", build.Id, logId);
this.logger.LogTrace("Getting logs for build {BuildId}, log {LogId} from rest api", build.Id, logId);
var buildHttpClient = vssConnection.GetClient<BuildHttpClient>();
var response = await buildHttpClient.GetBuildLogLinesAsync(build.Project.Id, build.Id, logId);
BuildHttpClient buildHttpClient = this.vssConnection.GetClient<BuildHttpClient>();
List<string> response = await buildHttpClient.GetBuildLogLinesAsync(build.Project.Id, build.Id, logId);
logger.LogTrace("Received {CharacterCount} characters in {LineCount} lines for build {BuildId}, log {LogId}", response.Sum(x => x.Length), response.Count, build.Id, logId);
this.logger.LogTrace("Received {CharacterCount} characters in {LineCount} lines for build {BuildId}, log {LogId}", response.Sum(x => x.Length), response.Count, build.Id, logId);
return response;
}
public virtual async Task<Stream> GetLogStreamAsync(string projectName, int buildId, int logId)
{
logger.LogTrace("Getting logs for build {BuildId}, log {LogId} from rest api", buildId, logId);
this.logger.LogTrace("Getting logs for build {BuildId}, log {LogId} from rest api", buildId, logId);
var buildHttpClient = vssConnection.GetClient<BuildHttpClient>();
var stream = await buildHttpClient.GetBuildLogAsync(projectName, buildId, logId);
BuildHttpClient buildHttpClient = this.vssConnection.GetClient<BuildHttpClient>();
Stream stream = await buildHttpClient.GetBuildLogAsync(projectName, buildId, logId);
return stream;
}

18
tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/EnhancedBuildHttpClient.cs
Просмотреть файл

@ -14,23 +14,23 @@ namespace Azure.Sdk.Tools.PipelineWitness.Services
{
public EnhancedBuildHttpClient(Uri baseUrl, VssCredentials credentials)
: base(baseUrl, credentials)
{}
{ }
public EnhancedBuildHttpClient(Uri baseUrl, VssCredentials credentials, VssHttpRequestSettings settings)
: base(baseUrl, credentials, settings)
{}
{ }
public EnhancedBuildHttpClient(Uri baseUrl, VssCredentials credentials, params DelegatingHandler[] handlers)
: base(baseUrl, credentials, handlers)
{}
{ }
public EnhancedBuildHttpClient(Uri baseUrl, VssCredentials credentials, VssHttpRequestSettings settings, params DelegatingHandler[] handlers)
: base(baseUrl, credentials, settings, handlers)
{}
{ }
public EnhancedBuildHttpClient(Uri baseUrl, HttpMessageHandler pipeline, bool disposeHandler)
: base(baseUrl, pipeline, disposeHandler)
{}
{ }
public override async Task<Stream> GetArtifactContentZipAsync(
Guid project,
@ -39,7 +39,7 @@ namespace Azure.Sdk.Tools.PipelineWitness.Services
object userState = null,
CancellationToken cancellationToken = default)
{
var artifact = await base.GetArtifactAsync(project, buildId, artifactName, userState, cancellationToken);
BuildArtifact artifact = await base.GetArtifactAsync(project, buildId, artifactName, userState, cancellationToken);
return await GetArtifactContentZipAsync(artifact, cancellationToken);
}
@ -50,19 +50,19 @@ namespace Azure.Sdk.Tools.PipelineWitness.Services
object userState = null,
CancellationToken cancellationToken = default)
{
var artifact = await base.GetArtifactAsync(project, buildId, artifactName, userState, cancellationToken);
BuildArtifact artifact = await base.GetArtifactAsync(project, buildId, artifactName, userState, cancellationToken);
return await GetArtifactContentZipAsync(artifact, cancellationToken);
}
private async Task<Stream> GetArtifactContentZipAsync(BuildArtifact artifact, CancellationToken cancellationToken)
{
var downloadUrl = artifact?.Resource?.DownloadUrl;
string downloadUrl = artifact?.Resource?.DownloadUrl;
if (string.IsNullOrWhiteSpace(downloadUrl))
{
throw new InvalidArtifactDataException("Artifact contained no download url");
}
var responseStream = await Client.GetStreamAsync(downloadUrl, cancellationToken);
Stream responseStream = await Client.GetStreamAsync(downloadUrl, cancellationToken);
return responseStream;
}
}

37
tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/AzureArtifactsServiceUnavailableClassifier.cs
Просмотреть файл

@ -1,37 +0,0 @@
using Microsoft.TeamFoundation.Build.WebApi;
using System.Linq;
using System.Threading.Tasks;
namespace Azure.Sdk.Tools.PipelineWitness.Services.FailureAnalysis
{
public class AzureArtifactsServiceUnavailableClassifier : IFailureClassifier
{
public AzureArtifactsServiceUnavailableClassifier(BuildLogProvider buildLogProvider)
{
this.buildLogProvider = buildLogProvider;
}
private readonly BuildLogProvider buildLogProvider;
public async Task ClassifyAsync(FailureAnalyzerContext context)
{
var failedTasks = from r in context.Timeline.Records
where r.Result == TaskResult.Failed
where r.RecordType == "Task"
where r.Task != null
where r.Name == "Publish to Java Dev Feed"
where r.Log != null
select r;
foreach (var failedTask in failedTasks)
{
var lines = await this.buildLogProvider.GetLogLinesAsync(context.Build, failedTask.Log.Id);
if (lines.Any(line => line.Contains("Transfer failed for https://pkgs.dev.azure.com") && line.Contains("503 Service Unavailable")))
{
context.AddFailure(failedTask, "Azure Artifacts Service Unavailable");
}
}
}
}
}

26
tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/AzurePipelinesPoolOutageClassifier.cs
Просмотреть файл

@ -1,26 +0,0 @@
using System.Linq;
using System.Threading.Tasks;
namespace Azure.Sdk.Tools.PipelineWitness.Services.FailureAnalysis
{
public class AzurePipelinesPoolOutageClassifier : IFailureClassifier
{
public Task ClassifyAsync(FailureAnalyzerContext context)
{
var jobs = from r in context.Timeline.Records
where r.RecordType == "Job"
where r.Issues.Any(i => i.Message.Contains("abandoned due to an infrastructure failure"))
select r;
if (jobs.Count() > 0)
{
foreach (var job in jobs)
{
context.AddFailure(job, "Azure Pipelines Pool Outage");
}
}
return Task.CompletedTask;
}
}
}

31
tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/AzurePowerShellModuleInstallationFailureClassifier.cs
Просмотреть файл

@ -1,31 +0,0 @@
using Microsoft.TeamFoundation.Build.WebApi;
using System.Linq;
using System.Threading.Tasks;
namespace Azure.Sdk.Tools.PipelineWitness.Services.FailureAnalysis
{
public class AzurePowerShellModuleInstallationFailureClassifier : IFailureClassifier
{
public Task ClassifyAsync(FailureAnalyzerContext context)
{
if (context.Build.Definition.Name.EndsWith("- tests"))
{
var failedTasks = from r in context.Timeline.Records
where r.Result == TaskResult.Failed
where r.RecordType == "Task"
where r.Name == "Install Azure PowerShell module"
select r;
if (failedTasks.Count() > 0)
{
foreach (var failedTask in failedTasks)
{
context.AddFailure(failedTask, "Azure PS Module");
}
}
}
return Task.CompletedTask;
}
}
}

28
tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/AzuriteInstallFailureClassifier.cs
Просмотреть файл

@ -1,28 +0,0 @@
using Microsoft.TeamFoundation.Build.WebApi;
using System.Linq;
using System.Threading.Tasks;
namespace Azure.Sdk.Tools.PipelineWitness.Services.FailureAnalysis
{
public class AzuriteInstallFailureClassifier : IFailureClassifier
{
public Task ClassifyAsync(FailureAnalyzerContext context)
{
var failedTasks = from r in context.Timeline.Records
where r.Result == TaskResult.Failed
where r.RecordType == "Task"
where r.Name == "Install Azurite"
select r;
if (failedTasks.Count() > 0)
{
foreach (var failedTask in failedTasks)
{
context.AddFailure(failedTask, "Azurite Install");
}
}
return Task.CompletedTask;
}
}
}

75
tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/CacheFailureClassifier.cs
Просмотреть файл

@ -1,75 +0,0 @@
using Microsoft.TeamFoundation.Build.WebApi;
using Microsoft.VisualStudio.Services.WebApi;
using System.Linq;
using System.Threading.Tasks;
namespace Azure.Sdk.Tools.PipelineWitness.Services.FailureAnalysis
{
public class CacheFailureClassifier : IFailureClassifier
{
private class FailureClassifier
{
public readonly string MessageContains;
public readonly string FailureName;
public FailureClassifier(string messageContains, string failureName)
{
this.MessageContains = messageContains;
this.FailureName = failureName;
}
public bool IsFailure(string message)
{
return message.StartsWith(MessageContains);
}
}
private static readonly FailureClassifier[] failureClassifiers = new FailureClassifier[]
{
new FailureClassifier("Chunks are not arriving in order or sizes are not matched up", "Cache Chunk Ordering" ),
new FailureClassifier("The task has timed out", "Cache Task Timeout" ),
new FailureClassifier("Service Unavailable", "Cache Service Unavailable"),
new FailureClassifier("The HTTP request timed out after", "Cache Service HTTP Timeout"),
new FailureClassifier("Access to the path", "Cache Cannot Access Path"),
};
public CacheFailureClassifier(VssConnection vssConnection)
{
this.vssConnection = vssConnection;
buildClient = vssConnection.GetClient<BuildHttpClient>();
}
private VssConnection vssConnection;
private BuildHttpClient buildClient;
public Task ClassifyAsync(FailureAnalyzerContext context)
{
var failedTasks = from r in context.Timeline.Records
where r.Result == TaskResult.Failed
where r.RecordType == "Task"
where r.Task != null
where r.Task.Name == "Cache"
where r.Log != null
select r;
var classificationFound = false;
foreach (var failedTask in failedTasks)
{
foreach (var classifier in failureClassifiers) {
if (failedTask.Issues.Any(i => classifier.IsFailure(i.Message)))
{
context.AddFailure(failedTask, classifier.FailureName);
classificationFound = true;
}
}
if (!classificationFound)
{
context.AddFailure(failedTask, "Cache Failure Other");
}
}
return Task.CompletedTask;
}
}
}

27
tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/CancelledTaskClassifier.cs
Просмотреть файл

@ -1,27 +0,0 @@
using Microsoft.TeamFoundation.Build.WebApi;
using System.Linq;
using System.Threading.Tasks;
namespace Azure.Sdk.Tools.PipelineWitness.Services.FailureAnalysis
{
public class CancelledTaskClassifier : IFailureClassifier
{
public Task ClassifyAsync(FailureAnalyzerContext context)
{
var timedOutTestTasks = from r in context.Timeline.Records
where r.RecordType == "Task"
where r.Result == TaskResult.Canceled
select r;
if (timedOutTestTasks.Count() > 0)
{
foreach (var timedOutTestTask in timedOutTestTasks)
{
context.AddFailure(timedOutTestTask, "Cancelled Task");
}
}
return Task.CompletedTask;
}
}
}

30
tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/CodeSigningFailureClassifier.cs
Просмотреть файл

@ -1,30 +0,0 @@
using System.Linq;
using System.Threading.Tasks;
using Microsoft.TeamFoundation.Build.WebApi;
namespace Azure.Sdk.Tools.PipelineWitness.Services.FailureAnalysis
{
public class CodeSigningFailureClassifier : IFailureClassifier
{
public Task ClassifyAsync(FailureAnalyzerContext context)
{
var failedTasks = from r in context.Timeline.Records
where r.Result == TaskResult.Failed
where r.RecordType == "Task"
where r.Task != null
where r.Task.Name == "EsrpCodeSigning"
where r.Log != null
select r;
if (failedTasks.Count() > 0)
{
foreach (var failedTask in failedTasks)
{
context.AddFailure(failedTask, "Code Signing");
}
}
return Task.CompletedTask;
}
}
}

28
tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/CosmosDbEmulatorStartFailureClassifier.cs
Просмотреть файл

@ -1,28 +0,0 @@
using System.Linq;
using System.Threading.Tasks;
using Microsoft.TeamFoundation.Build.WebApi;
namespace Azure.Sdk.Tools.PipelineWitness.Services.FailureAnalysis
{
public class CosmosDbEmulatorStartFailureClassifier : IFailureClassifier
{
public Task ClassifyAsync(FailureAnalyzerContext context)
{
var failedTasks = from r in context.Timeline.Records
where r.RecordType == "Task"
where r.Name == "Start Cosmos DB Emulator"
where r.Result == TaskResult.Failed
select r;
if (failedTasks.Count() > 0)
{
foreach (var failedTask in failedTasks)
{
context.AddFailure(failedTask, "Cosmos DB Emulator Failure");
}
}
return Task.CompletedTask;
}
}
}

45
tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/DnsResolutionFailureClassifier.cs
Просмотреть файл

@ -1,45 +0,0 @@
using Microsoft.TeamFoundation.Build.WebApi;
using System;
using System.Linq;
using System.Threading.Tasks;
namespace Azure.Sdk.Tools.PipelineWitness.Services.FailureAnalysis
{
public class DnsResolutionFailureClassifier : IFailureClassifier
{
public DnsResolutionFailureClassifier(BuildLogProvider buildLogProvider)
{
this.buildLogProvider = buildLogProvider;
}
private readonly BuildLogProvider buildLogProvider;
private bool IsDnsResolutionFailure(string line)
{
return line.Contains("EAI_AGAIN", StringComparison.OrdinalIgnoreCase)
|| line.Contains("getaddrinfo", StringComparison.OrdinalIgnoreCase)
|| line.Contains("Temporary failure in name resolution", StringComparison.OrdinalIgnoreCase)
|| line.Contains("No such host is known", StringComparison.OrdinalIgnoreCase)
|| line.Contains("Couldn't resolve host name", StringComparison.OrdinalIgnoreCase);
}
public async Task ClassifyAsync(FailureAnalyzerContext context)
{
var failedTasks = from r in context.Timeline.Records
where r.Result == TaskResult.Failed
where r.RecordType == "Task"
where r.Log != null
select r;
foreach (var failedTask in failedTasks)
{
var lines = await buildLogProvider.GetLogLinesAsync(context.Build, failedTask.Log.Id);
if (lines.Any(line => IsDnsResolutionFailure(line)))
{
context.AddFailure(failedTask, "DNS Resolution Failure");
}
}
}
}
}

31
tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/DotnetPipelineTestFailureClassifier.cs
Просмотреть файл

@ -1,31 +0,0 @@
using Microsoft.TeamFoundation.Build.WebApi;
using System.Linq;
using System.Threading.Tasks;
namespace Azure.Sdk.Tools.PipelineWitness.Services.FailureAnalysis
{
public class DotnetPipelineTestFailureClassifier : IFailureClassifier
{
public Task ClassifyAsync(FailureAnalyzerContext context)
{
if (context.Build.Definition.Name.StartsWith("net - "))
{
var failedTasks = from r in context.Timeline.Records
where r.Result == TaskResult.Failed
where r.RecordType == "Task"
where r.Name.StartsWith("Build & Test")
select r;
if (failedTasks.Count() > 0)
{
foreach (var failedTask in failedTasks)
{
context.AddFailure(failedTask, "Test Failure");
}
}
}
return Task.CompletedTask;
}
}
}

28
tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/DownloadSecretsFailureClassifier.cs
Просмотреть файл

@ -1,28 +0,0 @@
using Microsoft.TeamFoundation.Build.WebApi;
using System.Linq;
using System.Threading.Tasks;
namespace Azure.Sdk.Tools.PipelineWitness.Services.FailureAnalysis
{
public class DownloadSecretsFailureClassifier : IFailureClassifier
{
public Task ClassifyAsync(FailureAnalyzerContext context)
{
var failedTasks = from r in context.Timeline.Records
where r.RecordType == "Task"
where r.Result == TaskResult.Failed
where r.Name.Contains("Download secrets")
select r;
if (failedTasks.Count() > 0)
{
foreach (var failedTask in failedTasks)
{
context.AddFailure(failedTask, "Secrets Failure");
}
}
return Task.CompletedTask;
}
}
}

46
tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/FailureAnalyzer.cs
Просмотреть файл

@ -1,46 +0,0 @@
using Azure.Sdk.Tools.PipelineWitness.Entities.AzurePipelines;
using Microsoft.TeamFoundation.Build.WebApi;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
namespace Azure.Sdk.Tools.PipelineWitness.Services.FailureAnalysis
{
public class FailureAnalyzer : IFailureAnalyzer
{
public FailureAnalyzer(IEnumerable<IFailureClassifier> classifiers)
{
this.classifiers = classifiers.ToArray();
}
private IFailureClassifier[] classifiers;
public async Task<IEnumerable<Failure>> AnalyzeFailureAsync(Build build, Timeline timeline)
{
var failures = new List<Failure>();
var context = new FailureAnalyzerContext(build, timeline, failures);
foreach (var classifier in classifiers)
{
await classifier.ClassifyAsync(context);
}
if (failures.Count == 0)
{
if (build.Result != BuildResult.Succeeded &&
build.Result != BuildResult.Canceled)
{
foreach (var record in timeline.Records.Where(x => x.ParentId.HasValue == false))
{
if (record.Result == TaskResult.Failed)
{
failures.Add(new Failure(record, "Unknown"));
}
}
}
}
return failures;
}
}
}

57
tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/FailureAnalyzerContext.cs
Просмотреть файл

@ -1,57 +0,0 @@
using Azure.Sdk.Tools.PipelineWitness.Entities.AzurePipelines;
using Microsoft.TeamFoundation.Build.WebApi;
using Microsoft.VisualStudio.Services.Common;
using System.Collections.Generic;
using System.Linq;
using System.Text;
namespace Azure.Sdk.Tools.PipelineWitness.Services.FailureAnalysis
{
public class FailureAnalyzerContext
{
public FailureAnalyzerContext(Build build, Timeline timeline, IList<Failure> failures)
{
Build = build;
Timeline = timeline;
this.failures = failures;
}
public Build Build { get; private set; }
public Timeline Timeline { get; private set; }
private IList<Failure> failures;
private string GetScope(TimelineRecord record)
{
var timelineStack = new Stack<TimelineRecord>();
var current = record;
while (true)
{
timelineStack.Push(current);
var parent = Timeline.Records.Where(r => r.Id == current.ParentId).SingleOrDefault();
if (parent == null)
{
break;
}
else
{
current = parent;
}
}
var scopeBuilder = new StringBuilder();
timelineStack.ForEach(r => scopeBuilder.Append($"/{r.RecordType}:{r.Name}"));
var scope = scopeBuilder.ToString();
return scope;
}
public void AddFailure(TimelineRecord record, string classification)
{
var failure = new Failure(record, classification);
failures.Add(failure);
}
}
}

27
tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/GitCheckoutFailureClassifier.cs
Просмотреть файл

@ -1,27 +0,0 @@
using System.Linq;
using System.Threading.Tasks;
namespace Azure.Sdk.Tools.PipelineWitness.Services.FailureAnalysis
{
public class GitCheckoutFailureClassifier : IFailureClassifier
{
public Task ClassifyAsync(FailureAnalyzerContext context)
{
var tasks = from r in context.Timeline.Records
where r.RecordType == "Task"
where r.Issues.Any(i => i.Message.Contains("Git fetch failed with exit code: 128"))
select r;
if (tasks.Count() > 0)
{
foreach (var task in tasks)
{
context.AddFailure(task
, "Git Checkout");
}
}
return Task.CompletedTask;
}
}
}

12
tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/IFailureAnalyzer.cs
Просмотреть файл

@ -1,12 +0,0 @@
using Azure.Sdk.Tools.PipelineWitness.Entities.AzurePipelines;
using Microsoft.TeamFoundation.Build.WebApi;
using System.Collections.Generic;
using System.Threading.Tasks;
namespace Azure.Sdk.Tools.PipelineWitness.Services.FailureAnalysis
{
public interface IFailureAnalyzer
{
Task<IEnumerable<Failure>> AnalyzeFailureAsync(Build build, Timeline timeline);
}
}

9
tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/IFailureClassifier.cs
Просмотреть файл

@ -1,9 +0,0 @@
using System.Threading.Tasks;
namespace Azure.Sdk.Tools.PipelineWitness.Services.FailureAnalysis
{
public interface IFailureClassifier
{
Task ClassifyAsync(FailureAnalyzerContext context);
}
}

31
tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/JavaPipelineTestFailureClassifier.cs
Просмотреть файл

@ -1,31 +0,0 @@
using Microsoft.TeamFoundation.Build.WebApi;
using System.Linq;
using System.Threading.Tasks;
namespace Azure.Sdk.Tools.PipelineWitness.Services.FailureAnalysis
{
public class JavaPipelineTestFailureClassifier : IFailureClassifier
{
public Task ClassifyAsync(FailureAnalyzerContext context)
{
if (context.Build.Definition.Name.StartsWith("java - "))
{
var failedTasks = from r in context.Timeline.Records
where r.Result == TaskResult.Failed
where r.RecordType == "Task"
where r.Name.StartsWith("Run tests")
select r;
if (failedTasks.Count() > 0)
{
foreach (var failedTask in failedTasks)
{
context.AddFailure(failedTask, "Test Failure");
}
}
}
return Task.CompletedTask;
}
}
}

31
tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/JavaScriptLiveTestFailureClassifier.cs
Просмотреть файл

@ -1,31 +0,0 @@
using System.Linq;
using System.Threading.Tasks;
using Microsoft.TeamFoundation.Build.WebApi;
namespace Azure.Sdk.Tools.PipelineWitness.Services.FailureAnalysis
{
public class JavaScriptLiveTestFailureClassifier : IFailureClassifier
{
public Task ClassifyAsync(FailureAnalyzerContext context)
{
if (context.Build.Definition.Name.StartsWith("js - ") && context.Build.Definition.Name.EndsWith(" - tests"))
{
var failedTasks = from r in context.Timeline.Records
where r.RecordType == "Task"
where r.Name == "Integration test libraries"
where r.Result == TaskResult.Failed
select r;
if (failedTasks.Count() > 0)
{
foreach (var failedTask in failedTasks)
{
context.AddFailure(failedTask, "Test Failure");
}
}
}
return Task.CompletedTask;
}
}
}

31
tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/JsDevFeedPublishingFailureClassifier.cs
Просмотреть файл

@ -1,31 +0,0 @@
using Microsoft.TeamFoundation.Build.WebApi;
using System.Linq;
using System.Threading.Tasks;
namespace Azure.Sdk.Tools.PipelineWitness.Services.FailureAnalysis
{
public class JsDevFeedPublishingFailureClassifier : IFailureClassifier
{
public Task ClassifyAsync(FailureAnalyzerContext context)
{
if (context.Build.Definition.Name.StartsWith("js -"))
{
var failedJobs = from r in context.Timeline.Records
where r.Name == "Publish package to daily feed"
where r.RecordType == "Job"
where r.Result == TaskResult.Failed
select r;
if (failedJobs.Count() > 0)
{
foreach (var failedJob in failedJobs)
{
context.AddFailure(failedJob, "Publish Failure");
}
}
}
return Task.CompletedTask;
}
}
}

30
tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/JsSamplesExecutionFailureClassifier.cs
Просмотреть файл

@ -1,30 +0,0 @@
using Microsoft.TeamFoundation.Build.WebApi;
using System.Linq;
using System.Threading.Tasks;
namespace Azure.Sdk.Tools.PipelineWitness.Services.FailureAnalysis
{
public class JsSamplesExecutionFailureClassifier : IFailureClassifier
{
public Task ClassifyAsync(FailureAnalyzerContext context)
{
if (context.Build.Definition.Name.StartsWith("js -"))
{
var failedTasks = from r in context.Timeline.Records
where r.Name == "Execute Samples"
where r.Result == TaskResult.Failed
select r;
if (failedTasks.Count() > 0)
{
foreach (var failedTask in failedTasks)
{
context.AddFailure(failedTask, "Sample Execution");
}
}
}
return Task.CompletedTask;
}
}
}

37
tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/MavenBrokenPipeFailureClassifier.cs
Просмотреть файл

@ -1,37 +0,0 @@
using Microsoft.TeamFoundation.Build.WebApi;
using System.Linq;
using System.Threading.Tasks;
namespace Azure.Sdk.Tools.PipelineWitness.Services.FailureAnalysis
{
public class MavenBrokenPipeFailureClassifier : IFailureClassifier
{
public MavenBrokenPipeFailureClassifier(BuildLogProvider buildLogProvider)
{
this.buildLogProvider = buildLogProvider;
}
private readonly BuildLogProvider buildLogProvider;
public async Task ClassifyAsync(FailureAnalyzerContext context)
{
var failedTasks = from r in context.Timeline.Records
where r.Result == TaskResult.Failed
where r.RecordType == "Task"
where r.Task != null
where r.Task.Name == "Maven"
where r.Log != null
select r;
foreach (var failedTask in failedTasks)
{
var lines = await buildLogProvider.GetLogLinesAsync(context.Build, failedTask.Log.Id);
if (lines.Any(line => line.Contains("Connection reset") || line.Contains("Connection timed out") || line.Contains("504 Gateway Timeout")))
{
context.AddFailure(failedTask, "Maven Broken Pipe");
}
}
}
}
}

31
tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/PythonPipelineTestFailureClassifier.cs
Просмотреть файл

@ -1,31 +0,0 @@
using Microsoft.TeamFoundation.Build.WebApi;
using System.Linq;
using System.Threading.Tasks;
namespace Azure.Sdk.Tools.PipelineWitness.Services.FailureAnalysis
{
public class PythonPipelineTestFailureClassifier : IFailureClassifier
{
public Task ClassifyAsync(FailureAnalyzerContext context)
{
if (context.Build.Definition.Name.StartsWith("python - "))
{
var failedTasks = from r in context.Timeline.Records
where r.Result == TaskResult.Failed
where r.RecordType == "Task"
where r.Name == "Run Tests"
select r;
if (failedTasks.Count() > 0)
{
foreach (var failedTask in failedTasks)
{
context.AddFailure(failedTask, "Test Failure");
}
}
}
return Task.CompletedTask;
}
}
}

27
tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/TestResourcesDeploymentFailureClassifier.cs
Просмотреть файл

@ -1,27 +0,0 @@
using Microsoft.TeamFoundation.Build.WebApi;
using System.Linq;
using System.Threading.Tasks;
namespace Azure.Sdk.Tools.PipelineWitness.Services.FailureAnalysis
{
public class TestResourcesDeploymentFailureClassifier : IFailureClassifier
{
public Task ClassifyAsync(FailureAnalyzerContext context)
{
var failedTasks = from r in context.Timeline.Records
where r.Name.StartsWith("Deploy test resources")
where r.Result == TaskResult.Failed
select r;
if (failedTasks.Count() > 0)
{
foreach (var failedTask in failedTasks)
{
context.AddFailure(failedTask, "Test Resource Failure");
}
}
return Task.CompletedTask;
}
}
}

135
tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/QueueWorkerBackgroundService.cs
Просмотреть файл

@ -1,4 +1,4 @@
using System;
using System;
using System.Diagnostics;
using System.Threading;
using System.Threading.Tasks;
@ -9,6 +9,7 @@ using Azure.Storage.Queues.Models;
using Microsoft.ApplicationInsights;
using Microsoft.ApplicationInsights.DataContracts;
using Microsoft.ApplicationInsights.Extensibility;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
@ -18,7 +19,7 @@ namespace Azure.Sdk.Tools.PipelineWitness.Services
internal abstract class QueueWorkerBackgroundService : BackgroundService
{
private const string ActivitySourceName = "Azure.Sdk.Tools.PipelineWitness.Queue";
private static readonly ActivitySource activitySource = new ActivitySource(ActivitySourceName);
private static readonly ActivitySource activitySource = new(ActivitySourceName);
private readonly ILogger logger;
private readonly QueueServiceClient queueServiceClient;
@ -38,7 +39,7 @@ namespace Azure.Sdk.Tools.PipelineWitness.Services
this.queueServiceClient = queueServiceClient ?? throw new ArgumentNullException(nameof(options));
this.options = options ?? throw new ArgumentNullException(nameof(options));
if(string.IsNullOrWhiteSpace(queueName))
if (string.IsNullOrWhiteSpace(queueName))
{
throw new ArgumentException("Parameter cannot be null or whitespace", nameof(queueName));
}
@ -50,22 +51,22 @@ namespace Azure.Sdk.Tools.PipelineWitness.Services
{
this.logger.LogInformation("Starting ExecuteAsync for {TypeName}", this.GetType().Name);
var poisonQueueName = $"{this.queueName}-poison";
string poisonQueueName = $"{this.queueName}-poison";
var queueClient = this.queueServiceClient.GetQueueClient(this.queueName);
var poisonQueueClient = this.queueServiceClient.GetQueueClient(poisonQueueName);
QueueClient queueClient = this.queueServiceClient.GetQueueClient(this.queueName);
QueueClient poisonQueueClient = this.queueServiceClient.GetQueueClient(poisonQueueName);
await queueClient.CreateIfNotExistsAsync();
await poisonQueueClient.CreateIfNotExistsAsync();
await queueClient.CreateIfNotExistsAsync(cancellationToken: stoppingToken);
await poisonQueueClient.CreateIfNotExistsAsync(cancellationToken: stoppingToken);
while (true)
{
using var loopActivity = activitySource.CreateActivity("MessageLoopIteration", ActivityKind.Internal) ?? new Activity("MessageLoopIteration");
using Activity loopActivity = activitySource.CreateActivity("MessageLoopIteration", ActivityKind.Internal) ?? new Activity("MessageLoopIteration");
loopActivity?.AddBaggage("QueueName", queueClient.Name);
using var loopOperation = this.telemetryClient.StartOperation<RequestTelemetry>(loopActivity);
var options = this.options.CurrentValue;
using IOperationHolder<RequestTelemetry> loopOperation = this.telemetryClient.StartOperation<RequestTelemetry>(loopActivity);
PipelineWitnessSettings options = this.options.CurrentValue;
this.logger.LogDebug("Getting next message from queue {QueueName}", queueClient.Name);
@ -74,7 +75,7 @@ namespace Azure.Sdk.Tools.PipelineWitness.Services
// We consider a message leased when it's made invisible in the queue and the current process has a
// valid PopReceipt for the message. The PopReceipt is used to perform subsequent operations on the
// "leased" message.
QueueMessage message = await queueClient.ReceiveMessageAsync(options.MessageLeasePeriod);
QueueMessage message = await queueClient.ReceiveMessageAsync(options.MessageLeasePeriod, stoppingToken);
if (message == null)
{
@ -92,65 +93,63 @@ namespace Azure.Sdk.Tools.PipelineWitness.Services
});
}
using (var activity = activitySource.CreateActivity("ProcessMessage", ActivityKind.Internal) ?? new Activity("ProcessMessage"))
using Activity activity = activitySource.CreateActivity("ProcessMessage", ActivityKind.Internal) ?? new Activity("ProcessMessage");
activity?.AddBaggage("MessageId", message.MessageId);
using IOperationHolder<RequestTelemetry> operation = this.telemetryClient.StartOperation<RequestTelemetry>(activity);
try
{
activity?.AddBaggage("MessageId", message.MessageId);
this.logger.LogDebug("The queue returned a message.\n Queue: {Queue}\n Message: {MessageId}\n Dequeue Count: {DequeueCount}\n Pop Receipt: {PopReceipt}", queueClient.Name, message.MessageId, message.DequeueCount, message.PopReceipt);
using var operation = this.telemetryClient.StartOperation<RequestTelemetry>(activity);
using CancellationTokenSource cts = CancellationTokenSource.CreateLinkedTokenSource(stoppingToken);
try
// Because processing a message may take longer than our initial lease period, we want to continually
// renew our lease until processing completes.
Task<string> renewTask = RenewMessageLeaseAsync(queueClient, message, cts.Token);
Task<bool> processTask = SafelyProcessMessageAsync(message, cts.Token);
Task[] tasks = new Task[] { renewTask, processTask };
Task.WaitAny(tasks, CancellationToken.None);
cts.Cancel();
Task.WaitAll(tasks, CancellationToken.None);
// if the renew task doesn't complete successfully, we can't trust the PopReceipt on the message and must abort.
string latestPopReceipt = await renewTask;
if (processTask.IsCompletedSuccessfully && processTask.Result == true)
{
this.logger.LogDebug("The queue returned a message.\n Queue: {Queue}\n Message: {MessageId}\n Dequeue Count: {DequeueCount}\n Pop Receipt: {PopReceipt}", queueClient.Name, message.MessageId, message.DequeueCount, message.PopReceipt);
using var cts = CancellationTokenSource.CreateLinkedTokenSource(stoppingToken);
// Because processing a message may take longer than our initial lease period, we want to continually
// renew our lease until processing completes.
var renewTask = RenewMessageLeaseAsync(queueClient, message, cts.Token);
var processTask = SafelyProcessMessageAsync(message, cts.Token);
var tasks = new Task[] { renewTask, processTask };
Task.WaitAny(tasks, CancellationToken.None);
cts.Cancel();
Task.WaitAll(tasks, CancellationToken.None);
// if the renew task doesn't complete successfully, we can't trust the PopReceipt on the message and must abort.
var latestPopReceipt = await renewTask;
if (processTask.IsCompletedSuccessfully && processTask.Result == true)
this.logger.LogDebug("Message processed successfully. Removing message from queue.\n MessageId: {MessageId}\n Queue: {QueueName}\n PopReceipt: {PopReceipt}", message.MessageId, queueClient.Name, latestPopReceipt);
await queueClient.DeleteMessageAsync(message.MessageId, latestPopReceipt, stoppingToken);
activity?.SetStatus(ActivityStatusCode.Ok);
operation.Telemetry.Success = true;
}
else
{
activity?.SetStatus(ActivityStatusCode.Error);
operation.Telemetry.Success = false;
if (message.DequeueCount > options.MaxDequeueCount)
{
this.logger.LogDebug("Message processed successfully. Removing message from queue.\n MessageId: {MessageId}\n Queue: {QueueName}\n PopReceipt: {PopReceipt}", message.MessageId, queueClient.Name, latestPopReceipt);
this.logger.LogError("Message {MessageId} exceeded maximum dequeue count. Moving to poison queue {QueueName}", message.MessageId, poisonQueueClient.Name);
await poisonQueueClient.SendMessageAsync(message.Body, cancellationToken: stoppingToken);
this.logger.LogDebug("Removing message from queue.\n MessageId: {MessageId}\n Queue: {QueueName}\n PopReceipt: {PopReceipt}", message.MessageId, queueClient.Name, latestPopReceipt);
await queueClient.DeleteMessageAsync(message.MessageId, latestPopReceipt, stoppingToken);
activity?.SetStatus(ActivityStatusCode.Ok);
operation.Telemetry.Success = true;
}
else
{
activity?.SetStatus(ActivityStatusCode.Error);
operation.Telemetry.Success = false;
if (message.DequeueCount > options.MaxDequeueCount)
{
this.logger.LogError("Message {MessageId} exceeded maximum dequeue count. Moving to poison queue {QueueName}", message.MessageId, poisonQueueClient.Name);
await poisonQueueClient.SendMessageAsync(message.Body, cancellationToken: stoppingToken);
this.logger.LogDebug("Removing message from queue.\n MessageId: {MessageId}\n Queue: {QueueName}\n PopReceipt: {PopReceipt}", message.MessageId, queueClient.Name, latestPopReceipt);
await queueClient.DeleteMessageAsync(message.MessageId, latestPopReceipt, stoppingToken);
}
else
{
this.logger.LogError("Resetting message visibility timeout to {SleepPeriod}.\n MessageId: {MessageId}\n Queue: {QueueName}\n PopReceipt: {PopReceipt}", options.MessageErrorSleepPeriod, message.MessageId, queueClient.Name, latestPopReceipt);
await queueClient.UpdateMessageAsync(message.MessageId, latestPopReceipt, message.Body, options.MessageErrorSleepPeriod, cancellationToken: stoppingToken);
}
this.logger.LogError("Resetting message visibility timeout to {SleepPeriod}.\n MessageId: {MessageId}\n Queue: {QueueName}\n PopReceipt: {PopReceipt}", options.MessageErrorSleepPeriod, message.MessageId, queueClient.Name, latestPopReceipt);
await queueClient.UpdateMessageAsync(message.MessageId, latestPopReceipt, message.Body, options.MessageErrorSleepPeriod, cancellationToken: stoppingToken);
}
}
catch (Exception ex)
{
this.logger.LogError(ex, "Exception thrown while procesing queue message.");
activity?.SetStatus(ActivityStatusCode.Error);
operation.Telemetry.Success = false;
}
}
catch (Exception ex)
{
this.logger.LogError(ex, "Exception thrown while procesing queue message.");
activity?.SetStatus(ActivityStatusCode.Error);
operation.Telemetry.Success = false;
}
}
catch (Exception ex)
@ -172,12 +171,12 @@ namespace Azure.Sdk.Tools.PipelineWitness.Services
/// <returns>the current pop receipt (optimistic concurrency control) for the message.</returns>
private async Task<string> RenewMessageLeaseAsync(QueueClient queueClient, QueueMessage message, CancellationToken cancellationToken)
{
var leasePeriod = this.options.CurrentValue.MessageLeasePeriod;
var halfLife = new TimeSpan(leasePeriod.Ticks / 2);
var queueName = queueClient.Name;
var messageId = message.MessageId;
var popReceipt = message.PopReceipt;
var nextVisibleOn = message.NextVisibleOn;
TimeSpan leasePeriod = this.options.CurrentValue.MessageLeasePeriod;
TimeSpan halfLife = new(leasePeriod.Ticks / 2);
string queueName = queueClient.Name;
string messageId = message.MessageId;
string popReceipt = message.PopReceipt;
DateTimeOffset? nextVisibleOn = message.NextVisibleOn;
try
{
@ -191,7 +190,7 @@ namespace Azure.Sdk.Tools.PipelineWitness.Services
this.logger.LogDebug("Extending visibility timeout for message.\n Queue: {Queue}\n Message: {MessageId}\n Pop Receipt: {PopReceipt}\n Visible in: {VisibleIn}", queueName, messageId, popReceipt, nextVisibleOn - DateTimeOffset.UtcNow);
UpdateReceipt receipt = await queueClient.UpdateMessageAsync(messageId, popReceipt, visibilityTimeout: leasePeriod, cancellationToken: cancellationToken);
var oldPopReceipt = popReceipt;
string oldPopReceipt = popReceipt;
popReceipt = receipt.PopReceipt;
nextVisibleOn = receipt.NextVisibleOn;

6
tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/WorkTokens/CosmosAsyncLock.cs
Просмотреть файл

@ -33,7 +33,7 @@ namespace Azure.Sdk.Tools.PipelineWitness.Services.WorkTokens
{
await this.container.DeleteItemAsync<CosmosLockDocument>(this.id, this.partitionKey, new ItemRequestOptions { IfMatchEtag = this.etag });
}
catch(CosmosException ex) when (ex.StatusCode == HttpStatusCode.NotFound)
catch (CosmosException ex) when (ex.StatusCode == HttpStatusCode.NotFound)
{
}
}
@ -43,7 +43,7 @@ namespace Azure.Sdk.Tools.PipelineWitness.Services.WorkTokens
{
try
{
var response = await this.container.ReplaceItemAsync(
ItemResponse<CosmosLockDocument> response = await this.container.ReplaceItemAsync(
new CosmosLockDocument(this.id, this.duration),
this.id,
this.partitionKey,
@ -56,7 +56,7 @@ namespace Azure.Sdk.Tools.PipelineWitness.Services.WorkTokens
return true;
}
}
catch (CosmosException ex) when(ex.StatusCode == HttpStatusCode.Conflict)
catch (CosmosException ex) when (ex.StatusCode == HttpStatusCode.Conflict)
{
}

10
tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/WorkTokens/CosmosAsyncLockProvider.cs
Просмотреть файл

@ -23,10 +23,10 @@ namespace Azure.Sdk.Tools.PipelineWitness.Services.WorkTokens
public async Task<IAsyncLock> GetLockAsync(string id, TimeSpan duration, CancellationToken cancellationToken)
{
var partitionKey = new PartitionKey(id);
PartitionKey partitionKey = new(id);
ItemResponse<CosmosLockDocument> response;
try
{
response = await this.container.ReadItemAsync<CosmosLockDocument>(id, partitionKey, cancellationToken: cancellationToken);
@ -36,13 +36,13 @@ namespace Azure.Sdk.Tools.PipelineWitness.Services.WorkTokens
return await CreateLockAsync(id, duration, cancellationToken);
}
var existingLock = response.Resource;
CosmosLockDocument existingLock = response.Resource;
if (existingLock.Expiration >= DateTime.UtcNow)
{
return null;
}
try
{
response = await this.container.ReplaceItemAsync(
@ -68,7 +68,7 @@ namespace Azure.Sdk.Tools.PipelineWitness.Services.WorkTokens
{
try
{
var response = await this.container.CreateItemAsync(
ItemResponse<CosmosLockDocument> response = await this.container.CreateItemAsync(
new CosmosLockDocument(id, duration),
new PartitionKey(id),
cancellationToken: cancellationToken);

39
tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Startup.cs
Просмотреть файл

@ -6,7 +6,6 @@ using Azure.Identity;
using Azure.Sdk.Tools.PipelineWitness.ApplicationInsights;
using Azure.Sdk.Tools.PipelineWitness.Configuration;
using Azure.Sdk.Tools.PipelineWitness.Services;
using Azure.Sdk.Tools.PipelineWitness.Services.FailureAnalysis;
using Azure.Sdk.Tools.PipelineWitness.Services.WorkTokens;
using Microsoft.ApplicationInsights.Extensibility;
@ -28,9 +27,9 @@ namespace Azure.Sdk.Tools.PipelineWitness
{
public static void Configure(WebApplicationBuilder builder)
{
var azureCredential = new DefaultAzureCredential();
var settings = new PipelineWitnessSettings();
var settingsSection = builder.Configuration.GetSection("PipelineWitness");
DefaultAzureCredential azureCredential = new();
PipelineWitnessSettings settings = new();
IConfigurationSection settingsSection = builder.Configuration.GetSection("PipelineWitness");
settingsSection.Bind(settings);
builder.Services.AddApplicationInsightsTelemetry(builder.Configuration);
@ -57,26 +56,6 @@ namespace Azure.Sdk.Tools.PipelineWitness
builder.Services.AddMemoryCache();
builder.Services.AddSingleton<BlobUploadProcessor>();
builder.Services.AddSingleton<BuildLogProvider>();
builder.Services.AddSingleton<IFailureAnalyzer, FailureAnalyzer>();
builder.Services.AddSingleton<IFailureClassifier, AzuriteInstallFailureClassifier>();
builder.Services.AddSingleton<IFailureClassifier, CancelledTaskClassifier>();
builder.Services.AddSingleton<IFailureClassifier, CosmosDbEmulatorStartFailureClassifier>();
builder.Services.AddSingleton<IFailureClassifier, AzurePipelinesPoolOutageClassifier>();
builder.Services.AddSingleton<IFailureClassifier, PythonPipelineTestFailureClassifier>();
builder.Services.AddSingleton<IFailureClassifier, JavaScriptLiveTestFailureClassifier>();
builder.Services.AddSingleton<IFailureClassifier, TestResourcesDeploymentFailureClassifier>();
builder.Services.AddSingleton<IFailureClassifier, DotnetPipelineTestFailureClassifier>();
builder.Services.AddSingleton<IFailureClassifier, JavaPipelineTestFailureClassifier>();
builder.Services.AddSingleton<IFailureClassifier, JsSamplesExecutionFailureClassifier>();
builder.Services.AddSingleton<IFailureClassifier, JsDevFeedPublishingFailureClassifier>();
builder.Services.AddSingleton<IFailureClassifier, DownloadSecretsFailureClassifier>();
builder.Services.AddSingleton<IFailureClassifier, GitCheckoutFailureClassifier>();
builder.Services.AddSingleton<IFailureClassifier, AzuriteInstallFailureClassifier>();
builder.Services.AddSingleton<IFailureClassifier, MavenBrokenPipeFailureClassifier>();
builder.Services.AddSingleton<IFailureClassifier, CodeSigningFailureClassifier>();
builder.Services.AddSingleton<IFailureClassifier, AzureArtifactsServiceUnavailableClassifier>();
builder.Services.AddSingleton<IFailureClassifier, DnsResolutionFailureClassifier>();
builder.Services.AddSingleton<IFailureClassifier, CacheFailureClassifier>();
builder.Services.Configure<PipelineWitnessSettings>(settingsSection);
builder.Services.AddSingleton<TokenCredential, DefaultAzureCredential>();
@ -87,7 +66,7 @@ namespace Azure.Sdk.Tools.PipelineWitness
private static void AddHostedService<T>(this IServiceCollection services, int instanceCount) where T : class, IHostedService
{
for (var i = 0; i < instanceCount; i++)
for (int i = 0; i < instanceCount; i++)
{
services.AddSingleton<IHostedService, T>();
}
@ -107,13 +86,13 @@ namespace Azure.Sdk.Tools.PipelineWitness
private static VssConnection CreateVssConnection(IServiceProvider provider)
{
TokenCredential azureCredential = provider.GetRequiredService<TokenCredential>();
TokenRequestContext tokenRequestContext = new (VssAadSettings.DefaultScopes);
TokenRequestContext tokenRequestContext = new(VssAadSettings.DefaultScopes);
AccessToken token = azureCredential.GetToken(tokenRequestContext, CancellationToken.None);
Uri organizationUrl = new ("https://dev.azure.com/azure-sdk");
VssAadCredential vssCredential = new (new VssAadToken("Bearer", token.Token));
Uri organizationUrl = new("https://dev.azure.com/azure-sdk");
VssAadCredential vssCredential = new(new VssAadToken("Bearer", token.Token));
VssHttpRequestSettings settings = VssClientHttpRequestSettings.Default.Clone();
return new VssConnection(organizationUrl, vssCredential, settings);
}
}

70
tools/pipeline-witness/infrastructure/Assign-StoragePermissions.ps1 Normal file
Просмотреть файл

@ -0,0 +1,70 @@
<#
.SYNOPSIS
Assign appropriate storage permissions to the Azure SDK Engineering System Team for local debugging.
#>
param(
[Parameter(Mandatory)]
[validateSet('staging', 'test')]
[string]$target
)
function Invoke([string]$command) {
Write-Host "> $command"
Invoke-Expression $command
}
Push-Location $PSScriptRoot
try {
$subscriptionName = $target -eq 'test' ? 'Azure SDK Developer Playground' : 'Azure SDK Engineering System'
Write-Host "Setting subscription to '$subscriptionName'"
Invoke "az account set --subscription '$subscriptionName' --output none"
$parametersFile = "./bicep/parameters.$target.json"
Write-Host "Reading parameters from $parametersFile"
$parameters = (Get-Content -Path $parametersFile -Raw | ConvertFrom-Json).parameters
$appResourceGroupName = $parameters.appResourceGroupName.value
$appStorageAccountName = $parameters.appStorageAccountName.value
$cosmosAccountName = $parameters.cosmosAccountName.value
$logsResourceGroupName = $parameters.logsResourceGroupName.value
$logsStorageAccountName = $parameters.logsStorageAccountName.value
Write-Host "Adding Azure SDK Engineering System Team RBAC access to storage resources:`n" + `
" Blob: $logsResourceGroupName/$logsStorageAccountName`n" + `
" Queue: `n" + `
" Cosmos: $appResourceGroupName/$cosmosAccountName`n"
Write-Host "Getting group id for Azure SDK Engineering System Team"
$azAdGroupId = Invoke "az ad group show --group 'Azure SDK Engineering System Team' --query id --output tsv"
Write-Host "Granting 'read/write' access to $appResourceGroupName/$cosmosAccountName"
Invoke "az cosmosdb sql role assignment create --resource-group '$appResourceGroupName' --account-name '$cosmosAccountName' --scope '/' --role-definition-id '00000000-0000-0000-0000-000000000002' --principal-id '$azAdGroupId' --output none"
if ($LASTEXITCODE -ne 0) {
Write-Output $output
Write-Error "Failed to grant access"
exit 1
}
Write-Host "Granting 'Storage Blob Data Contributor' access to $logsResourceGroupName/$logsStorageAccountName"
$scope = "/subscriptions/$subscriptionId/resourceGroups/$logsResourceGroupName/providers/Microsoft.Storage/storageAccounts/$logsStorageAccountName"
$output = Invoke "az role assignment create --assignee '$azAdGroupId' --role 'Storage Blob Data Contributor' --scope '$scope' --output none"
if ($LASTEXITCODE -ne 0) {
Write-Output $output
Write-Error "Failed to grant access"
exit 1
}
Write-Host "Granting 'Storage Queue Data Contributor' access to $appResourceGroupName/$appStorageAccountName"
$scope = "/subscriptions/$subscriptionId/resourceGroups/$appResourceGroupName/providers/Microsoft.Storage/storageAccounts/$appStorageAccountName"
$output = Invoke "az role assignment create --assignee '$azAdGroupId' --role 'Storage Queue Data Contributor' --scope '$scope' --output none"
if ($LASTEXITCODE -ne 0) {
Write-Output $output
Write-Error "Failed to grant access"
exit 1
}
}
finally {
Pop-Location
}

9
tools/pipeline-witness/infrastructure/README.md Normal file
Просмотреть файл

@ -0,0 +1,9 @@
# Bicep
The bicep templates in the [bicep](./bicep) folder are used to provision the PipelineWitness resource group, web app and storage account as well as the PipelineLogs resource group, storage account, kusto cluster and all of the resources required to make continuous ingestion from storage to kusto work.
#### Deployment permissions
The bicep templates contain RBAC assignments between the web app's managed identity and the storage accounts it will be accessing. For the CI pipeline to be able to successfully set these RBAC assignments, it's service connection principal must be granted `Microsoft.Authorization/roleAssignments/write` to the storage resources. This permission is included in the `Owner`, `User Access Administrator` and `Role Based Access Control Administrator` roles.
# Kusto
The scripts in the kusto folder are deployed to the staging and production databases during ci pipeline runs. They are merged into a single `.kql` by [`deploy.ps1`](./deploy.ps1) using the [`Merge-KustoScripts.ps1`](./Merge-KustoScripts.ps1) script. To extract objects from an existing kusto database into the kusto folder, use the [`Extract-KustoScripts.ps1`](./Extract-KustoScripts.ps1) script.

36
tools/pipeline-witness/infrastructure/deploy.ps1
Просмотреть файл

@ -7,7 +7,7 @@ param(
[validateSet('production', 'staging', 'test')]
[string]$target,
[switch]$replaceRoleAssignments
[switch]$removeRoleAssignments
)
function Invoke([string]$command) {
@ -79,7 +79,7 @@ try {
" App Resource Group: $appResourceGroupName`n" + `
" Location: $location`n"
if ($replaceRoleAssignments) {
if ($removeRoleAssignments) {
RemoveStorageRoleAssignments $subscriptionId $logsResourceGroupName $logsStorageAccountName
RemoveStorageRoleAssignments $subscriptionId $appResourceGroupName $appStorageAccountName
RemoveCosmosRoleAssignments $subscriptionId $appResourceGroupName $cosmosAccountName
@ -90,38 +90,6 @@ try {
Write-Error "Failed to deploy resource groups"
exit 1
}
if ($target -ne 'production') {
$azAdGroupId = az ad group show --group "Azure SDK Engineering System Team" --query id --output tsv
Write-Host "Granting the Azure SDK Engineering System Team read/write access to cosmos account"
Invoke "az cosmosdb sql role assignment create --resource-group '$appResourceGroupName' --account-name '$cosmosAccountName' --scope '/' --role-definition-id '00000000-0000-0000-0000-000000000002' --principal-id '$azAdGroupId' --output none"
if ($LASTEXITCODE -ne 0) {
Write-Output $output
Write-Error "Failed to grant access to cosmos account"
exit 1
}
Write-Host "Granting the Azure SDK Engineering System Team access to storage accounts"
$scope = "/subscriptions/$subscriptionId/resourceGroups/$logsResourceGroupName/providers/Microsoft.Storage/storageAccounts/$logsStorageAccountName"
$output = Invoke "az role assignment create --assignee '$azAdGroupId' --role 'Storage Blob Data Contributor' --scope '$scope' --output none"
if ($LASTEXITCODE -ne 0) {
Write-Output $output
Write-Error "Failed to grant access to logs storage account"
exit 1
}
$scope = "/subscriptions/$subscriptionId/resourceGroups/$appResourceGroupName/providers/Microsoft.Storage/storageAccounts/$appStorageAccountName"
$output = Invoke "az role assignment create --assignee '$azAdGroupId' --role 'Storage Queue Data Contributor' --scope '$scope' --output none"
if ($LASTEXITCODE -ne 0) {
Write-Output $output
Write-Error "Failed to grant access to app storage account"
exit 1
}
}
}
finally {
Pop-Location