Azure
/
azure-sdk-tools
зеркало из https://github.com/Azure/azure-sdk-tools.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

Remove failure analysis and apply code cleanup (#8153)

This commit is contained in:
Patrick Hallisey 2024-04-26 10:49:12 -07:00 коммит произвёл GitHub
Родитель a8b08796fb
Коммит 478ff5c2d2
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: B5690EEEBB952194
45 изменённых файлов: 377 добавлений и 1207 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

9
.editorconfig
Просмотреть файл

@ -81,9 +81,9 @@ dotnet_naming_style.prefix_underscore.capitalization = camel_case
############################### ###############################
[*.cs] [*.cs]
# var preferences # var preferences
csharp_style_var_for_built_in_types = true:silent csharp_style_var_for_built_in_types = false:silent
csharp_style_var_when_type_is_apparent = true:silent csharp_style_var_when_type_is_apparent = false:silent
csharp_style_var_elsewhere = true:silent csharp_style_var_elsewhere = false:silent
# Expression-bodied members # Expression-bodied members
csharp_style_expression_bodied_methods = false:silent csharp_style_expression_bodied_methods = false:silent
csharp_style_expression_bodied_constructors = false:silent csharp_style_expression_bodied_constructors = false:silent
@ -105,6 +105,8 @@ csharp_style_deconstructed_variable_declaration = true:suggestion
csharp_prefer_simple_default_expression = true:suggestion csharp_prefer_simple_default_expression = true:suggestion
csharp_style_pattern_local_over_anonymous_function = true:suggestion csharp_style_pattern_local_over_anonymous_function = true:suggestion
csharp_style_inlined_variable_declaration = true:suggestion csharp_style_inlined_variable_declaration = true:suggestion
# one class per file
csharp_style_single_file_classes = true:suggestion
############################### ###############################
# C# Formatting Rules # # C# Formatting Rules #
@ -136,3 +138,4 @@ csharp_space_between_method_call_empty_parameter_list_parentheses = false
# Wrapping preferences # Wrapping preferences
csharp_preserve_single_line_statements = true csharp_preserve_single_line_statements = true
csharp_preserve_single_line_blocks = true csharp_preserve_single_line_blocks = true

50
tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness.Tests/BlobUploadProcessorIntegrationTests.cs
Просмотреть файл

@ -17,53 +17,53 @@ namespace Azure.Sdk.Tools.PipelineWitness.Tests
{ {
public class BlobUploadProcessorIntegrationTests public class BlobUploadProcessorIntegrationTests
{ {
private VssCredentials VisualStudioCredentials; private const string TARGET_ACCOUNT_ID = "azure-sdk";
private VssConnection VisualStudioConnection; private const string TARGET_PROJECT_ID = "29ec6040-b234-4e31-b139-33dc4287b756";
private string TARGET_ACCOUNT_ID = "azure-sdk"; private const int TARGET_DEFINITION_ID = 297;
private Guid TARGET_PROJECT_ID = new Guid("29ec6040-b234-4e31-b139-33dc4287b756"); private const string DEVOPS_PATH = "https://dev.azure.com/azure-sdk";
private int TARGET_DEFINITION_ID = 297;
private string DEVOPS_PATH = "https://dev.azure.com/azure-sdk"; private readonly VssCredentials visualStudioCredentials;
private PipelineWitnessSettings TestSettings = new PipelineWitnessSettings() private readonly VssConnection visualStudioConnection;
private readonly PipelineWitnessSettings testSettings = new()
{ {
PipelineOwnersDefinitionId = 5112, PipelineOwnersDefinitionId = 5112,
PipelineOwnersFilePath = "pipelineOwners/pipelineOwners.json", PipelineOwnersFilePath = "pipelineOwners/pipelineOwners.json",
PipelineOwnersArtifactName = "pipelineOwners" PipelineOwnersArtifactName = "pipelineOwners"
}; };
public BlobUploadProcessorIntegrationTests() public BlobUploadProcessorIntegrationTests()
{ {
var pat = Environment.GetEnvironmentVariable("AZURESDK_DEVOPS_TOKEN"); string pat = Environment.GetEnvironmentVariable("AZURESDK_DEVOPS_TOKEN");
var blobUri = Environment.GetEnvironmentVariable("AZURESDK_BLOB_CS"); string blobUri = Environment.GetEnvironmentVariable("AZURESDK_BLOB_CS");
if (!string.IsNullOrWhiteSpace(pat) && !string.IsNullOrWhiteSpace(blobUri) ) if (!string.IsNullOrWhiteSpace(pat) && !string.IsNullOrWhiteSpace(blobUri))
{ {
VisualStudioCredentials = new VssBasicCredential("nobody", pat); this.visualStudioCredentials = new VssBasicCredential("nobody", pat);
VisualStudioConnection = new VssConnection(new Uri(DEVOPS_PATH), VisualStudioCredentials); this.visualStudioConnection = new VssConnection(new Uri(DEVOPS_PATH), this.visualStudioCredentials);
} }
} }
[EnvironmentConditionalSkipFact] [EnvironmentConditionalSkipFact]
public async Task BasicBlobProcessInvokesSuccessfully() public async Task BasicBlobProcessInvokesSuccessfully()
{ {
var buildLogProvider = new BuildLogProvider(logger: new NullLogger<BuildLogProvider>(), VisualStudioConnection); BuildLogProvider buildLogProvider = new(logger: new NullLogger<BuildLogProvider>(), this.visualStudioConnection);
var blobServiceClient = new BlobServiceClient(Environment.GetEnvironmentVariable("AZURESDK_BLOB_CS")); BlobServiceClient blobServiceClient = new(Environment.GetEnvironmentVariable("AZURESDK_BLOB_CS"));
var buildHttpClient = VisualStudioConnection.GetClient<BuildHttpClient>(); BuildHttpClient buildHttpClient = this.visualStudioConnection.GetClient<BuildHttpClient>();
var testResultsBuiltClient = VisualStudioConnection.GetClient<TestResultsHttpClient>(); TestResultsHttpClient testResultsBuiltClient = this.visualStudioConnection.GetClient<TestResultsHttpClient>();
List<Build> recentBuilds = await buildHttpClient.GetBuildsAsync(TARGET_PROJECT_ID, definitions: new[] { TARGET_DEFINITION_ID }, resultFilter: BuildResult.Succeeded, statusFilter: BuildStatus.Completed, top: 1, queryOrder: BuildQueryOrder.FinishTimeDescending); List<Build> recentBuilds = await buildHttpClient.GetBuildsAsync(TARGET_PROJECT_ID, definitions: new[] { TARGET_DEFINITION_ID }, resultFilter: BuildResult.Succeeded, statusFilter: BuildStatus.Completed, top: 1, queryOrder: BuildQueryOrder.FinishTimeDescending);
Assert.True(recentBuilds.Count > 0); Assert.True(recentBuilds.Count > 0);
var targetBuildId = recentBuilds.First().Id; int targetBuildId = recentBuilds.First().Id;
BlobUploadProcessor processor = new BlobUploadProcessor(logger: new NullLogger<BlobUploadProcessor>(), BlobUploadProcessor processor = new(logger: new NullLogger<BlobUploadProcessor>(),
logProvider: buildLogProvider, logProvider: buildLogProvider,
blobServiceClient: blobServiceClient, blobServiceClient: blobServiceClient,
buildClient: buildHttpClient, buildClient: buildHttpClient,
testResultsClient: testResultsBuiltClient, testResultsClient: testResultsBuiltClient,
options: Options.Create<PipelineWitnessSettings>(TestSettings), options: Options.Create<PipelineWitnessSettings>(this.testSettings));
failureAnalyzer: new PassThroughFailureAnalyzer());
await processor.UploadBuildBlobsAsync(TARGET_ACCOUNT_ID, TARGET_PROJECT_ID, targetBuildId); await processor.UploadBuildBlobsAsync(TARGET_ACCOUNT_ID, new Guid(TARGET_PROJECT_ID), targetBuildId);
} }
[Theory] [Theory]
@ -73,7 +73,7 @@ namespace Azure.Sdk.Tools.PipelineWitness.Tests
[InlineData(0, 10000, 0)] [InlineData(0, 10000, 0)]
public void TestBatching(int startingNumber, int batchSize, int expectedBatchNumber) public void TestBatching(int startingNumber, int batchSize, int expectedBatchNumber)
{ {
var numberOfBatches = BlobUploadProcessor.CalculateBatches(startingNumber, batchSize); int numberOfBatches = BlobUploadProcessor.CalculateBatches(startingNumber, batchSize);
Assert.Equal(expectedBatchNumber, numberOfBatches); Assert.Equal(expectedBatchNumber, numberOfBatches);
} }

4
tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness.Tests/EnvironmentConditionalSkipFact.cs
Просмотреть файл

@ -11,8 +11,8 @@ namespace Azure.Sdk.Tools.PipelineWitness.Tests
{ {
public EnvironmentConditionalSkipFact() public EnvironmentConditionalSkipFact()
{ {
var devopsPat = Environment.GetEnvironmentVariable("AZURESDK_DEVOPS_TOKEN"); string devopsPat = Environment.GetEnvironmentVariable("AZURESDK_DEVOPS_TOKEN");
var blobToken = Environment.GetEnvironmentVariable("AZURESDK_BLOB_CS"); string blobToken = Environment.GetEnvironmentVariable("AZURESDK_BLOB_CS");
// and if we don't, skip this test // and if we don't, skip this test
if (string.IsNullOrWhiteSpace(devopsPat) || string.IsNullOrWhiteSpace(blobToken)) if (string.IsNullOrWhiteSpace(devopsPat) || string.IsNullOrWhiteSpace(blobToken))

16
tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness.Tests/PassThroughFailureAnalyzer.cs
Просмотреть файл

@ -1,16 +0,0 @@
using System.Collections.Generic;
using System.Threading.Tasks;
using Azure.Sdk.Tools.PipelineWitness.Entities.AzurePipelines;
using Azure.Sdk.Tools.PipelineWitness.Services.FailureAnalysis;
using Microsoft.TeamFoundation.Build.WebApi;
namespace Azure.Sdk.Tools.PipelineWitness.Tests
{
internal class PassThroughFailureAnalyzer : IFailureAnalyzer
{
public Task<IEnumerable<Failure>> AnalyzeFailureAsync(Build build, Timeline timeline)
{
return Task.FromResult<IEnumerable<Failure>>(new List<Failure>());
}
}
}

25
tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness.Tests/TestLogger.cs
Просмотреть файл

@ -24,29 +24,4 @@ namespace Azure.Sdk.Tools.PipelineWitness.Tests
Logs.Add(state); Logs.Add(state);
} }
} }
public class TestLoggingFactory : ILoggerFactory
{
private readonly TestLogger _logger;
public TestLoggingFactory(TestLogger logger)
{
_logger = logger;
}
public void Dispose()
{
throw new NotImplementedException();
}
public void AddProvider(ILoggerProvider provider)
{
throw new NotImplementedException();
}
public ILogger CreateLogger(string categoryName)
{
return _logger;
}
}
} }

19
tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/ApplicationInsights/ApplicationVersionTelemetryInitializer.cs
Просмотреть файл

@ -1,31 +1,32 @@
using System.Reflection; using System.Reflection;
using Microsoft.ApplicationInsights.Channel; using Microsoft.ApplicationInsights.Channel;
using Microsoft.ApplicationInsights.Extensibility; using Microsoft.ApplicationInsights.Extensibility;
using Microsoft.ApplicationInsights.Extensibility.Implementation;
namespace Azure.Sdk.Tools.PipelineWitness.ApplicationInsights namespace Azure.Sdk.Tools.PipelineWitness.ApplicationInsights
{ {
public class ApplicationVersionTelemetryInitializer : ITelemetryInitializer public class ApplicationVersionTelemetryInitializer : ITelemetryInitializer
{ {
private static string _version = GetVersion(); private static readonly string version = GetVersion();
public void Initialize(ITelemetry telemetry) public void Initialize(ITelemetry telemetry)
{ {
if (!string.IsNullOrEmpty(_version)) if (!string.IsNullOrEmpty(version))
{ {
var component = telemetry.Context?.Component; ComponentContext component = telemetry.Context?.Component;
if (component != null) if (component != null)
{ {
component.Version = _version; component.Version = version;
} }
} }
} }
private static string GetVersion() private static string GetVersion()
{ {
var assembly = typeof(ApplicationVersionTelemetryInitializer).Assembly; Assembly assembly = typeof(ApplicationVersionTelemetryInitializer).Assembly;
var version = assembly.GetCustomAttribute<AssemblyInformationalVersionAttribute>()?.InformationalVersion string version = assembly.GetCustomAttribute<AssemblyInformationalVersionAttribute>()?.InformationalVersion
?? assembly.GetName().Version?.ToString(); ?? assembly.GetName().Version?.ToString();
return version; return version;

8
tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/ApplicationInsights/BlobNotFoundTelemetryProcessor.cs
Просмотреть файл

@ -13,14 +13,14 @@ namespace Azure.Sdk.Tools.PipelineWitness.ApplicationInsights
{ {
this.next = next; this.next = next;
} }
public void Process(ITelemetry telemetry) public void Process(ITelemetry telemetry)
{ {
if (telemetry is DependencyTelemetry { Success: false, Type: "Azure blob" or "Microsoft.Storage" } blobRequestTelemetry) if (telemetry is DependencyTelemetry { Success: false, Type: "Azure blob" or "Microsoft.Storage" } blobRequestTelemetry)
{ {
blobRequestTelemetry.Properties.TryGetValue("Error", out var errorProperty); blobRequestTelemetry.Properties.TryGetValue("Error", out string errorProperty);
var isNotFound = blobRequestTelemetry.ResultCode is "404" or "409" bool isNotFound = blobRequestTelemetry.ResultCode is "404" or "409"
|| (blobRequestTelemetry.ResultCode == "" && errorProperty?.Contains("Status: 404") == true); || (blobRequestTelemetry.ResultCode == "" && errorProperty?.Contains("Status: 404") == true);
if (isNotFound) if (isNotFound)

292
tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/BlobUploadProcessor.cs
Просмотреть файл

@ -1,44 +1,45 @@
using System;
using System.Collections.Generic;
using System.Diagnostics.CodeAnalysis;
using System.IO;
using System.IO.Compression;
using System.Linq;
using System.Net;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using Azure.Sdk.Tools.PipelineWitness.Configuration;
using Azure.Sdk.Tools.PipelineWitness.Services;
using Azure.Storage.Blobs;
using Azure.Storage.Blobs.Models;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using Microsoft.TeamFoundation.Build.WebApi;
using Microsoft.TeamFoundation.TestManagement.WebApi;
using Microsoft.VisualStudio.Services.TestResults.WebApi;
using Microsoft.VisualStudio.Services.WebApi;
using Newtonsoft.Json;
using Newtonsoft.Json.Converters;
using Newtonsoft.Json.Serialization;
namespace Azure.Sdk.Tools.PipelineWitness namespace Azure.Sdk.Tools.PipelineWitness
{ {
using System; [SuppressMessage("Style", "IDE0037:Use inferred member name", Justification = "Explicit member names are added to json export objects for clarity")]
using System.Collections.Generic;
using System.IO;
using System.IO.Compression;
using System.Linq;
using System.Net;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using Azure.Sdk.Tools.PipelineWitness.Configuration;
using Azure.Sdk.Tools.PipelineWitness.Services;
using Azure.Sdk.Tools.PipelineWitness.Services.FailureAnalysis;
using Azure.Storage.Blobs;
using Azure.Storage.Blobs.Models;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using Microsoft.TeamFoundation.Build.WebApi;
using Microsoft.TeamFoundation.TestManagement.WebApi;
using Microsoft.VisualStudio.Services.TestResults.WebApi;
using Newtonsoft.Json;
using Newtonsoft.Json.Converters;
using Newtonsoft.Json.Serialization;
public class BlobUploadProcessor public class BlobUploadProcessor
{ {
private const string BuildsContainerName = "builds"; private const string BuildsContainerName = "builds";
private const string BuildLogLinesContainerName = "buildloglines"; private const string BuildLogLinesContainerName = "buildloglines";
private const string BuildTimelineRecordsContainerName = "buildtimelinerecords"; private const string BuildTimelineRecordsContainerName = "buildtimelinerecords";
private const string BuildDefinitionsContainerName = "builddefinitions"; private const string BuildDefinitionsContainerName = "builddefinitions";
private const string BuildFailuresContainerName = "buildfailures";
private const string PipelineOwnersContainerName = "pipelineowners"; private const string PipelineOwnersContainerName = "pipelineowners";
private const string TestRunsContainerName = "testruns"; private const string TestRunsContainerName = "testruns";
private const string TestResultsContainerName = "testrunresults"; private const string TestResultsContainerName = "testrunresults";
private const int ApiBatchSize = 10000; private const int ApiBatchSize = 10000;
private const string TimeFormat = @"yyyy-MM-dd\THH:mm:ss.fffffff\Z"; private const string TimeFormat = @"yyyy-MM-dd\THH:mm:ss.fffffff\Z";
private static readonly JsonSerializerSettings jsonSettings = new JsonSerializerSettings() private static readonly JsonSerializerSettings jsonSettings = new()
{ {
ContractResolver = new CamelCasePropertyNamesContractResolver(), ContractResolver = new CamelCasePropertyNamesContractResolver(),
Converters = { new StringEnumConverter(new CamelCaseNamingStrategy()) }, Converters = { new StringEnumConverter(new CamelCaseNamingStrategy()) },
@ -55,11 +56,9 @@ namespace Azure.Sdk.Tools.PipelineWitness
private readonly BlobContainerClient testRunsContainerClient; private readonly BlobContainerClient testRunsContainerClient;
private readonly BlobContainerClient testResultsContainerClient; private readonly BlobContainerClient testResultsContainerClient;
private readonly BlobContainerClient buildDefinitionsContainerClient; private readonly BlobContainerClient buildDefinitionsContainerClient;
private readonly BlobContainerClient buildFailuresContainerClient;
private readonly BlobContainerClient pipelineOwnersContainerClient; private readonly BlobContainerClient pipelineOwnersContainerClient;
private readonly IOptions<PipelineWitnessSettings> options; private readonly IOptions<PipelineWitnessSettings> options;
private readonly Dictionary<string, int?> cachedDefinitionRevisions = new(); private readonly Dictionary<string, int?> cachedDefinitionRevisions = new();
private readonly IFailureAnalyzer failureAnalyzer;
public BlobUploadProcessor( public BlobUploadProcessor(
ILogger<BlobUploadProcessor> logger, ILogger<BlobUploadProcessor> logger,
@ -67,8 +66,7 @@ namespace Azure.Sdk.Tools.PipelineWitness
BlobServiceClient blobServiceClient, BlobServiceClient blobServiceClient,
BuildHttpClient buildClient, BuildHttpClient buildClient,
TestResultsHttpClient testResultsClient, TestResultsHttpClient testResultsClient,
IOptions<PipelineWitnessSettings> options, IOptions<PipelineWitnessSettings> options)
IFailureAnalyzer failureAnalyzer)
{ {
if (blobServiceClient == null) if (blobServiceClient == null)
{ {
@ -84,17 +82,15 @@ namespace Azure.Sdk.Tools.PipelineWitness
this.buildTimelineRecordsContainerClient = blobServiceClient.GetBlobContainerClient(BuildTimelineRecordsContainerName); this.buildTimelineRecordsContainerClient = blobServiceClient.GetBlobContainerClient(BuildTimelineRecordsContainerName);
this.buildLogLinesContainerClient = blobServiceClient.GetBlobContainerClient(BuildLogLinesContainerName); this.buildLogLinesContainerClient = blobServiceClient.GetBlobContainerClient(BuildLogLinesContainerName);
this.buildDefinitionsContainerClient = blobServiceClient.GetBlobContainerClient(BuildDefinitionsContainerName); this.buildDefinitionsContainerClient = blobServiceClient.GetBlobContainerClient(BuildDefinitionsContainerName);
this.buildFailuresContainerClient = blobServiceClient.GetBlobContainerClient(BuildFailuresContainerName);
this.testRunsContainerClient = blobServiceClient.GetBlobContainerClient(TestRunsContainerName); this.testRunsContainerClient = blobServiceClient.GetBlobContainerClient(TestRunsContainerName);
this.testResultsContainerClient = blobServiceClient.GetBlobContainerClient(TestResultsContainerName); this.testResultsContainerClient = blobServiceClient.GetBlobContainerClient(TestResultsContainerName);
this.buildDefinitionsContainerClient = blobServiceClient.GetBlobContainerClient(BuildDefinitionsContainerName); this.buildDefinitionsContainerClient = blobServiceClient.GetBlobContainerClient(BuildDefinitionsContainerName);
this.pipelineOwnersContainerClient = blobServiceClient.GetBlobContainerClient(PipelineOwnersContainerName); this.pipelineOwnersContainerClient = blobServiceClient.GetBlobContainerClient(PipelineOwnersContainerName);
this.failureAnalyzer = failureAnalyzer;
} }
public async Task UploadBuildBlobsAsync(string account, Guid projectId, int buildId) public async Task UploadBuildBlobsAsync(string account, Guid projectId, int buildId)
{ {
var build = await GetBuildAsync(projectId, buildId); Build build = await GetBuildAsync(projectId, buildId);
if (build == null) if (build == null)
{ {
@ -102,7 +98,7 @@ namespace Azure.Sdk.Tools.PipelineWitness
return; return;
} }
var skipBuild = false; bool skipBuild = false;
// Project name is used in blob paths and cannot be empty // Project name is used in blob paths and cannot be empty
if (build.Project == null) if (build.Project == null)
@ -157,34 +153,33 @@ namespace Azure.Sdk.Tools.PipelineWitness
await UploadTestRunBlobsAsync(account, build); await UploadTestRunBlobsAsync(account, build);
var timeline = await this.buildClient.GetBuildTimelineAsync(projectId, buildId); Timeline timeline = await this.buildClient.GetBuildTimelineAsync(projectId, buildId);
if (timeline == null) if (timeline == null)
{ {
logger.LogWarning("No timeline available for build {Project}: {BuildId}", build.Project.Name, build.Id); this.logger.LogWarning("No timeline available for build {Project}: {BuildId}", build.Project.Name, build.Id);
} }
else else
{ {
await UploadTimelineBlobAsync(account, build, timeline); await UploadTimelineBlobAsync(account, build, timeline);
await UploadBuildFailureBlobAsync(account, build, timeline);
} }
var logs = await buildClient.GetBuildLogsAsync(build.Project.Id, build.Id); List<BuildLog> logs = await this.buildClient.GetBuildLogsAsync(build.Project.Id, build.Id);
if (logs == null || logs.Count == 0) if (logs == null || logs.Count == 0)
{ {
logger.LogWarning("No logs available for build {Project}: {BuildId}", build.Project.Name, build.Id); this.logger.LogWarning("No logs available for build {Project}: {BuildId}", build.Project.Name, build.Id);
return; return;
} }
var buildLogInfos = GetBuildLogInfos(account, build, timeline, logs); List<BuildLogInfo> buildLogInfos = GetBuildLogInfos(build, timeline, logs);
foreach (var log in buildLogInfos) foreach (BuildLogInfo log in buildLogInfos)
{ {
await UploadLogLinesBlobAsync(account, build, log); await UploadLogLinesBlobAsync(account, build, log);
} }
if (build.Definition.Id == options.Value.PipelineOwnersDefinitionId) if (build.Definition.Id == this.options.Value.PipelineOwnersDefinitionId)
{ {
await UploadPipelineOwnersBlobAsync(account, build, timeline); await UploadPipelineOwnersBlobAsync(account, build, timeline);
} }
@ -194,8 +189,8 @@ namespace Azure.Sdk.Tools.PipelineWitness
{ {
try try
{ {
var blobPath = $"{build.Project.Name}/{build.FinishTime:yyyy/MM/dd}/{build.Id}-{timeline.ChangeId}.jsonl"; string blobPath = $"{build.Project.Name}/{build.FinishTime:yyyy/MM/dd}/{build.Id}-{timeline.ChangeId}.jsonl";
var blobClient = this.pipelineOwnersContainerClient.GetBlobClient(blobPath); BlobClient blobClient = this.pipelineOwnersContainerClient.GetBlobClient(blobPath);
if (await blobClient.ExistsAsync()) if (await blobClient.ExistsAsync())
{ {
@ -203,7 +198,7 @@ namespace Azure.Sdk.Tools.PipelineWitness
return; return;
} }
var owners = await GetOwnersFromBuildArtifactAsync(build); Dictionary<int, string[]> owners = await GetOwnersFromBuildArtifactAsync(build);
if (owners == null) if (owners == null)
{ {
@ -213,11 +208,11 @@ namespace Azure.Sdk.Tools.PipelineWitness
this.logger.LogInformation("Creating owners blob for build {DefinitionId} change {ChangeId}", build.Id, timeline.ChangeId); this.logger.LogInformation("Creating owners blob for build {DefinitionId} change {ChangeId}", build.Id, timeline.ChangeId);
var stringBuilder = new StringBuilder(); StringBuilder stringBuilder = new();
foreach (var owner in owners) foreach (KeyValuePair<int, string[]> owner in owners)
{ {
var contentLine = JsonConvert.SerializeObject(new string contentLine = JsonConvert.SerializeObject(new
{ {
OrganizationName = account, OrganizationName = account,
BuildDefinitionId = owner.Key, BuildDefinitionId = owner.Key,
@ -244,15 +239,15 @@ namespace Azure.Sdk.Tools.PipelineWitness
private async Task<Dictionary<int, string[]>> GetOwnersFromBuildArtifactAsync(Build build) private async Task<Dictionary<int, string[]>> GetOwnersFromBuildArtifactAsync(Build build)
{ {
var artifactName = this.options.Value.PipelineOwnersArtifactName; string artifactName = this.options.Value.PipelineOwnersArtifactName;
var filePath = this.options.Value.PipelineOwnersFilePath; string filePath = this.options.Value.PipelineOwnersFilePath;
try try
{ {
await using var artifactStream = await this.buildClient.GetArtifactContentZipAsync(build.Project.Id, build.Id, artifactName); await using Stream artifactStream = await this.buildClient.GetArtifactContentZipAsync(build.Project.Id, build.Id, artifactName);
using var zip = new ZipArchive(artifactStream); using ZipArchive zip = new(artifactStream);
var fileEntry = zip.GetEntry(filePath); ZipArchiveEntry fileEntry = zip.GetEntry(filePath);
if (fileEntry == null) if (fileEntry == null)
{ {
@ -260,9 +255,9 @@ namespace Azure.Sdk.Tools.PipelineWitness
return null; return null;
} }
await using var contentStream = fileEntry.Open(); await using Stream contentStream = fileEntry.Open();
using var contentReader = new StreamReader(contentStream); using StreamReader contentReader = new(contentStream);
var content = await contentReader.ReadToEndAsync(); string content = await contentReader.ReadToEndAsync();
if (string.IsNullOrEmpty(content)) if (string.IsNullOrEmpty(content))
{ {
@ -270,7 +265,7 @@ namespace Azure.Sdk.Tools.PipelineWitness
return null; return null;
} }
var ownersDictionary = JsonConvert.DeserializeObject<Dictionary<int, string[]>>(content); Dictionary<int, string[]> ownersDictionary = JsonConvert.DeserializeObject<Dictionary<int, string[]>>(content);
if (ownersDictionary == null) if (ownersDictionary == null)
{ {
@ -298,72 +293,15 @@ namespace Azure.Sdk.Tools.PipelineWitness
return null; return null;
} }
private async Task UploadBuildFailureBlobAsync(string account, Build build, Timeline timeline)
{
try
{
var blobPath = $"{build.Project.Name}/{build.FinishTime:yyyy/MM/dd}/{build.Id}-{timeline.ChangeId}.jsonl";
var blobClient = this.buildFailuresContainerClient.GetBlobClient(blobPath);
if (await blobClient.ExistsAsync())
{
this.logger.LogInformation("Skipping existing build failure blob for build {BuildId}", build.Id);
return;
}
var failures = await this.failureAnalyzer.AnalyzeFailureAsync(build, timeline);
if (!failures.Any())
{
return;
}
this.logger.LogInformation("Creating failure blob for build {DefinitionId} change {ChangeId}", build.Id, timeline.ChangeId);
var stringBuilder = new StringBuilder();
foreach (var failure in failures)
{
var contentLine = JsonConvert.SerializeObject(new
{
OrganizationName = account,
ProjectId = build.Project.Id,
ProjectName = build.Project.Name,
BuildDefinitionId = build.Definition.Id,
BuildDefinitionName = build.Definition.Name,
BuildId = build.Id,
BuildFinishTime = build.FinishTime,
RecordFinishTime = failure.Record.FinishTime,
ChangeId = timeline.ChangeId,
RecordId = failure.Record.Id,
BuildTimelineId = timeline.Id,
ErrorClassification = failure.Classification,
EtlIngestDate = DateTimeOffset.UtcNow
}, jsonSettings);
stringBuilder.AppendLine(contentLine);
}
await blobClient.UploadAsync(new BinaryData(stringBuilder.ToString()));
}
catch (RequestFailedException ex) when (ex.Status == (int)HttpStatusCode.Conflict)
{
this.logger.LogInformation("Ignoring exception from existing failure blob for build {BuildId}", build.Id);
}
catch (Exception ex)
{
this.logger.LogError(ex, "Error processing build failure blob for build {BuildId}", build.Id);
throw;
}
}
public async Task UploadBuildDefinitionBlobsAsync(string account, string projectName) public async Task UploadBuildDefinitionBlobsAsync(string account, string projectName)
{ {
var definitions = await buildClient.GetFullDefinitionsAsync2(project: projectName); IPagedList<BuildDefinition> definitions = await this.buildClient.GetFullDefinitionsAsync2(project: projectName);
foreach (var definition in definitions) foreach (BuildDefinition definition in definitions)
{ {
var cacheKey = $"{definition.Project.Id}:{definition.Id}"; string cacheKey = $"{definition.Project.Id}:{definition.Id}";
if (!this.cachedDefinitionRevisions.TryGetValue(cacheKey, out var cachedRevision) || cachedRevision != definition.Revision) if (!this.cachedDefinitionRevisions.TryGetValue(cacheKey, out int? cachedRevision) || cachedRevision != definition.Revision)
{ {
await UploadBuildDefinitionBlobAsync(account, definition); await UploadBuildDefinitionBlobAsync(account, definition);
} }
@ -371,14 +309,14 @@ namespace Azure.Sdk.Tools.PipelineWitness
this.cachedDefinitionRevisions[cacheKey] = definition.Revision; this.cachedDefinitionRevisions[cacheKey] = definition.Revision;
} }
} }
private async Task UploadBuildDefinitionBlobAsync(string account, BuildDefinition definition) private async Task UploadBuildDefinitionBlobAsync(string account, BuildDefinition definition)
{ {
var blobPath = $"{definition.Project.Name}/{definition.Id}-{definition.Revision}.jsonl"; string blobPath = $"{definition.Project.Name}/{definition.Id}-{definition.Revision}.jsonl";
try try
{ {
var blobClient = this.buildDefinitionsContainerClient.GetBlobClient(blobPath); BlobClient blobClient = this.buildDefinitionsContainerClient.GetBlobClient(blobPath);
if (await blobClient.ExistsAsync()) if (await blobClient.ExistsAsync())
{ {
@ -388,7 +326,7 @@ namespace Azure.Sdk.Tools.PipelineWitness
this.logger.LogInformation("Creating blob for build definition {DefinitionId} revision {Revision} project {Project}", definition.Id, definition.Revision, definition.Project.Name); this.logger.LogInformation("Creating blob for build definition {DefinitionId} revision {Revision} project {Project}", definition.Id, definition.Revision, definition.Project.Name);
var content = JsonConvert.SerializeObject(new string content = JsonConvert.SerializeObject(new
{ {
OrganizationName = account, OrganizationName = account,
ProjectId = definition.Project.Id, ProjectId = definition.Project.Id,
@ -446,32 +384,32 @@ namespace Azure.Sdk.Tools.PipelineWitness
} }
} }
private List<BuildLogInfo> GetBuildLogInfos(string account, Build build, Timeline timeline, List<BuildLog> logs) private List<BuildLogInfo> GetBuildLogInfos(Build build, Timeline timeline, List<BuildLog> logs)
{ {
var logsById = logs.ToDictionary(l => l.Id); Dictionary<int, BuildLog> logsById = logs.ToDictionary(l => l.Id);
var buildLogInfos = new List<BuildLogInfo>(); List<BuildLogInfo> buildLogInfos = new();
foreach (var log in logs) foreach (BuildLog log in logs)
{ {
var logRecords = timeline.Records.Where(x => x.Log?.Id == log.Id).ToArray(); TimelineRecord[] logRecords = timeline.Records.Where(x => x.Log?.Id == log.Id).ToArray();
if(logRecords.Length > 1) if (logRecords.Length > 1)
{ {
this.logger.LogWarning("Found multiple timeline records for build {BuildId}, log {LogId}", build.Id, log.Id); this.logger.LogWarning("Found multiple timeline records for build {BuildId}, log {LogId}", build.Id, log.Id);
} }
var logRecord = logRecords.FirstOrDefault(); TimelineRecord logRecord = logRecords.FirstOrDefault();
// Job logs are typically just a duplication of their child task logs with the addition of extra start and end lines. // Job logs are typically just a duplication of their child task logs with the addition of extra start and end lines.
// If we can, we skip the redundant lines. // If we can, we skip the redundant lines.
if (string.Equals(logRecord?.RecordType, "job", StringComparison.OrdinalIgnoreCase)) if (string.Equals(logRecord?.RecordType, "job", StringComparison.OrdinalIgnoreCase))
{ {
// find all of the child task records // find all of the child task records
var childRecords = timeline.Records.Where(x => x.ParentId == logRecord.Id); IEnumerable<TimelineRecord> childRecords = timeline.Records.Where(x => x.ParentId == logRecord.Id);
// sum the line counts for all of the child task records // sum the line counts for all of the child task records
var childLineCount = childRecords long childLineCount = childRecords
.Where(x => x.Log != null && logsById.ContainsKey(x.Log.Id)) .Where(x => x.Log != null && logsById.ContainsKey(x.Log.Id))
.Sum(x => logsById[x.Log.Id].LineCount); .Sum(x => logsById[x.Log.Id].LineCount);
@ -502,8 +440,8 @@ namespace Azure.Sdk.Tools.PipelineWitness
try try
{ {
long changeTime = ((DateTimeOffset)build.LastChangedDate).ToUnixTimeSeconds(); long changeTime = ((DateTimeOffset)build.LastChangedDate).ToUnixTimeSeconds();
var blobPath = $"{build.Project.Name}/{build.FinishTime:yyyy/MM/dd}/{build.Id}-{changeTime}.jsonl"; string blobPath = $"{build.Project.Name}/{build.FinishTime:yyyy/MM/dd}/{build.Id}-{changeTime}.jsonl";
var blobClient = this.buildsContainerClient.GetBlobClient(blobPath); BlobClient blobClient = this.buildsContainerClient.GetBlobClient(blobPath);
if (await blobClient.ExistsAsync()) if (await blobClient.ExistsAsync())
{ {
@ -511,7 +449,7 @@ namespace Azure.Sdk.Tools.PipelineWitness
return; return;
} }
var content = JsonConvert.SerializeObject(new string content = JsonConvert.SerializeObject(new
{ {
OrganizationName = account, OrganizationName = account,
ProjectId = build.Project?.Id, ProjectId = build.Project?.Id,
@ -582,8 +520,8 @@ namespace Azure.Sdk.Tools.PipelineWitness
return; return;
} }
var blobPath = $"{build.Project.Name}/{build.FinishTime:yyyy/MM/dd}/{build.Id}-{timeline.ChangeId}.jsonl"; string blobPath = $"{build.Project.Name}/{build.FinishTime:yyyy/MM/dd}/{build.Id}-{timeline.ChangeId}.jsonl";
var blobClient = this.buildTimelineRecordsContainerClient.GetBlobClient(blobPath); BlobClient blobClient = this.buildTimelineRecordsContainerClient.GetBlobClient(blobPath);
if (await blobClient.ExistsAsync()) if (await blobClient.ExistsAsync())
{ {
@ -591,8 +529,8 @@ namespace Azure.Sdk.Tools.PipelineWitness
return; return;
} }
var builder = new StringBuilder(); StringBuilder builder = new();
foreach (var record in timeline.Records) foreach (TimelineRecord record in timeline.Records)
{ {
builder.AppendLine(JsonConvert.SerializeObject( builder.AppendLine(JsonConvert.SerializeObject(
new new
@ -656,8 +594,8 @@ namespace Azure.Sdk.Tools.PipelineWitness
{ {
// we don't use FinishTime in the logs blob path to prevent duplicating logs when processing retries. // we don't use FinishTime in the logs blob path to prevent duplicating logs when processing retries.
// i.e. logs with a given buildid/logid are immutable and retries only add new logs. // i.e. logs with a given buildid/logid are immutable and retries only add new logs.
var blobPath = $"{build.Project.Name}/{build.QueueTime:yyyy/MM/dd}/{build.Id}-{log.LogId}.jsonl"; string blobPath = $"{build.Project.Name}/{build.QueueTime:yyyy/MM/dd}/{build.Id}-{log.LogId}.jsonl";
var blobClient = this.buildLogLinesContainerClient.GetBlobClient(blobPath); BlobClient blobClient = this.buildLogLinesContainerClient.GetBlobClient(blobPath);
if (await blobClient.ExistsAsync()) if (await blobClient.ExistsAsync())
{ {
@ -667,44 +605,44 @@ namespace Azure.Sdk.Tools.PipelineWitness
this.logger.LogInformation("Processing log for build {BuildId}, record {RecordId}, log {LogId}", build.Id, log.RecordId, log.LogId); this.logger.LogInformation("Processing log for build {BuildId}, record {RecordId}, log {LogId}", build.Id, log.RecordId, log.LogId);
var lineNumber = 0; int lineNumber = 0;
var characterCount = 0; int characterCount = 0;
// Over an open read stream and an open write stream, one line at a time, read, process, and write to // Over an open read stream and an open write stream, one line at a time, read, process, and write to
// blob storage // blob storage
using (var logStream = await this.logProvider.GetLogStreamAsync(build.Project.Name, build.Id, log.LogId)) using (Stream logStream = await this.logProvider.GetLogStreamAsync(build.Project.Name, build.Id, log.LogId))
using (var logReader = new StreamReader(logStream)) using (StreamReader logReader = new(logStream))
using (var blobStream = await blobClient.OpenWriteAsync(overwrite: true, new BlobOpenWriteOptions())) using (Stream blobStream = await blobClient.OpenWriteAsync(overwrite: true, new BlobOpenWriteOptions()))
using (var blobWriter = new StreamWriter(blobStream)) using (StreamWriter blobWriter = new(blobStream))
{ {
var lastTimeStamp = log.LogCreatedOn; DateTimeOffset lastTimeStamp = log.LogCreatedOn;
while (true) while (true)
{ {
var line = await logReader.ReadLineAsync(); string line = await logReader.ReadLineAsync();
if (line == null) if (line == null)
{ {
break; break;
} }
var isLastLine = logReader.EndOfStream; bool isLastLine = logReader.EndOfStream;
lineNumber += 1; lineNumber += 1;
characterCount += line.Length; characterCount += line.Length;
// log lines usually follow the format: // log lines usually follow the format:
// 2022-03-30T21:38:38.7007903Z Downloading task: AzureKeyVault (1.200.0) // 2022-03-30T21:38:38.7007903Z Downloading task: AzureKeyVault (1.200.0)
// Sometimes, there's no leading timestamp, so we'll use the last timestamp we saw. // Sometimes, there's no leading timestamp, so we'll use the last timestamp we saw.
var match = Regex.Match(line, @"^([^Z]{20,28}Z) (.*)$"); Match match = Regex.Match(line, @"^([^Z]{20,28}Z) (.*)$");
var timestamp = match.Success DateTimeOffset timestamp = match.Success
? DateTime.ParseExact(match.Groups[1].Value, TimeFormat, null, ? DateTime.ParseExact(match.Groups[1].Value, TimeFormat, null,
System.Globalization.DateTimeStyles.AssumeUniversal).ToUniversalTime() System.Globalization.DateTimeStyles.AssumeUniversal).ToUniversalTime()
: lastTimeStamp; : lastTimeStamp;
lastTimeStamp = timestamp; lastTimeStamp = timestamp;
var message = match.Success ? match.Groups[2].Value : line; string message = match.Success ? match.Groups[2].Value : line;
await blobWriter.WriteLineAsync(JsonConvert.SerializeObject(new await blobWriter.WriteLineAsync(JsonConvert.SerializeObject(new
{ {
@ -725,7 +663,7 @@ namespace Azure.Sdk.Tools.PipelineWitness
} }
} }
logger.LogInformation("Processed {CharacterCount} characters and {LineCount} lines for build {BuildId}, record {RecordId}, log {LogId}", characterCount, lineNumber, build.Id, log.RecordId, log.LogId); this.logger.LogInformation("Processed {CharacterCount} characters and {LineCount} lines for build {BuildId}, record {RecordId}, log {LogId}", characterCount, lineNumber, build.Id, log.RecordId, log.LogId);
} }
catch (RequestFailedException ex) when (ex.Status == (int)HttpStatusCode.Conflict) catch (RequestFailedException ex) when (ex.Status == (int)HttpStatusCode.Conflict)
{ {
@ -737,31 +675,31 @@ namespace Azure.Sdk.Tools.PipelineWitness
throw; throw;
} }
} }
private async Task UploadTestRunBlobsAsync(string account, Build build) private async Task UploadTestRunBlobsAsync(string account, Build build)
{ {
try try
{ {
var continuationToken = string.Empty; string continuationToken = string.Empty;
var buildIds = new[] { build.Id }; int[] buildIds = new[] { build.Id };
var minLastUpdatedDate = build.QueueTime.Value.AddHours(-1); DateTime minLastUpdatedDate = build.QueueTime.Value.AddHours(-1);
var maxLastUpdatedDate = build.FinishTime.Value.AddHours(1); DateTime maxLastUpdatedDate = build.FinishTime.Value.AddHours(1);
var rangeStart = minLastUpdatedDate; DateTime rangeStart = minLastUpdatedDate;
while(rangeStart < maxLastUpdatedDate) while (rangeStart < maxLastUpdatedDate)
{ {
// Ado limits test run queries to a 7 day range, so we'll chunk on 6 days. // Ado limits test run queries to a 7 day range, so we'll chunk on 6 days.
var rangeEnd = rangeStart.AddDays(6); DateTime rangeEnd = rangeStart.AddDays(6);
if(rangeEnd > maxLastUpdatedDate) if (rangeEnd > maxLastUpdatedDate)
{ {
rangeEnd = maxLastUpdatedDate; rangeEnd = maxLastUpdatedDate;
} }
do do
{ {
var page = await testResultsClient.QueryTestRunsAsync2( IPagedList<TestRun> page = await this.testResultsClient.QueryTestRunsAsync2(
build.Project.Id, build.Project.Id,
rangeStart, rangeStart,
rangeEnd, rangeEnd,
@ -769,7 +707,7 @@ namespace Azure.Sdk.Tools.PipelineWitness
buildIds: buildIds buildIds: buildIds
); );
foreach (var testRun in page) foreach (TestRun testRun in page)
{ {
await UploadTestRunBlobAsync(account, build, testRun); await UploadTestRunBlobAsync(account, build, testRun);
await UploadTestRunResultBlobAsync(account, build, testRun); await UploadTestRunResultBlobAsync(account, build, testRun);
@ -792,8 +730,8 @@ namespace Azure.Sdk.Tools.PipelineWitness
{ {
try try
{ {
var blobPath = $"{build.Project.Name}/{testRun.CompletedDate:yyyy/MM/dd}/{testRun.Id}.jsonl"; string blobPath = $"{build.Project.Name}/{testRun.CompletedDate:yyyy/MM/dd}/{testRun.Id}.jsonl";
var blobClient = this.testRunsContainerClient.GetBlobClient(blobPath); BlobClient blobClient = this.testRunsContainerClient.GetBlobClient(blobPath);
if (await blobClient.ExistsAsync()) if (await blobClient.ExistsAsync())
{ {
@ -801,9 +739,9 @@ namespace Azure.Sdk.Tools.PipelineWitness
return; return;
} }
var stats = testRun.RunStatistics.ToDictionary(x => x.Outcome, x => x.Count); Dictionary<string, int> stats = testRun.RunStatistics.ToDictionary(x => x.Outcome, x => x.Count);
var content = JsonConvert.SerializeObject(new string content = JsonConvert.SerializeObject(new
{ {
OrganizationName = account, OrganizationName = account,
ProjectId = build.Project?.Id, ProjectId = build.Project?.Id,
@ -821,7 +759,7 @@ namespace Azure.Sdk.Tools.PipelineWitness
BranchName = build.SourceBranch, BranchName = build.SourceBranch,
HasDetail = default(bool?), HasDetail = default(bool?),
IsAutomated = testRun.IsAutomated, IsAutomated = testRun.IsAutomated,
ResultAbortedCount = stats.TryGetValue("Aborted", out var value) ? value : 0, ResultAbortedCount = stats.TryGetValue("Aborted", out int value) ? value : 0,
ResultBlockedCount = stats.TryGetValue("Blocked", out value) ? value : 0, ResultBlockedCount = stats.TryGetValue("Blocked", out value) ? value : 0,
ResultCount = testRun.TotalTests, ResultCount = testRun.TotalTests,
ResultErrorCount = stats.TryGetValue("Error", out value) ? value : 0, ResultErrorCount = stats.TryGetValue("Error", out value) ? value : 0,
@ -875,8 +813,8 @@ namespace Azure.Sdk.Tools.PipelineWitness
{ {
try try
{ {
var blobPath = $"{build.Project.Name}/{testRun.CompletedDate:yyyy/MM/dd}/{testRun.Id}.jsonl"; string blobPath = $"{build.Project.Name}/{testRun.CompletedDate:yyyy/MM/dd}/{testRun.Id}.jsonl";
var blobClient = this.testResultsContainerClient.GetBlobClient(blobPath); BlobClient blobClient = this.testResultsContainerClient.GetBlobClient(blobPath);
if (await blobClient.ExistsAsync()) if (await blobClient.ExistsAsync())
{ {
@ -884,14 +822,14 @@ namespace Azure.Sdk.Tools.PipelineWitness
return; return;
} }
var builder = new StringBuilder(); StringBuilder builder = new();
int batchCount = BlobUploadProcessor.CalculateBatches(testRun.TotalTests, batchSize: ApiBatchSize); int batchCount = BlobUploadProcessor.CalculateBatches(testRun.TotalTests, batchSize: ApiBatchSize);
for(int batchMultiplier = 0; batchMultiplier < batchCount; batchMultiplier++) for (int batchMultiplier = 0; batchMultiplier < batchCount; batchMultiplier++)
{ {
var data = await testResultsClient.GetTestResultsAsync(build.Project.Id, testRun.Id, top: ApiBatchSize, skip: batchMultiplier * ApiBatchSize); List<TestCaseResult> data = await this.testResultsClient.GetTestResultsAsync(build.Project.Id, testRun.Id, top: ApiBatchSize, skip: batchMultiplier * ApiBatchSize);
foreach (var record in data) foreach (TestCaseResult record in data)
{ {
builder.AppendLine(JsonConvert.SerializeObject( builder.AppendLine(JsonConvert.SerializeObject(
new new
@ -936,7 +874,7 @@ namespace Azure.Sdk.Tools.PipelineWitness
try try
{ {
build = await buildClient.GetBuildAsync(projectId, buildId); build = await this.buildClient.GetBuildAsync(projectId, buildId);
} }
catch (BuildNotFoundException) catch (BuildNotFoundException)
{ {

10
tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Entities/BuildLogBundle.cs
Просмотреть файл

@ -14,15 +14,15 @@ namespace Azure.Sdk.Tools.PipelineWitness
public int BuildId { get; set; } public int BuildId { get; set; }
public DateTimeOffset StartTime { get; set; } public DateTimeOffset StartTime { get; set; }
public DateTimeOffset FinishTime { get; set; } public DateTimeOffset FinishTime { get; set; }
public DateTimeOffset QueueTime { get; set; } public DateTimeOffset QueueTime { get; set; }
public int DefinitionId { get; set; } public int DefinitionId { get; set; }
public string DefinitionPath { get; set; } public string DefinitionPath { get; set; }
public string DefinitionName { get; set; } public string DefinitionName { get; set; }
public List<BuildLogInfo> TimelineLogs { get; } = new List<BuildLogInfo>(); public List<BuildLogInfo> TimelineLogs { get; } = new List<BuildLogInfo>();

10
tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Entities/BuildLogInfo.cs
Просмотреть файл

@ -1,7 +1,7 @@
namespace Azure.Sdk.Tools.PipelineWitness using System;
{
using System;
namespace Azure.Sdk.Tools.PipelineWitness
{
public class BuildLogInfo public class BuildLogInfo
{ {
public int LogId { get; set; } public int LogId { get; set; }
@ -11,9 +11,9 @@
public DateTimeOffset LogCreatedOn { get; set; } public DateTimeOffset LogCreatedOn { get; set; }
public string RecordType { get; set; } public string RecordType { get; set; }
public Guid? RecordId { get; set; } public Guid? RecordId { get; set; }
public Guid? ParentRecordId { get; set; } public Guid? ParentRecordId { get; set; }
} }
} }

6
tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Program.cs
Просмотреть файл

@ -13,8 +13,8 @@ namespace Azure.Sdk.Tools.PipelineWitness
// per queue storage performance docs, set the default connection limit to >= 100 // per queue storage performance docs, set the default connection limit to >= 100
// https://learn.microsoft.com/en-us/azure/storage/queues/storage-performance-checklist#increase-default-connection-limit // https://learn.microsoft.com/en-us/azure/storage/queues/storage-performance-checklist#increase-default-connection-limit
ServicePointManager.DefaultConnectionLimit = 100; ServicePointManager.DefaultConnectionLimit = 100;
var builder = WebApplication.CreateBuilder(args); WebApplicationBuilder builder = WebApplication.CreateBuilder(args);
// Add services to the container. // Add services to the container.
builder.Services.AddControllers(); builder.Services.AddControllers();
@ -26,7 +26,7 @@ namespace Azure.Sdk.Tools.PipelineWitness
Startup.Configure(builder); Startup.Configure(builder);
var app = builder.Build(); WebApplication app = builder.Build();
// Configure the HTTP request pipeline. // Configure the HTTP request pipeline.
if (app.Environment.IsDevelopment()) if (app.Environment.IsDevelopment())

22
tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/AzurePipelinesBuildDefinitionWorker.cs
Просмотреть файл

@ -1,12 +1,12 @@
using System; using System;
using System.Diagnostics;
using System.Threading; using System.Threading;
using System.Threading.Tasks; using System.Threading.Tasks;
using Azure.Sdk.Tools.PipelineWitness.Configuration;
using Azure.Sdk.Tools.PipelineWitness.Services.WorkTokens;
using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging;
using System.Diagnostics;
using Microsoft.Extensions.Options; using Microsoft.Extensions.Options;
using Azure.Sdk.Tools.PipelineWitness.Services.WorkTokens;
using Azure.Sdk.Tools.PipelineWitness.Configuration;
namespace Azure.Sdk.Tools.PipelineWitness.Services namespace Azure.Sdk.Tools.PipelineWitness.Services
{ {
@ -15,7 +15,7 @@ namespace Azure.Sdk.Tools.PipelineWitness.Services
private readonly ILogger<AzurePipelinesBuildDefinitionWorker> logger; private readonly ILogger<AzurePipelinesBuildDefinitionWorker> logger;
private readonly BlobUploadProcessor runProcessor; private readonly BlobUploadProcessor runProcessor;
private readonly IOptions<PipelineWitnessSettings> options; private readonly IOptions<PipelineWitnessSettings> options;
private IAsyncLockProvider asyncLockProvider; private readonly IAsyncLockProvider asyncLockProvider;
public AzurePipelinesBuildDefinitionWorker( public AzurePipelinesBuildDefinitionWorker(
ILogger<AzurePipelinesBuildDefinitionWorker> logger, ILogger<AzurePipelinesBuildDefinitionWorker> logger,
@ -31,32 +31,32 @@ namespace Azure.Sdk.Tools.PipelineWitness.Services
protected override async Task ExecuteAsync(CancellationToken stoppingToken) protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{ {
var processEvery = TimeSpan.FromMinutes(60); TimeSpan processEvery = TimeSpan.FromMinutes(60);
while (true) while (true)
{ {
var stopWatch = Stopwatch.StartNew(); Stopwatch stopWatch = Stopwatch.StartNew();
var settings = this.options.Value; PipelineWitnessSettings settings = this.options.Value;
try try
{ {
await using var asyncLock = await this.asyncLockProvider.GetLockAsync("UpdateBuildDefinitions", processEvery, stoppingToken); await using IAsyncLock asyncLock = await this.asyncLockProvider.GetLockAsync("UpdateBuildDefinitions", processEvery, stoppingToken);
// if there's no asyncLock, this process has alread completed in the last hour // if there's no asyncLock, this process has alread completed in the last hour
if (asyncLock != null) if (asyncLock != null)
{ {
foreach (var project in settings.Projects) foreach (string project in settings.Projects)
{ {
await this.runProcessor.UploadBuildDefinitionBlobsAsync(settings.Account, project); await this.runProcessor.UploadBuildDefinitionBlobsAsync(settings.Account, project);
} }
} }
} }
catch(Exception ex) catch (Exception ex)
{ {
this.logger.LogError(ex, "Error processing build definitions"); this.logger.LogError(ex, "Error processing build definitions");
} }
var duration = settings.BuildDefinitionLoopPeriod - stopWatch.Elapsed; TimeSpan duration = settings.BuildDefinitionLoopPeriod - stopWatch.Elapsed;
if (duration > TimeSpan.Zero) if (duration > TimeSpan.Zero)
{ {
await Task.Delay(duration, stoppingToken); await Task.Delay(duration, stoppingToken);

21
tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/BuildCompleteQueueWorker.cs
Просмотреть файл

@ -1,3 +1,4 @@
using System;
using System.Text.RegularExpressions; using System.Text.RegularExpressions;
using System.Threading; using System.Threading;
using System.Threading.Tasks; using System.Threading.Tasks;
@ -17,7 +18,6 @@ namespace Azure.Sdk.Tools.PipelineWitness.Services
internal class BuildCompleteQueueWorker : QueueWorkerBackgroundService internal class BuildCompleteQueueWorker : QueueWorkerBackgroundService
{ {
private readonly ILogger logger; private readonly ILogger logger;
private readonly TelemetryClient telemetryClient;
private readonly BlobUploadProcessor runProcessor; private readonly BlobUploadProcessor runProcessor;
public BuildCompleteQueueWorker( public BuildCompleteQueueWorker(
@ -35,16 +35,15 @@ namespace Azure.Sdk.Tools.PipelineWitness.Services
{ {
this.logger = logger; this.logger = logger;
this.runProcessor = runProcessor; this.runProcessor = runProcessor;
this.telemetryClient = telemetryClient;
} }
internal override async Task ProcessMessageAsync(QueueMessage message, CancellationToken cancellationToken) internal override async Task ProcessMessageAsync(QueueMessage message, CancellationToken cancellationToken)
{ {
this.logger.LogInformation("Processing build.complete event: {MessageText}", message.MessageText); this.logger.LogInformation("Processing build.complete event: {MessageText}", message.MessageText);
var devopsEvent = JObject.Parse(message.MessageText); JObject devopsEvent = JObject.Parse(message.MessageText);
var buildUrl = devopsEvent["resource"]?.Value<string>("url"); string buildUrl = devopsEvent["resource"]?.Value<string>("url");
if (buildUrl == null) if (buildUrl == null)
{ {
@ -52,7 +51,7 @@ namespace Azure.Sdk.Tools.PipelineWitness.Services
return; return;
} }
var match = Regex.Match(buildUrl, @"^https://dev.azure.com/(?<account>[\w-]+)/(?<project>[0-9a-fA-F-]+)/_apis/build/Builds/(?<build>\d+)$"); Match match = Regex.Match(buildUrl, @"^https://dev.azure.com/(?<account>[\w-]+)/(?<project>[0-9a-fA-F-]+)/_apis/build/Builds/(?<build>\d+)$");
if (!match.Success) if (!match.Success)
{ {
@ -60,23 +59,23 @@ namespace Azure.Sdk.Tools.PipelineWitness.Services
return; return;
} }
var account = match.Groups["account"].Value; string account = match.Groups["account"].Value;
var projectIdString = match.Groups["project"].Value; string projectIdString = match.Groups["project"].Value;
var buildIdString = match.Groups["build"].Value; string buildIdString = match.Groups["build"].Value;
if (!System.Guid.TryParse(projectIdString, out var projectId)) if (!Guid.TryParse(projectIdString, out Guid projectId))
{ {
this.logger.LogError("Could not parse project id as a guid '{ProjectId}'", projectIdString); this.logger.LogError("Could not parse project id as a guid '{ProjectId}'", projectIdString);
return; return;
} }
if (!int.TryParse(buildIdString, out var buildId)) if (!int.TryParse(buildIdString, out int buildId))
{ {
this.logger.LogError("Could not parse build id as a guid '{BuildId}'", buildIdString); this.logger.LogError("Could not parse build id as a guid '{BuildId}'", buildIdString);
return; return;
} }
await runProcessor.UploadBuildBlobsAsync(account, projectId, buildId); await this.runProcessor.UploadBuildBlobsAsync(account, projectId, buildId);
} }
} }
} }

14
tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/BuildLogProvider.cs
Просмотреть файл

@ -22,22 +22,22 @@ namespace Azure.Sdk.Tools.PipelineWitness.Services
public virtual async Task<IReadOnlyList<string>> GetLogLinesAsync(Build build, int logId) public virtual async Task<IReadOnlyList<string>> GetLogLinesAsync(Build build, int logId)
{ {
logger.LogTrace("Getting logs for build {BuildId}, log {LogId} from rest api", build.Id, logId); this.logger.LogTrace("Getting logs for build {BuildId}, log {LogId} from rest api", build.Id, logId);
var buildHttpClient = vssConnection.GetClient<BuildHttpClient>(); BuildHttpClient buildHttpClient = this.vssConnection.GetClient<BuildHttpClient>();
var response = await buildHttpClient.GetBuildLogLinesAsync(build.Project.Id, build.Id, logId); List<string> response = await buildHttpClient.GetBuildLogLinesAsync(build.Project.Id, build.Id, logId);
logger.LogTrace("Received {CharacterCount} characters in {LineCount} lines for build {BuildId}, log {LogId}", response.Sum(x => x.Length), response.Count, build.Id, logId); this.logger.LogTrace("Received {CharacterCount} characters in {LineCount} lines for build {BuildId}, log {LogId}", response.Sum(x => x.Length), response.Count, build.Id, logId);
return response; return response;
} }
public virtual async Task<Stream> GetLogStreamAsync(string projectName, int buildId, int logId) public virtual async Task<Stream> GetLogStreamAsync(string projectName, int buildId, int logId)
{ {
logger.LogTrace("Getting logs for build {BuildId}, log {LogId} from rest api", buildId, logId); this.logger.LogTrace("Getting logs for build {BuildId}, log {LogId} from rest api", buildId, logId);
var buildHttpClient = vssConnection.GetClient<BuildHttpClient>(); BuildHttpClient buildHttpClient = this.vssConnection.GetClient<BuildHttpClient>();
var stream = await buildHttpClient.GetBuildLogAsync(projectName, buildId, logId); Stream stream = await buildHttpClient.GetBuildLogAsync(projectName, buildId, logId);
return stream; return stream;
} }

18
tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/EnhancedBuildHttpClient.cs
Просмотреть файл

@ -14,23 +14,23 @@ namespace Azure.Sdk.Tools.PipelineWitness.Services
{ {
public EnhancedBuildHttpClient(Uri baseUrl, VssCredentials credentials) public EnhancedBuildHttpClient(Uri baseUrl, VssCredentials credentials)
: base(baseUrl, credentials) : base(baseUrl, credentials)
{} { }
public EnhancedBuildHttpClient(Uri baseUrl, VssCredentials credentials, VssHttpRequestSettings settings) public EnhancedBuildHttpClient(Uri baseUrl, VssCredentials credentials, VssHttpRequestSettings settings)
: base(baseUrl, credentials, settings) : base(baseUrl, credentials, settings)
{} { }
public EnhancedBuildHttpClient(Uri baseUrl, VssCredentials credentials, params DelegatingHandler[] handlers) public EnhancedBuildHttpClient(Uri baseUrl, VssCredentials credentials, params DelegatingHandler[] handlers)
: base(baseUrl, credentials, handlers) : base(baseUrl, credentials, handlers)
{} { }
public EnhancedBuildHttpClient(Uri baseUrl, VssCredentials credentials, VssHttpRequestSettings settings, params DelegatingHandler[] handlers) public EnhancedBuildHttpClient(Uri baseUrl, VssCredentials credentials, VssHttpRequestSettings settings, params DelegatingHandler[] handlers)
: base(baseUrl, credentials, settings, handlers) : base(baseUrl, credentials, settings, handlers)
{} { }
public EnhancedBuildHttpClient(Uri baseUrl, HttpMessageHandler pipeline, bool disposeHandler) public EnhancedBuildHttpClient(Uri baseUrl, HttpMessageHandler pipeline, bool disposeHandler)
: base(baseUrl, pipeline, disposeHandler) : base(baseUrl, pipeline, disposeHandler)
{} { }
public override async Task<Stream> GetArtifactContentZipAsync( public override async Task<Stream> GetArtifactContentZipAsync(
Guid project, Guid project,
@ -39,7 +39,7 @@ namespace Azure.Sdk.Tools.PipelineWitness.Services
object userState = null, object userState = null,
CancellationToken cancellationToken = default) CancellationToken cancellationToken = default)
{ {
var artifact = await base.GetArtifactAsync(project, buildId, artifactName, userState, cancellationToken); BuildArtifact artifact = await base.GetArtifactAsync(project, buildId, artifactName, userState, cancellationToken);
return await GetArtifactContentZipAsync(artifact, cancellationToken); return await GetArtifactContentZipAsync(artifact, cancellationToken);
} }
@ -50,19 +50,19 @@ namespace Azure.Sdk.Tools.PipelineWitness.Services
object userState = null, object userState = null,
CancellationToken cancellationToken = default) CancellationToken cancellationToken = default)
{ {
var artifact = await base.GetArtifactAsync(project, buildId, artifactName, userState, cancellationToken); BuildArtifact artifact = await base.GetArtifactAsync(project, buildId, artifactName, userState, cancellationToken);
return await GetArtifactContentZipAsync(artifact, cancellationToken); return await GetArtifactContentZipAsync(artifact, cancellationToken);
} }
private async Task<Stream> GetArtifactContentZipAsync(BuildArtifact artifact, CancellationToken cancellationToken) private async Task<Stream> GetArtifactContentZipAsync(BuildArtifact artifact, CancellationToken cancellationToken)
{ {
var downloadUrl = artifact?.Resource?.DownloadUrl; string downloadUrl = artifact?.Resource?.DownloadUrl;
if (string.IsNullOrWhiteSpace(downloadUrl)) if (string.IsNullOrWhiteSpace(downloadUrl))
{ {
throw new InvalidArtifactDataException("Artifact contained no download url"); throw new InvalidArtifactDataException("Artifact contained no download url");
} }
var responseStream = await Client.GetStreamAsync(downloadUrl, cancellationToken); Stream responseStream = await Client.GetStreamAsync(downloadUrl, cancellationToken);
return responseStream; return responseStream;
} }
} }

37
tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/AzureArtifactsServiceUnavailableClassifier.cs
Просмотреть файл

@ -1,37 +0,0 @@
using Microsoft.TeamFoundation.Build.WebApi;
using System.Linq;
using System.Threading.Tasks;
namespace Azure.Sdk.Tools.PipelineWitness.Services.FailureAnalysis
{
public class AzureArtifactsServiceUnavailableClassifier : IFailureClassifier
{
public AzureArtifactsServiceUnavailableClassifier(BuildLogProvider buildLogProvider)
{
this.buildLogProvider = buildLogProvider;
}
private readonly BuildLogProvider buildLogProvider;
public async Task ClassifyAsync(FailureAnalyzerContext context)
{
var failedTasks = from r in context.Timeline.Records
where r.Result == TaskResult.Failed
where r.RecordType == "Task"
where r.Task != null
where r.Name == "Publish to Java Dev Feed"
where r.Log != null
select r;
foreach (var failedTask in failedTasks)
{
var lines = await this.buildLogProvider.GetLogLinesAsync(context.Build, failedTask.Log.Id);
if (lines.Any(line => line.Contains("Transfer failed for https://pkgs.dev.azure.com") && line.Contains("503 Service Unavailable")))
{
context.AddFailure(failedTask, "Azure Artifacts Service Unavailable");
}
}
}
}
}

26
tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/AzurePipelinesPoolOutageClassifier.cs
Просмотреть файл

@ -1,26 +0,0 @@
using System.Linq;
using System.Threading.Tasks;
namespace Azure.Sdk.Tools.PipelineWitness.Services.FailureAnalysis
{
public class AzurePipelinesPoolOutageClassifier : IFailureClassifier
{
public Task ClassifyAsync(FailureAnalyzerContext context)
{
var jobs = from r in context.Timeline.Records
where r.RecordType == "Job"
where r.Issues.Any(i => i.Message.Contains("abandoned due to an infrastructure failure"))
select r;
if (jobs.Count() > 0)
{
foreach (var job in jobs)
{
context.AddFailure(job, "Azure Pipelines Pool Outage");
}
}
return Task.CompletedTask;
}
}
}

31
tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/AzurePowerShellModuleInstallationFailureClassifier.cs
Просмотреть файл

@ -1,31 +0,0 @@
using Microsoft.TeamFoundation.Build.WebApi;
using System.Linq;
using System.Threading.Tasks;
namespace Azure.Sdk.Tools.PipelineWitness.Services.FailureAnalysis
{
public class AzurePowerShellModuleInstallationFailureClassifier : IFailureClassifier
{
public Task ClassifyAsync(FailureAnalyzerContext context)
{
if (context.Build.Definition.Name.EndsWith("- tests"))
{
var failedTasks = from r in context.Timeline.Records
where r.Result == TaskResult.Failed
where r.RecordType == "Task"
where r.Name == "Install Azure PowerShell module"
select r;
if (failedTasks.Count() > 0)
{
foreach (var failedTask in failedTasks)
{
context.AddFailure(failedTask, "Azure PS Module");
}
}
}
return Task.CompletedTask;
}
}
}

28
tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/AzuriteInstallFailureClassifier.cs
Просмотреть файл

@ -1,28 +0,0 @@
using Microsoft.TeamFoundation.Build.WebApi;
using System.Linq;
using System.Threading.Tasks;
namespace Azure.Sdk.Tools.PipelineWitness.Services.FailureAnalysis
{
public class AzuriteInstallFailureClassifier : IFailureClassifier
{
public Task ClassifyAsync(FailureAnalyzerContext context)
{
var failedTasks = from r in context.Timeline.Records
where r.Result == TaskResult.Failed
where r.RecordType == "Task"
where r.Name == "Install Azurite"
select r;
if (failedTasks.Count() > 0)
{
foreach (var failedTask in failedTasks)
{
context.AddFailure(failedTask, "Azurite Install");
}
}
return Task.CompletedTask;
}
}
}

75
tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/CacheFailureClassifier.cs
Просмотреть файл

@ -1,75 +0,0 @@
using Microsoft.TeamFoundation.Build.WebApi;
using Microsoft.VisualStudio.Services.WebApi;
using System.Linq;
using System.Threading.Tasks;
namespace Azure.Sdk.Tools.PipelineWitness.Services.FailureAnalysis
{
public class CacheFailureClassifier : IFailureClassifier
{
private class FailureClassifier
{
public readonly string MessageContains;
public readonly string FailureName;
public FailureClassifier(string messageContains, string failureName)
{
this.MessageContains = messageContains;
this.FailureName = failureName;
}
public bool IsFailure(string message)
{
return message.StartsWith(MessageContains);
}
}
private static readonly FailureClassifier[] failureClassifiers = new FailureClassifier[]
{
new FailureClassifier("Chunks are not arriving in order or sizes are not matched up", "Cache Chunk Ordering" ),
new FailureClassifier("The task has timed out", "Cache Task Timeout" ),
new FailureClassifier("Service Unavailable", "Cache Service Unavailable"),
new FailureClassifier("The HTTP request timed out after", "Cache Service HTTP Timeout"),
new FailureClassifier("Access to the path", "Cache Cannot Access Path"),
};
public CacheFailureClassifier(VssConnection vssConnection)
{
this.vssConnection = vssConnection;
buildClient = vssConnection.GetClient<BuildHttpClient>();
}
private VssConnection vssConnection;
private BuildHttpClient buildClient;
public Task ClassifyAsync(FailureAnalyzerContext context)
{
var failedTasks = from r in context.Timeline.Records
where r.Result == TaskResult.Failed
where r.RecordType == "Task"
where r.Task != null
where r.Task.Name == "Cache"
where r.Log != null
select r;
var classificationFound = false;
foreach (var failedTask in failedTasks)
{
foreach (var classifier in failureClassifiers) {
if (failedTask.Issues.Any(i => classifier.IsFailure(i.Message)))
{
context.AddFailure(failedTask, classifier.FailureName);
classificationFound = true;
}
}
if (!classificationFound)
{
context.AddFailure(failedTask, "Cache Failure Other");
}
}
return Task.CompletedTask;
}
}
}

27
tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/CancelledTaskClassifier.cs
Просмотреть файл

@ -1,27 +0,0 @@
using Microsoft.TeamFoundation.Build.WebApi;
using System.Linq;
using System.Threading.Tasks;
namespace Azure.Sdk.Tools.PipelineWitness.Services.FailureAnalysis
{
public class CancelledTaskClassifier : IFailureClassifier
{
public Task ClassifyAsync(FailureAnalyzerContext context)
{
var timedOutTestTasks = from r in context.Timeline.Records
where r.RecordType == "Task"
where r.Result == TaskResult.Canceled
select r;
if (timedOutTestTasks.Count() > 0)
{
foreach (var timedOutTestTask in timedOutTestTasks)
{
context.AddFailure(timedOutTestTask, "Cancelled Task");
}
}
return Task.CompletedTask;
}
}
}

30
tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/CodeSigningFailureClassifier.cs
Просмотреть файл

@ -1,30 +0,0 @@
using System.Linq;
using System.Threading.Tasks;
using Microsoft.TeamFoundation.Build.WebApi;
namespace Azure.Sdk.Tools.PipelineWitness.Services.FailureAnalysis
{
public class CodeSigningFailureClassifier : IFailureClassifier
{
public Task ClassifyAsync(FailureAnalyzerContext context)
{
var failedTasks = from r in context.Timeline.Records
where r.Result == TaskResult.Failed
where r.RecordType == "Task"
where r.Task != null
where r.Task.Name == "EsrpCodeSigning"
where r.Log != null
select r;
if (failedTasks.Count() > 0)
{
foreach (var failedTask in failedTasks)
{
context.AddFailure(failedTask, "Code Signing");
}
}
return Task.CompletedTask;
}
}
}

28
tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/CosmosDbEmulatorStartFailureClassifier.cs
Просмотреть файл

@ -1,28 +0,0 @@
using System.Linq;
using System.Threading.Tasks;
using Microsoft.TeamFoundation.Build.WebApi;
namespace Azure.Sdk.Tools.PipelineWitness.Services.FailureAnalysis
{
public class CosmosDbEmulatorStartFailureClassifier : IFailureClassifier
{
public Task ClassifyAsync(FailureAnalyzerContext context)
{
var failedTasks = from r in context.Timeline.Records
where r.RecordType == "Task"
where r.Name == "Start Cosmos DB Emulator"
where r.Result == TaskResult.Failed
select r;
if (failedTasks.Count() > 0)
{
foreach (var failedTask in failedTasks)
{
context.AddFailure(failedTask, "Cosmos DB Emulator Failure");
}
}
return Task.CompletedTask;
}
}
}

45
tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/DnsResolutionFailureClassifier.cs
Просмотреть файл

@ -1,45 +0,0 @@
using Microsoft.TeamFoundation.Build.WebApi;
using System;
using System.Linq;
using System.Threading.Tasks;
namespace Azure.Sdk.Tools.PipelineWitness.Services.FailureAnalysis
{
public class DnsResolutionFailureClassifier : IFailureClassifier
{
public DnsResolutionFailureClassifier(BuildLogProvider buildLogProvider)
{
this.buildLogProvider = buildLogProvider;
}
private readonly BuildLogProvider buildLogProvider;
private bool IsDnsResolutionFailure(string line)
{
return line.Contains("EAI_AGAIN", StringComparison.OrdinalIgnoreCase)
|| line.Contains("getaddrinfo", StringComparison.OrdinalIgnoreCase)
|| line.Contains("Temporary failure in name resolution", StringComparison.OrdinalIgnoreCase)
|| line.Contains("No such host is known", StringComparison.OrdinalIgnoreCase)
|| line.Contains("Couldn't resolve host name", StringComparison.OrdinalIgnoreCase);
}
public async Task ClassifyAsync(FailureAnalyzerContext context)
{
var failedTasks = from r in context.Timeline.Records
where r.Result == TaskResult.Failed
where r.RecordType == "Task"
where r.Log != null
select r;
foreach (var failedTask in failedTasks)
{
var lines = await buildLogProvider.GetLogLinesAsync(context.Build, failedTask.Log.Id);
if (lines.Any(line => IsDnsResolutionFailure(line)))
{
context.AddFailure(failedTask, "DNS Resolution Failure");
}
}
}
}
}

31
tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/DotnetPipelineTestFailureClassifier.cs
Просмотреть файл

@ -1,31 +0,0 @@
using Microsoft.TeamFoundation.Build.WebApi;
using System.Linq;
using System.Threading.Tasks;
namespace Azure.Sdk.Tools.PipelineWitness.Services.FailureAnalysis
{
public class DotnetPipelineTestFailureClassifier : IFailureClassifier
{
public Task ClassifyAsync(FailureAnalyzerContext context)
{
if (context.Build.Definition.Name.StartsWith("net - "))
{
var failedTasks = from r in context.Timeline.Records
where r.Result == TaskResult.Failed
where r.RecordType == "Task"
where r.Name.StartsWith("Build & Test")
select r;
if (failedTasks.Count() > 0)
{
foreach (var failedTask in failedTasks)
{
context.AddFailure(failedTask, "Test Failure");
}
}
}
return Task.CompletedTask;
}
}
}

28
tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/DownloadSecretsFailureClassifier.cs
Просмотреть файл

@ -1,28 +0,0 @@
using Microsoft.TeamFoundation.Build.WebApi;
using System.Linq;
using System.Threading.Tasks;
namespace Azure.Sdk.Tools.PipelineWitness.Services.FailureAnalysis
{
public class DownloadSecretsFailureClassifier : IFailureClassifier
{
public Task ClassifyAsync(FailureAnalyzerContext context)
{
var failedTasks = from r in context.Timeline.Records
where r.RecordType == "Task"
where r.Result == TaskResult.Failed
where r.Name.Contains("Download secrets")
select r;
if (failedTasks.Count() > 0)
{
foreach (var failedTask in failedTasks)
{
context.AddFailure(failedTask, "Secrets Failure");
}
}
return Task.CompletedTask;
}
}
}

46
tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/FailureAnalyzer.cs
Просмотреть файл

@ -1,46 +0,0 @@
using Azure.Sdk.Tools.PipelineWitness.Entities.AzurePipelines;
using Microsoft.TeamFoundation.Build.WebApi;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
namespace Azure.Sdk.Tools.PipelineWitness.Services.FailureAnalysis
{
public class FailureAnalyzer : IFailureAnalyzer
{
public FailureAnalyzer(IEnumerable<IFailureClassifier> classifiers)
{
this.classifiers = classifiers.ToArray();
}
private IFailureClassifier[] classifiers;
public async Task<IEnumerable<Failure>> AnalyzeFailureAsync(Build build, Timeline timeline)
{
var failures = new List<Failure>();
var context = new FailureAnalyzerContext(build, timeline, failures);
foreach (var classifier in classifiers)
{
await classifier.ClassifyAsync(context);
}
if (failures.Count == 0)
{
if (build.Result != BuildResult.Succeeded &&
build.Result != BuildResult.Canceled)
{
foreach (var record in timeline.Records.Where(x => x.ParentId.HasValue == false))
{
if (record.Result == TaskResult.Failed)
{
failures.Add(new Failure(record, "Unknown"));
}
}
}
}
return failures;
}
}
}

57
tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/FailureAnalyzerContext.cs
Просмотреть файл

@ -1,57 +0,0 @@
using Azure.Sdk.Tools.PipelineWitness.Entities.AzurePipelines;
using Microsoft.TeamFoundation.Build.WebApi;
using Microsoft.VisualStudio.Services.Common;
using System.Collections.Generic;
using System.Linq;
using System.Text;
namespace Azure.Sdk.Tools.PipelineWitness.Services.FailureAnalysis
{
public class FailureAnalyzerContext
{
public FailureAnalyzerContext(Build build, Timeline timeline, IList<Failure> failures)
{
Build = build;
Timeline = timeline;
this.failures = failures;
}
public Build Build { get; private set; }
public Timeline Timeline { get; private set; }
private IList<Failure> failures;
private string GetScope(TimelineRecord record)
{
var timelineStack = new Stack<TimelineRecord>();
var current = record;
while (true)
{
timelineStack.Push(current);
var parent = Timeline.Records.Where(r => r.Id == current.ParentId).SingleOrDefault();
if (parent == null)
{
break;
}
else
{
current = parent;
}
}
var scopeBuilder = new StringBuilder();
timelineStack.ForEach(r => scopeBuilder.Append($"/{r.RecordType}:{r.Name}"));
var scope = scopeBuilder.ToString();
return scope;
}
public void AddFailure(TimelineRecord record, string classification)
{
var failure = new Failure(record, classification);
failures.Add(failure);
}
}
}

27
tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/GitCheckoutFailureClassifier.cs
Просмотреть файл

@ -1,27 +0,0 @@
using System.Linq;
using System.Threading.Tasks;
namespace Azure.Sdk.Tools.PipelineWitness.Services.FailureAnalysis
{
public class GitCheckoutFailureClassifier : IFailureClassifier
{
public Task ClassifyAsync(FailureAnalyzerContext context)
{
var tasks = from r in context.Timeline.Records
where r.RecordType == "Task"
where r.Issues.Any(i => i.Message.Contains("Git fetch failed with exit code: 128"))
select r;
if (tasks.Count() > 0)
{
foreach (var task in tasks)
{
context.AddFailure(task
, "Git Checkout");
}
}
return Task.CompletedTask;
}
}
}

12
tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/IFailureAnalyzer.cs
Просмотреть файл

@ -1,12 +0,0 @@
using Azure.Sdk.Tools.PipelineWitness.Entities.AzurePipelines;
using Microsoft.TeamFoundation.Build.WebApi;
using System.Collections.Generic;
using System.Threading.Tasks;
namespace Azure.Sdk.Tools.PipelineWitness.Services.FailureAnalysis
{
public interface IFailureAnalyzer
{
Task<IEnumerable<Failure>> AnalyzeFailureAsync(Build build, Timeline timeline);
}
}

9
tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/IFailureClassifier.cs
Просмотреть файл

@ -1,9 +0,0 @@
using System.Threading.Tasks;
namespace Azure.Sdk.Tools.PipelineWitness.Services.FailureAnalysis
{
public interface IFailureClassifier
{
Task ClassifyAsync(FailureAnalyzerContext context);
}
}

31
tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/JavaPipelineTestFailureClassifier.cs
Просмотреть файл

@ -1,31 +0,0 @@
using Microsoft.TeamFoundation.Build.WebApi;
using System.Linq;
using System.Threading.Tasks;
namespace Azure.Sdk.Tools.PipelineWitness.Services.FailureAnalysis
{
public class JavaPipelineTestFailureClassifier : IFailureClassifier
{
public Task ClassifyAsync(FailureAnalyzerContext context)
{
if (context.Build.Definition.Name.StartsWith("java - "))
{
var failedTasks = from r in context.Timeline.Records
where r.Result == TaskResult.Failed
where r.RecordType == "Task"
where r.Name.StartsWith("Run tests")
select r;
if (failedTasks.Count() > 0)
{
foreach (var failedTask in failedTasks)
{
context.AddFailure(failedTask, "Test Failure");
}
}
}
return Task.CompletedTask;
}
}
}

31
tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/JavaScriptLiveTestFailureClassifier.cs
Просмотреть файл

@ -1,31 +0,0 @@
using System.Linq;
using System.Threading.Tasks;
using Microsoft.TeamFoundation.Build.WebApi;
namespace Azure.Sdk.Tools.PipelineWitness.Services.FailureAnalysis
{
public class JavaScriptLiveTestFailureClassifier : IFailureClassifier
{
public Task ClassifyAsync(FailureAnalyzerContext context)
{
if (context.Build.Definition.Name.StartsWith("js - ") && context.Build.Definition.Name.EndsWith(" - tests"))
{
var failedTasks = from r in context.Timeline.Records
where r.RecordType == "Task"
where r.Name == "Integration test libraries"
where r.Result == TaskResult.Failed
select r;
if (failedTasks.Count() > 0)
{
foreach (var failedTask in failedTasks)
{
context.AddFailure(failedTask, "Test Failure");
}
}
}
return Task.CompletedTask;
}
}
}

31
tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/JsDevFeedPublishingFailureClassifier.cs
Просмотреть файл

@ -1,31 +0,0 @@
using Microsoft.TeamFoundation.Build.WebApi;
using System.Linq;
using System.Threading.Tasks;
namespace Azure.Sdk.Tools.PipelineWitness.Services.FailureAnalysis
{
public class JsDevFeedPublishingFailureClassifier : IFailureClassifier
{
public Task ClassifyAsync(FailureAnalyzerContext context)
{
if (context.Build.Definition.Name.StartsWith("js -"))
{
var failedJobs = from r in context.Timeline.Records
where r.Name == "Publish package to daily feed"
where r.RecordType == "Job"
where r.Result == TaskResult.Failed
select r;
if (failedJobs.Count() > 0)
{
foreach (var failedJob in failedJobs)
{
context.AddFailure(failedJob, "Publish Failure");
}
}
}
return Task.CompletedTask;
}
}
}

30
tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/JsSamplesExecutionFailureClassifier.cs
Просмотреть файл

@ -1,30 +0,0 @@
using Microsoft.TeamFoundation.Build.WebApi;
using System.Linq;
using System.Threading.Tasks;
namespace Azure.Sdk.Tools.PipelineWitness.Services.FailureAnalysis
{
public class JsSamplesExecutionFailureClassifier : IFailureClassifier
{
public Task ClassifyAsync(FailureAnalyzerContext context)
{
if (context.Build.Definition.Name.StartsWith("js -"))
{
var failedTasks = from r in context.Timeline.Records
where r.Name == "Execute Samples"
where r.Result == TaskResult.Failed
select r;
if (failedTasks.Count() > 0)
{
foreach (var failedTask in failedTasks)
{
context.AddFailure(failedTask, "Sample Execution");
}
}
}
return Task.CompletedTask;
}
}
}

37
tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/MavenBrokenPipeFailureClassifier.cs
Просмотреть файл

@ -1,37 +0,0 @@
using Microsoft.TeamFoundation.Build.WebApi;
using System.Linq;
using System.Threading.Tasks;
namespace Azure.Sdk.Tools.PipelineWitness.Services.FailureAnalysis
{
public class MavenBrokenPipeFailureClassifier : IFailureClassifier
{
public MavenBrokenPipeFailureClassifier(BuildLogProvider buildLogProvider)
{
this.buildLogProvider = buildLogProvider;
}
private readonly BuildLogProvider buildLogProvider;
public async Task ClassifyAsync(FailureAnalyzerContext context)
{
var failedTasks = from r in context.Timeline.Records
where r.Result == TaskResult.Failed
where r.RecordType == "Task"
where r.Task != null
where r.Task.Name == "Maven"
where r.Log != null
select r;
foreach (var failedTask in failedTasks)
{
var lines = await buildLogProvider.GetLogLinesAsync(context.Build, failedTask.Log.Id);
if (lines.Any(line => line.Contains("Connection reset") || line.Contains("Connection timed out") || line.Contains("504 Gateway Timeout")))
{
context.AddFailure(failedTask, "Maven Broken Pipe");
}
}
}
}
}

31
tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/PythonPipelineTestFailureClassifier.cs
Просмотреть файл

@ -1,31 +0,0 @@
using Microsoft.TeamFoundation.Build.WebApi;
using System.Linq;
using System.Threading.Tasks;
namespace Azure.Sdk.Tools.PipelineWitness.Services.FailureAnalysis
{
public class PythonPipelineTestFailureClassifier : IFailureClassifier
{
public Task ClassifyAsync(FailureAnalyzerContext context)
{
if (context.Build.Definition.Name.StartsWith("python - "))
{
var failedTasks = from r in context.Timeline.Records
where r.Result == TaskResult.Failed
where r.RecordType == "Task"
where r.Name == "Run Tests"
select r;
if (failedTasks.Count() > 0)
{
foreach (var failedTask in failedTasks)
{
context.AddFailure(failedTask, "Test Failure");
}
}
}
return Task.CompletedTask;
}
}
}

27
tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/TestResourcesDeploymentFailureClassifier.cs
Просмотреть файл

@ -1,27 +0,0 @@
using Microsoft.TeamFoundation.Build.WebApi;
using System.Linq;
using System.Threading.Tasks;
namespace Azure.Sdk.Tools.PipelineWitness.Services.FailureAnalysis
{
public class TestResourcesDeploymentFailureClassifier : IFailureClassifier
{
public Task ClassifyAsync(FailureAnalyzerContext context)
{
var failedTasks = from r in context.Timeline.Records
where r.Name.StartsWith("Deploy test resources")
where r.Result == TaskResult.Failed
select r;
if (failedTasks.Count() > 0)
{
foreach (var failedTask in failedTasks)
{
context.AddFailure(failedTask, "Test Resource Failure");
}
}
return Task.CompletedTask;
}
}
}

135
tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/QueueWorkerBackgroundService.cs
Просмотреть файл

@ -1,4 +1,4 @@
using System; using System;
using System.Diagnostics; using System.Diagnostics;
using System.Threading; using System.Threading;
using System.Threading.Tasks; using System.Threading.Tasks;
@ -9,6 +9,7 @@ using Azure.Storage.Queues.Models;
using Microsoft.ApplicationInsights; using Microsoft.ApplicationInsights;
using Microsoft.ApplicationInsights.DataContracts; using Microsoft.ApplicationInsights.DataContracts;
using Microsoft.ApplicationInsights.Extensibility;
using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options; using Microsoft.Extensions.Options;
@ -18,7 +19,7 @@ namespace Azure.Sdk.Tools.PipelineWitness.Services
internal abstract class QueueWorkerBackgroundService : BackgroundService internal abstract class QueueWorkerBackgroundService : BackgroundService
{ {
private const string ActivitySourceName = "Azure.Sdk.Tools.PipelineWitness.Queue"; private const string ActivitySourceName = "Azure.Sdk.Tools.PipelineWitness.Queue";
private static readonly ActivitySource activitySource = new ActivitySource(ActivitySourceName); private static readonly ActivitySource activitySource = new(ActivitySourceName);
private readonly ILogger logger; private readonly ILogger logger;
private readonly QueueServiceClient queueServiceClient; private readonly QueueServiceClient queueServiceClient;
@ -38,7 +39,7 @@ namespace Azure.Sdk.Tools.PipelineWitness.Services
this.queueServiceClient = queueServiceClient ?? throw new ArgumentNullException(nameof(options)); this.queueServiceClient = queueServiceClient ?? throw new ArgumentNullException(nameof(options));
this.options = options ?? throw new ArgumentNullException(nameof(options)); this.options = options ?? throw new ArgumentNullException(nameof(options));
if(string.IsNullOrWhiteSpace(queueName)) if (string.IsNullOrWhiteSpace(queueName))
{ {
throw new ArgumentException("Parameter cannot be null or whitespace", nameof(queueName)); throw new ArgumentException("Parameter cannot be null or whitespace", nameof(queueName));
} }
@ -50,22 +51,22 @@ namespace Azure.Sdk.Tools.PipelineWitness.Services
{ {
this.logger.LogInformation("Starting ExecuteAsync for {TypeName}", this.GetType().Name); this.logger.LogInformation("Starting ExecuteAsync for {TypeName}", this.GetType().Name);
var poisonQueueName = $"{this.queueName}-poison"; string poisonQueueName = $"{this.queueName}-poison";
var queueClient = this.queueServiceClient.GetQueueClient(this.queueName); QueueClient queueClient = this.queueServiceClient.GetQueueClient(this.queueName);
var poisonQueueClient = this.queueServiceClient.GetQueueClient(poisonQueueName); QueueClient poisonQueueClient = this.queueServiceClient.GetQueueClient(poisonQueueName);
await queueClient.CreateIfNotExistsAsync(); await queueClient.CreateIfNotExistsAsync(cancellationToken: stoppingToken);
await poisonQueueClient.CreateIfNotExistsAsync(); await poisonQueueClient.CreateIfNotExistsAsync(cancellationToken: stoppingToken);
while (true) while (true)
{ {
using var loopActivity = activitySource.CreateActivity("MessageLoopIteration", ActivityKind.Internal) ?? new Activity("MessageLoopIteration"); using Activity loopActivity = activitySource.CreateActivity("MessageLoopIteration", ActivityKind.Internal) ?? new Activity("MessageLoopIteration");
loopActivity?.AddBaggage("QueueName", queueClient.Name); loopActivity?.AddBaggage("QueueName", queueClient.Name);
using var loopOperation = this.telemetryClient.StartOperation<RequestTelemetry>(loopActivity);
var options = this.options.CurrentValue; using IOperationHolder<RequestTelemetry> loopOperation = this.telemetryClient.StartOperation<RequestTelemetry>(loopActivity);
PipelineWitnessSettings options = this.options.CurrentValue;
this.logger.LogDebug("Getting next message from queue {QueueName}", queueClient.Name); this.logger.LogDebug("Getting next message from queue {QueueName}", queueClient.Name);
@ -74,7 +75,7 @@ namespace Azure.Sdk.Tools.PipelineWitness.Services
// We consider a message leased when it's made invisible in the queue and the current process has a // We consider a message leased when it's made invisible in the queue and the current process has a
// valid PopReceipt for the message. The PopReceipt is used to perform subsequent operations on the // valid PopReceipt for the message. The PopReceipt is used to perform subsequent operations on the
// "leased" message. // "leased" message.
QueueMessage message = await queueClient.ReceiveMessageAsync(options.MessageLeasePeriod); QueueMessage message = await queueClient.ReceiveMessageAsync(options.MessageLeasePeriod, stoppingToken);
if (message == null) if (message == null)
{ {
@ -92,65 +93,63 @@ namespace Azure.Sdk.Tools.PipelineWitness.Services
}); });
} }
using (var activity = activitySource.CreateActivity("ProcessMessage", ActivityKind.Internal) ?? new Activity("ProcessMessage")) using Activity activity = activitySource.CreateActivity("ProcessMessage", ActivityKind.Internal) ?? new Activity("ProcessMessage");
activity?.AddBaggage("MessageId", message.MessageId);
using IOperationHolder<RequestTelemetry> operation = this.telemetryClient.StartOperation<RequestTelemetry>(activity);
try
{ {
activity?.AddBaggage("MessageId", message.MessageId); this.logger.LogDebug("The queue returned a message.\n Queue: {Queue}\n Message: {MessageId}\n Dequeue Count: {DequeueCount}\n Pop Receipt: {PopReceipt}", queueClient.Name, message.MessageId, message.DequeueCount, message.PopReceipt);
using var operation = this.telemetryClient.StartOperation<RequestTelemetry>(activity); using CancellationTokenSource cts = CancellationTokenSource.CreateLinkedTokenSource(stoppingToken);
try // Because processing a message may take longer than our initial lease period, we want to continually
// renew our lease until processing completes.
Task<string> renewTask = RenewMessageLeaseAsync(queueClient, message, cts.Token);
Task<bool> processTask = SafelyProcessMessageAsync(message, cts.Token);
Task[] tasks = new Task[] { renewTask, processTask };
Task.WaitAny(tasks, CancellationToken.None);
cts.Cancel();
Task.WaitAll(tasks, CancellationToken.None);
// if the renew task doesn't complete successfully, we can't trust the PopReceipt on the message and must abort.
string latestPopReceipt = await renewTask;
if (processTask.IsCompletedSuccessfully && processTask.Result == true)
{ {
this.logger.LogDebug("The queue returned a message.\n Queue: {Queue}\n Message: {MessageId}\n Dequeue Count: {DequeueCount}\n Pop Receipt: {PopReceipt}", queueClient.Name, message.MessageId, message.DequeueCount, message.PopReceipt); this.logger.LogDebug("Message processed successfully. Removing message from queue.\n MessageId: {MessageId}\n Queue: {QueueName}\n PopReceipt: {PopReceipt}", message.MessageId, queueClient.Name, latestPopReceipt);
await queueClient.DeleteMessageAsync(message.MessageId, latestPopReceipt, stoppingToken);
using var cts = CancellationTokenSource.CreateLinkedTokenSource(stoppingToken); activity?.SetStatus(ActivityStatusCode.Ok);
operation.Telemetry.Success = true;
// Because processing a message may take longer than our initial lease period, we want to continually }
// renew our lease until processing completes. else
var renewTask = RenewMessageLeaseAsync(queueClient, message, cts.Token); {
var processTask = SafelyProcessMessageAsync(message, cts.Token); activity?.SetStatus(ActivityStatusCode.Error);
operation.Telemetry.Success = false;
var tasks = new Task[] { renewTask, processTask }; if (message.DequeueCount > options.MaxDequeueCount)
Task.WaitAny(tasks, CancellationToken.None);
cts.Cancel();
Task.WaitAll(tasks, CancellationToken.None);
// if the renew task doesn't complete successfully, we can't trust the PopReceipt on the message and must abort.
var latestPopReceipt = await renewTask;
if (processTask.IsCompletedSuccessfully && processTask.Result == true)
{ {
this.logger.LogDebug("Message processed successfully. Removing message from queue.\n MessageId: {MessageId}\n Queue: {QueueName}\n PopReceipt: {PopReceipt}", message.MessageId, queueClient.Name, latestPopReceipt); this.logger.LogError("Message {MessageId} exceeded maximum dequeue count. Moving to poison queue {QueueName}", message.MessageId, poisonQueueClient.Name);
await poisonQueueClient.SendMessageAsync(message.Body, cancellationToken: stoppingToken);
this.logger.LogDebug("Removing message from queue.\n MessageId: {MessageId}\n Queue: {QueueName}\n PopReceipt: {PopReceipt}", message.MessageId, queueClient.Name, latestPopReceipt);
await queueClient.DeleteMessageAsync(message.MessageId, latestPopReceipt, stoppingToken); await queueClient.DeleteMessageAsync(message.MessageId, latestPopReceipt, stoppingToken);
activity?.SetStatus(ActivityStatusCode.Ok);
operation.Telemetry.Success = true;
} }
else else
{ {
activity?.SetStatus(ActivityStatusCode.Error); this.logger.LogError("Resetting message visibility timeout to {SleepPeriod}.\n MessageId: {MessageId}\n Queue: {QueueName}\n PopReceipt: {PopReceipt}", options.MessageErrorSleepPeriod, message.MessageId, queueClient.Name, latestPopReceipt);
operation.Telemetry.Success = false; await queueClient.UpdateMessageAsync(message.MessageId, latestPopReceipt, message.Body, options.MessageErrorSleepPeriod, cancellationToken: stoppingToken);
if (message.DequeueCount > options.MaxDequeueCount)
{
this.logger.LogError("Message {MessageId} exceeded maximum dequeue count. Moving to poison queue {QueueName}", message.MessageId, poisonQueueClient.Name);
await poisonQueueClient.SendMessageAsync(message.Body, cancellationToken: stoppingToken);
this.logger.LogDebug("Removing message from queue.\n MessageId: {MessageId}\n Queue: {QueueName}\n PopReceipt: {PopReceipt}", message.MessageId, queueClient.Name, latestPopReceipt);
await queueClient.DeleteMessageAsync(message.MessageId, latestPopReceipt, stoppingToken);
}
else
{
this.logger.LogError("Resetting message visibility timeout to {SleepPeriod}.\n MessageId: {MessageId}\n Queue: {QueueName}\n PopReceipt: {PopReceipt}", options.MessageErrorSleepPeriod, message.MessageId, queueClient.Name, latestPopReceipt);
await queueClient.UpdateMessageAsync(message.MessageId, latestPopReceipt, message.Body, options.MessageErrorSleepPeriod, cancellationToken: stoppingToken);
}
} }
} }
catch (Exception ex) }
{ catch (Exception ex)
this.logger.LogError(ex, "Exception thrown while procesing queue message."); {
activity?.SetStatus(ActivityStatusCode.Error); this.logger.LogError(ex, "Exception thrown while procesing queue message.");
operation.Telemetry.Success = false; activity?.SetStatus(ActivityStatusCode.Error);
} operation.Telemetry.Success = false;
} }
} }
catch (Exception ex) catch (Exception ex)
@ -172,12 +171,12 @@ namespace Azure.Sdk.Tools.PipelineWitness.Services
/// <returns>the current pop receipt (optimistic concurrency control) for the message.</returns> /// <returns>the current pop receipt (optimistic concurrency control) for the message.</returns>
private async Task<string> RenewMessageLeaseAsync(QueueClient queueClient, QueueMessage message, CancellationToken cancellationToken) private async Task<string> RenewMessageLeaseAsync(QueueClient queueClient, QueueMessage message, CancellationToken cancellationToken)
{ {
var leasePeriod = this.options.CurrentValue.MessageLeasePeriod; TimeSpan leasePeriod = this.options.CurrentValue.MessageLeasePeriod;
var halfLife = new TimeSpan(leasePeriod.Ticks / 2); TimeSpan halfLife = new(leasePeriod.Ticks / 2);
var queueName = queueClient.Name; string queueName = queueClient.Name;
var messageId = message.MessageId; string messageId = message.MessageId;
var popReceipt = message.PopReceipt; string popReceipt = message.PopReceipt;
var nextVisibleOn = message.NextVisibleOn; DateTimeOffset? nextVisibleOn = message.NextVisibleOn;
try try
{ {
@ -191,7 +190,7 @@ namespace Azure.Sdk.Tools.PipelineWitness.Services
this.logger.LogDebug("Extending visibility timeout for message.\n Queue: {Queue}\n Message: {MessageId}\n Pop Receipt: {PopReceipt}\n Visible in: {VisibleIn}", queueName, messageId, popReceipt, nextVisibleOn - DateTimeOffset.UtcNow); this.logger.LogDebug("Extending visibility timeout for message.\n Queue: {Queue}\n Message: {MessageId}\n Pop Receipt: {PopReceipt}\n Visible in: {VisibleIn}", queueName, messageId, popReceipt, nextVisibleOn - DateTimeOffset.UtcNow);
UpdateReceipt receipt = await queueClient.UpdateMessageAsync(messageId, popReceipt, visibilityTimeout: leasePeriod, cancellationToken: cancellationToken); UpdateReceipt receipt = await queueClient.UpdateMessageAsync(messageId, popReceipt, visibilityTimeout: leasePeriod, cancellationToken: cancellationToken);
var oldPopReceipt = popReceipt; string oldPopReceipt = popReceipt;
popReceipt = receipt.PopReceipt; popReceipt = receipt.PopReceipt;
nextVisibleOn = receipt.NextVisibleOn; nextVisibleOn = receipt.NextVisibleOn;

6
tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/WorkTokens/CosmosAsyncLock.cs
Просмотреть файл

@ -33,7 +33,7 @@ namespace Azure.Sdk.Tools.PipelineWitness.Services.WorkTokens
{ {
await this.container.DeleteItemAsync<CosmosLockDocument>(this.id, this.partitionKey, new ItemRequestOptions { IfMatchEtag = this.etag }); await this.container.DeleteItemAsync<CosmosLockDocument>(this.id, this.partitionKey, new ItemRequestOptions { IfMatchEtag = this.etag });
} }
catch(CosmosException ex) when (ex.StatusCode == HttpStatusCode.NotFound) catch (CosmosException ex) when (ex.StatusCode == HttpStatusCode.NotFound)
{ {
} }
} }
@ -43,7 +43,7 @@ namespace Azure.Sdk.Tools.PipelineWitness.Services.WorkTokens
{ {
try try
{ {
var response = await this.container.ReplaceItemAsync( ItemResponse<CosmosLockDocument> response = await this.container.ReplaceItemAsync(
new CosmosLockDocument(this.id, this.duration), new CosmosLockDocument(this.id, this.duration),
this.id, this.id,
this.partitionKey, this.partitionKey,
@ -56,7 +56,7 @@ namespace Azure.Sdk.Tools.PipelineWitness.Services.WorkTokens
return true; return true;
} }
} }
catch (CosmosException ex) when(ex.StatusCode == HttpStatusCode.Conflict) catch (CosmosException ex) when (ex.StatusCode == HttpStatusCode.Conflict)
{ {
} }

10
tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/WorkTokens/CosmosAsyncLockProvider.cs
Просмотреть файл

@ -23,10 +23,10 @@ namespace Azure.Sdk.Tools.PipelineWitness.Services.WorkTokens
public async Task<IAsyncLock> GetLockAsync(string id, TimeSpan duration, CancellationToken cancellationToken) public async Task<IAsyncLock> GetLockAsync(string id, TimeSpan duration, CancellationToken cancellationToken)
{ {
var partitionKey = new PartitionKey(id); PartitionKey partitionKey = new(id);
ItemResponse<CosmosLockDocument> response; ItemResponse<CosmosLockDocument> response;
try try
{ {
response = await this.container.ReadItemAsync<CosmosLockDocument>(id, partitionKey, cancellationToken: cancellationToken); response = await this.container.ReadItemAsync<CosmosLockDocument>(id, partitionKey, cancellationToken: cancellationToken);
@ -36,13 +36,13 @@ namespace Azure.Sdk.Tools.PipelineWitness.Services.WorkTokens
return await CreateLockAsync(id, duration, cancellationToken); return await CreateLockAsync(id, duration, cancellationToken);
} }
var existingLock = response.Resource; CosmosLockDocument existingLock = response.Resource;
if (existingLock.Expiration >= DateTime.UtcNow) if (existingLock.Expiration >= DateTime.UtcNow)
{ {
return null; return null;
} }
try try
{ {
response = await this.container.ReplaceItemAsync( response = await this.container.ReplaceItemAsync(
@ -68,7 +68,7 @@ namespace Azure.Sdk.Tools.PipelineWitness.Services.WorkTokens
{ {
try try
{ {
var response = await this.container.CreateItemAsync( ItemResponse<CosmosLockDocument> response = await this.container.CreateItemAsync(
new CosmosLockDocument(id, duration), new CosmosLockDocument(id, duration),
new PartitionKey(id), new PartitionKey(id),
cancellationToken: cancellationToken); cancellationToken: cancellationToken);

39
tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Startup.cs
Просмотреть файл

@ -6,7 +6,6 @@ using Azure.Identity;
using Azure.Sdk.Tools.PipelineWitness.ApplicationInsights; using Azure.Sdk.Tools.PipelineWitness.ApplicationInsights;
using Azure.Sdk.Tools.PipelineWitness.Configuration; using Azure.Sdk.Tools.PipelineWitness.Configuration;
using Azure.Sdk.Tools.PipelineWitness.Services; using Azure.Sdk.Tools.PipelineWitness.Services;
using Azure.Sdk.Tools.PipelineWitness.Services.FailureAnalysis;
using Azure.Sdk.Tools.PipelineWitness.Services.WorkTokens; using Azure.Sdk.Tools.PipelineWitness.Services.WorkTokens;
using Microsoft.ApplicationInsights.Extensibility; using Microsoft.ApplicationInsights.Extensibility;
@ -28,9 +27,9 @@ namespace Azure.Sdk.Tools.PipelineWitness
{ {
public static void Configure(WebApplicationBuilder builder) public static void Configure(WebApplicationBuilder builder)
{ {
var azureCredential = new DefaultAzureCredential(); DefaultAzureCredential azureCredential = new();
var settings = new PipelineWitnessSettings(); PipelineWitnessSettings settings = new();
var settingsSection = builder.Configuration.GetSection("PipelineWitness"); IConfigurationSection settingsSection = builder.Configuration.GetSection("PipelineWitness");
settingsSection.Bind(settings); settingsSection.Bind(settings);
builder.Services.AddApplicationInsightsTelemetry(builder.Configuration); builder.Services.AddApplicationInsightsTelemetry(builder.Configuration);
@ -57,26 +56,6 @@ namespace Azure.Sdk.Tools.PipelineWitness
builder.Services.AddMemoryCache(); builder.Services.AddMemoryCache();
builder.Services.AddSingleton<BlobUploadProcessor>(); builder.Services.AddSingleton<BlobUploadProcessor>();
builder.Services.AddSingleton<BuildLogProvider>(); builder.Services.AddSingleton<BuildLogProvider>();
builder.Services.AddSingleton<IFailureAnalyzer, FailureAnalyzer>();
builder.Services.AddSingleton<IFailureClassifier, AzuriteInstallFailureClassifier>();
builder.Services.AddSingleton<IFailureClassifier, CancelledTaskClassifier>();
builder.Services.AddSingleton<IFailureClassifier, CosmosDbEmulatorStartFailureClassifier>();
builder.Services.AddSingleton<IFailureClassifier, AzurePipelinesPoolOutageClassifier>();
builder.Services.AddSingleton<IFailureClassifier, PythonPipelineTestFailureClassifier>();
builder.Services.AddSingleton<IFailureClassifier, JavaScriptLiveTestFailureClassifier>();
builder.Services.AddSingleton<IFailureClassifier, TestResourcesDeploymentFailureClassifier>();
builder.Services.AddSingleton<IFailureClassifier, DotnetPipelineTestFailureClassifier>();
builder.Services.AddSingleton<IFailureClassifier, JavaPipelineTestFailureClassifier>();
builder.Services.AddSingleton<IFailureClassifier, JsSamplesExecutionFailureClassifier>();
builder.Services.AddSingleton<IFailureClassifier, JsDevFeedPublishingFailureClassifier>();
builder.Services.AddSingleton<IFailureClassifier, DownloadSecretsFailureClassifier>();
builder.Services.AddSingleton<IFailureClassifier, GitCheckoutFailureClassifier>();
builder.Services.AddSingleton<IFailureClassifier, AzuriteInstallFailureClassifier>();
builder.Services.AddSingleton<IFailureClassifier, MavenBrokenPipeFailureClassifier>();
builder.Services.AddSingleton<IFailureClassifier, CodeSigningFailureClassifier>();
builder.Services.AddSingleton<IFailureClassifier, AzureArtifactsServiceUnavailableClassifier>();
builder.Services.AddSingleton<IFailureClassifier, DnsResolutionFailureClassifier>();
builder.Services.AddSingleton<IFailureClassifier, CacheFailureClassifier>();
builder.Services.Configure<PipelineWitnessSettings>(settingsSection); builder.Services.Configure<PipelineWitnessSettings>(settingsSection);
builder.Services.AddSingleton<TokenCredential, DefaultAzureCredential>(); builder.Services.AddSingleton<TokenCredential, DefaultAzureCredential>();
@ -87,7 +66,7 @@ namespace Azure.Sdk.Tools.PipelineWitness
private static void AddHostedService<T>(this IServiceCollection services, int instanceCount) where T : class, IHostedService private static void AddHostedService<T>(this IServiceCollection services, int instanceCount) where T : class, IHostedService
{ {
for (var i = 0; i < instanceCount; i++) for (int i = 0; i < instanceCount; i++)
{ {
services.AddSingleton<IHostedService, T>(); services.AddSingleton<IHostedService, T>();
} }
@ -107,13 +86,13 @@ namespace Azure.Sdk.Tools.PipelineWitness
private static VssConnection CreateVssConnection(IServiceProvider provider) private static VssConnection CreateVssConnection(IServiceProvider provider)
{ {
TokenCredential azureCredential = provider.GetRequiredService<TokenCredential>(); TokenCredential azureCredential = provider.GetRequiredService<TokenCredential>();
TokenRequestContext tokenRequestContext = new (VssAadSettings.DefaultScopes); TokenRequestContext tokenRequestContext = new(VssAadSettings.DefaultScopes);
AccessToken token = azureCredential.GetToken(tokenRequestContext, CancellationToken.None); AccessToken token = azureCredential.GetToken(tokenRequestContext, CancellationToken.None);
Uri organizationUrl = new ("https://dev.azure.com/azure-sdk"); Uri organizationUrl = new("https://dev.azure.com/azure-sdk");
VssAadCredential vssCredential = new (new VssAadToken("Bearer", token.Token)); VssAadCredential vssCredential = new(new VssAadToken("Bearer", token.Token));
VssHttpRequestSettings settings = VssClientHttpRequestSettings.Default.Clone(); VssHttpRequestSettings settings = VssClientHttpRequestSettings.Default.Clone();
return new VssConnection(organizationUrl, vssCredential, settings); return new VssConnection(organizationUrl, vssCredential, settings);
} }
} }

70
tools/pipeline-witness/infrastructure/Assign-StoragePermissions.ps1 Normal file
Просмотреть файл

@ -0,0 +1,70 @@
<#
.SYNOPSIS
Assign appropriate storage permissions to the Azure SDK Engineering System Team for local debugging.
#>
param(
[Parameter(Mandatory)]
[validateSet('staging', 'test')]
[string]$target
)
function Invoke([string]$command) {
Write-Host "> $command"
Invoke-Expression $command
}
Push-Location $PSScriptRoot
try {
$subscriptionName = $target -eq 'test' ? 'Azure SDK Developer Playground' : 'Azure SDK Engineering System'
Write-Host "Setting subscription to '$subscriptionName'"
Invoke "az account set --subscription '$subscriptionName' --output none"
$parametersFile = "./bicep/parameters.$target.json"
Write-Host "Reading parameters from $parametersFile"
$parameters = (Get-Content -Path $parametersFile -Raw | ConvertFrom-Json).parameters
$appResourceGroupName = $parameters.appResourceGroupName.value
$appStorageAccountName = $parameters.appStorageAccountName.value
$cosmosAccountName = $parameters.cosmosAccountName.value
$logsResourceGroupName = $parameters.logsResourceGroupName.value
$logsStorageAccountName = $parameters.logsStorageAccountName.value
Write-Host "Adding Azure SDK Engineering System Team RBAC access to storage resources:`n" + `
" Blob: $logsResourceGroupName/$logsStorageAccountName`n" + `
" Queue: `n" + `
" Cosmos: $appResourceGroupName/$cosmosAccountName`n"
Write-Host "Getting group id for Azure SDK Engineering System Team"
$azAdGroupId = Invoke "az ad group show --group 'Azure SDK Engineering System Team' --query id --output tsv"
Write-Host "Granting 'read/write' access to $appResourceGroupName/$cosmosAccountName"
Invoke "az cosmosdb sql role assignment create --resource-group '$appResourceGroupName' --account-name '$cosmosAccountName' --scope '/' --role-definition-id '00000000-0000-0000-0000-000000000002' --principal-id '$azAdGroupId' --output none"
if ($LASTEXITCODE -ne 0) {
Write-Output $output
Write-Error "Failed to grant access"
exit 1
}
Write-Host "Granting 'Storage Blob Data Contributor' access to $logsResourceGroupName/$logsStorageAccountName"
$scope = "/subscriptions/$subscriptionId/resourceGroups/$logsResourceGroupName/providers/Microsoft.Storage/storageAccounts/$logsStorageAccountName"
$output = Invoke "az role assignment create --assignee '$azAdGroupId' --role 'Storage Blob Data Contributor' --scope '$scope' --output none"
if ($LASTEXITCODE -ne 0) {
Write-Output $output
Write-Error "Failed to grant access"
exit 1
}
Write-Host "Granting 'Storage Queue Data Contributor' access to $appResourceGroupName/$appStorageAccountName"
$scope = "/subscriptions/$subscriptionId/resourceGroups/$appResourceGroupName/providers/Microsoft.Storage/storageAccounts/$appStorageAccountName"
$output = Invoke "az role assignment create --assignee '$azAdGroupId' --role 'Storage Queue Data Contributor' --scope '$scope' --output none"
if ($LASTEXITCODE -ne 0) {
Write-Output $output
Write-Error "Failed to grant access"
exit 1
}
}
finally {
Pop-Location
}

9
tools/pipeline-witness/infrastructure/README.md Normal file
Просмотреть файл

@ -0,0 +1,9 @@
# Bicep
The bicep templates in the [bicep](./bicep) folder are used to provision the PipelineWitness resource group, web app and storage account as well as the PipelineLogs resource group, storage account, kusto cluster and all of the resources required to make continuous ingestion from storage to kusto work.
#### Deployment permissions
The bicep templates contain RBAC assignments between the web app's managed identity and the storage accounts it will be accessing. For the CI pipeline to be able to successfully set these RBAC assignments, it's service connection principal must be granted `Microsoft.Authorization/roleAssignments/write` to the storage resources. This permission is included in the `Owner`, `User Access Administrator` and `Role Based Access Control Administrator` roles.
# Kusto
The scripts in the kusto folder are deployed to the staging and production databases during ci pipeline runs. They are merged into a single `.kql` by [`deploy.ps1`](./deploy.ps1) using the [`Merge-KustoScripts.ps1`](./Merge-KustoScripts.ps1) script. To extract objects from an existing kusto database into the kusto folder, use the [`Extract-KustoScripts.ps1`](./Extract-KustoScripts.ps1) script.

36
tools/pipeline-witness/infrastructure/deploy.ps1
Просмотреть файл

@ -7,7 +7,7 @@ param(
[validateSet('production', 'staging', 'test')] [validateSet('production', 'staging', 'test')]
[string]$target, [string]$target,
[switch]$replaceRoleAssignments [switch]$removeRoleAssignments
) )
function Invoke([string]$command) { function Invoke([string]$command) {
@ -79,7 +79,7 @@ try {
" App Resource Group: $appResourceGroupName`n" + ` " App Resource Group: $appResourceGroupName`n" + `
" Location: $location`n" " Location: $location`n"
if ($replaceRoleAssignments) { if ($removeRoleAssignments) {
RemoveStorageRoleAssignments $subscriptionId $logsResourceGroupName $logsStorageAccountName RemoveStorageRoleAssignments $subscriptionId $logsResourceGroupName $logsStorageAccountName
RemoveStorageRoleAssignments $subscriptionId $appResourceGroupName $appStorageAccountName RemoveStorageRoleAssignments $subscriptionId $appResourceGroupName $appStorageAccountName
RemoveCosmosRoleAssignments $subscriptionId $appResourceGroupName $cosmosAccountName RemoveCosmosRoleAssignments $subscriptionId $appResourceGroupName $cosmosAccountName
@ -90,38 +90,6 @@ try {
Write-Error "Failed to deploy resource groups" Write-Error "Failed to deploy resource groups"
exit 1 exit 1
} }
if ($target -ne 'production') {
$azAdGroupId = az ad group show --group "Azure SDK Engineering System Team" --query id --output tsv
Write-Host "Granting the Azure SDK Engineering System Team read/write access to cosmos account"
Invoke "az cosmosdb sql role assignment create --resource-group '$appResourceGroupName' --account-name '$cosmosAccountName' --scope '/' --role-definition-id '00000000-0000-0000-0000-000000000002' --principal-id '$azAdGroupId' --output none"
if ($LASTEXITCODE -ne 0) {
Write-Output $output
Write-Error "Failed to grant access to cosmos account"
exit 1
}
Write-Host "Granting the Azure SDK Engineering System Team access to storage accounts"
$scope = "/subscriptions/$subscriptionId/resourceGroups/$logsResourceGroupName/providers/Microsoft.Storage/storageAccounts/$logsStorageAccountName"
$output = Invoke "az role assignment create --assignee '$azAdGroupId' --role 'Storage Blob Data Contributor' --scope '$scope' --output none"
if ($LASTEXITCODE -ne 0) {
Write-Output $output
Write-Error "Failed to grant access to logs storage account"
exit 1
}
$scope = "/subscriptions/$subscriptionId/resourceGroups/$appResourceGroupName/providers/Microsoft.Storage/storageAccounts/$appStorageAccountName"
$output = Invoke "az role assignment create --assignee '$azAdGroupId' --role 'Storage Queue Data Contributor' --scope '$scope' --output none"
if ($LASTEXITCODE -ne 0) {
Write-Output $output
Write-Error "Failed to grant access to app storage account"
exit 1
}
}
} }
finally { finally {
Pop-Location Pop-Location