From 478ff5c2d240f2f9105999572a573d537cf87b19 Mon Sep 17 00:00:00 2001 From: Patrick Hallisey Date: Fri, 26 Apr 2024 10:49:12 -0700 Subject: [PATCH] Remove failure analysis and apply code cleanup (#8153) --- .editorconfig | 9 +- .../BlobUploadProcessorIntegrationTests.cs | 50 +-- .../EnvironmentConditionalSkipFact.cs | 4 +- .../PassThroughFailureAnalyzer.cs | 16 - .../TestLogger.cs | 25 -- .../ApplicationVersionTelemetryInitializer.cs | 19 +- .../BlobNotFoundTelemetryProcessor.cs | 8 +- .../BlobUploadProcessor.cs | 292 +++++++----------- .../Entities/BuildLogBundle.cs | 10 +- .../Entities/BuildLogInfo.cs | 10 +- .../Program.cs | 6 +- .../AzurePipelinesBuildDefinitionWorker.cs | 22 +- .../Services/BuildCompleteQueueWorker.cs | 21 +- .../Services/BuildLogProvider.cs | 14 +- .../Services/EnhancedBuildHttpClient.cs | 18 +- ...reArtifactsServiceUnavailableClassifier.cs | 37 --- .../AzurePipelinesPoolOutageClassifier.cs | 26 -- ...hellModuleInstallationFailureClassifier.cs | 31 -- .../AzuriteInstallFailureClassifier.cs | 28 -- .../FailureAnalysis/CacheFailureClassifier.cs | 75 ----- .../CancelledTaskClassifier.cs | 27 -- .../CodeSigningFailureClassifier.cs | 30 -- .../CosmosDbEmulatorStartFailureClassifier.cs | 28 -- .../DnsResolutionFailureClassifier.cs | 45 --- .../DotnetPipelineTestFailureClassifier.cs | 31 -- .../DownloadSecretsFailureClassifier.cs | 28 -- .../FailureAnalysis/FailureAnalyzer.cs | 46 --- .../FailureAnalysis/FailureAnalyzerContext.cs | 57 ---- .../GitCheckoutFailureClassifier.cs | 27 -- .../FailureAnalysis/IFailureAnalyzer.cs | 12 - .../FailureAnalysis/IFailureClassifier.cs | 9 - .../JavaPipelineTestFailureClassifier.cs | 31 -- .../JavaScriptLiveTestFailureClassifier.cs | 31 -- .../JsDevFeedPublishingFailureClassifier.cs | 31 -- .../JsSamplesExecutionFailureClassifier.cs | 30 -- .../MavenBrokenPipeFailureClassifier.cs | 37 --- .../PythonPipelineTestFailureClassifier.cs | 31 -- ...estResourcesDeploymentFailureClassifier.cs | 27 -- .../Services/QueueWorkerBackgroundService.cs | 135 ++++---- .../Services/WorkTokens/CosmosAsyncLock.cs | 6 +- .../WorkTokens/CosmosAsyncLockProvider.cs | 10 +- .../Startup.cs | 39 +-- .../Assign-StoragePermissions.ps1 | 70 +++++ .../pipeline-witness/infrastructure/README.md | 9 + .../infrastructure/deploy.ps1 | 36 +-- 45 files changed, 377 insertions(+), 1207 deletions(-) delete mode 100644 tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness.Tests/PassThroughFailureAnalyzer.cs delete mode 100644 tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/AzureArtifactsServiceUnavailableClassifier.cs delete mode 100644 tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/AzurePipelinesPoolOutageClassifier.cs delete mode 100644 tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/AzurePowerShellModuleInstallationFailureClassifier.cs delete mode 100644 tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/AzuriteInstallFailureClassifier.cs delete mode 100644 tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/CacheFailureClassifier.cs delete mode 100644 tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/CancelledTaskClassifier.cs delete mode 100644 tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/CodeSigningFailureClassifier.cs delete mode 100644 tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/CosmosDbEmulatorStartFailureClassifier.cs delete mode 100644 tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/DnsResolutionFailureClassifier.cs delete mode 100644 tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/DotnetPipelineTestFailureClassifier.cs delete mode 100644 tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/DownloadSecretsFailureClassifier.cs delete mode 100644 tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/FailureAnalyzer.cs delete mode 100644 tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/FailureAnalyzerContext.cs delete mode 100644 tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/GitCheckoutFailureClassifier.cs delete mode 100644 tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/IFailureAnalyzer.cs delete mode 100644 tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/IFailureClassifier.cs delete mode 100644 tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/JavaPipelineTestFailureClassifier.cs delete mode 100644 tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/JavaScriptLiveTestFailureClassifier.cs delete mode 100644 tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/JsDevFeedPublishingFailureClassifier.cs delete mode 100644 tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/JsSamplesExecutionFailureClassifier.cs delete mode 100644 tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/MavenBrokenPipeFailureClassifier.cs delete mode 100644 tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/PythonPipelineTestFailureClassifier.cs delete mode 100644 tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/TestResourcesDeploymentFailureClassifier.cs create mode 100644 tools/pipeline-witness/infrastructure/Assign-StoragePermissions.ps1 create mode 100644 tools/pipeline-witness/infrastructure/README.md diff --git a/.editorconfig b/.editorconfig index 4ea6d3bc8..066b03324 100644 --- a/.editorconfig +++ b/.editorconfig @@ -81,9 +81,9 @@ dotnet_naming_style.prefix_underscore.capitalization = camel_case ############################### [*.cs] # var preferences -csharp_style_var_for_built_in_types = true:silent -csharp_style_var_when_type_is_apparent = true:silent -csharp_style_var_elsewhere = true:silent +csharp_style_var_for_built_in_types = false:silent +csharp_style_var_when_type_is_apparent = false:silent +csharp_style_var_elsewhere = false:silent # Expression-bodied members csharp_style_expression_bodied_methods = false:silent csharp_style_expression_bodied_constructors = false:silent @@ -105,6 +105,8 @@ csharp_style_deconstructed_variable_declaration = true:suggestion csharp_prefer_simple_default_expression = true:suggestion csharp_style_pattern_local_over_anonymous_function = true:suggestion csharp_style_inlined_variable_declaration = true:suggestion +# one class per file +csharp_style_single_file_classes = true:suggestion ############################### # C# Formatting Rules # @@ -136,3 +138,4 @@ csharp_space_between_method_call_empty_parameter_list_parentheses = false # Wrapping preferences csharp_preserve_single_line_statements = true csharp_preserve_single_line_blocks = true + diff --git a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness.Tests/BlobUploadProcessorIntegrationTests.cs b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness.Tests/BlobUploadProcessorIntegrationTests.cs index 73b3de74e..bbc16678f 100644 --- a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness.Tests/BlobUploadProcessorIntegrationTests.cs +++ b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness.Tests/BlobUploadProcessorIntegrationTests.cs @@ -17,53 +17,53 @@ namespace Azure.Sdk.Tools.PipelineWitness.Tests { public class BlobUploadProcessorIntegrationTests { - private VssCredentials VisualStudioCredentials; - private VssConnection VisualStudioConnection; - private string TARGET_ACCOUNT_ID = "azure-sdk"; - private Guid TARGET_PROJECT_ID = new Guid("29ec6040-b234-4e31-b139-33dc4287b756"); - private int TARGET_DEFINITION_ID = 297; - private string DEVOPS_PATH = "https://dev.azure.com/azure-sdk"; - private PipelineWitnessSettings TestSettings = new PipelineWitnessSettings() + private const string TARGET_ACCOUNT_ID = "azure-sdk"; + private const string TARGET_PROJECT_ID = "29ec6040-b234-4e31-b139-33dc4287b756"; + private const int TARGET_DEFINITION_ID = 297; + private const string DEVOPS_PATH = "https://dev.azure.com/azure-sdk"; + + private readonly VssCredentials visualStudioCredentials; + private readonly VssConnection visualStudioConnection; + private readonly PipelineWitnessSettings testSettings = new() { PipelineOwnersDefinitionId = 5112, - PipelineOwnersFilePath = "pipelineOwners/pipelineOwners.json", - PipelineOwnersArtifactName = "pipelineOwners" - }; + PipelineOwnersFilePath = "pipelineOwners/pipelineOwners.json", + PipelineOwnersArtifactName = "pipelineOwners" + }; public BlobUploadProcessorIntegrationTests() { - var pat = Environment.GetEnvironmentVariable("AZURESDK_DEVOPS_TOKEN"); - var blobUri = Environment.GetEnvironmentVariable("AZURESDK_BLOB_CS"); + string pat = Environment.GetEnvironmentVariable("AZURESDK_DEVOPS_TOKEN"); + string blobUri = Environment.GetEnvironmentVariable("AZURESDK_BLOB_CS"); - if (!string.IsNullOrWhiteSpace(pat) && !string.IsNullOrWhiteSpace(blobUri) ) + if (!string.IsNullOrWhiteSpace(pat) && !string.IsNullOrWhiteSpace(blobUri)) { - VisualStudioCredentials = new VssBasicCredential("nobody", pat); - VisualStudioConnection = new VssConnection(new Uri(DEVOPS_PATH), VisualStudioCredentials); + this.visualStudioCredentials = new VssBasicCredential("nobody", pat); + this.visualStudioConnection = new VssConnection(new Uri(DEVOPS_PATH), this.visualStudioCredentials); } } [EnvironmentConditionalSkipFact] public async Task BasicBlobProcessInvokesSuccessfully() { - var buildLogProvider = new BuildLogProvider(logger: new NullLogger(), VisualStudioConnection); - var blobServiceClient = new BlobServiceClient(Environment.GetEnvironmentVariable("AZURESDK_BLOB_CS")); - var buildHttpClient = VisualStudioConnection.GetClient(); - var testResultsBuiltClient = VisualStudioConnection.GetClient(); + BuildLogProvider buildLogProvider = new(logger: new NullLogger(), this.visualStudioConnection); + BlobServiceClient blobServiceClient = new(Environment.GetEnvironmentVariable("AZURESDK_BLOB_CS")); + BuildHttpClient buildHttpClient = this.visualStudioConnection.GetClient(); + TestResultsHttpClient testResultsBuiltClient = this.visualStudioConnection.GetClient(); List recentBuilds = await buildHttpClient.GetBuildsAsync(TARGET_PROJECT_ID, definitions: new[] { TARGET_DEFINITION_ID }, resultFilter: BuildResult.Succeeded, statusFilter: BuildStatus.Completed, top: 1, queryOrder: BuildQueryOrder.FinishTimeDescending); Assert.True(recentBuilds.Count > 0); - var targetBuildId = recentBuilds.First().Id; + int targetBuildId = recentBuilds.First().Id; - BlobUploadProcessor processor = new BlobUploadProcessor(logger: new NullLogger(), + BlobUploadProcessor processor = new(logger: new NullLogger(), logProvider: buildLogProvider, blobServiceClient: blobServiceClient, buildClient: buildHttpClient, testResultsClient: testResultsBuiltClient, - options: Options.Create(TestSettings), - failureAnalyzer: new PassThroughFailureAnalyzer()); + options: Options.Create(this.testSettings)); - await processor.UploadBuildBlobsAsync(TARGET_ACCOUNT_ID, TARGET_PROJECT_ID, targetBuildId); + await processor.UploadBuildBlobsAsync(TARGET_ACCOUNT_ID, new Guid(TARGET_PROJECT_ID), targetBuildId); } [Theory] @@ -73,7 +73,7 @@ namespace Azure.Sdk.Tools.PipelineWitness.Tests [InlineData(0, 10000, 0)] public void TestBatching(int startingNumber, int batchSize, int expectedBatchNumber) { - var numberOfBatches = BlobUploadProcessor.CalculateBatches(startingNumber, batchSize); + int numberOfBatches = BlobUploadProcessor.CalculateBatches(startingNumber, batchSize); Assert.Equal(expectedBatchNumber, numberOfBatches); } diff --git a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness.Tests/EnvironmentConditionalSkipFact.cs b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness.Tests/EnvironmentConditionalSkipFact.cs index 0151e731a..bffb88d10 100644 --- a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness.Tests/EnvironmentConditionalSkipFact.cs +++ b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness.Tests/EnvironmentConditionalSkipFact.cs @@ -11,8 +11,8 @@ namespace Azure.Sdk.Tools.PipelineWitness.Tests { public EnvironmentConditionalSkipFact() { - var devopsPat = Environment.GetEnvironmentVariable("AZURESDK_DEVOPS_TOKEN"); - var blobToken = Environment.GetEnvironmentVariable("AZURESDK_BLOB_CS"); + string devopsPat = Environment.GetEnvironmentVariable("AZURESDK_DEVOPS_TOKEN"); + string blobToken = Environment.GetEnvironmentVariable("AZURESDK_BLOB_CS"); // and if we don't, skip this test if (string.IsNullOrWhiteSpace(devopsPat) || string.IsNullOrWhiteSpace(blobToken)) diff --git a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness.Tests/PassThroughFailureAnalyzer.cs b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness.Tests/PassThroughFailureAnalyzer.cs deleted file mode 100644 index c4c58790f..000000000 --- a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness.Tests/PassThroughFailureAnalyzer.cs +++ /dev/null @@ -1,16 +0,0 @@ -using System.Collections.Generic; -using System.Threading.Tasks; -using Azure.Sdk.Tools.PipelineWitness.Entities.AzurePipelines; -using Azure.Sdk.Tools.PipelineWitness.Services.FailureAnalysis; -using Microsoft.TeamFoundation.Build.WebApi; - -namespace Azure.Sdk.Tools.PipelineWitness.Tests -{ - internal class PassThroughFailureAnalyzer : IFailureAnalyzer - { - public Task> AnalyzeFailureAsync(Build build, Timeline timeline) - { - return Task.FromResult>(new List()); - } - } -} diff --git a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness.Tests/TestLogger.cs b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness.Tests/TestLogger.cs index 1832718ca..9b5072fd6 100644 --- a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness.Tests/TestLogger.cs +++ b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness.Tests/TestLogger.cs @@ -24,29 +24,4 @@ namespace Azure.Sdk.Tools.PipelineWitness.Tests Logs.Add(state); } } - - public class TestLoggingFactory : ILoggerFactory - { - private readonly TestLogger _logger; - - public TestLoggingFactory(TestLogger logger) - { - _logger = logger; - } - - public void Dispose() - { - throw new NotImplementedException(); - } - - public void AddProvider(ILoggerProvider provider) - { - throw new NotImplementedException(); - } - - public ILogger CreateLogger(string categoryName) - { - return _logger; - } - } } diff --git a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/ApplicationInsights/ApplicationVersionTelemetryInitializer.cs b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/ApplicationInsights/ApplicationVersionTelemetryInitializer.cs index 94efb0546..5e522577e 100644 --- a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/ApplicationInsights/ApplicationVersionTelemetryInitializer.cs +++ b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/ApplicationInsights/ApplicationVersionTelemetryInitializer.cs @@ -1,31 +1,32 @@ -using System.Reflection; +using System.Reflection; using Microsoft.ApplicationInsights.Channel; using Microsoft.ApplicationInsights.Extensibility; +using Microsoft.ApplicationInsights.Extensibility.Implementation; namespace Azure.Sdk.Tools.PipelineWitness.ApplicationInsights { public class ApplicationVersionTelemetryInitializer : ITelemetryInitializer { - private static string _version = GetVersion(); + private static readonly string version = GetVersion(); public void Initialize(ITelemetry telemetry) { - if (!string.IsNullOrEmpty(_version)) + if (!string.IsNullOrEmpty(version)) { - var component = telemetry.Context?.Component; + ComponentContext component = telemetry.Context?.Component; if (component != null) { - component.Version = _version; + component.Version = version; } } } - + private static string GetVersion() { - var assembly = typeof(ApplicationVersionTelemetryInitializer).Assembly; - - var version = assembly.GetCustomAttribute()?.InformationalVersion + Assembly assembly = typeof(ApplicationVersionTelemetryInitializer).Assembly; + + string version = assembly.GetCustomAttribute()?.InformationalVersion ?? assembly.GetName().Version?.ToString(); return version; diff --git a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/ApplicationInsights/BlobNotFoundTelemetryProcessor.cs b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/ApplicationInsights/BlobNotFoundTelemetryProcessor.cs index 573ed2fd4..989927902 100644 --- a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/ApplicationInsights/BlobNotFoundTelemetryProcessor.cs +++ b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/ApplicationInsights/BlobNotFoundTelemetryProcessor.cs @@ -13,14 +13,14 @@ namespace Azure.Sdk.Tools.PipelineWitness.ApplicationInsights { this.next = next; } - + public void Process(ITelemetry telemetry) { if (telemetry is DependencyTelemetry { Success: false, Type: "Azure blob" or "Microsoft.Storage" } blobRequestTelemetry) { - blobRequestTelemetry.Properties.TryGetValue("Error", out var errorProperty); - - var isNotFound = blobRequestTelemetry.ResultCode is "404" or "409" + blobRequestTelemetry.Properties.TryGetValue("Error", out string errorProperty); + + bool isNotFound = blobRequestTelemetry.ResultCode is "404" or "409" || (blobRequestTelemetry.ResultCode == "" && errorProperty?.Contains("Status: 404") == true); if (isNotFound) diff --git a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/BlobUploadProcessor.cs b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/BlobUploadProcessor.cs index 7fbc438fd..91a121daf 100644 --- a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/BlobUploadProcessor.cs +++ b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/BlobUploadProcessor.cs @@ -1,44 +1,45 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.IO; +using System.IO.Compression; +using System.Linq; +using System.Net; +using System.Text; +using System.Text.RegularExpressions; +using System.Threading.Tasks; + +using Azure.Sdk.Tools.PipelineWitness.Configuration; +using Azure.Sdk.Tools.PipelineWitness.Services; +using Azure.Storage.Blobs; +using Azure.Storage.Blobs.Models; + +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Microsoft.TeamFoundation.Build.WebApi; +using Microsoft.TeamFoundation.TestManagement.WebApi; +using Microsoft.VisualStudio.Services.TestResults.WebApi; +using Microsoft.VisualStudio.Services.WebApi; +using Newtonsoft.Json; +using Newtonsoft.Json.Converters; +using Newtonsoft.Json.Serialization; + namespace Azure.Sdk.Tools.PipelineWitness { - using System; - using System.Collections.Generic; - using System.IO; - using System.IO.Compression; - using System.Linq; - using System.Net; - using System.Text; - using System.Text.RegularExpressions; - using System.Threading.Tasks; - - using Azure.Sdk.Tools.PipelineWitness.Configuration; - using Azure.Sdk.Tools.PipelineWitness.Services; - using Azure.Sdk.Tools.PipelineWitness.Services.FailureAnalysis; - using Azure.Storage.Blobs; - using Azure.Storage.Blobs.Models; - - using Microsoft.Extensions.Logging; - using Microsoft.Extensions.Options; - using Microsoft.TeamFoundation.Build.WebApi; - using Microsoft.TeamFoundation.TestManagement.WebApi; - using Microsoft.VisualStudio.Services.TestResults.WebApi; - using Newtonsoft.Json; - using Newtonsoft.Json.Converters; - using Newtonsoft.Json.Serialization; - + [SuppressMessage("Style", "IDE0037:Use inferred member name", Justification = "Explicit member names are added to json export objects for clarity")] public class BlobUploadProcessor { private const string BuildsContainerName = "builds"; private const string BuildLogLinesContainerName = "buildloglines"; private const string BuildTimelineRecordsContainerName = "buildtimelinerecords"; private const string BuildDefinitionsContainerName = "builddefinitions"; - private const string BuildFailuresContainerName = "buildfailures"; private const string PipelineOwnersContainerName = "pipelineowners"; private const string TestRunsContainerName = "testruns"; private const string TestResultsContainerName = "testrunresults"; private const int ApiBatchSize = 10000; private const string TimeFormat = @"yyyy-MM-dd\THH:mm:ss.fffffff\Z"; - private static readonly JsonSerializerSettings jsonSettings = new JsonSerializerSettings() + private static readonly JsonSerializerSettings jsonSettings = new() { ContractResolver = new CamelCasePropertyNamesContractResolver(), Converters = { new StringEnumConverter(new CamelCaseNamingStrategy()) }, @@ -55,11 +56,9 @@ namespace Azure.Sdk.Tools.PipelineWitness private readonly BlobContainerClient testRunsContainerClient; private readonly BlobContainerClient testResultsContainerClient; private readonly BlobContainerClient buildDefinitionsContainerClient; - private readonly BlobContainerClient buildFailuresContainerClient; private readonly BlobContainerClient pipelineOwnersContainerClient; private readonly IOptions options; private readonly Dictionary cachedDefinitionRevisions = new(); - private readonly IFailureAnalyzer failureAnalyzer; public BlobUploadProcessor( ILogger logger, @@ -67,8 +66,7 @@ namespace Azure.Sdk.Tools.PipelineWitness BlobServiceClient blobServiceClient, BuildHttpClient buildClient, TestResultsHttpClient testResultsClient, - IOptions options, - IFailureAnalyzer failureAnalyzer) + IOptions options) { if (blobServiceClient == null) { @@ -84,17 +82,15 @@ namespace Azure.Sdk.Tools.PipelineWitness this.buildTimelineRecordsContainerClient = blobServiceClient.GetBlobContainerClient(BuildTimelineRecordsContainerName); this.buildLogLinesContainerClient = blobServiceClient.GetBlobContainerClient(BuildLogLinesContainerName); this.buildDefinitionsContainerClient = blobServiceClient.GetBlobContainerClient(BuildDefinitionsContainerName); - this.buildFailuresContainerClient = blobServiceClient.GetBlobContainerClient(BuildFailuresContainerName); this.testRunsContainerClient = blobServiceClient.GetBlobContainerClient(TestRunsContainerName); this.testResultsContainerClient = blobServiceClient.GetBlobContainerClient(TestResultsContainerName); this.buildDefinitionsContainerClient = blobServiceClient.GetBlobContainerClient(BuildDefinitionsContainerName); this.pipelineOwnersContainerClient = blobServiceClient.GetBlobContainerClient(PipelineOwnersContainerName); - this.failureAnalyzer = failureAnalyzer; } public async Task UploadBuildBlobsAsync(string account, Guid projectId, int buildId) { - var build = await GetBuildAsync(projectId, buildId); + Build build = await GetBuildAsync(projectId, buildId); if (build == null) { @@ -102,7 +98,7 @@ namespace Azure.Sdk.Tools.PipelineWitness return; } - var skipBuild = false; + bool skipBuild = false; // Project name is used in blob paths and cannot be empty if (build.Project == null) @@ -157,34 +153,33 @@ namespace Azure.Sdk.Tools.PipelineWitness await UploadTestRunBlobsAsync(account, build); - var timeline = await this.buildClient.GetBuildTimelineAsync(projectId, buildId); + Timeline timeline = await this.buildClient.GetBuildTimelineAsync(projectId, buildId); if (timeline == null) { - logger.LogWarning("No timeline available for build {Project}: {BuildId}", build.Project.Name, build.Id); + this.logger.LogWarning("No timeline available for build {Project}: {BuildId}", build.Project.Name, build.Id); } else { await UploadTimelineBlobAsync(account, build, timeline); - await UploadBuildFailureBlobAsync(account, build, timeline); } - var logs = await buildClient.GetBuildLogsAsync(build.Project.Id, build.Id); + List logs = await this.buildClient.GetBuildLogsAsync(build.Project.Id, build.Id); if (logs == null || logs.Count == 0) { - logger.LogWarning("No logs available for build {Project}: {BuildId}", build.Project.Name, build.Id); + this.logger.LogWarning("No logs available for build {Project}: {BuildId}", build.Project.Name, build.Id); return; } - var buildLogInfos = GetBuildLogInfos(account, build, timeline, logs); + List buildLogInfos = GetBuildLogInfos(build, timeline, logs); - foreach (var log in buildLogInfos) + foreach (BuildLogInfo log in buildLogInfos) { await UploadLogLinesBlobAsync(account, build, log); } - if (build.Definition.Id == options.Value.PipelineOwnersDefinitionId) + if (build.Definition.Id == this.options.Value.PipelineOwnersDefinitionId) { await UploadPipelineOwnersBlobAsync(account, build, timeline); } @@ -194,8 +189,8 @@ namespace Azure.Sdk.Tools.PipelineWitness { try { - var blobPath = $"{build.Project.Name}/{build.FinishTime:yyyy/MM/dd}/{build.Id}-{timeline.ChangeId}.jsonl"; - var blobClient = this.pipelineOwnersContainerClient.GetBlobClient(blobPath); + string blobPath = $"{build.Project.Name}/{build.FinishTime:yyyy/MM/dd}/{build.Id}-{timeline.ChangeId}.jsonl"; + BlobClient blobClient = this.pipelineOwnersContainerClient.GetBlobClient(blobPath); if (await blobClient.ExistsAsync()) { @@ -203,7 +198,7 @@ namespace Azure.Sdk.Tools.PipelineWitness return; } - var owners = await GetOwnersFromBuildArtifactAsync(build); + Dictionary owners = await GetOwnersFromBuildArtifactAsync(build); if (owners == null) { @@ -213,11 +208,11 @@ namespace Azure.Sdk.Tools.PipelineWitness this.logger.LogInformation("Creating owners blob for build {DefinitionId} change {ChangeId}", build.Id, timeline.ChangeId); - var stringBuilder = new StringBuilder(); + StringBuilder stringBuilder = new(); - foreach (var owner in owners) + foreach (KeyValuePair owner in owners) { - var contentLine = JsonConvert.SerializeObject(new + string contentLine = JsonConvert.SerializeObject(new { OrganizationName = account, BuildDefinitionId = owner.Key, @@ -244,15 +239,15 @@ namespace Azure.Sdk.Tools.PipelineWitness private async Task> GetOwnersFromBuildArtifactAsync(Build build) { - var artifactName = this.options.Value.PipelineOwnersArtifactName; - var filePath = this.options.Value.PipelineOwnersFilePath; + string artifactName = this.options.Value.PipelineOwnersArtifactName; + string filePath = this.options.Value.PipelineOwnersFilePath; try { - await using var artifactStream = await this.buildClient.GetArtifactContentZipAsync(build.Project.Id, build.Id, artifactName); - using var zip = new ZipArchive(artifactStream); + await using Stream artifactStream = await this.buildClient.GetArtifactContentZipAsync(build.Project.Id, build.Id, artifactName); + using ZipArchive zip = new(artifactStream); - var fileEntry = zip.GetEntry(filePath); + ZipArchiveEntry fileEntry = zip.GetEntry(filePath); if (fileEntry == null) { @@ -260,9 +255,9 @@ namespace Azure.Sdk.Tools.PipelineWitness return null; } - await using var contentStream = fileEntry.Open(); - using var contentReader = new StreamReader(contentStream); - var content = await contentReader.ReadToEndAsync(); + await using Stream contentStream = fileEntry.Open(); + using StreamReader contentReader = new(contentStream); + string content = await contentReader.ReadToEndAsync(); if (string.IsNullOrEmpty(content)) { @@ -270,7 +265,7 @@ namespace Azure.Sdk.Tools.PipelineWitness return null; } - var ownersDictionary = JsonConvert.DeserializeObject>(content); + Dictionary ownersDictionary = JsonConvert.DeserializeObject>(content); if (ownersDictionary == null) { @@ -298,72 +293,15 @@ namespace Azure.Sdk.Tools.PipelineWitness return null; } - private async Task UploadBuildFailureBlobAsync(string account, Build build, Timeline timeline) - { - try - { - var blobPath = $"{build.Project.Name}/{build.FinishTime:yyyy/MM/dd}/{build.Id}-{timeline.ChangeId}.jsonl"; - var blobClient = this.buildFailuresContainerClient.GetBlobClient(blobPath); - - if (await blobClient.ExistsAsync()) - { - this.logger.LogInformation("Skipping existing build failure blob for build {BuildId}", build.Id); - return; - } - - var failures = await this.failureAnalyzer.AnalyzeFailureAsync(build, timeline); - if (!failures.Any()) - { - return; - } - - this.logger.LogInformation("Creating failure blob for build {DefinitionId} change {ChangeId}", build.Id, timeline.ChangeId); - - var stringBuilder = new StringBuilder(); - - foreach (var failure in failures) - { - var contentLine = JsonConvert.SerializeObject(new - { - OrganizationName = account, - ProjectId = build.Project.Id, - ProjectName = build.Project.Name, - BuildDefinitionId = build.Definition.Id, - BuildDefinitionName = build.Definition.Name, - BuildId = build.Id, - BuildFinishTime = build.FinishTime, - RecordFinishTime = failure.Record.FinishTime, - ChangeId = timeline.ChangeId, - RecordId = failure.Record.Id, - BuildTimelineId = timeline.Id, - ErrorClassification = failure.Classification, - EtlIngestDate = DateTimeOffset.UtcNow - }, jsonSettings); - stringBuilder.AppendLine(contentLine); - } - - await blobClient.UploadAsync(new BinaryData(stringBuilder.ToString())); - } - catch (RequestFailedException ex) when (ex.Status == (int)HttpStatusCode.Conflict) - { - this.logger.LogInformation("Ignoring exception from existing failure blob for build {BuildId}", build.Id); - } - catch (Exception ex) - { - this.logger.LogError(ex, "Error processing build failure blob for build {BuildId}", build.Id); - throw; - } - } - public async Task UploadBuildDefinitionBlobsAsync(string account, string projectName) { - var definitions = await buildClient.GetFullDefinitionsAsync2(project: projectName); + IPagedList definitions = await this.buildClient.GetFullDefinitionsAsync2(project: projectName); - foreach (var definition in definitions) + foreach (BuildDefinition definition in definitions) { - var cacheKey = $"{definition.Project.Id}:{definition.Id}"; + string cacheKey = $"{definition.Project.Id}:{definition.Id}"; - if (!this.cachedDefinitionRevisions.TryGetValue(cacheKey, out var cachedRevision) || cachedRevision != definition.Revision) + if (!this.cachedDefinitionRevisions.TryGetValue(cacheKey, out int? cachedRevision) || cachedRevision != definition.Revision) { await UploadBuildDefinitionBlobAsync(account, definition); } @@ -371,14 +309,14 @@ namespace Azure.Sdk.Tools.PipelineWitness this.cachedDefinitionRevisions[cacheKey] = definition.Revision; } } - + private async Task UploadBuildDefinitionBlobAsync(string account, BuildDefinition definition) { - var blobPath = $"{definition.Project.Name}/{definition.Id}-{definition.Revision}.jsonl"; + string blobPath = $"{definition.Project.Name}/{definition.Id}-{definition.Revision}.jsonl"; try { - var blobClient = this.buildDefinitionsContainerClient.GetBlobClient(blobPath); + BlobClient blobClient = this.buildDefinitionsContainerClient.GetBlobClient(blobPath); if (await blobClient.ExistsAsync()) { @@ -388,7 +326,7 @@ namespace Azure.Sdk.Tools.PipelineWitness this.logger.LogInformation("Creating blob for build definition {DefinitionId} revision {Revision} project {Project}", definition.Id, definition.Revision, definition.Project.Name); - var content = JsonConvert.SerializeObject(new + string content = JsonConvert.SerializeObject(new { OrganizationName = account, ProjectId = definition.Project.Id, @@ -446,32 +384,32 @@ namespace Azure.Sdk.Tools.PipelineWitness } } - private List GetBuildLogInfos(string account, Build build, Timeline timeline, List logs) + private List GetBuildLogInfos(Build build, Timeline timeline, List logs) { - var logsById = logs.ToDictionary(l => l.Id); + Dictionary logsById = logs.ToDictionary(l => l.Id); - var buildLogInfos = new List(); + List buildLogInfos = new(); - foreach (var log in logs) + foreach (BuildLog log in logs) { - var logRecords = timeline.Records.Where(x => x.Log?.Id == log.Id).ToArray(); + TimelineRecord[] logRecords = timeline.Records.Where(x => x.Log?.Id == log.Id).ToArray(); - if(logRecords.Length > 1) + if (logRecords.Length > 1) { this.logger.LogWarning("Found multiple timeline records for build {BuildId}, log {LogId}", build.Id, log.Id); } - var logRecord = logRecords.FirstOrDefault(); + TimelineRecord logRecord = logRecords.FirstOrDefault(); // Job logs are typically just a duplication of their child task logs with the addition of extra start and end lines. // If we can, we skip the redundant lines. if (string.Equals(logRecord?.RecordType, "job", StringComparison.OrdinalIgnoreCase)) { // find all of the child task records - var childRecords = timeline.Records.Where(x => x.ParentId == logRecord.Id); + IEnumerable childRecords = timeline.Records.Where(x => x.ParentId == logRecord.Id); // sum the line counts for all of the child task records - var childLineCount = childRecords + long childLineCount = childRecords .Where(x => x.Log != null && logsById.ContainsKey(x.Log.Id)) .Sum(x => logsById[x.Log.Id].LineCount); @@ -502,8 +440,8 @@ namespace Azure.Sdk.Tools.PipelineWitness try { long changeTime = ((DateTimeOffset)build.LastChangedDate).ToUnixTimeSeconds(); - var blobPath = $"{build.Project.Name}/{build.FinishTime:yyyy/MM/dd}/{build.Id}-{changeTime}.jsonl"; - var blobClient = this.buildsContainerClient.GetBlobClient(blobPath); + string blobPath = $"{build.Project.Name}/{build.FinishTime:yyyy/MM/dd}/{build.Id}-{changeTime}.jsonl"; + BlobClient blobClient = this.buildsContainerClient.GetBlobClient(blobPath); if (await blobClient.ExistsAsync()) { @@ -511,7 +449,7 @@ namespace Azure.Sdk.Tools.PipelineWitness return; } - var content = JsonConvert.SerializeObject(new + string content = JsonConvert.SerializeObject(new { OrganizationName = account, ProjectId = build.Project?.Id, @@ -582,8 +520,8 @@ namespace Azure.Sdk.Tools.PipelineWitness return; } - var blobPath = $"{build.Project.Name}/{build.FinishTime:yyyy/MM/dd}/{build.Id}-{timeline.ChangeId}.jsonl"; - var blobClient = this.buildTimelineRecordsContainerClient.GetBlobClient(blobPath); + string blobPath = $"{build.Project.Name}/{build.FinishTime:yyyy/MM/dd}/{build.Id}-{timeline.ChangeId}.jsonl"; + BlobClient blobClient = this.buildTimelineRecordsContainerClient.GetBlobClient(blobPath); if (await blobClient.ExistsAsync()) { @@ -591,8 +529,8 @@ namespace Azure.Sdk.Tools.PipelineWitness return; } - var builder = new StringBuilder(); - foreach (var record in timeline.Records) + StringBuilder builder = new(); + foreach (TimelineRecord record in timeline.Records) { builder.AppendLine(JsonConvert.SerializeObject( new @@ -656,8 +594,8 @@ namespace Azure.Sdk.Tools.PipelineWitness { // we don't use FinishTime in the logs blob path to prevent duplicating logs when processing retries. // i.e. logs with a given buildid/logid are immutable and retries only add new logs. - var blobPath = $"{build.Project.Name}/{build.QueueTime:yyyy/MM/dd}/{build.Id}-{log.LogId}.jsonl"; - var blobClient = this.buildLogLinesContainerClient.GetBlobClient(blobPath); + string blobPath = $"{build.Project.Name}/{build.QueueTime:yyyy/MM/dd}/{build.Id}-{log.LogId}.jsonl"; + BlobClient blobClient = this.buildLogLinesContainerClient.GetBlobClient(blobPath); if (await blobClient.ExistsAsync()) { @@ -667,44 +605,44 @@ namespace Azure.Sdk.Tools.PipelineWitness this.logger.LogInformation("Processing log for build {BuildId}, record {RecordId}, log {LogId}", build.Id, log.RecordId, log.LogId); - var lineNumber = 0; - var characterCount = 0; + int lineNumber = 0; + int characterCount = 0; // Over an open read stream and an open write stream, one line at a time, read, process, and write to // blob storage - using (var logStream = await this.logProvider.GetLogStreamAsync(build.Project.Name, build.Id, log.LogId)) - using (var logReader = new StreamReader(logStream)) - using (var blobStream = await blobClient.OpenWriteAsync(overwrite: true, new BlobOpenWriteOptions())) - using (var blobWriter = new StreamWriter(blobStream)) + using (Stream logStream = await this.logProvider.GetLogStreamAsync(build.Project.Name, build.Id, log.LogId)) + using (StreamReader logReader = new(logStream)) + using (Stream blobStream = await blobClient.OpenWriteAsync(overwrite: true, new BlobOpenWriteOptions())) + using (StreamWriter blobWriter = new(blobStream)) { - var lastTimeStamp = log.LogCreatedOn; + DateTimeOffset lastTimeStamp = log.LogCreatedOn; while (true) { - var line = await logReader.ReadLineAsync(); + string line = await logReader.ReadLineAsync(); if (line == null) { break; } - var isLastLine = logReader.EndOfStream; + bool isLastLine = logReader.EndOfStream; lineNumber += 1; characterCount += line.Length; // log lines usually follow the format: // 2022-03-30T21:38:38.7007903Z Downloading task: AzureKeyVault (1.200.0) // Sometimes, there's no leading timestamp, so we'll use the last timestamp we saw. - var match = Regex.Match(line, @"^([^Z]{20,28}Z) (.*)$"); + Match match = Regex.Match(line, @"^([^Z]{20,28}Z) (.*)$"); - var timestamp = match.Success + DateTimeOffset timestamp = match.Success ? DateTime.ParseExact(match.Groups[1].Value, TimeFormat, null, System.Globalization.DateTimeStyles.AssumeUniversal).ToUniversalTime() : lastTimeStamp; lastTimeStamp = timestamp; - var message = match.Success ? match.Groups[2].Value : line; + string message = match.Success ? match.Groups[2].Value : line; await blobWriter.WriteLineAsync(JsonConvert.SerializeObject(new { @@ -725,7 +663,7 @@ namespace Azure.Sdk.Tools.PipelineWitness } } - logger.LogInformation("Processed {CharacterCount} characters and {LineCount} lines for build {BuildId}, record {RecordId}, log {LogId}", characterCount, lineNumber, build.Id, log.RecordId, log.LogId); + this.logger.LogInformation("Processed {CharacterCount} characters and {LineCount} lines for build {BuildId}, record {RecordId}, log {LogId}", characterCount, lineNumber, build.Id, log.RecordId, log.LogId); } catch (RequestFailedException ex) when (ex.Status == (int)HttpStatusCode.Conflict) { @@ -737,31 +675,31 @@ namespace Azure.Sdk.Tools.PipelineWitness throw; } } - + private async Task UploadTestRunBlobsAsync(string account, Build build) { try { - var continuationToken = string.Empty; - var buildIds = new[] { build.Id }; + string continuationToken = string.Empty; + int[] buildIds = new[] { build.Id }; - var minLastUpdatedDate = build.QueueTime.Value.AddHours(-1); - var maxLastUpdatedDate = build.FinishTime.Value.AddHours(1); + DateTime minLastUpdatedDate = build.QueueTime.Value.AddHours(-1); + DateTime maxLastUpdatedDate = build.FinishTime.Value.AddHours(1); - var rangeStart = minLastUpdatedDate; + DateTime rangeStart = minLastUpdatedDate; - while(rangeStart < maxLastUpdatedDate) + while (rangeStart < maxLastUpdatedDate) { // Ado limits test run queries to a 7 day range, so we'll chunk on 6 days. - var rangeEnd = rangeStart.AddDays(6); - if(rangeEnd > maxLastUpdatedDate) + DateTime rangeEnd = rangeStart.AddDays(6); + if (rangeEnd > maxLastUpdatedDate) { rangeEnd = maxLastUpdatedDate; } do { - var page = await testResultsClient.QueryTestRunsAsync2( + IPagedList page = await this.testResultsClient.QueryTestRunsAsync2( build.Project.Id, rangeStart, rangeEnd, @@ -769,7 +707,7 @@ namespace Azure.Sdk.Tools.PipelineWitness buildIds: buildIds ); - foreach (var testRun in page) + foreach (TestRun testRun in page) { await UploadTestRunBlobAsync(account, build, testRun); await UploadTestRunResultBlobAsync(account, build, testRun); @@ -792,8 +730,8 @@ namespace Azure.Sdk.Tools.PipelineWitness { try { - var blobPath = $"{build.Project.Name}/{testRun.CompletedDate:yyyy/MM/dd}/{testRun.Id}.jsonl"; - var blobClient = this.testRunsContainerClient.GetBlobClient(blobPath); + string blobPath = $"{build.Project.Name}/{testRun.CompletedDate:yyyy/MM/dd}/{testRun.Id}.jsonl"; + BlobClient blobClient = this.testRunsContainerClient.GetBlobClient(blobPath); if (await blobClient.ExistsAsync()) { @@ -801,9 +739,9 @@ namespace Azure.Sdk.Tools.PipelineWitness return; } - var stats = testRun.RunStatistics.ToDictionary(x => x.Outcome, x => x.Count); + Dictionary stats = testRun.RunStatistics.ToDictionary(x => x.Outcome, x => x.Count); - var content = JsonConvert.SerializeObject(new + string content = JsonConvert.SerializeObject(new { OrganizationName = account, ProjectId = build.Project?.Id, @@ -821,7 +759,7 @@ namespace Azure.Sdk.Tools.PipelineWitness BranchName = build.SourceBranch, HasDetail = default(bool?), IsAutomated = testRun.IsAutomated, - ResultAbortedCount = stats.TryGetValue("Aborted", out var value) ? value : 0, + ResultAbortedCount = stats.TryGetValue("Aborted", out int value) ? value : 0, ResultBlockedCount = stats.TryGetValue("Blocked", out value) ? value : 0, ResultCount = testRun.TotalTests, ResultErrorCount = stats.TryGetValue("Error", out value) ? value : 0, @@ -875,8 +813,8 @@ namespace Azure.Sdk.Tools.PipelineWitness { try { - var blobPath = $"{build.Project.Name}/{testRun.CompletedDate:yyyy/MM/dd}/{testRun.Id}.jsonl"; - var blobClient = this.testResultsContainerClient.GetBlobClient(blobPath); + string blobPath = $"{build.Project.Name}/{testRun.CompletedDate:yyyy/MM/dd}/{testRun.Id}.jsonl"; + BlobClient blobClient = this.testResultsContainerClient.GetBlobClient(blobPath); if (await blobClient.ExistsAsync()) { @@ -884,14 +822,14 @@ namespace Azure.Sdk.Tools.PipelineWitness return; } - var builder = new StringBuilder(); + StringBuilder builder = new(); int batchCount = BlobUploadProcessor.CalculateBatches(testRun.TotalTests, batchSize: ApiBatchSize); - for(int batchMultiplier = 0; batchMultiplier < batchCount; batchMultiplier++) + for (int batchMultiplier = 0; batchMultiplier < batchCount; batchMultiplier++) { - var data = await testResultsClient.GetTestResultsAsync(build.Project.Id, testRun.Id, top: ApiBatchSize, skip: batchMultiplier * ApiBatchSize); + List data = await this.testResultsClient.GetTestResultsAsync(build.Project.Id, testRun.Id, top: ApiBatchSize, skip: batchMultiplier * ApiBatchSize); - foreach (var record in data) + foreach (TestCaseResult record in data) { builder.AppendLine(JsonConvert.SerializeObject( new @@ -936,7 +874,7 @@ namespace Azure.Sdk.Tools.PipelineWitness try { - build = await buildClient.GetBuildAsync(projectId, buildId); + build = await this.buildClient.GetBuildAsync(projectId, buildId); } catch (BuildNotFoundException) { diff --git a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Entities/BuildLogBundle.cs b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Entities/BuildLogBundle.cs index b12158acd..7ad749c75 100644 --- a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Entities/BuildLogBundle.cs +++ b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Entities/BuildLogBundle.cs @@ -14,15 +14,15 @@ namespace Azure.Sdk.Tools.PipelineWitness public int BuildId { get; set; } public DateTimeOffset StartTime { get; set; } - + public DateTimeOffset FinishTime { get; set; } - + public DateTimeOffset QueueTime { get; set; } - + public int DefinitionId { get; set; } - + public string DefinitionPath { get; set; } - + public string DefinitionName { get; set; } public List TimelineLogs { get; } = new List(); diff --git a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Entities/BuildLogInfo.cs b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Entities/BuildLogInfo.cs index 79f0bae02..b657b1d15 100644 --- a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Entities/BuildLogInfo.cs +++ b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Entities/BuildLogInfo.cs @@ -1,7 +1,7 @@ -namespace Azure.Sdk.Tools.PipelineWitness -{ - using System; +using System; +namespace Azure.Sdk.Tools.PipelineWitness +{ public class BuildLogInfo { public int LogId { get; set; } @@ -11,9 +11,9 @@ public DateTimeOffset LogCreatedOn { get; set; } public string RecordType { get; set; } - + public Guid? RecordId { get; set; } - + public Guid? ParentRecordId { get; set; } } } diff --git a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Program.cs b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Program.cs index 3e99350f8..f5460e275 100644 --- a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Program.cs +++ b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Program.cs @@ -13,8 +13,8 @@ namespace Azure.Sdk.Tools.PipelineWitness // per queue storage performance docs, set the default connection limit to >= 100 // https://learn.microsoft.com/en-us/azure/storage/queues/storage-performance-checklist#increase-default-connection-limit ServicePointManager.DefaultConnectionLimit = 100; - - var builder = WebApplication.CreateBuilder(args); + + WebApplicationBuilder builder = WebApplication.CreateBuilder(args); // Add services to the container. builder.Services.AddControllers(); @@ -26,7 +26,7 @@ namespace Azure.Sdk.Tools.PipelineWitness Startup.Configure(builder); - var app = builder.Build(); + WebApplication app = builder.Build(); // Configure the HTTP request pipeline. if (app.Environment.IsDevelopment()) diff --git a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/AzurePipelinesBuildDefinitionWorker.cs b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/AzurePipelinesBuildDefinitionWorker.cs index 6a14b773c..6ca01daba 100644 --- a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/AzurePipelinesBuildDefinitionWorker.cs +++ b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/AzurePipelinesBuildDefinitionWorker.cs @@ -1,12 +1,12 @@ using System; +using System.Diagnostics; using System.Threading; using System.Threading.Tasks; +using Azure.Sdk.Tools.PipelineWitness.Configuration; +using Azure.Sdk.Tools.PipelineWitness.Services.WorkTokens; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; -using System.Diagnostics; using Microsoft.Extensions.Options; -using Azure.Sdk.Tools.PipelineWitness.Services.WorkTokens; -using Azure.Sdk.Tools.PipelineWitness.Configuration; namespace Azure.Sdk.Tools.PipelineWitness.Services { @@ -15,7 +15,7 @@ namespace Azure.Sdk.Tools.PipelineWitness.Services private readonly ILogger logger; private readonly BlobUploadProcessor runProcessor; private readonly IOptions options; - private IAsyncLockProvider asyncLockProvider; + private readonly IAsyncLockProvider asyncLockProvider; public AzurePipelinesBuildDefinitionWorker( ILogger logger, @@ -31,32 +31,32 @@ namespace Azure.Sdk.Tools.PipelineWitness.Services protected override async Task ExecuteAsync(CancellationToken stoppingToken) { - var processEvery = TimeSpan.FromMinutes(60); + TimeSpan processEvery = TimeSpan.FromMinutes(60); while (true) { - var stopWatch = Stopwatch.StartNew(); - var settings = this.options.Value; + Stopwatch stopWatch = Stopwatch.StartNew(); + PipelineWitnessSettings settings = this.options.Value; try { - await using var asyncLock = await this.asyncLockProvider.GetLockAsync("UpdateBuildDefinitions", processEvery, stoppingToken); + await using IAsyncLock asyncLock = await this.asyncLockProvider.GetLockAsync("UpdateBuildDefinitions", processEvery, stoppingToken); // if there's no asyncLock, this process has alread completed in the last hour if (asyncLock != null) { - foreach (var project in settings.Projects) + foreach (string project in settings.Projects) { await this.runProcessor.UploadBuildDefinitionBlobsAsync(settings.Account, project); } } } - catch(Exception ex) + catch (Exception ex) { this.logger.LogError(ex, "Error processing build definitions"); } - var duration = settings.BuildDefinitionLoopPeriod - stopWatch.Elapsed; + TimeSpan duration = settings.BuildDefinitionLoopPeriod - stopWatch.Elapsed; if (duration > TimeSpan.Zero) { await Task.Delay(duration, stoppingToken); diff --git a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/BuildCompleteQueueWorker.cs b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/BuildCompleteQueueWorker.cs index 8fa7ed547..c115a118d 100644 --- a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/BuildCompleteQueueWorker.cs +++ b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/BuildCompleteQueueWorker.cs @@ -1,3 +1,4 @@ +using System; using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; @@ -17,7 +18,6 @@ namespace Azure.Sdk.Tools.PipelineWitness.Services internal class BuildCompleteQueueWorker : QueueWorkerBackgroundService { private readonly ILogger logger; - private readonly TelemetryClient telemetryClient; private readonly BlobUploadProcessor runProcessor; public BuildCompleteQueueWorker( @@ -35,16 +35,15 @@ namespace Azure.Sdk.Tools.PipelineWitness.Services { this.logger = logger; this.runProcessor = runProcessor; - this.telemetryClient = telemetryClient; } internal override async Task ProcessMessageAsync(QueueMessage message, CancellationToken cancellationToken) { this.logger.LogInformation("Processing build.complete event: {MessageText}", message.MessageText); - var devopsEvent = JObject.Parse(message.MessageText); + JObject devopsEvent = JObject.Parse(message.MessageText); - var buildUrl = devopsEvent["resource"]?.Value("url"); + string buildUrl = devopsEvent["resource"]?.Value("url"); if (buildUrl == null) { @@ -52,7 +51,7 @@ namespace Azure.Sdk.Tools.PipelineWitness.Services return; } - var match = Regex.Match(buildUrl, @"^https://dev.azure.com/(?[\w-]+)/(?[0-9a-fA-F-]+)/_apis/build/Builds/(?\d+)$"); + Match match = Regex.Match(buildUrl, @"^https://dev.azure.com/(?[\w-]+)/(?[0-9a-fA-F-]+)/_apis/build/Builds/(?\d+)$"); if (!match.Success) { @@ -60,23 +59,23 @@ namespace Azure.Sdk.Tools.PipelineWitness.Services return; } - var account = match.Groups["account"].Value; - var projectIdString = match.Groups["project"].Value; - var buildIdString = match.Groups["build"].Value; + string account = match.Groups["account"].Value; + string projectIdString = match.Groups["project"].Value; + string buildIdString = match.Groups["build"].Value; - if (!System.Guid.TryParse(projectIdString, out var projectId)) + if (!Guid.TryParse(projectIdString, out Guid projectId)) { this.logger.LogError("Could not parse project id as a guid '{ProjectId}'", projectIdString); return; } - if (!int.TryParse(buildIdString, out var buildId)) + if (!int.TryParse(buildIdString, out int buildId)) { this.logger.LogError("Could not parse build id as a guid '{BuildId}'", buildIdString); return; } - await runProcessor.UploadBuildBlobsAsync(account, projectId, buildId); + await this.runProcessor.UploadBuildBlobsAsync(account, projectId, buildId); } } } diff --git a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/BuildLogProvider.cs b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/BuildLogProvider.cs index 1c054223a..20f435f71 100644 --- a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/BuildLogProvider.cs +++ b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/BuildLogProvider.cs @@ -22,22 +22,22 @@ namespace Azure.Sdk.Tools.PipelineWitness.Services public virtual async Task> GetLogLinesAsync(Build build, int logId) { - logger.LogTrace("Getting logs for build {BuildId}, log {LogId} from rest api", build.Id, logId); + this.logger.LogTrace("Getting logs for build {BuildId}, log {LogId} from rest api", build.Id, logId); - var buildHttpClient = vssConnection.GetClient(); - var response = await buildHttpClient.GetBuildLogLinesAsync(build.Project.Id, build.Id, logId); + BuildHttpClient buildHttpClient = this.vssConnection.GetClient(); + List response = await buildHttpClient.GetBuildLogLinesAsync(build.Project.Id, build.Id, logId); - logger.LogTrace("Received {CharacterCount} characters in {LineCount} lines for build {BuildId}, log {LogId}", response.Sum(x => x.Length), response.Count, build.Id, logId); + this.logger.LogTrace("Received {CharacterCount} characters in {LineCount} lines for build {BuildId}, log {LogId}", response.Sum(x => x.Length), response.Count, build.Id, logId); return response; } public virtual async Task GetLogStreamAsync(string projectName, int buildId, int logId) { - logger.LogTrace("Getting logs for build {BuildId}, log {LogId} from rest api", buildId, logId); + this.logger.LogTrace("Getting logs for build {BuildId}, log {LogId} from rest api", buildId, logId); - var buildHttpClient = vssConnection.GetClient(); - var stream = await buildHttpClient.GetBuildLogAsync(projectName, buildId, logId); + BuildHttpClient buildHttpClient = this.vssConnection.GetClient(); + Stream stream = await buildHttpClient.GetBuildLogAsync(projectName, buildId, logId); return stream; } diff --git a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/EnhancedBuildHttpClient.cs b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/EnhancedBuildHttpClient.cs index 8fabcb06b..bbb91ac11 100644 --- a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/EnhancedBuildHttpClient.cs +++ b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/EnhancedBuildHttpClient.cs @@ -14,23 +14,23 @@ namespace Azure.Sdk.Tools.PipelineWitness.Services { public EnhancedBuildHttpClient(Uri baseUrl, VssCredentials credentials) : base(baseUrl, credentials) - {} + { } public EnhancedBuildHttpClient(Uri baseUrl, VssCredentials credentials, VssHttpRequestSettings settings) : base(baseUrl, credentials, settings) - {} + { } public EnhancedBuildHttpClient(Uri baseUrl, VssCredentials credentials, params DelegatingHandler[] handlers) : base(baseUrl, credentials, handlers) - {} + { } public EnhancedBuildHttpClient(Uri baseUrl, VssCredentials credentials, VssHttpRequestSettings settings, params DelegatingHandler[] handlers) : base(baseUrl, credentials, settings, handlers) - {} + { } public EnhancedBuildHttpClient(Uri baseUrl, HttpMessageHandler pipeline, bool disposeHandler) : base(baseUrl, pipeline, disposeHandler) - {} + { } public override async Task GetArtifactContentZipAsync( Guid project, @@ -39,7 +39,7 @@ namespace Azure.Sdk.Tools.PipelineWitness.Services object userState = null, CancellationToken cancellationToken = default) { - var artifact = await base.GetArtifactAsync(project, buildId, artifactName, userState, cancellationToken); + BuildArtifact artifact = await base.GetArtifactAsync(project, buildId, artifactName, userState, cancellationToken); return await GetArtifactContentZipAsync(artifact, cancellationToken); } @@ -50,19 +50,19 @@ namespace Azure.Sdk.Tools.PipelineWitness.Services object userState = null, CancellationToken cancellationToken = default) { - var artifact = await base.GetArtifactAsync(project, buildId, artifactName, userState, cancellationToken); + BuildArtifact artifact = await base.GetArtifactAsync(project, buildId, artifactName, userState, cancellationToken); return await GetArtifactContentZipAsync(artifact, cancellationToken); } private async Task GetArtifactContentZipAsync(BuildArtifact artifact, CancellationToken cancellationToken) { - var downloadUrl = artifact?.Resource?.DownloadUrl; + string downloadUrl = artifact?.Resource?.DownloadUrl; if (string.IsNullOrWhiteSpace(downloadUrl)) { throw new InvalidArtifactDataException("Artifact contained no download url"); } - var responseStream = await Client.GetStreamAsync(downloadUrl, cancellationToken); + Stream responseStream = await Client.GetStreamAsync(downloadUrl, cancellationToken); return responseStream; } } diff --git a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/AzureArtifactsServiceUnavailableClassifier.cs b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/AzureArtifactsServiceUnavailableClassifier.cs deleted file mode 100644 index c1768ef81..000000000 --- a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/AzureArtifactsServiceUnavailableClassifier.cs +++ /dev/null @@ -1,37 +0,0 @@ -using Microsoft.TeamFoundation.Build.WebApi; -using System.Linq; -using System.Threading.Tasks; - -namespace Azure.Sdk.Tools.PipelineWitness.Services.FailureAnalysis -{ - public class AzureArtifactsServiceUnavailableClassifier : IFailureClassifier - { - public AzureArtifactsServiceUnavailableClassifier(BuildLogProvider buildLogProvider) - { - this.buildLogProvider = buildLogProvider; - } - - private readonly BuildLogProvider buildLogProvider; - - public async Task ClassifyAsync(FailureAnalyzerContext context) - { - var failedTasks = from r in context.Timeline.Records - where r.Result == TaskResult.Failed - where r.RecordType == "Task" - where r.Task != null - where r.Name == "Publish to Java Dev Feed" - where r.Log != null - select r; - - foreach (var failedTask in failedTasks) - { - var lines = await this.buildLogProvider.GetLogLinesAsync(context.Build, failedTask.Log.Id); - - if (lines.Any(line => line.Contains("Transfer failed for https://pkgs.dev.azure.com") && line.Contains("503 Service Unavailable"))) - { - context.AddFailure(failedTask, "Azure Artifacts Service Unavailable"); - } - } - } - } -} diff --git a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/AzurePipelinesPoolOutageClassifier.cs b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/AzurePipelinesPoolOutageClassifier.cs deleted file mode 100644 index 9d6de528e..000000000 --- a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/AzurePipelinesPoolOutageClassifier.cs +++ /dev/null @@ -1,26 +0,0 @@ -using System.Linq; -using System.Threading.Tasks; - -namespace Azure.Sdk.Tools.PipelineWitness.Services.FailureAnalysis -{ - public class AzurePipelinesPoolOutageClassifier : IFailureClassifier - { - public Task ClassifyAsync(FailureAnalyzerContext context) - { - var jobs = from r in context.Timeline.Records - where r.RecordType == "Job" - where r.Issues.Any(i => i.Message.Contains("abandoned due to an infrastructure failure")) - select r; - - if (jobs.Count() > 0) - { - foreach (var job in jobs) - { - context.AddFailure(job, "Azure Pipelines Pool Outage"); - } - } - - return Task.CompletedTask; - } - } -} diff --git a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/AzurePowerShellModuleInstallationFailureClassifier.cs b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/AzurePowerShellModuleInstallationFailureClassifier.cs deleted file mode 100644 index fafc0b811..000000000 --- a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/AzurePowerShellModuleInstallationFailureClassifier.cs +++ /dev/null @@ -1,31 +0,0 @@ -using Microsoft.TeamFoundation.Build.WebApi; -using System.Linq; -using System.Threading.Tasks; - -namespace Azure.Sdk.Tools.PipelineWitness.Services.FailureAnalysis -{ - public class AzurePowerShellModuleInstallationFailureClassifier : IFailureClassifier - { - public Task ClassifyAsync(FailureAnalyzerContext context) - { - if (context.Build.Definition.Name.EndsWith("- tests")) - { - var failedTasks = from r in context.Timeline.Records - where r.Result == TaskResult.Failed - where r.RecordType == "Task" - where r.Name == "Install Azure PowerShell module" - select r; - - if (failedTasks.Count() > 0) - { - foreach (var failedTask in failedTasks) - { - context.AddFailure(failedTask, "Azure PS Module"); - } - } - } - - return Task.CompletedTask; - } - } -} diff --git a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/AzuriteInstallFailureClassifier.cs b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/AzuriteInstallFailureClassifier.cs deleted file mode 100644 index 4110f6009..000000000 --- a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/AzuriteInstallFailureClassifier.cs +++ /dev/null @@ -1,28 +0,0 @@ -using Microsoft.TeamFoundation.Build.WebApi; -using System.Linq; -using System.Threading.Tasks; - -namespace Azure.Sdk.Tools.PipelineWitness.Services.FailureAnalysis -{ - public class AzuriteInstallFailureClassifier : IFailureClassifier - { - public Task ClassifyAsync(FailureAnalyzerContext context) - { - var failedTasks = from r in context.Timeline.Records - where r.Result == TaskResult.Failed - where r.RecordType == "Task" - where r.Name == "Install Azurite" - select r; - - if (failedTasks.Count() > 0) - { - foreach (var failedTask in failedTasks) - { - context.AddFailure(failedTask, "Azurite Install"); - } - } - - return Task.CompletedTask; - } - } -} diff --git a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/CacheFailureClassifier.cs b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/CacheFailureClassifier.cs deleted file mode 100644 index cf20bb6df..000000000 --- a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/CacheFailureClassifier.cs +++ /dev/null @@ -1,75 +0,0 @@ -using Microsoft.TeamFoundation.Build.WebApi; -using Microsoft.VisualStudio.Services.WebApi; -using System.Linq; -using System.Threading.Tasks; - -namespace Azure.Sdk.Tools.PipelineWitness.Services.FailureAnalysis -{ - public class CacheFailureClassifier : IFailureClassifier - { - private class FailureClassifier - { - public readonly string MessageContains; - public readonly string FailureName; - - public FailureClassifier(string messageContains, string failureName) - { - this.MessageContains = messageContains; - this.FailureName = failureName; - } - - public bool IsFailure(string message) - { - return message.StartsWith(MessageContains); - } - } - - private static readonly FailureClassifier[] failureClassifiers = new FailureClassifier[] - { - new FailureClassifier("Chunks are not arriving in order or sizes are not matched up", "Cache Chunk Ordering" ), - new FailureClassifier("The task has timed out", "Cache Task Timeout" ), - new FailureClassifier("Service Unavailable", "Cache Service Unavailable"), - new FailureClassifier("The HTTP request timed out after", "Cache Service HTTP Timeout"), - new FailureClassifier("Access to the path", "Cache Cannot Access Path"), - }; - - public CacheFailureClassifier(VssConnection vssConnection) - { - this.vssConnection = vssConnection; - buildClient = vssConnection.GetClient(); - } - - private VssConnection vssConnection; - private BuildHttpClient buildClient; - - public Task ClassifyAsync(FailureAnalyzerContext context) - { - var failedTasks = from r in context.Timeline.Records - where r.Result == TaskResult.Failed - where r.RecordType == "Task" - where r.Task != null - where r.Task.Name == "Cache" - where r.Log != null - select r; - - var classificationFound = false; - foreach (var failedTask in failedTasks) - { - foreach (var classifier in failureClassifiers) { - if (failedTask.Issues.Any(i => classifier.IsFailure(i.Message))) - { - context.AddFailure(failedTask, classifier.FailureName); - classificationFound = true; - } - } - - if (!classificationFound) - { - context.AddFailure(failedTask, "Cache Failure Other"); - } - } - - return Task.CompletedTask; - } - } -} diff --git a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/CancelledTaskClassifier.cs b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/CancelledTaskClassifier.cs deleted file mode 100644 index 7197f33dc..000000000 --- a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/CancelledTaskClassifier.cs +++ /dev/null @@ -1,27 +0,0 @@ -using Microsoft.TeamFoundation.Build.WebApi; -using System.Linq; -using System.Threading.Tasks; - -namespace Azure.Sdk.Tools.PipelineWitness.Services.FailureAnalysis -{ - public class CancelledTaskClassifier : IFailureClassifier - { - public Task ClassifyAsync(FailureAnalyzerContext context) - { - var timedOutTestTasks = from r in context.Timeline.Records - where r.RecordType == "Task" - where r.Result == TaskResult.Canceled - select r; - - if (timedOutTestTasks.Count() > 0) - { - foreach (var timedOutTestTask in timedOutTestTasks) - { - context.AddFailure(timedOutTestTask, "Cancelled Task"); - } - } - - return Task.CompletedTask; - } - } -} diff --git a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/CodeSigningFailureClassifier.cs b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/CodeSigningFailureClassifier.cs deleted file mode 100644 index c03f311b5..000000000 --- a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/CodeSigningFailureClassifier.cs +++ /dev/null @@ -1,30 +0,0 @@ -using System.Linq; -using System.Threading.Tasks; -using Microsoft.TeamFoundation.Build.WebApi; - -namespace Azure.Sdk.Tools.PipelineWitness.Services.FailureAnalysis -{ - public class CodeSigningFailureClassifier : IFailureClassifier - { - public Task ClassifyAsync(FailureAnalyzerContext context) - { - var failedTasks = from r in context.Timeline.Records - where r.Result == TaskResult.Failed - where r.RecordType == "Task" - where r.Task != null - where r.Task.Name == "EsrpCodeSigning" - where r.Log != null - select r; - - if (failedTasks.Count() > 0) - { - foreach (var failedTask in failedTasks) - { - context.AddFailure(failedTask, "Code Signing"); - } - } - - return Task.CompletedTask; - } - } -} diff --git a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/CosmosDbEmulatorStartFailureClassifier.cs b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/CosmosDbEmulatorStartFailureClassifier.cs deleted file mode 100644 index 0ca916974..000000000 --- a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/CosmosDbEmulatorStartFailureClassifier.cs +++ /dev/null @@ -1,28 +0,0 @@ -using System.Linq; -using System.Threading.Tasks; -using Microsoft.TeamFoundation.Build.WebApi; - -namespace Azure.Sdk.Tools.PipelineWitness.Services.FailureAnalysis -{ - public class CosmosDbEmulatorStartFailureClassifier : IFailureClassifier - { - public Task ClassifyAsync(FailureAnalyzerContext context) - { - var failedTasks = from r in context.Timeline.Records - where r.RecordType == "Task" - where r.Name == "Start Cosmos DB Emulator" - where r.Result == TaskResult.Failed - select r; - - if (failedTasks.Count() > 0) - { - foreach (var failedTask in failedTasks) - { - context.AddFailure(failedTask, "Cosmos DB Emulator Failure"); - } - } - - return Task.CompletedTask; - } - } -} diff --git a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/DnsResolutionFailureClassifier.cs b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/DnsResolutionFailureClassifier.cs deleted file mode 100644 index 88e0b9c1d..000000000 --- a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/DnsResolutionFailureClassifier.cs +++ /dev/null @@ -1,45 +0,0 @@ -using Microsoft.TeamFoundation.Build.WebApi; -using System; -using System.Linq; -using System.Threading.Tasks; - -namespace Azure.Sdk.Tools.PipelineWitness.Services.FailureAnalysis -{ - public class DnsResolutionFailureClassifier : IFailureClassifier - { - public DnsResolutionFailureClassifier(BuildLogProvider buildLogProvider) - { - this.buildLogProvider = buildLogProvider; - } - - private readonly BuildLogProvider buildLogProvider; - - private bool IsDnsResolutionFailure(string line) - { - return line.Contains("EAI_AGAIN", StringComparison.OrdinalIgnoreCase) - || line.Contains("getaddrinfo", StringComparison.OrdinalIgnoreCase) - || line.Contains("Temporary failure in name resolution", StringComparison.OrdinalIgnoreCase) - || line.Contains("No such host is known", StringComparison.OrdinalIgnoreCase) - || line.Contains("Couldn't resolve host name", StringComparison.OrdinalIgnoreCase); - } - - public async Task ClassifyAsync(FailureAnalyzerContext context) - { - var failedTasks = from r in context.Timeline.Records - where r.Result == TaskResult.Failed - where r.RecordType == "Task" - where r.Log != null - select r; - - foreach (var failedTask in failedTasks) - { - var lines = await buildLogProvider.GetLogLinesAsync(context.Build, failedTask.Log.Id); - - if (lines.Any(line => IsDnsResolutionFailure(line))) - { - context.AddFailure(failedTask, "DNS Resolution Failure"); - } - } - } - } -} diff --git a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/DotnetPipelineTestFailureClassifier.cs b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/DotnetPipelineTestFailureClassifier.cs deleted file mode 100644 index 0424e5bdc..000000000 --- a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/DotnetPipelineTestFailureClassifier.cs +++ /dev/null @@ -1,31 +0,0 @@ -using Microsoft.TeamFoundation.Build.WebApi; -using System.Linq; -using System.Threading.Tasks; - -namespace Azure.Sdk.Tools.PipelineWitness.Services.FailureAnalysis -{ - public class DotnetPipelineTestFailureClassifier : IFailureClassifier - { - public Task ClassifyAsync(FailureAnalyzerContext context) - { - if (context.Build.Definition.Name.StartsWith("net - ")) - { - var failedTasks = from r in context.Timeline.Records - where r.Result == TaskResult.Failed - where r.RecordType == "Task" - where r.Name.StartsWith("Build & Test") - select r; - - if (failedTasks.Count() > 0) - { - foreach (var failedTask in failedTasks) - { - context.AddFailure(failedTask, "Test Failure"); - } - } - } - - return Task.CompletedTask; - } - } -} diff --git a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/DownloadSecretsFailureClassifier.cs b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/DownloadSecretsFailureClassifier.cs deleted file mode 100644 index 2fa78a545..000000000 --- a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/DownloadSecretsFailureClassifier.cs +++ /dev/null @@ -1,28 +0,0 @@ -using Microsoft.TeamFoundation.Build.WebApi; -using System.Linq; -using System.Threading.Tasks; - -namespace Azure.Sdk.Tools.PipelineWitness.Services.FailureAnalysis -{ - public class DownloadSecretsFailureClassifier : IFailureClassifier - { - public Task ClassifyAsync(FailureAnalyzerContext context) - { - var failedTasks = from r in context.Timeline.Records - where r.RecordType == "Task" - where r.Result == TaskResult.Failed - where r.Name.Contains("Download secrets") - select r; - - if (failedTasks.Count() > 0) - { - foreach (var failedTask in failedTasks) - { - context.AddFailure(failedTask, "Secrets Failure"); - } - } - - return Task.CompletedTask; - } - } -} diff --git a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/FailureAnalyzer.cs b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/FailureAnalyzer.cs deleted file mode 100644 index 692cf6b60..000000000 --- a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/FailureAnalyzer.cs +++ /dev/null @@ -1,46 +0,0 @@ -using Azure.Sdk.Tools.PipelineWitness.Entities.AzurePipelines; -using Microsoft.TeamFoundation.Build.WebApi; -using System.Collections.Generic; -using System.Linq; -using System.Threading.Tasks; - -namespace Azure.Sdk.Tools.PipelineWitness.Services.FailureAnalysis -{ - public class FailureAnalyzer : IFailureAnalyzer - { - public FailureAnalyzer(IEnumerable classifiers) - { - this.classifiers = classifiers.ToArray(); - } - - private IFailureClassifier[] classifiers; - - public async Task> AnalyzeFailureAsync(Build build, Timeline timeline) - { - var failures = new List(); - - var context = new FailureAnalyzerContext(build, timeline, failures); - foreach (var classifier in classifiers) - { - await classifier.ClassifyAsync(context); - } - - if (failures.Count == 0) - { - if (build.Result != BuildResult.Succeeded && - build.Result != BuildResult.Canceled) - { - foreach (var record in timeline.Records.Where(x => x.ParentId.HasValue == false)) - { - if (record.Result == TaskResult.Failed) - { - failures.Add(new Failure(record, "Unknown")); - } - } - } - } - - return failures; - } - } -} diff --git a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/FailureAnalyzerContext.cs b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/FailureAnalyzerContext.cs deleted file mode 100644 index 6b72af16d..000000000 --- a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/FailureAnalyzerContext.cs +++ /dev/null @@ -1,57 +0,0 @@ -using Azure.Sdk.Tools.PipelineWitness.Entities.AzurePipelines; -using Microsoft.TeamFoundation.Build.WebApi; -using Microsoft.VisualStudio.Services.Common; -using System.Collections.Generic; -using System.Linq; -using System.Text; - -namespace Azure.Sdk.Tools.PipelineWitness.Services.FailureAnalysis -{ - public class FailureAnalyzerContext - { - public FailureAnalyzerContext(Build build, Timeline timeline, IList failures) - { - Build = build; - Timeline = timeline; - this.failures = failures; - } - - public Build Build { get; private set; } - public Timeline Timeline { get; private set; } - - private IList failures; - - private string GetScope(TimelineRecord record) - { - var timelineStack = new Stack(); - - var current = record; - while (true) - { - timelineStack.Push(current); - - var parent = Timeline.Records.Where(r => r.Id == current.ParentId).SingleOrDefault(); - if (parent == null) - { - break; - } - else - { - current = parent; - } - } - - var scopeBuilder = new StringBuilder(); - timelineStack.ForEach(r => scopeBuilder.Append($"/{r.RecordType}:{r.Name}")); - - var scope = scopeBuilder.ToString(); - return scope; - } - - public void AddFailure(TimelineRecord record, string classification) - { - var failure = new Failure(record, classification); - failures.Add(failure); - } - } -} diff --git a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/GitCheckoutFailureClassifier.cs b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/GitCheckoutFailureClassifier.cs deleted file mode 100644 index 275fddcf4..000000000 --- a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/GitCheckoutFailureClassifier.cs +++ /dev/null @@ -1,27 +0,0 @@ -using System.Linq; -using System.Threading.Tasks; - -namespace Azure.Sdk.Tools.PipelineWitness.Services.FailureAnalysis -{ - public class GitCheckoutFailureClassifier : IFailureClassifier - { - public Task ClassifyAsync(FailureAnalyzerContext context) - { - var tasks = from r in context.Timeline.Records - where r.RecordType == "Task" - where r.Issues.Any(i => i.Message.Contains("Git fetch failed with exit code: 128")) - select r; - - if (tasks.Count() > 0) - { - foreach (var task in tasks) - { - context.AddFailure(task - , "Git Checkout"); - } - } - - return Task.CompletedTask; - } - } -} diff --git a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/IFailureAnalyzer.cs b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/IFailureAnalyzer.cs deleted file mode 100644 index d9677650e..000000000 --- a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/IFailureAnalyzer.cs +++ /dev/null @@ -1,12 +0,0 @@ -using Azure.Sdk.Tools.PipelineWitness.Entities.AzurePipelines; -using Microsoft.TeamFoundation.Build.WebApi; -using System.Collections.Generic; -using System.Threading.Tasks; - -namespace Azure.Sdk.Tools.PipelineWitness.Services.FailureAnalysis -{ - public interface IFailureAnalyzer - { - Task> AnalyzeFailureAsync(Build build, Timeline timeline); - } -} diff --git a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/IFailureClassifier.cs b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/IFailureClassifier.cs deleted file mode 100644 index d3424d739..000000000 --- a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/IFailureClassifier.cs +++ /dev/null @@ -1,9 +0,0 @@ -using System.Threading.Tasks; - -namespace Azure.Sdk.Tools.PipelineWitness.Services.FailureAnalysis -{ - public interface IFailureClassifier - { - Task ClassifyAsync(FailureAnalyzerContext context); - } -} diff --git a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/JavaPipelineTestFailureClassifier.cs b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/JavaPipelineTestFailureClassifier.cs deleted file mode 100644 index 79fb3c46c..000000000 --- a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/JavaPipelineTestFailureClassifier.cs +++ /dev/null @@ -1,31 +0,0 @@ -using Microsoft.TeamFoundation.Build.WebApi; -using System.Linq; -using System.Threading.Tasks; - -namespace Azure.Sdk.Tools.PipelineWitness.Services.FailureAnalysis -{ - public class JavaPipelineTestFailureClassifier : IFailureClassifier - { - public Task ClassifyAsync(FailureAnalyzerContext context) - { - if (context.Build.Definition.Name.StartsWith("java - ")) - { - var failedTasks = from r in context.Timeline.Records - where r.Result == TaskResult.Failed - where r.RecordType == "Task" - where r.Name.StartsWith("Run tests") - select r; - - if (failedTasks.Count() > 0) - { - foreach (var failedTask in failedTasks) - { - context.AddFailure(failedTask, "Test Failure"); - } - } - } - - return Task.CompletedTask; - } - } -} diff --git a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/JavaScriptLiveTestFailureClassifier.cs b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/JavaScriptLiveTestFailureClassifier.cs deleted file mode 100644 index 4d6f17880..000000000 --- a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/JavaScriptLiveTestFailureClassifier.cs +++ /dev/null @@ -1,31 +0,0 @@ -using System.Linq; -using System.Threading.Tasks; -using Microsoft.TeamFoundation.Build.WebApi; - -namespace Azure.Sdk.Tools.PipelineWitness.Services.FailureAnalysis -{ - public class JavaScriptLiveTestFailureClassifier : IFailureClassifier - { - public Task ClassifyAsync(FailureAnalyzerContext context) - { - if (context.Build.Definition.Name.StartsWith("js - ") && context.Build.Definition.Name.EndsWith(" - tests")) - { - var failedTasks = from r in context.Timeline.Records - where r.RecordType == "Task" - where r.Name == "Integration test libraries" - where r.Result == TaskResult.Failed - select r; - - if (failedTasks.Count() > 0) - { - foreach (var failedTask in failedTasks) - { - context.AddFailure(failedTask, "Test Failure"); - } - } - } - - return Task.CompletedTask; - } - } -} diff --git a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/JsDevFeedPublishingFailureClassifier.cs b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/JsDevFeedPublishingFailureClassifier.cs deleted file mode 100644 index eb5a6750d..000000000 --- a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/JsDevFeedPublishingFailureClassifier.cs +++ /dev/null @@ -1,31 +0,0 @@ -using Microsoft.TeamFoundation.Build.WebApi; -using System.Linq; -using System.Threading.Tasks; - -namespace Azure.Sdk.Tools.PipelineWitness.Services.FailureAnalysis -{ - public class JsDevFeedPublishingFailureClassifier : IFailureClassifier - { - public Task ClassifyAsync(FailureAnalyzerContext context) - { - if (context.Build.Definition.Name.StartsWith("js -")) - { - var failedJobs = from r in context.Timeline.Records - where r.Name == "Publish package to daily feed" - where r.RecordType == "Job" - where r.Result == TaskResult.Failed - select r; - - if (failedJobs.Count() > 0) - { - foreach (var failedJob in failedJobs) - { - context.AddFailure(failedJob, "Publish Failure"); - } - } - } - - return Task.CompletedTask; - } - } -} diff --git a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/JsSamplesExecutionFailureClassifier.cs b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/JsSamplesExecutionFailureClassifier.cs deleted file mode 100644 index 9c9d6575c..000000000 --- a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/JsSamplesExecutionFailureClassifier.cs +++ /dev/null @@ -1,30 +0,0 @@ -using Microsoft.TeamFoundation.Build.WebApi; -using System.Linq; -using System.Threading.Tasks; - -namespace Azure.Sdk.Tools.PipelineWitness.Services.FailureAnalysis -{ - public class JsSamplesExecutionFailureClassifier : IFailureClassifier - { - public Task ClassifyAsync(FailureAnalyzerContext context) - { - if (context.Build.Definition.Name.StartsWith("js -")) - { - var failedTasks = from r in context.Timeline.Records - where r.Name == "Execute Samples" - where r.Result == TaskResult.Failed - select r; - - if (failedTasks.Count() > 0) - { - foreach (var failedTask in failedTasks) - { - context.AddFailure(failedTask, "Sample Execution"); - } - } - } - - return Task.CompletedTask; - } - } -} diff --git a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/MavenBrokenPipeFailureClassifier.cs b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/MavenBrokenPipeFailureClassifier.cs deleted file mode 100644 index fb45943fb..000000000 --- a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/MavenBrokenPipeFailureClassifier.cs +++ /dev/null @@ -1,37 +0,0 @@ -using Microsoft.TeamFoundation.Build.WebApi; -using System.Linq; -using System.Threading.Tasks; - -namespace Azure.Sdk.Tools.PipelineWitness.Services.FailureAnalysis -{ - public class MavenBrokenPipeFailureClassifier : IFailureClassifier - { - public MavenBrokenPipeFailureClassifier(BuildLogProvider buildLogProvider) - { - this.buildLogProvider = buildLogProvider; - } - - private readonly BuildLogProvider buildLogProvider; - - public async Task ClassifyAsync(FailureAnalyzerContext context) - { - var failedTasks = from r in context.Timeline.Records - where r.Result == TaskResult.Failed - where r.RecordType == "Task" - where r.Task != null - where r.Task.Name == "Maven" - where r.Log != null - select r; - - foreach (var failedTask in failedTasks) - { - var lines = await buildLogProvider.GetLogLinesAsync(context.Build, failedTask.Log.Id); - - if (lines.Any(line => line.Contains("Connection reset") || line.Contains("Connection timed out") || line.Contains("504 Gateway Timeout"))) - { - context.AddFailure(failedTask, "Maven Broken Pipe"); - } - } - } - } -} diff --git a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/PythonPipelineTestFailureClassifier.cs b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/PythonPipelineTestFailureClassifier.cs deleted file mode 100644 index 19acf38dc..000000000 --- a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/PythonPipelineTestFailureClassifier.cs +++ /dev/null @@ -1,31 +0,0 @@ -using Microsoft.TeamFoundation.Build.WebApi; -using System.Linq; -using System.Threading.Tasks; - -namespace Azure.Sdk.Tools.PipelineWitness.Services.FailureAnalysis -{ - public class PythonPipelineTestFailureClassifier : IFailureClassifier - { - public Task ClassifyAsync(FailureAnalyzerContext context) - { - if (context.Build.Definition.Name.StartsWith("python - ")) - { - var failedTasks = from r in context.Timeline.Records - where r.Result == TaskResult.Failed - where r.RecordType == "Task" - where r.Name == "Run Tests" - select r; - - if (failedTasks.Count() > 0) - { - foreach (var failedTask in failedTasks) - { - context.AddFailure(failedTask, "Test Failure"); - } - } - } - - return Task.CompletedTask; - } - } -} diff --git a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/TestResourcesDeploymentFailureClassifier.cs b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/TestResourcesDeploymentFailureClassifier.cs deleted file mode 100644 index 08bde33fa..000000000 --- a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/FailureAnalysis/TestResourcesDeploymentFailureClassifier.cs +++ /dev/null @@ -1,27 +0,0 @@ -using Microsoft.TeamFoundation.Build.WebApi; -using System.Linq; -using System.Threading.Tasks; - -namespace Azure.Sdk.Tools.PipelineWitness.Services.FailureAnalysis -{ - public class TestResourcesDeploymentFailureClassifier : IFailureClassifier - { - public Task ClassifyAsync(FailureAnalyzerContext context) - { - var failedTasks = from r in context.Timeline.Records - where r.Name.StartsWith("Deploy test resources") - where r.Result == TaskResult.Failed - select r; - - if (failedTasks.Count() > 0) - { - foreach (var failedTask in failedTasks) - { - context.AddFailure(failedTask, "Test Resource Failure"); - } - } - - return Task.CompletedTask; - } - } -} diff --git a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/QueueWorkerBackgroundService.cs b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/QueueWorkerBackgroundService.cs index e24da5519..3d2ab6cf0 100644 --- a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/QueueWorkerBackgroundService.cs +++ b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/QueueWorkerBackgroundService.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Diagnostics; using System.Threading; using System.Threading.Tasks; @@ -9,6 +9,7 @@ using Azure.Storage.Queues.Models; using Microsoft.ApplicationInsights; using Microsoft.ApplicationInsights.DataContracts; +using Microsoft.ApplicationInsights.Extensibility; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; @@ -18,7 +19,7 @@ namespace Azure.Sdk.Tools.PipelineWitness.Services internal abstract class QueueWorkerBackgroundService : BackgroundService { private const string ActivitySourceName = "Azure.Sdk.Tools.PipelineWitness.Queue"; - private static readonly ActivitySource activitySource = new ActivitySource(ActivitySourceName); + private static readonly ActivitySource activitySource = new(ActivitySourceName); private readonly ILogger logger; private readonly QueueServiceClient queueServiceClient; @@ -38,7 +39,7 @@ namespace Azure.Sdk.Tools.PipelineWitness.Services this.queueServiceClient = queueServiceClient ?? throw new ArgumentNullException(nameof(options)); this.options = options ?? throw new ArgumentNullException(nameof(options)); - if(string.IsNullOrWhiteSpace(queueName)) + if (string.IsNullOrWhiteSpace(queueName)) { throw new ArgumentException("Parameter cannot be null or whitespace", nameof(queueName)); } @@ -50,22 +51,22 @@ namespace Azure.Sdk.Tools.PipelineWitness.Services { this.logger.LogInformation("Starting ExecuteAsync for {TypeName}", this.GetType().Name); - var poisonQueueName = $"{this.queueName}-poison"; + string poisonQueueName = $"{this.queueName}-poison"; - var queueClient = this.queueServiceClient.GetQueueClient(this.queueName); - var poisonQueueClient = this.queueServiceClient.GetQueueClient(poisonQueueName); + QueueClient queueClient = this.queueServiceClient.GetQueueClient(this.queueName); + QueueClient poisonQueueClient = this.queueServiceClient.GetQueueClient(poisonQueueName); - await queueClient.CreateIfNotExistsAsync(); - await poisonQueueClient.CreateIfNotExistsAsync(); + await queueClient.CreateIfNotExistsAsync(cancellationToken: stoppingToken); + await poisonQueueClient.CreateIfNotExistsAsync(cancellationToken: stoppingToken); while (true) { - using var loopActivity = activitySource.CreateActivity("MessageLoopIteration", ActivityKind.Internal) ?? new Activity("MessageLoopIteration"); + using Activity loopActivity = activitySource.CreateActivity("MessageLoopIteration", ActivityKind.Internal) ?? new Activity("MessageLoopIteration"); loopActivity?.AddBaggage("QueueName", queueClient.Name); - - using var loopOperation = this.telemetryClient.StartOperation(loopActivity); - var options = this.options.CurrentValue; + using IOperationHolder loopOperation = this.telemetryClient.StartOperation(loopActivity); + + PipelineWitnessSettings options = this.options.CurrentValue; this.logger.LogDebug("Getting next message from queue {QueueName}", queueClient.Name); @@ -74,7 +75,7 @@ namespace Azure.Sdk.Tools.PipelineWitness.Services // We consider a message leased when it's made invisible in the queue and the current process has a // valid PopReceipt for the message. The PopReceipt is used to perform subsequent operations on the // "leased" message. - QueueMessage message = await queueClient.ReceiveMessageAsync(options.MessageLeasePeriod); + QueueMessage message = await queueClient.ReceiveMessageAsync(options.MessageLeasePeriod, stoppingToken); if (message == null) { @@ -92,65 +93,63 @@ namespace Azure.Sdk.Tools.PipelineWitness.Services }); } - using (var activity = activitySource.CreateActivity("ProcessMessage", ActivityKind.Internal) ?? new Activity("ProcessMessage")) + using Activity activity = activitySource.CreateActivity("ProcessMessage", ActivityKind.Internal) ?? new Activity("ProcessMessage"); + activity?.AddBaggage("MessageId", message.MessageId); + + using IOperationHolder operation = this.telemetryClient.StartOperation(activity); + + try { - activity?.AddBaggage("MessageId", message.MessageId); + this.logger.LogDebug("The queue returned a message.\n Queue: {Queue}\n Message: {MessageId}\n Dequeue Count: {DequeueCount}\n Pop Receipt: {PopReceipt}", queueClient.Name, message.MessageId, message.DequeueCount, message.PopReceipt); - using var operation = this.telemetryClient.StartOperation(activity); + using CancellationTokenSource cts = CancellationTokenSource.CreateLinkedTokenSource(stoppingToken); - try + // Because processing a message may take longer than our initial lease period, we want to continually + // renew our lease until processing completes. + Task renewTask = RenewMessageLeaseAsync(queueClient, message, cts.Token); + Task processTask = SafelyProcessMessageAsync(message, cts.Token); + + Task[] tasks = new Task[] { renewTask, processTask }; + + Task.WaitAny(tasks, CancellationToken.None); + + cts.Cancel(); + + Task.WaitAll(tasks, CancellationToken.None); + + // if the renew task doesn't complete successfully, we can't trust the PopReceipt on the message and must abort. + string latestPopReceipt = await renewTask; + + if (processTask.IsCompletedSuccessfully && processTask.Result == true) { - this.logger.LogDebug("The queue returned a message.\n Queue: {Queue}\n Message: {MessageId}\n Dequeue Count: {DequeueCount}\n Pop Receipt: {PopReceipt}", queueClient.Name, message.MessageId, message.DequeueCount, message.PopReceipt); - - using var cts = CancellationTokenSource.CreateLinkedTokenSource(stoppingToken); - - // Because processing a message may take longer than our initial lease period, we want to continually - // renew our lease until processing completes. - var renewTask = RenewMessageLeaseAsync(queueClient, message, cts.Token); - var processTask = SafelyProcessMessageAsync(message, cts.Token); - - var tasks = new Task[] { renewTask, processTask }; - - Task.WaitAny(tasks, CancellationToken.None); - - cts.Cancel(); - - Task.WaitAll(tasks, CancellationToken.None); - - // if the renew task doesn't complete successfully, we can't trust the PopReceipt on the message and must abort. - var latestPopReceipt = await renewTask; - - if (processTask.IsCompletedSuccessfully && processTask.Result == true) + this.logger.LogDebug("Message processed successfully. Removing message from queue.\n MessageId: {MessageId}\n Queue: {QueueName}\n PopReceipt: {PopReceipt}", message.MessageId, queueClient.Name, latestPopReceipt); + await queueClient.DeleteMessageAsync(message.MessageId, latestPopReceipt, stoppingToken); + activity?.SetStatus(ActivityStatusCode.Ok); + operation.Telemetry.Success = true; + } + else + { + activity?.SetStatus(ActivityStatusCode.Error); + operation.Telemetry.Success = false; + if (message.DequeueCount > options.MaxDequeueCount) { - this.logger.LogDebug("Message processed successfully. Removing message from queue.\n MessageId: {MessageId}\n Queue: {QueueName}\n PopReceipt: {PopReceipt}", message.MessageId, queueClient.Name, latestPopReceipt); + this.logger.LogError("Message {MessageId} exceeded maximum dequeue count. Moving to poison queue {QueueName}", message.MessageId, poisonQueueClient.Name); + await poisonQueueClient.SendMessageAsync(message.Body, cancellationToken: stoppingToken); + this.logger.LogDebug("Removing message from queue.\n MessageId: {MessageId}\n Queue: {QueueName}\n PopReceipt: {PopReceipt}", message.MessageId, queueClient.Name, latestPopReceipt); await queueClient.DeleteMessageAsync(message.MessageId, latestPopReceipt, stoppingToken); - activity?.SetStatus(ActivityStatusCode.Ok); - operation.Telemetry.Success = true; } else { - activity?.SetStatus(ActivityStatusCode.Error); - operation.Telemetry.Success = false; - if (message.DequeueCount > options.MaxDequeueCount) - { - this.logger.LogError("Message {MessageId} exceeded maximum dequeue count. Moving to poison queue {QueueName}", message.MessageId, poisonQueueClient.Name); - await poisonQueueClient.SendMessageAsync(message.Body, cancellationToken: stoppingToken); - this.logger.LogDebug("Removing message from queue.\n MessageId: {MessageId}\n Queue: {QueueName}\n PopReceipt: {PopReceipt}", message.MessageId, queueClient.Name, latestPopReceipt); - await queueClient.DeleteMessageAsync(message.MessageId, latestPopReceipt, stoppingToken); - } - else - { - this.logger.LogError("Resetting message visibility timeout to {SleepPeriod}.\n MessageId: {MessageId}\n Queue: {QueueName}\n PopReceipt: {PopReceipt}", options.MessageErrorSleepPeriod, message.MessageId, queueClient.Name, latestPopReceipt); - await queueClient.UpdateMessageAsync(message.MessageId, latestPopReceipt, message.Body, options.MessageErrorSleepPeriod, cancellationToken: stoppingToken); - } + this.logger.LogError("Resetting message visibility timeout to {SleepPeriod}.\n MessageId: {MessageId}\n Queue: {QueueName}\n PopReceipt: {PopReceipt}", options.MessageErrorSleepPeriod, message.MessageId, queueClient.Name, latestPopReceipt); + await queueClient.UpdateMessageAsync(message.MessageId, latestPopReceipt, message.Body, options.MessageErrorSleepPeriod, cancellationToken: stoppingToken); } } - catch (Exception ex) - { - this.logger.LogError(ex, "Exception thrown while procesing queue message."); - activity?.SetStatus(ActivityStatusCode.Error); - operation.Telemetry.Success = false; - } + } + catch (Exception ex) + { + this.logger.LogError(ex, "Exception thrown while procesing queue message."); + activity?.SetStatus(ActivityStatusCode.Error); + operation.Telemetry.Success = false; } } catch (Exception ex) @@ -172,12 +171,12 @@ namespace Azure.Sdk.Tools.PipelineWitness.Services /// the current pop receipt (optimistic concurrency control) for the message. private async Task RenewMessageLeaseAsync(QueueClient queueClient, QueueMessage message, CancellationToken cancellationToken) { - var leasePeriod = this.options.CurrentValue.MessageLeasePeriod; - var halfLife = new TimeSpan(leasePeriod.Ticks / 2); - var queueName = queueClient.Name; - var messageId = message.MessageId; - var popReceipt = message.PopReceipt; - var nextVisibleOn = message.NextVisibleOn; + TimeSpan leasePeriod = this.options.CurrentValue.MessageLeasePeriod; + TimeSpan halfLife = new(leasePeriod.Ticks / 2); + string queueName = queueClient.Name; + string messageId = message.MessageId; + string popReceipt = message.PopReceipt; + DateTimeOffset? nextVisibleOn = message.NextVisibleOn; try { @@ -191,7 +190,7 @@ namespace Azure.Sdk.Tools.PipelineWitness.Services this.logger.LogDebug("Extending visibility timeout for message.\n Queue: {Queue}\n Message: {MessageId}\n Pop Receipt: {PopReceipt}\n Visible in: {VisibleIn}", queueName, messageId, popReceipt, nextVisibleOn - DateTimeOffset.UtcNow); UpdateReceipt receipt = await queueClient.UpdateMessageAsync(messageId, popReceipt, visibilityTimeout: leasePeriod, cancellationToken: cancellationToken); - var oldPopReceipt = popReceipt; + string oldPopReceipt = popReceipt; popReceipt = receipt.PopReceipt; nextVisibleOn = receipt.NextVisibleOn; diff --git a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/WorkTokens/CosmosAsyncLock.cs b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/WorkTokens/CosmosAsyncLock.cs index 91d98f652..c0177481d 100644 --- a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/WorkTokens/CosmosAsyncLock.cs +++ b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/WorkTokens/CosmosAsyncLock.cs @@ -33,7 +33,7 @@ namespace Azure.Sdk.Tools.PipelineWitness.Services.WorkTokens { await this.container.DeleteItemAsync(this.id, this.partitionKey, new ItemRequestOptions { IfMatchEtag = this.etag }); } - catch(CosmosException ex) when (ex.StatusCode == HttpStatusCode.NotFound) + catch (CosmosException ex) when (ex.StatusCode == HttpStatusCode.NotFound) { } } @@ -43,7 +43,7 @@ namespace Azure.Sdk.Tools.PipelineWitness.Services.WorkTokens { try { - var response = await this.container.ReplaceItemAsync( + ItemResponse response = await this.container.ReplaceItemAsync( new CosmosLockDocument(this.id, this.duration), this.id, this.partitionKey, @@ -56,7 +56,7 @@ namespace Azure.Sdk.Tools.PipelineWitness.Services.WorkTokens return true; } } - catch (CosmosException ex) when(ex.StatusCode == HttpStatusCode.Conflict) + catch (CosmosException ex) when (ex.StatusCode == HttpStatusCode.Conflict) { } diff --git a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/WorkTokens/CosmosAsyncLockProvider.cs b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/WorkTokens/CosmosAsyncLockProvider.cs index 10bc490f4..2cd5299ac 100644 --- a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/WorkTokens/CosmosAsyncLockProvider.cs +++ b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Services/WorkTokens/CosmosAsyncLockProvider.cs @@ -23,10 +23,10 @@ namespace Azure.Sdk.Tools.PipelineWitness.Services.WorkTokens public async Task GetLockAsync(string id, TimeSpan duration, CancellationToken cancellationToken) { - var partitionKey = new PartitionKey(id); + PartitionKey partitionKey = new(id); ItemResponse response; - + try { response = await this.container.ReadItemAsync(id, partitionKey, cancellationToken: cancellationToken); @@ -36,13 +36,13 @@ namespace Azure.Sdk.Tools.PipelineWitness.Services.WorkTokens return await CreateLockAsync(id, duration, cancellationToken); } - var existingLock = response.Resource; + CosmosLockDocument existingLock = response.Resource; if (existingLock.Expiration >= DateTime.UtcNow) { return null; } - + try { response = await this.container.ReplaceItemAsync( @@ -68,7 +68,7 @@ namespace Azure.Sdk.Tools.PipelineWitness.Services.WorkTokens { try { - var response = await this.container.CreateItemAsync( + ItemResponse response = await this.container.CreateItemAsync( new CosmosLockDocument(id, duration), new PartitionKey(id), cancellationToken: cancellationToken); diff --git a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Startup.cs b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Startup.cs index b0a42cbc9..700236420 100644 --- a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Startup.cs +++ b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Startup.cs @@ -6,7 +6,6 @@ using Azure.Identity; using Azure.Sdk.Tools.PipelineWitness.ApplicationInsights; using Azure.Sdk.Tools.PipelineWitness.Configuration; using Azure.Sdk.Tools.PipelineWitness.Services; -using Azure.Sdk.Tools.PipelineWitness.Services.FailureAnalysis; using Azure.Sdk.Tools.PipelineWitness.Services.WorkTokens; using Microsoft.ApplicationInsights.Extensibility; @@ -28,9 +27,9 @@ namespace Azure.Sdk.Tools.PipelineWitness { public static void Configure(WebApplicationBuilder builder) { - var azureCredential = new DefaultAzureCredential(); - var settings = new PipelineWitnessSettings(); - var settingsSection = builder.Configuration.GetSection("PipelineWitness"); + DefaultAzureCredential azureCredential = new(); + PipelineWitnessSettings settings = new(); + IConfigurationSection settingsSection = builder.Configuration.GetSection("PipelineWitness"); settingsSection.Bind(settings); builder.Services.AddApplicationInsightsTelemetry(builder.Configuration); @@ -57,26 +56,6 @@ namespace Azure.Sdk.Tools.PipelineWitness builder.Services.AddMemoryCache(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); - builder.Services.AddSingleton(); - builder.Services.AddSingleton(); - builder.Services.AddSingleton(); - builder.Services.AddSingleton(); - builder.Services.AddSingleton(); - builder.Services.AddSingleton(); - builder.Services.AddSingleton(); - builder.Services.AddSingleton(); - builder.Services.AddSingleton(); - builder.Services.AddSingleton(); - builder.Services.AddSingleton(); - builder.Services.AddSingleton(); - builder.Services.AddSingleton(); - builder.Services.AddSingleton(); - builder.Services.AddSingleton(); - builder.Services.AddSingleton(); - builder.Services.AddSingleton(); - builder.Services.AddSingleton(); - builder.Services.AddSingleton(); - builder.Services.AddSingleton(); builder.Services.Configure(settingsSection); builder.Services.AddSingleton(); @@ -87,7 +66,7 @@ namespace Azure.Sdk.Tools.PipelineWitness private static void AddHostedService(this IServiceCollection services, int instanceCount) where T : class, IHostedService { - for (var i = 0; i < instanceCount; i++) + for (int i = 0; i < instanceCount; i++) { services.AddSingleton(); } @@ -107,13 +86,13 @@ namespace Azure.Sdk.Tools.PipelineWitness private static VssConnection CreateVssConnection(IServiceProvider provider) { TokenCredential azureCredential = provider.GetRequiredService(); - TokenRequestContext tokenRequestContext = new (VssAadSettings.DefaultScopes); + TokenRequestContext tokenRequestContext = new(VssAadSettings.DefaultScopes); AccessToken token = azureCredential.GetToken(tokenRequestContext, CancellationToken.None); - - Uri organizationUrl = new ("https://dev.azure.com/azure-sdk"); - VssAadCredential vssCredential = new (new VssAadToken("Bearer", token.Token)); + + Uri organizationUrl = new("https://dev.azure.com/azure-sdk"); + VssAadCredential vssCredential = new(new VssAadToken("Bearer", token.Token)); VssHttpRequestSettings settings = VssClientHttpRequestSettings.Default.Clone(); - + return new VssConnection(organizationUrl, vssCredential, settings); } } diff --git a/tools/pipeline-witness/infrastructure/Assign-StoragePermissions.ps1 b/tools/pipeline-witness/infrastructure/Assign-StoragePermissions.ps1 new file mode 100644 index 000000000..b920fc689 --- /dev/null +++ b/tools/pipeline-witness/infrastructure/Assign-StoragePermissions.ps1 @@ -0,0 +1,70 @@ +<# +.SYNOPSIS + Assign appropriate storage permissions to the Azure SDK Engineering System Team for local debugging. +#> +param( + [Parameter(Mandatory)] + [validateSet('staging', 'test')] + [string]$target +) + +function Invoke([string]$command) { + Write-Host "> $command" + Invoke-Expression $command +} + +Push-Location $PSScriptRoot +try { + $subscriptionName = $target -eq 'test' ? 'Azure SDK Developer Playground' : 'Azure SDK Engineering System' + Write-Host "Setting subscription to '$subscriptionName'" + Invoke "az account set --subscription '$subscriptionName' --output none" + + $parametersFile = "./bicep/parameters.$target.json" + Write-Host "Reading parameters from $parametersFile" + $parameters = (Get-Content -Path $parametersFile -Raw | ConvertFrom-Json).parameters + $appResourceGroupName = $parameters.appResourceGroupName.value + $appStorageAccountName = $parameters.appStorageAccountName.value + $cosmosAccountName = $parameters.cosmosAccountName.value + $logsResourceGroupName = $parameters.logsResourceGroupName.value + $logsStorageAccountName = $parameters.logsStorageAccountName.value + + Write-Host "Adding Azure SDK Engineering System Team RBAC access to storage resources:`n" + ` + " Blob: $logsResourceGroupName/$logsStorageAccountName`n" + ` + " Queue: `n" + ` + " Cosmos: $appResourceGroupName/$cosmosAccountName`n" + + Write-Host "Getting group id for Azure SDK Engineering System Team" + $azAdGroupId = Invoke "az ad group show --group 'Azure SDK Engineering System Team' --query id --output tsv" + + Write-Host "Granting 'read/write' access to $appResourceGroupName/$cosmosAccountName" + Invoke "az cosmosdb sql role assignment create --resource-group '$appResourceGroupName' --account-name '$cosmosAccountName' --scope '/' --role-definition-id '00000000-0000-0000-0000-000000000002' --principal-id '$azAdGroupId' --output none" + + if ($LASTEXITCODE -ne 0) { + Write-Output $output + Write-Error "Failed to grant access" + exit 1 + } + + Write-Host "Granting 'Storage Blob Data Contributor' access to $logsResourceGroupName/$logsStorageAccountName" + $scope = "/subscriptions/$subscriptionId/resourceGroups/$logsResourceGroupName/providers/Microsoft.Storage/storageAccounts/$logsStorageAccountName" + $output = Invoke "az role assignment create --assignee '$azAdGroupId' --role 'Storage Blob Data Contributor' --scope '$scope' --output none" + + if ($LASTEXITCODE -ne 0) { + Write-Output $output + Write-Error "Failed to grant access" + exit 1 + } + + Write-Host "Granting 'Storage Queue Data Contributor' access to $appResourceGroupName/$appStorageAccountName" + $scope = "/subscriptions/$subscriptionId/resourceGroups/$appResourceGroupName/providers/Microsoft.Storage/storageAccounts/$appStorageAccountName" + $output = Invoke "az role assignment create --assignee '$azAdGroupId' --role 'Storage Queue Data Contributor' --scope '$scope' --output none" + + if ($LASTEXITCODE -ne 0) { + Write-Output $output + Write-Error "Failed to grant access" + exit 1 + } +} +finally { + Pop-Location +} \ No newline at end of file diff --git a/tools/pipeline-witness/infrastructure/README.md b/tools/pipeline-witness/infrastructure/README.md new file mode 100644 index 000000000..b9ddb07b5 --- /dev/null +++ b/tools/pipeline-witness/infrastructure/README.md @@ -0,0 +1,9 @@ +# Bicep +The bicep templates in the [bicep](./bicep) folder are used to provision the PipelineWitness resource group, web app and storage account as well as the PipelineLogs resource group, storage account, kusto cluster and all of the resources required to make continuous ingestion from storage to kusto work. + +#### Deployment permissions +The bicep templates contain RBAC assignments between the web app's managed identity and the storage accounts it will be accessing. For the CI pipeline to be able to successfully set these RBAC assignments, it's service connection principal must be granted `Microsoft.Authorization/roleAssignments/write` to the storage resources. This permission is included in the `Owner`, `User Access Administrator` and `Role Based Access Control Administrator` roles. + + +# Kusto +The scripts in the kusto folder are deployed to the staging and production databases during ci pipeline runs. They are merged into a single `.kql` by [`deploy.ps1`](./deploy.ps1) using the [`Merge-KustoScripts.ps1`](./Merge-KustoScripts.ps1) script. To extract objects from an existing kusto database into the kusto folder, use the [`Extract-KustoScripts.ps1`](./Extract-KustoScripts.ps1) script. diff --git a/tools/pipeline-witness/infrastructure/deploy.ps1 b/tools/pipeline-witness/infrastructure/deploy.ps1 index 23d307757..5903e241b 100644 --- a/tools/pipeline-witness/infrastructure/deploy.ps1 +++ b/tools/pipeline-witness/infrastructure/deploy.ps1 @@ -7,7 +7,7 @@ param( [validateSet('production', 'staging', 'test')] [string]$target, - [switch]$replaceRoleAssignments + [switch]$removeRoleAssignments ) function Invoke([string]$command) { @@ -79,7 +79,7 @@ try { " App Resource Group: $appResourceGroupName`n" + ` " Location: $location`n" - if ($replaceRoleAssignments) { + if ($removeRoleAssignments) { RemoveStorageRoleAssignments $subscriptionId $logsResourceGroupName $logsStorageAccountName RemoveStorageRoleAssignments $subscriptionId $appResourceGroupName $appStorageAccountName RemoveCosmosRoleAssignments $subscriptionId $appResourceGroupName $cosmosAccountName @@ -90,38 +90,6 @@ try { Write-Error "Failed to deploy resource groups" exit 1 } - - if ($target -ne 'production') { - $azAdGroupId = az ad group show --group "Azure SDK Engineering System Team" --query id --output tsv - - Write-Host "Granting the Azure SDK Engineering System Team read/write access to cosmos account" - Invoke "az cosmosdb sql role assignment create --resource-group '$appResourceGroupName' --account-name '$cosmosAccountName' --scope '/' --role-definition-id '00000000-0000-0000-0000-000000000002' --principal-id '$azAdGroupId' --output none" - - if ($LASTEXITCODE -ne 0) { - Write-Output $output - Write-Error "Failed to grant access to cosmos account" - exit 1 - } - - Write-Host "Granting the Azure SDK Engineering System Team access to storage accounts" - $scope = "/subscriptions/$subscriptionId/resourceGroups/$logsResourceGroupName/providers/Microsoft.Storage/storageAccounts/$logsStorageAccountName" - $output = Invoke "az role assignment create --assignee '$azAdGroupId' --role 'Storage Blob Data Contributor' --scope '$scope' --output none" - - if ($LASTEXITCODE -ne 0) { - Write-Output $output - Write-Error "Failed to grant access to logs storage account" - exit 1 - } - - $scope = "/subscriptions/$subscriptionId/resourceGroups/$appResourceGroupName/providers/Microsoft.Storage/storageAccounts/$appStorageAccountName" - $output = Invoke "az role assignment create --assignee '$azAdGroupId' --role 'Storage Queue Data Contributor' --scope '$scope' --output none" - - if ($LASTEXITCODE -ne 0) { - Write-Output $output - Write-Error "Failed to grant access to app storage account" - exit 1 - } - } } finally { Pop-Location