[Pipelines] Add Integration/E2E Retry (#3642)

* Added trx logging and test publishing * Fixed test reporting path * changed to vstest * rollback build change * Fixed newline issue * Fixed export tests to use VSTest * Changed over last dotnet test * Fixed export tests * pipeline test fix * formatting fix * Fixed test filter typo * pinning vstest to fix issue * trying different version of vstest * Testing batch size to fix data test issues * Testing custom batch size fix * removed vstest batching * Testing changing the TestClassWithFixtureArgumentsTypeInfo name * testing blame timeout * Testing new blame params * Changing SQL exception number * Removed restriction for testing message only in Win * Added publishing of test files on failure on error to help debugging * Updated upload test failure task * Fixed integration test artifact name * Added fix for sql test * Fox normalize in SqlServerSchemaUpgradeTests * Fixed SQL int test newline integration * Updated dump type / timeout for tests * Reduced blame back down to 7 * Added pool name to windows jobs * Changing to correct windows pool
2024-02-09 15:36:36 -08:00 · 2024-02-09 15:36:36 -08:00 · d23bf8fd07
--- a/build/build-variables.yml
+++ b/build/build-variables.yml
@ -31,6 +31,7 @@ variables:
  LinuxVmImage: 'ubuntu-latest'  
  TestApplicationResource: 'https://$(DeploymentEnvironmentName).$(tenantDomain)'
  # The following is set by a build Pipeline variable:
-  # DefaultLinuxPool: 'Azure Pipelines'
-  # SharedLinuxPool: 'Azure Pipelines'
+  # DefaultLinuxPool: 'DevOps1ESSharedPool'
+  # SharedLinuxPool: 'DevOps1ESSharedPool'
+  # DefaultWindowsPool: 'mshealthagents'
  #-----------------------------------------------------------------------------------------
--- a/build/jobs/e2e-tests.yml
+++ b/build/jobs/e2e-tests.yml
@ -94,13 +94,18 @@ steps:

        dotnet dev-certs https

-  - task: DotNetCoreCLI@2
+  - task: VSTest@3
    displayName: 'E2E ${{ parameters.version }} ${{parameters.appServiceType}}'
    inputs:
-      command: test
-      arguments: '"$(Agent.TempDirectory)/E2ETests/**/*${{ parameters.version }}.Tests.E2E*.dll" --blame-hang-timeout 7m --filter "FullyQualifiedName~${{parameters.appServiceType}}&Category!=ExportLongRunning"'
-      workingDirectory: "$(System.ArtifactsDirectory)"
+      testSelector: 'testAssemblies'
+      testAssemblyVer2: '**/*${{ parameters.version }}.Tests.E2E*.dll'
+      searchFolder: '$(Agent.TempDirectory)/E2ETests/'
+      testFiltercriteria: 'FullyQualifiedName~${{parameters.appServiceType}}&Category!=ExportLongRunning'
      testRunTitle: '${{ parameters.version }} ${{parameters.appServiceType}}'
+      otherConsoleOptions: '/Blame:"CollectDump;DumpType=Mini;CollectHangDump;TestTimeout=7min;HangDumpType=Mini"'
+      rerunFailedTests: true
+      rerunType: 'basedOnTestFailurePercentage'
+      rerunFailedThreshold: 10
    env:
      'TestEnvironmentUrl': $(TestEnvironmentUrl)
      'TestEnvironmentUrl_${{ parameters.version }}': $(TestEnvironmentUrl_${{ parameters.version }})
@ -137,3 +142,12 @@ steps:
      'user_globalReaderUser_secret': $(user_globalReaderUser_secret)
      'user_globalWriterUser_id': $(user_globalWriterUser_id)
      'user_globalWriterUser_secret': $(user_globalWriterUser_secret)
+
+  # Publish the crash dump and XML files as build artifacts on failure only
+  - task: PublishPipelineArtifact@1
+    displayName: 'Publish E2E ${{ parameters.version }} ${{parameters.appServiceType}} Failures'
+    condition: failed()
+    continueOnError: true
+    inputs:
+      targetPath: '$(Agent.TempDirectory)/TestResults'
+      artifactName: 'E2E ${{ parameters.version }} ${{parameters.appServiceType}} Test Failures'
--- a/build/jobs/run-export-tests.yml
+++ b/build/jobs/run-export-tests.yml
@ -8,8 +8,7 @@ jobs:
 - job: 'cosmosE2eTests'
  dependsOn: []
  pool:
-    name: '$(DefaultLinuxPool)'
-    vmImage: '$(LinuxVmImage)'
+    name: '$(DefaultWindowsPool)'
  steps:
  - template: e2e-setup.yml
  - template: e2e-tests-extract.yml
@ -72,13 +71,18 @@ jobs:

        dotnet dev-certs https

-  - task: DotNetCoreCLI@2
+  - task: VSTest@3
    displayName: 'Export E2E ${{ parameters.version }} CosmosDB'
    inputs:
-      command: test
-      arguments: '"$(Agent.TempDirectory)/E2ETests/**/*${{ parameters.version }}.Tests.E2E*.dll" --filter "FullyQualifiedName~CosmosDb&Category=ExportLongRunning"'
-      workingDirectory: "$(System.ArtifactsDirectory)"
-      testRunTitle: 'Export ${{ parameters.version }} CosmosDB'
+      testSelector: 'testAssemblies'
+      testAssemblyVer2: '"$(Agent.TempDirectory)/E2ETests/**/*${{ parameters.version }}.Tests.E2E*.dll"'
+      searchFolder: '$(System.ArtifactsDirectory)'
+      testFiltercriteria: 'FullyQualifiedName~CosmosDb&Category=ExportLongRunning'
+      testRunTitle: '${{ parameters.version }} CosmosDB Export'
+      otherConsoleOptions: '/Blame:"CollectDump;DumpType=Mini;CollectHangDump;TestTimeout=7min;HangDumpType=Mini"'
+      rerunFailedTests: true
+      rerunType: 'basedOnTestFailurePercentage'
+      rerunFailedThreshold: 10
    env:
      'TestEnvironmentUrl': $(TestEnvironmentUrl)
      'TestEnvironmentUrl_${{ parameters.version }}': $(TestEnvironmentUrl_${{ parameters.version }})
@ -106,11 +110,19 @@ jobs:
      'user_globalWriterUser_id': $(user_globalWriterUser_id)
      'user_globalWriterUser_secret': $(user_globalWriterUser_secret)

+  # Publish the crash dump and XML files as build artifacts on failure only
+  - task: PublishPipelineArtifact@1
+    displayName: 'Publish Export E2E Cosmos Test Failures'
+    condition: failed()
+    continueOnError: true
+    inputs:
+      targetPath: '$(Agent.TempDirectory)/TestResults'
+      artifactName: 'Export E2E Cosmos Test Failures'
+
 - job: 'sqlE2eTests'
  dependsOn: []
  pool:
-    name: '$(DefaultLinuxPool)'
-    vmImage: '$(LinuxVmImage)'
+    name: '$(DefaultWindowsPool)'
  steps:
  - template: e2e-setup.yml
  - template: e2e-tests-extract.yml
@ -173,13 +185,18 @@ jobs:

        dotnet dev-certs https

-  - task: DotNetCoreCLI@2
+  - task: VSTest@3
    displayName: 'Export E2E ${{ parameters.version }} SQL'
    inputs:
-      command: test
-      arguments: '"$(Agent.TempDirectory)/E2ETests/**/*${{ parameters.version }}.Tests.E2E*.dll" --filter "FullyQualifiedName~SqlServer&Category=ExportLongRunning"'
-      workingDirectory: "$(System.ArtifactsDirectory)"
-      testRunTitle: 'Export ${{ parameters.version }} SQL'
+      testSelector: 'testAssemblies'
+      testAssemblyVer2: '"$(Agent.TempDirectory)/E2ETests/**/*${{ parameters.version }}.Tests.E2E*.dll"'
+      searchFolder: '$(System.ArtifactsDirectory)'
+      testFiltercriteria: 'FullyQualifiedName~SqlServer&Category=ExportLongRunning'
+      testRunTitle: '${{ parameters.version }} SQL Export'
+      otherConsoleOptions: '/Blame:"CollectDump;DumpType=Mini;CollectHangDump;TestTimeout=7min;HangDumpType=Mini"'
+      rerunFailedTests: true
+      rerunType: 'basedOnTestFailurePercentage'
+      rerunFailedThreshold: 10
    env:
      'TestEnvironmentUrl_Sql': $(TestEnvironmentUrl_Sql)
      'TestEnvironmentUrl_${{ parameters.version }}_Sql': $(TestEnvironmentUrl_${{ parameters.version }}_Sql)
@ -206,4 +223,12 @@ jobs:
      'user_globalReaderUser_secret': $(user_globalReaderUser_secret)
      'user_globalWriterUser_id': $(user_globalWriterUser_id)
      'user_globalWriterUser_secret': $(user_globalWriterUser_secret)
-      
+
+  # Publish the crash dump and XML files as build artifacts on failure only
+  - task: PublishPipelineArtifact@1
+    displayName: 'Publish Export E2E SQL Test Failures'
+    condition: failed()
+    continueOnError: true
+    inputs:
+      targetPath: '$(Agent.TempDirectory)/TestResults'
+      artifactName: 'Export E2E SQL Test Failures'
--- a/build/jobs/run-tests.yml
+++ b/build/jobs/run-tests.yml
@ -8,8 +8,7 @@ parameters:
 jobs:
 - job: "integrationTests"
  pool:
-    name: '$(SharedLinuxPool)'
-    vmImage: '$(LinuxVmImage)'
+    name: '$(DefaultWindowsPool)'
  steps:
  - task: DownloadBuildArtifacts@0
    inputs:
@ -40,23 +39,35 @@ jobs:
      azureSubscription: $(ConnectedServiceName)
      KeyVaultName: '${{ parameters.keyVaultName }}-sql'

-  - task: DotNetCoreCLI@2
+  - task: VSTest@3
    displayName: 'Run Integration Tests'
    inputs:
-      command: test
-      arguments: '"$(Agent.TempDirectory)/IntegrationTests/**/*${{ parameters.version }}.Tests.Integration*.dll" --blame-hang-timeout 15m'
-      workingDirectory: "$(System.ArtifactsDirectory)"
+      testSelector: 'testAssemblies'
+      testAssemblyVer2: '**/*${{ parameters.version }}.Tests.Integration*.dll'
+      searchFolder: '$(Agent.TempDirectory)/IntegrationTests/'
      testRunTitle: '${{ parameters.version }} Integration'
+      otherConsoleOptions: '/Blame:"CollectDump;DumpType=Mini;CollectHangDump;TestTimeout=7min;HangDumpType=Mini"'
+      rerunFailedTests: true
+      rerunType: 'basedOnTestFailurePercentage'
+      rerunFailedThreshold: 10
    env:
      'CosmosDb:Host': $(CosmosDb--Host)
      'CosmosDb:Key': $(CosmosDb--Key)
      'SqlServer:ConnectionString': $(SqlServer--ConnectionString)

+  # Publish the crash dump and XML files as build artifacts on failure only
+  - task: PublishPipelineArtifact@1
+    displayName: 'Publish Integration Test Failures'
+    condition: failed()
+    continueOnError: true
+    inputs:
+      targetPath: '$(Agent.TempDirectory)/TestResults'
+      artifactName: '${{ parameters.version }} Integration Test Failures'
+
 - job: 'cosmosE2eTests'
  dependsOn: []
  pool:
-    name: '$(SharedLinuxPool)'
-    vmImage: '$(LinuxVmImage)'
+    name: '$(DefaultWindowsPool)'
  steps:
  - template: e2e-setup.yml
  - template: e2e-tests.yml
@ -68,8 +79,7 @@ jobs:
 - job: 'sqlE2eTests'
  dependsOn: []
  pool:
-    name: '$(SharedLinuxPool)'
-    vmImage: '$(LinuxVmImage)'
+    name: '$(DefaultWindowsPool)'
  steps:
  - template: e2e-setup.yml
  - template: e2e-tests.yml
--- a/src/Microsoft.Health.Extensions.Xunit/TestClassWithFixtureArgumentsTypeInfo.cs
+++ b/src/Microsoft.Health.Extensions.Xunit/TestClassWithFixtureArgumentsTypeInfo.cs
@ -27,7 +27,7 @@ namespace Microsoft.Health.Extensions.Xunit

            _typeInfoImplementation = typeInfoImplementation;
            FixtureArguments = fixtureArguments;
-            Name = $"{typeInfoImplementation.Name}({string.Join(", ", fixtureArguments.Select(v => $"{v.EnumValue}"))})";
+            Name = $"{typeInfoImplementation.Name}[{string.Join(", ", fixtureArguments.Select(v => $"{v.EnumValue}"))}]";
        }

        public IReadOnlyList<SingleFlag> FixtureArguments { get; }
--- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlRetryServiceTests.cs
+++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlRetryServiceTests.cs
@ -306,7 +306,7 @@ END
                        // On Windows we get correct error number.
                        return true;
                    }
-                    else if (sqlEx.Number == 0 && sqlEx.Message.Contains("connection", StringComparison.OrdinalIgnoreCase) && sqlEx.Message.Contains("error", StringComparison.OrdinalIgnoreCase))
+                    else if (sqlEx.Message.Contains("connection", StringComparison.OrdinalIgnoreCase) && sqlEx.Message.Contains("error", StringComparison.OrdinalIgnoreCase))
                    {
                        // On Linux we get 0 error number for various connection problems so we check message string as well.
                        return true;
@ -319,7 +319,7 @@ END
                        // On Windows we get correct error number.
                        return true;
                    }
-                    else if (sqlEx.Number == 0 && sqlEx.Message.Contains("transport", StringComparison.OrdinalIgnoreCase) && sqlEx.Message.Contains("error", StringComparison.OrdinalIgnoreCase))
+                    else if (sqlEx.Message.Contains("transport", StringComparison.OrdinalIgnoreCase) && sqlEx.Message.Contains("error", StringComparison.OrdinalIgnoreCase))
                    {
                        // On Linux we get 0 error number for various connection problems so we check message string as well.
                        return true;
--- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerSchemaUpgradeTests.cs
+++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerSchemaUpgradeTests.cs
@ -331,19 +331,30 @@ namespace Microsoft.Health.Fhir.Tests.Integration.Persistence

        private string Normalize(string text)
        {
+            // normalize newlines
+            text = text.Replace("\r\n", "\n").Replace("\r", "\n");
+
            // remove inline comments
-            while (text.IndexOf("--") > 0)
+            while (text.Contains("--"))
            {
                var indexStart = text.IndexOf("--");
-                var indexEnd = text.IndexOf(Environment.NewLine, indexStart);
-                text = text.Substring(0, indexStart) + text.Substring(indexEnd, text.Length - indexEnd);
+                var indexEnd = text.IndexOf("\n", indexStart);
+
+                if (indexEnd == -1)
+                {
+                    text = text.Substring(0, indexStart);
+                }
+                else
+                {
+                    text = text.Substring(0, indexStart) + text.Substring(indexEnd + 1);
+                }
            }

            return text.ToLowerInvariant()
-                       .Replace(Environment.NewLine, " ")
-                       .Replace("\r", " ")
-                       .Replace("\t", " ")
-                       .Replace(";", " ")
+                       .Replace("\n", string.Empty)
+                       .Replace("\r", string.Empty)
+                       .Replace("\t", string.Empty)
+                       .Replace(";", string.Empty)
                       .Replace(" output,", " out,")
                       .Replace(" output ", " out ")
                       .Replace(" inner join ", " join ")