[Pipelines] Add Integration/E2E Retry (#3642)

* Added trx logging and test publishing

* Fixed test reporting path

* changed to vstest

* rollback build change

* Fixed newline issue

* Fixed export tests to use VSTest

* Changed over last dotnet test

* Fixed export tests

* pipeline test fix

* formatting fix

* Fixed test filter typo

* pinning vstest to fix issue

* trying different version of vstest

* Testing batch size to fix data test issues

* Testing custom batch size fix

* removed vstest batching

* Testing changing the TestClassWithFixtureArgumentsTypeInfo name

* testing blame timeout

* Testing new blame params

* Changing SQL exception number

* Removed restriction for testing message only in Win

* Added publishing of test files on failure on error to help debugging

* Updated upload test failure task

* Fixed integration test artifact name

* Added fix for sql test

* Fox normalize in SqlServerSchemaUpgradeTests

* Fixed SQL int test newline integration

* Updated dump type / timeout for tests

* Reduced blame back down to 7

* Added pool name to windows jobs

* Changing to correct windows pool

build/build-variables.yml

@@ -31,6 +31,7 @@ variables:
   LinuxVmImage: 'ubuntu-latest'  
   TestApplicationResource: 'https://$(DeploymentEnvironmentName).$(tenantDomain)'
   # The following is set by a build Pipeline variable:
-  # DefaultLinuxPool: 'Azure Pipelines'
-  # SharedLinuxPool: 'Azure Pipelines'
+  # DefaultLinuxPool: 'DevOps1ESSharedPool'
+  # SharedLinuxPool: 'DevOps1ESSharedPool'
+  # DefaultWindowsPool: 'mshealthagents'
   #-----------------------------------------------------------------------------------------

build/jobs/e2e-tests.yml

@@ -94,13 +94,18 @@ steps:
 
         dotnet dev-certs https
 
-  - task: DotNetCoreCLI@2
+  - task: VSTest@3
     displayName: 'E2E ${{ parameters.version }} ${{parameters.appServiceType}}'
     inputs:
-      command: test
-      arguments: '"$(Agent.TempDirectory)/E2ETests/**/*${{ parameters.version }}.Tests.E2E*.dll" --blame-hang-timeout 7m --filter "FullyQualifiedName~${{parameters.appServiceType}}&Category!=ExportLongRunning"'
-      workingDirectory: "$(System.ArtifactsDirectory)"
+      testSelector: 'testAssemblies'
+      testAssemblyVer2: '**/*${{ parameters.version }}.Tests.E2E*.dll'
+      searchFolder: '$(Agent.TempDirectory)/E2ETests/'
+      testFiltercriteria: 'FullyQualifiedName~${{parameters.appServiceType}}&Category!=ExportLongRunning'
       testRunTitle: '${{ parameters.version }} ${{parameters.appServiceType}}'
+      otherConsoleOptions: '/Blame:"CollectDump;DumpType=Mini;CollectHangDump;TestTimeout=7min;HangDumpType=Mini"'
+      rerunFailedTests: true
+      rerunType: 'basedOnTestFailurePercentage'
+      rerunFailedThreshold: 10
     env:
       'TestEnvironmentUrl': $(TestEnvironmentUrl)
       'TestEnvironmentUrl_${{ parameters.version }}': $(TestEnvironmentUrl_${{ parameters.version }})
@@ -137,3 +142,12 @@ steps:
       'user_globalReaderUser_secret': $(user_globalReaderUser_secret)
       'user_globalWriterUser_id': $(user_globalWriterUser_id)
       'user_globalWriterUser_secret': $(user_globalWriterUser_secret)
+
+  # Publish the crash dump and XML files as build artifacts on failure only
+  - task: PublishPipelineArtifact@1
+    displayName: 'Publish E2E ${{ parameters.version }} ${{parameters.appServiceType}} Failures'
+    condition: failed()
+    continueOnError: true
+    inputs:
+      targetPath: '$(Agent.TempDirectory)/TestResults'
+      artifactName: 'E2E ${{ parameters.version }} ${{parameters.appServiceType}} Test Failures'

build/jobs/run-export-tests.yml

@@ -8,8 +8,7 @@ jobs:
 - job: 'cosmosE2eTests'
   dependsOn: []
   pool:
-    name: '$(DefaultLinuxPool)'
-    vmImage: '$(LinuxVmImage)'
+    name: '$(DefaultWindowsPool)'
   steps:
   - template: e2e-setup.yml
   - template: e2e-tests-extract.yml
@@ -72,13 +71,18 @@ jobs:
 
         dotnet dev-certs https
 
-  - task: DotNetCoreCLI@2
+  - task: VSTest@3
     displayName: 'Export E2E ${{ parameters.version }} CosmosDB'
     inputs:
-      command: test
-      arguments: '"$(Agent.TempDirectory)/E2ETests/**/*${{ parameters.version }}.Tests.E2E*.dll" --filter "FullyQualifiedName~CosmosDb&Category=ExportLongRunning"'
-      workingDirectory: "$(System.ArtifactsDirectory)"
-      testRunTitle: 'Export ${{ parameters.version }} CosmosDB'
+      testSelector: 'testAssemblies'
+      testAssemblyVer2: '"$(Agent.TempDirectory)/E2ETests/**/*${{ parameters.version }}.Tests.E2E*.dll"'
+      searchFolder: '$(System.ArtifactsDirectory)'
+      testFiltercriteria: 'FullyQualifiedName~CosmosDb&Category=ExportLongRunning'
+      testRunTitle: '${{ parameters.version }} CosmosDB Export'
+      otherConsoleOptions: '/Blame:"CollectDump;DumpType=Mini;CollectHangDump;TestTimeout=7min;HangDumpType=Mini"'
+      rerunFailedTests: true
+      rerunType: 'basedOnTestFailurePercentage'
+      rerunFailedThreshold: 10
     env:
       'TestEnvironmentUrl': $(TestEnvironmentUrl)
       'TestEnvironmentUrl_${{ parameters.version }}': $(TestEnvironmentUrl_${{ parameters.version }})
@@ -106,11 +110,19 @@ jobs:
       'user_globalWriterUser_id': $(user_globalWriterUser_id)
       'user_globalWriterUser_secret': $(user_globalWriterUser_secret)
 
+  # Publish the crash dump and XML files as build artifacts on failure only
+  - task: PublishPipelineArtifact@1
+    displayName: 'Publish Export E2E Cosmos Test Failures'
+    condition: failed()
+    continueOnError: true
+    inputs:
+      targetPath: '$(Agent.TempDirectory)/TestResults'
+      artifactName: 'Export E2E Cosmos Test Failures'
+
 - job: 'sqlE2eTests'
   dependsOn: []
   pool:
-    name: '$(DefaultLinuxPool)'
-    vmImage: '$(LinuxVmImage)'
+    name: '$(DefaultWindowsPool)'
   steps:
   - template: e2e-setup.yml
   - template: e2e-tests-extract.yml
@@ -173,13 +185,18 @@ jobs:
 
         dotnet dev-certs https
 
-  - task: DotNetCoreCLI@2
+  - task: VSTest@3
     displayName: 'Export E2E ${{ parameters.version }} SQL'
     inputs:
-      command: test
-      arguments: '"$(Agent.TempDirectory)/E2ETests/**/*${{ parameters.version }}.Tests.E2E*.dll" --filter "FullyQualifiedName~SqlServer&Category=ExportLongRunning"'
-      workingDirectory: "$(System.ArtifactsDirectory)"
-      testRunTitle: 'Export ${{ parameters.version }} SQL'
+      testSelector: 'testAssemblies'
+      testAssemblyVer2: '"$(Agent.TempDirectory)/E2ETests/**/*${{ parameters.version }}.Tests.E2E*.dll"'
+      searchFolder: '$(System.ArtifactsDirectory)'
+      testFiltercriteria: 'FullyQualifiedName~SqlServer&Category=ExportLongRunning'
+      testRunTitle: '${{ parameters.version }} SQL Export'
+      otherConsoleOptions: '/Blame:"CollectDump;DumpType=Mini;CollectHangDump;TestTimeout=7min;HangDumpType=Mini"'
+      rerunFailedTests: true
+      rerunType: 'basedOnTestFailurePercentage'
+      rerunFailedThreshold: 10
     env:
       'TestEnvironmentUrl_Sql': $(TestEnvironmentUrl_Sql)
       'TestEnvironmentUrl_${{ parameters.version }}_Sql': $(TestEnvironmentUrl_${{ parameters.version }}_Sql)
@@ -207,3 +224,11 @@ jobs:
       'user_globalWriterUser_id': $(user_globalWriterUser_id)
       'user_globalWriterUser_secret': $(user_globalWriterUser_secret)
 
+  # Publish the crash dump and XML files as build artifacts on failure only
+  - task: PublishPipelineArtifact@1
+    displayName: 'Publish Export E2E SQL Test Failures'
+    condition: failed()
+    continueOnError: true
+    inputs:
+      targetPath: '$(Agent.TempDirectory)/TestResults'
+      artifactName: 'Export E2E SQL Test Failures'

build/jobs/run-tests.yml

@@ -8,8 +8,7 @@ parameters:
 jobs:
 - job: "integrationTests"
   pool:
-    name: '$(SharedLinuxPool)'
-    vmImage: '$(LinuxVmImage)'
+    name: '$(DefaultWindowsPool)'
   steps:
   - task: DownloadBuildArtifacts@0
     inputs:
@@ -40,23 +39,35 @@ jobs:
       azureSubscription: $(ConnectedServiceName)
       KeyVaultName: '${{ parameters.keyVaultName }}-sql'
 
-  - task: DotNetCoreCLI@2
+  - task: VSTest@3
     displayName: 'Run Integration Tests'
     inputs:
-      command: test
-      arguments: '"$(Agent.TempDirectory)/IntegrationTests/**/*${{ parameters.version }}.Tests.Integration*.dll" --blame-hang-timeout 15m'
-      workingDirectory: "$(System.ArtifactsDirectory)"
+      testSelector: 'testAssemblies'
+      testAssemblyVer2: '**/*${{ parameters.version }}.Tests.Integration*.dll'
+      searchFolder: '$(Agent.TempDirectory)/IntegrationTests/'
       testRunTitle: '${{ parameters.version }} Integration'
+      otherConsoleOptions: '/Blame:"CollectDump;DumpType=Mini;CollectHangDump;TestTimeout=7min;HangDumpType=Mini"'
+      rerunFailedTests: true
+      rerunType: 'basedOnTestFailurePercentage'
+      rerunFailedThreshold: 10
     env:
       'CosmosDb:Host': $(CosmosDb--Host)
       'CosmosDb:Key': $(CosmosDb--Key)
       'SqlServer:ConnectionString': $(SqlServer--ConnectionString)
 
+  # Publish the crash dump and XML files as build artifacts on failure only
+  - task: PublishPipelineArtifact@1
+    displayName: 'Publish Integration Test Failures'
+    condition: failed()
+    continueOnError: true
+    inputs:
+      targetPath: '$(Agent.TempDirectory)/TestResults'
+      artifactName: '${{ parameters.version }} Integration Test Failures'
+
 - job: 'cosmosE2eTests'
   dependsOn: []
   pool:
-    name: '$(SharedLinuxPool)'
-    vmImage: '$(LinuxVmImage)'
+    name: '$(DefaultWindowsPool)'
   steps:
   - template: e2e-setup.yml
   - template: e2e-tests.yml
@@ -68,8 +79,7 @@ jobs:
 - job: 'sqlE2eTests'
   dependsOn: []
   pool:
-    name: '$(SharedLinuxPool)'
-    vmImage: '$(LinuxVmImage)'
+    name: '$(DefaultWindowsPool)'
   steps:
   - template: e2e-setup.yml
   - template: e2e-tests.yml

src/Microsoft.Health.Extensions.Xunit/TestClassWithFixtureArgumentsTypeInfo.cs

@@ -27,7 +27,7 @@ namespace Microsoft.Health.Extensions.Xunit
 
             _typeInfoImplementation = typeInfoImplementation;
             FixtureArguments = fixtureArguments;
-            Name = $"{typeInfoImplementation.Name}({string.Join(", ", fixtureArguments.Select(v => $"{v.EnumValue}"))})";
+            Name = $"{typeInfoImplementation.Name}[{string.Join(", ", fixtureArguments.Select(v => $"{v.EnumValue}"))}]";
         }
 
         public IReadOnlyList<SingleFlag> FixtureArguments { get; }

test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlRetryServiceTests.cs

@@ -306,7 +306,7 @@ END
                         // On Windows we get correct error number.
                         return true;
                     }
-                    else if (sqlEx.Number == 0 && sqlEx.Message.Contains("connection", StringComparison.OrdinalIgnoreCase) && sqlEx.Message.Contains("error", StringComparison.OrdinalIgnoreCase))
+                    else if (sqlEx.Message.Contains("connection", StringComparison.OrdinalIgnoreCase) && sqlEx.Message.Contains("error", StringComparison.OrdinalIgnoreCase))
                     {
                         // On Linux we get 0 error number for various connection problems so we check message string as well.
                         return true;
@@ -319,7 +319,7 @@ END
                         // On Windows we get correct error number.
                         return true;
                     }
-                    else if (sqlEx.Number == 0 && sqlEx.Message.Contains("transport", StringComparison.OrdinalIgnoreCase) && sqlEx.Message.Contains("error", StringComparison.OrdinalIgnoreCase))
+                    else if (sqlEx.Message.Contains("transport", StringComparison.OrdinalIgnoreCase) && sqlEx.Message.Contains("error", StringComparison.OrdinalIgnoreCase))
                     {
                         // On Linux we get 0 error number for various connection problems so we check message string as well.
                         return true;

test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerSchemaUpgradeTests.cs

@@ -331,19 +331,30 @@ namespace Microsoft.Health.Fhir.Tests.Integration.Persistence
 
         private string Normalize(string text)
         {
+            // normalize newlines
+            text = text.Replace("\r\n", "\n").Replace("\r", "\n");
+
             // remove inline comments
-            while (text.IndexOf("--") > 0)
+            while (text.Contains("--"))
             {
                 var indexStart = text.IndexOf("--");
-                var indexEnd = text.IndexOf(Environment.NewLine, indexStart);
-                text = text.Substring(0, indexStart) + text.Substring(indexEnd, text.Length - indexEnd);
+                var indexEnd = text.IndexOf("\n", indexStart);
+
+                if (indexEnd == -1)
+                {
+                    text = text.Substring(0, indexStart);
+                }
+                else
+                {
+                    text = text.Substring(0, indexStart) + text.Substring(indexEnd + 1);
+                }
             }
 
             return text.ToLowerInvariant()
-                       .Replace(Environment.NewLine, " ")
-                       .Replace("\r", " ")
-                       .Replace("\t", " ")
-                       .Replace(";", " ")
+                       .Replace("\n", string.Empty)
+                       .Replace("\r", string.Empty)
+                       .Replace("\t", string.Empty)
+                       .Replace(";", string.Empty)
                        .Replace(" output,", " out,")
                        .Replace(" output ", " out ")
                        .Replace(" inner join ", " join ")