Fixed nightly build failures in DB integration tests (#66)

build/test-integration.sh

@@ -13,10 +13,12 @@ source $REPO_DIR/build/__variables.sh
 # 	export STORAGEACCOUNTKEY=`az.cmd storage account keys list -n oryxautomation | grep key1 | awk '{print $NF}'`
 # fi
 
+testCaseFilter=${1:-'Category!=AKS'}
+
 echo
-echo "Running integration tests..."
+echo "Running integration tests with filter '$testCaseFilter'..."
 echo
 testProjectName="Oryx.Integration.Tests"
 cd "$TESTS_SRC_DIR/$testProjectName"
-# Run non-AKS tests temporarily
+
-dotnet test --filter E2E!=AKS --test-adapter-path:. --logger:"xunit;LogFilePath=$ARTIFACTS_DIR/testResults/$testProjectName.xml" -c $BUILD_CONFIGURATION
+dotnet test --filter $testCaseFilter --test-adapter-path:. --logger:"xunit;LogFilePath=$ARTIFACTS_DIR/testResults/$testProjectName.xml" -c $BUILD_CONFIGURATION

tests/Oryx.Integration.Tests/AksTests/SampleAppsTest.cs

@@ -147,7 +147,7 @@ namespace Microsoft.Oryx.Integration.Tests.AksTests
             }
         }
 
-        [Theory, Trait("E2E", "AKS")]
+        [Theory, Trait("Category", "AKS")]
         [InlineData("linxnodeexpress", "oryxdevms/node-4.4:latest", "nodejs", "4.4.7")]
         [InlineData("webfrontend", "oryxdevms/node-8.1:latest", "nodejs", "8.1")]
         [InlineData("soundcloud-ngrx", "oryxdevms/node-8.11:latest", "nodejs", "8.11")]

tests/Oryx.Integration.Tests/LocalDockerTests/DatabaseTestsBase.cs

@@ -19,27 +19,25 @@ namespace Microsoft.Oryx.Integration.Tests.LocalDockerTests
         protected readonly ITestOutputHelper _output;
         protected readonly Fixtures.DbContainerFixtureBase _dbFixture;
         private static readonly Random _rand = new Random();
-        protected readonly int _appPort;
+        protected readonly int _hostPort;
+        protected readonly HttpClient _httpClient = new HttpClient();
 
         protected DatabaseTestsBase(ITestOutputHelper outputHelper, Fixtures.DbContainerFixtureBase dbFixture)
         {
             _output = outputHelper;
             _dbFixture = dbFixture;
-            _appPort = 8080 + _rand.Next(100);
+            _hostPort = 8080 + _rand.Next(100);
             HostSamplesDir = Path.Combine(Directory.GetCurrentDirectory(), "SampleApps");
-            HttpClient = new HttpClient();
         }
 
         protected string HostSamplesDir { get; }
 
-        protected HttpClient HttpClient { get; }
-
         protected async Task RunTestAsync(string language, string languageVersion, string samplePath,
             int containerPort = 8000, bool specifyBindPortFlag = true)
         {
             var volume = DockerVolume.Create(samplePath);
             var appDir = volume.ContainerDir;
-            var portMapping = $"{_appPort}:{containerPort}";
+            var portMapping = $"{_hostPort}:{containerPort}";
             var entrypointScript = "./run.sh";
             var bindPortFlag = specifyBindPortFlag ? $"-bindPort {containerPort}" : string.Empty;
             var script = new ShellScriptBuilder()
@@ -66,7 +64,7 @@ namespace Microsoft.Oryx.Integration.Tests.LocalDockerTests
                 "/bin/sh", new[] { "-c", script },
                 async () =>
                 {
-                    var data = await HttpClient.GetStringAsync($"http://localhost:{_appPort}/");
+                    var data = await _httpClient.GetStringAsync($"http://localhost:{_hostPort}/");
                     Assert.Equal(_dbFixture.GetSampleDataAsJson(), data.Trim(), ignoreLineEndingDifferences: true, ignoreWhiteSpaceDifferences: true);
                 });
         }

tests/Oryx.Integration.Tests/LocalDockerTests/Fixtures/DbContainerFixtureBase.cs

@@ -31,7 +31,10 @@ namespace Microsoft.Oryx.Integration.Tests.LocalDockerTests.Fixtures
         public DbContainerFixtureBase()
         {
             DbServerContainerName = RunDbServerContainer().ContainerName;
-            WaitUntilDbServerIsUp();
+            if (!WaitUntilDbServerIsUp())
+            {
+                throw new Exception("Database not ready in time");
+            }
             InsertSampleData();
         }
 
@@ -64,7 +67,7 @@ namespace Microsoft.Oryx.Integration.Tests.LocalDockerTests.Fixtures
 
         protected abstract DockerRunCommandResult RunDbServerContainer();
 
-        protected abstract void WaitUntilDbServerIsUp();
+        protected abstract bool WaitUntilDbServerIsUp();
 
         protected virtual string GetSampleDataInsertionSql()
         {

tests/Oryx.Integration.Tests/LocalDockerTests/Fixtures/MySqlDbContainerFixture.cs

@@ -35,11 +35,11 @@ namespace Microsoft.Oryx.Integration.Tests.LocalDockerTests.Fixtures
             return runDbContainerResult;
         }
 
-        protected override void WaitUntilDbServerIsUp()
+        protected override bool WaitUntilDbServerIsUp()
         {
             // Try 30 times at most, with a constant 2s in between attempts
             var retry = Policy.HandleResult(result: false).WaitAndRetry(30, i => TimeSpan.FromSeconds(2));
-            retry.Execute(() =>
+            return retry.Execute(() =>
             {
                 // Based on https://hub.docker.com/r/mysql/mysql-server/#starting-a-mysql-server-instance
                 string status = _dockerCli.GetContainerStatus(DbServerContainerName);

tests/Oryx.Integration.Tests/LocalDockerTests/Fixtures/PostgreSqlDbContainerFixture.cs

@@ -34,12 +34,12 @@ namespace Microsoft.Oryx.Integration.Tests.LocalDockerTests.Fixtures
             return runDbContainerResult;
         }
 
-        protected override void WaitUntilDbServerIsUp()
+        protected override bool WaitUntilDbServerIsUp()
         {
-            // Try 30 times at most, with a constant 2s in between attempts
+            // Try 30 times at most, with a constant 3s in between attempts
-            var retry = Policy.HandleResult(result: false).WaitAndRetry(30, i => TimeSpan.FromSeconds(2));
+            var retry = Policy.HandleResult(result: false).WaitAndRetry(30, i => TimeSpan.FromSeconds(3));
-            retry.Execute(() => _dockerCli.GetContainerLogs(DbServerContainerName)
+            return retry.Execute(() => _dockerCli.GetContainerLogs(DbServerContainerName)
-                                          .Contains("database system is ready to accept connections"));
+                                                 .Contains("database system is ready to accept connections"));
         }
 
         protected override void InsertSampleData()

tests/Oryx.Integration.Tests/LocalDockerTests/Fixtures/SqlServerDbContainerFixture.cs

@@ -47,14 +47,12 @@ namespace Microsoft.Oryx.Integration.Tests.LocalDockerTests.Fixtures
             return runDbContainerResult;
         }
 
-        protected override void WaitUntilDbServerIsUp()
+        protected override bool WaitUntilDbServerIsUp()
         {
-            var interval = TimeSpan.FromSeconds(3);
             // Try 30 times at most, with a constant 3s in between attempts
-            var retry = Policy.HandleResult(result: false).WaitAndRetry(30, i => interval);
+            var retry = Policy.HandleResult(result: false).WaitAndRetry(30, i => TimeSpan.FromSeconds(3));
-            retry.Execute(() => _dockerCli.GetContainerLogs(DbServerContainerName)
+            return retry.Execute(() => _dockerCli.GetContainerLogs(DbServerContainerName)
-                                          .Contains("SQL Server is now ready for client connections"));
+                                                 .Contains("SQL Server is now ready for client connections"));
-            Thread.Sleep(interval);
         }
 
         protected override void InsertSampleData()

tests/Oryx.Integration.Tests/LocalDockerTests/MySqlIntegrationTests.cs

@@ -10,6 +10,7 @@ using Xunit.Abstractions;
 
 namespace Microsoft.Oryx.Integration.Tests.LocalDockerTests
 {
+    [Trait("Category", "DB")]
     public class MySqlIntegrationTests : DatabaseTestsBase, IClassFixture<Fixtures.MySqlDbContainerFixture>
     {
         public MySqlIntegrationTests(ITestOutputHelper output, Fixtures.MySqlDbContainerFixture dbFixture)
@@ -17,13 +18,13 @@ namespace Microsoft.Oryx.Integration.Tests.LocalDockerTests
         {
         }
 
-        [Fact(Skip = "Bug 832951")]
+        [Fact]
         public async Task NodeApp_MySqlDB()
         {
             await RunTestAsync("nodejs",  "10.14", Path.Combine(HostSamplesDir, "nodejs", "node-mysql"));
         }
 
-        [Theory(Skip = "Bug 832951")]
+        [Theory]
         [InlineData("mysql-pymysql-sample")]
         [InlineData("mysql-mysqlconnector-sample")]
         [InlineData("mysql-mysqlclient-sample")]
@@ -32,7 +33,7 @@ namespace Microsoft.Oryx.Integration.Tests.LocalDockerTests
             await RunTestAsync("python", "3.7", Path.Combine(HostSamplesDir, "python", sampleAppName));
         }
 
-        [Theory(Skip = "Bug 832951")]
+        [Theory]
         [InlineData("7.3")]
         [InlineData("7.2")]
         [InlineData("7.0")]

tests/Oryx.Integration.Tests/LocalDockerTests/PostgreSqlIntegrationTests.cs

@@ -10,6 +10,7 @@ using Xunit.Abstractions;
 
 namespace Microsoft.Oryx.Integration.Tests.LocalDockerTests
 {
+    [Trait("Category", "DB")]
     public class PostgreSqlIntegrationTests : DatabaseTestsBase, IClassFixture<Fixtures.PostgreSqlDbContainerFixture>
     {
         public PostgreSqlIntegrationTests(ITestOutputHelper output, Fixtures.PostgreSqlDbContainerFixture dbFixture)
@@ -17,19 +18,19 @@ namespace Microsoft.Oryx.Integration.Tests.LocalDockerTests
         {
         }
 
-        [Fact(Skip = "Bug 832951")]
+        [Fact]
         public async Task NodeApp_PostgreSqlDB()
         {
             await RunTestAsync("nodejs", "10.14", Path.Combine(HostSamplesDir, "nodejs", "node-postgres"));
         }
 
-        [Fact(Skip = "Bug 832951")]
+        [Fact]
         public async Task Python37App_PostgreSqlDB()
         {
             await RunTestAsync("python", "3.7", Path.Combine(HostSamplesDir, "python", "postgres-sample"));
         }
 
-        [Theory(Skip = "Bug 832951")]
+        [Theory]
         [InlineData("7.3")]
         [InlineData("7.2")]
         [InlineData("7.0")]

tests/Oryx.Integration.Tests/LocalDockerTests/SqlServerIntegrationTests.cs

@@ -10,6 +10,7 @@ using Xunit.Abstractions;
 
 namespace Microsoft.Oryx.Integration.Tests.LocalDockerTests
 {
+    [Trait("Category", "DB")]
     public class SqlServerIntegrationTests : DatabaseTestsBase, IClassFixture<Fixtures.SqlServerDbContainerFixture>
     {
         public SqlServerIntegrationTests(ITestOutputHelper output, Fixtures.SqlServerDbContainerFixture dbFixture)
@@ -17,19 +18,19 @@ namespace Microsoft.Oryx.Integration.Tests.LocalDockerTests
         {
         }
 
-        [Fact(Skip = "Bug 832951")]
+        [Fact]
         public async Task NodeApp_MicrosoftSqlServerDB()
         {
             await RunTestAsync("nodejs", "10.14", Path.Combine(HostSamplesDir, "nodejs", "node-mssql"));
         }
 
-        [Fact(Skip = "Bug 832951")]
+        [Fact]
         public async Task Python37App_MicrosoftSqlServerDB()
         {
             await RunTestAsync("python", "3.7", Path.Combine(HostSamplesDir, "python", "mssqlserver-sample"));
         }
 
-        [Theory(Skip = "Bug 832951")]
+        [Theory]
         [InlineData("7.3")]
         [InlineData("7.2")]
         // pdo_sqlsrv only supports PHP >= 7.1

tests/SampleApps/python/mysql-mysqlconnector-sample/requirements.txt

@@ -1,2 +1,2 @@
 Flask==1.0.2
-mysql-connector==2.1.6
+mysql-connector==2.1.7

vsts/pipelines/_buildParallel.yml

@@ -119,6 +119,7 @@ steps:
   displayName: 'Test integration'
   inputs:
     scriptPath: ./build/test-integration.sh
+    args: $(TestIntegrationCaseFilter)
   env:
     STORAGEACCOUNTKEY: $(STORAGEACCOUNTKEY)
     KUBECONFIG: $(KUBECONFIG)

vsts/pipelines/nightly.yml

@@ -47,8 +47,8 @@ jobs:
     displayName: 'Set variables'
   - template: _buildParallel.yml
 
-- job: Job_IntegrationTest
+- job: Job_IntegrationTests
-  displayName: Run Integration Tests
+  displayName: Test Integration
   dependsOn: 
     - Job_BuildImage
     - Job_RuntimeImages
@@ -60,6 +60,29 @@ jobs:
       echo "##vso[task.setvariable variable=BuildRuntimeImages;]false"
       echo "##vso[task.setvariable variable=TestBuildImages;]false"
       echo "##vso[task.setvariable variable=TestRuntimeImages;]false"
+      echo "##vso[task.setvariable variable=TestIntegrationCaseFilter;]Category!=AKS&Category!=DB"
+      echo "##vso[task.setvariable variable=TestIntegration;]true"
+      echo "##vso[task.setvariable variable=PushBuildImages;]false"
+      echo "##vso[task.setvariable variable=PushRuntimeImages;]false"
+      echo "##vso[task.setvariable variable=PushToDockerHub;]false"
+      echo "##vso[task.setvariable variable=EmbedBuildContextInImages;]false"
+    displayName: 'Set variables'
+  - template: _buildParallel.yml
+
+- job: Job_DbIntegrationTests
+  displayName: Test Database Integration
+  dependsOn: 
+    - Job_BuildImage
+    - Job_RuntimeImages
+  pool:
+    name: OryxLinux
+  steps:
+  - script: |
+      echo "##vso[task.setvariable variable=BuildBuildImages;]false"
+      echo "##vso[task.setvariable variable=BuildRuntimeImages;]false"
+      echo "##vso[task.setvariable variable=TestBuildImages;]false"
+      echo "##vso[task.setvariable variable=TestRuntimeImages;]false"
+      echo "##vso[task.setvariable variable=TestIntegrationCaseFilter;]Category=DB"
       echo "##vso[task.setvariable variable=TestIntegration;]true"
       echo "##vso[task.setvariable variable=PushBuildImages;]false"
       echo "##vso[task.setvariable variable=PushRuntimeImages;]false"