diff --git a/build/test-integration.sh b/build/test-integration.sh index 31182f48e..e41d8b8d3 100755 --- a/build/test-integration.sh +++ b/build/test-integration.sh @@ -13,10 +13,12 @@ source $REPO_DIR/build/__variables.sh # export STORAGEACCOUNTKEY=`az.cmd storage account keys list -n oryxautomation | grep key1 | awk '{print $NF}'` # fi +testCaseFilter=${1:-'Category!=AKS'} + echo -echo "Running integration tests..." +echo "Running integration tests with filter '$testCaseFilter'..." echo testProjectName="Oryx.Integration.Tests" cd "$TESTS_SRC_DIR/$testProjectName" -# Run non-AKS tests temporarily -dotnet test --filter E2E!=AKS --test-adapter-path:. --logger:"xunit;LogFilePath=$ARTIFACTS_DIR/testResults/$testProjectName.xml" -c $BUILD_CONFIGURATION + +dotnet test --filter $testCaseFilter --test-adapter-path:. --logger:"xunit;LogFilePath=$ARTIFACTS_DIR/testResults/$testProjectName.xml" -c $BUILD_CONFIGURATION diff --git a/tests/Oryx.Integration.Tests/AksTests/SampleAppsTest.cs b/tests/Oryx.Integration.Tests/AksTests/SampleAppsTest.cs index 7c83d92f1..1343c2cc0 100644 --- a/tests/Oryx.Integration.Tests/AksTests/SampleAppsTest.cs +++ b/tests/Oryx.Integration.Tests/AksTests/SampleAppsTest.cs @@ -147,7 +147,7 @@ namespace Microsoft.Oryx.Integration.Tests.AksTests } } - [Theory, Trait("E2E", "AKS")] + [Theory, Trait("Category", "AKS")] [InlineData("linxnodeexpress", "oryxdevms/node-4.4:latest", "nodejs", "4.4.7")] [InlineData("webfrontend", "oryxdevms/node-8.1:latest", "nodejs", "8.1")] [InlineData("soundcloud-ngrx", "oryxdevms/node-8.11:latest", "nodejs", "8.11")] diff --git a/tests/Oryx.Integration.Tests/LocalDockerTests/DatabaseTestsBase.cs b/tests/Oryx.Integration.Tests/LocalDockerTests/DatabaseTestsBase.cs index 29dacefad..189ecf704 100644 --- a/tests/Oryx.Integration.Tests/LocalDockerTests/DatabaseTestsBase.cs +++ b/tests/Oryx.Integration.Tests/LocalDockerTests/DatabaseTestsBase.cs @@ -19,27 +19,25 @@ namespace Microsoft.Oryx.Integration.Tests.LocalDockerTests protected readonly ITestOutputHelper _output; protected readonly Fixtures.DbContainerFixtureBase _dbFixture; private static readonly Random _rand = new Random(); - protected readonly int _appPort; + protected readonly int _hostPort; + protected readonly HttpClient _httpClient = new HttpClient(); protected DatabaseTestsBase(ITestOutputHelper outputHelper, Fixtures.DbContainerFixtureBase dbFixture) { _output = outputHelper; _dbFixture = dbFixture; - _appPort = 8080 + _rand.Next(100); + _hostPort = 8080 + _rand.Next(100); HostSamplesDir = Path.Combine(Directory.GetCurrentDirectory(), "SampleApps"); - HttpClient = new HttpClient(); } protected string HostSamplesDir { get; } - protected HttpClient HttpClient { get; } - protected async Task RunTestAsync(string language, string languageVersion, string samplePath, int containerPort = 8000, bool specifyBindPortFlag = true) { var volume = DockerVolume.Create(samplePath); var appDir = volume.ContainerDir; - var portMapping = $"{_appPort}:{containerPort}"; + var portMapping = $"{_hostPort}:{containerPort}"; var entrypointScript = "./run.sh"; var bindPortFlag = specifyBindPortFlag ? $"-bindPort {containerPort}" : string.Empty; var script = new ShellScriptBuilder() @@ -66,7 +64,7 @@ namespace Microsoft.Oryx.Integration.Tests.LocalDockerTests "/bin/sh", new[] { "-c", script }, async () => { - var data = await HttpClient.GetStringAsync($"http://localhost:{_appPort}/"); + var data = await _httpClient.GetStringAsync($"http://localhost:{_hostPort}/"); Assert.Equal(_dbFixture.GetSampleDataAsJson(), data.Trim(), ignoreLineEndingDifferences: true, ignoreWhiteSpaceDifferences: true); }); } diff --git a/tests/Oryx.Integration.Tests/LocalDockerTests/Fixtures/DbContainerFixtureBase.cs b/tests/Oryx.Integration.Tests/LocalDockerTests/Fixtures/DbContainerFixtureBase.cs index 4aefb3234..cd7f0821a 100644 --- a/tests/Oryx.Integration.Tests/LocalDockerTests/Fixtures/DbContainerFixtureBase.cs +++ b/tests/Oryx.Integration.Tests/LocalDockerTests/Fixtures/DbContainerFixtureBase.cs @@ -31,7 +31,10 @@ namespace Microsoft.Oryx.Integration.Tests.LocalDockerTests.Fixtures public DbContainerFixtureBase() { DbServerContainerName = RunDbServerContainer().ContainerName; - WaitUntilDbServerIsUp(); + if (!WaitUntilDbServerIsUp()) + { + throw new Exception("Database not ready in time"); + } InsertSampleData(); } @@ -64,7 +67,7 @@ namespace Microsoft.Oryx.Integration.Tests.LocalDockerTests.Fixtures protected abstract DockerRunCommandResult RunDbServerContainer(); - protected abstract void WaitUntilDbServerIsUp(); + protected abstract bool WaitUntilDbServerIsUp(); protected virtual string GetSampleDataInsertionSql() { diff --git a/tests/Oryx.Integration.Tests/LocalDockerTests/Fixtures/MySqlDbContainerFixture.cs b/tests/Oryx.Integration.Tests/LocalDockerTests/Fixtures/MySqlDbContainerFixture.cs index 42c6669c9..c2a63065a 100644 --- a/tests/Oryx.Integration.Tests/LocalDockerTests/Fixtures/MySqlDbContainerFixture.cs +++ b/tests/Oryx.Integration.Tests/LocalDockerTests/Fixtures/MySqlDbContainerFixture.cs @@ -35,11 +35,11 @@ namespace Microsoft.Oryx.Integration.Tests.LocalDockerTests.Fixtures return runDbContainerResult; } - protected override void WaitUntilDbServerIsUp() + protected override bool WaitUntilDbServerIsUp() { // Try 30 times at most, with a constant 2s in between attempts var retry = Policy.HandleResult(result: false).WaitAndRetry(30, i => TimeSpan.FromSeconds(2)); - retry.Execute(() => + return retry.Execute(() => { // Based on https://hub.docker.com/r/mysql/mysql-server/#starting-a-mysql-server-instance string status = _dockerCli.GetContainerStatus(DbServerContainerName); diff --git a/tests/Oryx.Integration.Tests/LocalDockerTests/Fixtures/PostgreSqlDbContainerFixture.cs b/tests/Oryx.Integration.Tests/LocalDockerTests/Fixtures/PostgreSqlDbContainerFixture.cs index 40a31053a..75c74b849 100644 --- a/tests/Oryx.Integration.Tests/LocalDockerTests/Fixtures/PostgreSqlDbContainerFixture.cs +++ b/tests/Oryx.Integration.Tests/LocalDockerTests/Fixtures/PostgreSqlDbContainerFixture.cs @@ -34,12 +34,12 @@ namespace Microsoft.Oryx.Integration.Tests.LocalDockerTests.Fixtures return runDbContainerResult; } - protected override void WaitUntilDbServerIsUp() + protected override bool WaitUntilDbServerIsUp() { - // Try 30 times at most, with a constant 2s in between attempts - var retry = Policy.HandleResult(result: false).WaitAndRetry(30, i => TimeSpan.FromSeconds(2)); - retry.Execute(() => _dockerCli.GetContainerLogs(DbServerContainerName) - .Contains("database system is ready to accept connections")); + // Try 30 times at most, with a constant 3s in between attempts + var retry = Policy.HandleResult(result: false).WaitAndRetry(30, i => TimeSpan.FromSeconds(3)); + return retry.Execute(() => _dockerCli.GetContainerLogs(DbServerContainerName) + .Contains("database system is ready to accept connections")); } protected override void InsertSampleData() diff --git a/tests/Oryx.Integration.Tests/LocalDockerTests/Fixtures/SqlServerDbContainerFixture.cs b/tests/Oryx.Integration.Tests/LocalDockerTests/Fixtures/SqlServerDbContainerFixture.cs index 353ce4fcc..419fa2122 100644 --- a/tests/Oryx.Integration.Tests/LocalDockerTests/Fixtures/SqlServerDbContainerFixture.cs +++ b/tests/Oryx.Integration.Tests/LocalDockerTests/Fixtures/SqlServerDbContainerFixture.cs @@ -47,14 +47,12 @@ namespace Microsoft.Oryx.Integration.Tests.LocalDockerTests.Fixtures return runDbContainerResult; } - protected override void WaitUntilDbServerIsUp() + protected override bool WaitUntilDbServerIsUp() { - var interval = TimeSpan.FromSeconds(3); // Try 30 times at most, with a constant 3s in between attempts - var retry = Policy.HandleResult(result: false).WaitAndRetry(30, i => interval); - retry.Execute(() => _dockerCli.GetContainerLogs(DbServerContainerName) - .Contains("SQL Server is now ready for client connections")); - Thread.Sleep(interval); + var retry = Policy.HandleResult(result: false).WaitAndRetry(30, i => TimeSpan.FromSeconds(3)); + return retry.Execute(() => _dockerCli.GetContainerLogs(DbServerContainerName) + .Contains("SQL Server is now ready for client connections")); } protected override void InsertSampleData() diff --git a/tests/Oryx.Integration.Tests/LocalDockerTests/MySqlIntegrationTests.cs b/tests/Oryx.Integration.Tests/LocalDockerTests/MySqlIntegrationTests.cs index 5d7b54cf9..089cebf13 100644 --- a/tests/Oryx.Integration.Tests/LocalDockerTests/MySqlIntegrationTests.cs +++ b/tests/Oryx.Integration.Tests/LocalDockerTests/MySqlIntegrationTests.cs @@ -10,6 +10,7 @@ using Xunit.Abstractions; namespace Microsoft.Oryx.Integration.Tests.LocalDockerTests { + [Trait("Category", "DB")] public class MySqlIntegrationTests : DatabaseTestsBase, IClassFixture { public MySqlIntegrationTests(ITestOutputHelper output, Fixtures.MySqlDbContainerFixture dbFixture) @@ -17,13 +18,13 @@ namespace Microsoft.Oryx.Integration.Tests.LocalDockerTests { } - [Fact(Skip = "Bug 832951")] + [Fact] public async Task NodeApp_MySqlDB() { await RunTestAsync("nodejs", "10.14", Path.Combine(HostSamplesDir, "nodejs", "node-mysql")); } - [Theory(Skip = "Bug 832951")] + [Theory] [InlineData("mysql-pymysql-sample")] [InlineData("mysql-mysqlconnector-sample")] [InlineData("mysql-mysqlclient-sample")] @@ -32,7 +33,7 @@ namespace Microsoft.Oryx.Integration.Tests.LocalDockerTests await RunTestAsync("python", "3.7", Path.Combine(HostSamplesDir, "python", sampleAppName)); } - [Theory(Skip = "Bug 832951")] + [Theory] [InlineData("7.3")] [InlineData("7.2")] [InlineData("7.0")] diff --git a/tests/Oryx.Integration.Tests/LocalDockerTests/PostgreSqlIntegrationTests.cs b/tests/Oryx.Integration.Tests/LocalDockerTests/PostgreSqlIntegrationTests.cs index 92482c3af..51a4f78da 100644 --- a/tests/Oryx.Integration.Tests/LocalDockerTests/PostgreSqlIntegrationTests.cs +++ b/tests/Oryx.Integration.Tests/LocalDockerTests/PostgreSqlIntegrationTests.cs @@ -10,6 +10,7 @@ using Xunit.Abstractions; namespace Microsoft.Oryx.Integration.Tests.LocalDockerTests { + [Trait("Category", "DB")] public class PostgreSqlIntegrationTests : DatabaseTestsBase, IClassFixture { public PostgreSqlIntegrationTests(ITestOutputHelper output, Fixtures.PostgreSqlDbContainerFixture dbFixture) @@ -17,19 +18,19 @@ namespace Microsoft.Oryx.Integration.Tests.LocalDockerTests { } - [Fact(Skip = "Bug 832951")] + [Fact] public async Task NodeApp_PostgreSqlDB() { await RunTestAsync("nodejs", "10.14", Path.Combine(HostSamplesDir, "nodejs", "node-postgres")); } - [Fact(Skip = "Bug 832951")] + [Fact] public async Task Python37App_PostgreSqlDB() { await RunTestAsync("python", "3.7", Path.Combine(HostSamplesDir, "python", "postgres-sample")); } - [Theory(Skip = "Bug 832951")] + [Theory] [InlineData("7.3")] [InlineData("7.2")] [InlineData("7.0")] diff --git a/tests/Oryx.Integration.Tests/LocalDockerTests/SqlServerIntegrationTests.cs b/tests/Oryx.Integration.Tests/LocalDockerTests/SqlServerIntegrationTests.cs index fc538dcaf..6519442d9 100644 --- a/tests/Oryx.Integration.Tests/LocalDockerTests/SqlServerIntegrationTests.cs +++ b/tests/Oryx.Integration.Tests/LocalDockerTests/SqlServerIntegrationTests.cs @@ -10,6 +10,7 @@ using Xunit.Abstractions; namespace Microsoft.Oryx.Integration.Tests.LocalDockerTests { + [Trait("Category", "DB")] public class SqlServerIntegrationTests : DatabaseTestsBase, IClassFixture { public SqlServerIntegrationTests(ITestOutputHelper output, Fixtures.SqlServerDbContainerFixture dbFixture) @@ -17,19 +18,19 @@ namespace Microsoft.Oryx.Integration.Tests.LocalDockerTests { } - [Fact(Skip = "Bug 832951")] + [Fact] public async Task NodeApp_MicrosoftSqlServerDB() { await RunTestAsync("nodejs", "10.14", Path.Combine(HostSamplesDir, "nodejs", "node-mssql")); } - [Fact(Skip = "Bug 832951")] + [Fact] public async Task Python37App_MicrosoftSqlServerDB() { await RunTestAsync("python", "3.7", Path.Combine(HostSamplesDir, "python", "mssqlserver-sample")); } - [Theory(Skip = "Bug 832951")] + [Theory] [InlineData("7.3")] [InlineData("7.2")] // pdo_sqlsrv only supports PHP >= 7.1 diff --git a/tests/SampleApps/python/mysql-mysqlconnector-sample/requirements.txt b/tests/SampleApps/python/mysql-mysqlconnector-sample/requirements.txt index dd14a52a6..f07cc7034 100644 --- a/tests/SampleApps/python/mysql-mysqlconnector-sample/requirements.txt +++ b/tests/SampleApps/python/mysql-mysqlconnector-sample/requirements.txt @@ -1,2 +1,2 @@ Flask==1.0.2 -mysql-connector==2.1.6 \ No newline at end of file +mysql-connector==2.1.7 diff --git a/vsts/pipelines/_buildParallel.yml b/vsts/pipelines/_buildParallel.yml index a0508da76..b7af479c2 100644 --- a/vsts/pipelines/_buildParallel.yml +++ b/vsts/pipelines/_buildParallel.yml @@ -119,6 +119,7 @@ steps: displayName: 'Test integration' inputs: scriptPath: ./build/test-integration.sh + args: $(TestIntegrationCaseFilter) env: STORAGEACCOUNTKEY: $(STORAGEACCOUNTKEY) KUBECONFIG: $(KUBECONFIG) diff --git a/vsts/pipelines/nightly.yml b/vsts/pipelines/nightly.yml index 16f336b74..b36595d92 100644 --- a/vsts/pipelines/nightly.yml +++ b/vsts/pipelines/nightly.yml @@ -47,8 +47,8 @@ jobs: displayName: 'Set variables' - template: _buildParallel.yml -- job: Job_IntegrationTest - displayName: Run Integration Tests +- job: Job_IntegrationTests + displayName: Test Integration dependsOn: - Job_BuildImage - Job_RuntimeImages @@ -60,6 +60,29 @@ jobs: echo "##vso[task.setvariable variable=BuildRuntimeImages;]false" echo "##vso[task.setvariable variable=TestBuildImages;]false" echo "##vso[task.setvariable variable=TestRuntimeImages;]false" + echo "##vso[task.setvariable variable=TestIntegrationCaseFilter;]Category!=AKS&Category!=DB" + echo "##vso[task.setvariable variable=TestIntegration;]true" + echo "##vso[task.setvariable variable=PushBuildImages;]false" + echo "##vso[task.setvariable variable=PushRuntimeImages;]false" + echo "##vso[task.setvariable variable=PushToDockerHub;]false" + echo "##vso[task.setvariable variable=EmbedBuildContextInImages;]false" + displayName: 'Set variables' + - template: _buildParallel.yml + +- job: Job_DbIntegrationTests + displayName: Test Database Integration + dependsOn: + - Job_BuildImage + - Job_RuntimeImages + pool: + name: OryxLinux + steps: + - script: | + echo "##vso[task.setvariable variable=BuildBuildImages;]false" + echo "##vso[task.setvariable variable=BuildRuntimeImages;]false" + echo "##vso[task.setvariable variable=TestBuildImages;]false" + echo "##vso[task.setvariable variable=TestRuntimeImages;]false" + echo "##vso[task.setvariable variable=TestIntegrationCaseFilter;]Category=DB" echo "##vso[task.setvariable variable=TestIntegration;]true" echo "##vso[task.setvariable variable=PushBuildImages;]false" echo "##vso[task.setvariable variable=PushRuntimeImages;]false"