From 8878da920fe56ef07711ae2c0891ce4c60f8da56 Mon Sep 17 00:00:00 2001 From: Ryan Brandenburg Date: Fri, 1 Sep 2017 16:52:03 -0700 Subject: [PATCH] Add docker support --- .travis.yml | 2 + files/KoreBuild/KoreBuild.sh | 8 +- files/KoreBuild/scripts/KoreBuild.psm1 | 2 +- scripts/bootstrapper/run.sh | 14 +- .../SimpleRepoTests.cs | 83 ++++++++ .../Utilities/TestApp.cs | 21 +- .../Commands/DockerBuildCommand.cs | 182 ++++++++++++++++++ .../Commands/DockerFiles/.dockerignore | 4 + .../Commands/DockerFiles/jessie.dockerfile | 16 ++ .../DockerFiles/winservercore.dockerfile | 18 ++ .../KoreBuild.Console/Commands/RootCommand.cs | 1 + .../KoreBuild.Console.csproj | 1 + 12 files changed, 337 insertions(+), 15 deletions(-) create mode 100644 tools/KoreBuild.Console/Commands/DockerBuildCommand.cs create mode 100644 tools/KoreBuild.Console/Commands/DockerFiles/.dockerignore create mode 100644 tools/KoreBuild.Console/Commands/DockerFiles/jessie.dockerfile create mode 100644 tools/KoreBuild.Console/Commands/DockerFiles/winservercore.dockerfile diff --git a/.travis.yml b/.travis.yml index ab94b40..1063375 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,7 @@ language: csharp sudo: false +services: + - docker dist: trusty env: global: diff --git a/files/KoreBuild/KoreBuild.sh b/files/KoreBuild/KoreBuild.sh index fa5ef4b..550f2de 100755 --- a/files/KoreBuild/KoreBuild.sh +++ b/files/KoreBuild/KoreBuild.sh @@ -5,6 +5,7 @@ __korebuild_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" source "$__korebuild_dir/scripts/common.sh" # functions +default_tools_source='https://aspnetcore.blob.core.windows.net/buildtools' set_korebuildsettings() { tools_source=$1 @@ -13,7 +14,7 @@ set_korebuildsettings() { local config_file="${4:-}" # optional. Not used yet. [ -z "${dot_net_home:-}" ] && dot_net_home="$HOME/.dotnet" - [ -z "${tools_source:-}" ] && tools_source='https://aspnetcore.blob.core.windows.net/buildtools' + [ -z "${tools_source:-}" ] && tools_source="$default_tools_source" return 0 } @@ -71,10 +72,11 @@ install_tools() { # Instructs MSBuild where to find .NET Framework reference assemblies export ReferenceAssemblyRoot="$tools_home/netfx/$netfx_version" - + # Call "sync" between "chmod" and execution to prevent "text file busy" error in Docker (aufs) chmod +x "$__korebuild_dir/scripts/get-netfx.sh"; sync - "$__korebuild_dir/scripts/get-netfx.sh" $verbose_flag $netfx_version "$tools_source" "$ReferenceAssemblyRoot" \ + # we don't include netfx in the BuildTools artifacts currently, it ends up on the blob store through other means, so we'll only look for it in the default_tools_source + "$__korebuild_dir/scripts/get-netfx.sh" $verbose_flag $netfx_version "$default_tools_source" "$ReferenceAssemblyRoot" \ || return 1 # Call "sync" between "chmod" and execution to prevent "text file busy" error in Docker (aufs) diff --git a/files/KoreBuild/scripts/KoreBuild.psm1 b/files/KoreBuild/scripts/KoreBuild.psm1 index 8bd3a5b..1a12541 100644 --- a/files/KoreBuild/scripts/KoreBuild.psm1 +++ b/files/KoreBuild/scripts/KoreBuild.psm1 @@ -327,7 +327,7 @@ function Invoke-KoreBuildCommand( --tools-source $global:KoreBuildSettings.ToolsSource ` --dotnet-home $global:KoreBuildSettings.DotNetHome ` --repo-path $global:KoreBuildSettings.RepoPath ` - $Arguments + @Arguments } } diff --git a/scripts/bootstrapper/run.sh b/scripts/bootstrapper/run.sh index 697508a..c278423 100755 --- a/scripts/bootstrapper/run.sh +++ b/scripts/bootstrapper/run.sh @@ -29,13 +29,13 @@ __usage() { echo " ... Arguments passed to the command. Variable number of arguments allowed." echo "" echo "Options:" - echo " --verbose Show verbose output." - echo " -c|--channel The channel of KoreBuild to download. Overrides the value from the config file.." - echo " --config-file TThe path to the configuration file that stores values. Defaults to korebuild.json." - echo " -d|--dotnet-home The directory where .NET Core tools will be stored. Defaults to '\$DOTNET_HOME' or '\$HOME/.dotnet." - echo " --path The directory to build. Defaults to the directory containing the script." - echo " -s|--tools-source The base url where build tools can be downloaded. Overrides the value from the config file." - echo " -u|--update Update to the latest KoreBuild even if the lock file is present." + echo " --verbose Show verbose output." + echo " -c|--channel The channel of KoreBuild to download. Overrides the value from the config file.." + echo " --config-file The path to the configuration file that stores values. Defaults to korebuild.json." + echo " -d|--dotnet-home The directory where .NET Core tools will be stored. Defaults to '\$DOTNET_HOME' or '\$HOME/.dotnet." + echo " --path The directory to build. Defaults to the directory containing the script." + echo " -s|--tools-source|-ToolsSource The base url where build tools can be downloaded. Overrides the value from the config file." + echo " -u|--update Update to the latest KoreBuild even if the lock file is present." echo "" echo "Description:" echo " This function will create a file \$DIR/korebuild-lock.txt. This lock file can be committed to source, but does not have to be." diff --git a/test/KoreBuild.FunctionalTests/SimpleRepoTests.cs b/test/KoreBuild.FunctionalTests/SimpleRepoTests.cs index d1b022c..54fe195 100644 --- a/test/KoreBuild.FunctionalTests/SimpleRepoTests.cs +++ b/test/KoreBuild.FunctionalTests/SimpleRepoTests.cs @@ -2,8 +2,11 @@ // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using System; +using System.ComponentModel; +using System.Diagnostics; using System.IO; using System.Threading.Tasks; +using System.Runtime.InteropServices; using Xunit; using Xunit.Abstractions; @@ -55,5 +58,85 @@ namespace KoreBuild.FunctionalTests Assert.Same(task, build); Assert.NotEqual(0, build.Result); } + + [DockerExistsFact] + public async Task DockerSuccessful() + { + var app = _fixture.CreateTestApp("SimpleRepo"); + var platform = "jessie"; + + var dockerPlatform = GetDockerPlatform(); + if (dockerPlatform == OSPlatform.Windows) + { + platform = "winservercore"; + } + + var build = app.ExecuteRun(_output, new string[]{ "docker-build", "-Path", app.WorkingDirectory}, platform, "/p:BuildNumber=0001"); + var task = await Task.WhenAny(build, Task.Delay(TimeSpan.FromMinutes(10))); + + Assert.Same(task, build); + + Assert.Equal(0, build.Result); + } + + private static OSPlatform GetDockerPlatform() + { + var startInfo = new ProcessStartInfo("docker", @"version -f ""{{ .Server.Os }}""") + { + RedirectStandardOutput = true + }; + + using (var process = Process.Start(startInfo)) + { + var output = process.StandardOutput.ReadToEnd().Trim(); + + OSPlatform result; + switch(output) + { + case "windows": + result = OSPlatform.Windows; + break; + case "linux": + result = OSPlatform.Linux; + break; + default: + throw new NotImplementedException($"No default for docker platform {output}"); + } + + return result; + } + } + } + + [AttributeUsage(AttributeTargets.Method, AllowMultiple = false)] + public class DockerExistsFactAttribute : FactAttribute + { + public DockerExistsFactAttribute() + { + if(!HasDocker()) + { + Skip = "Docker must be installed to run this test."; + } + } + + private static bool HasDocker() + { + try + { + var startInfo = new ProcessStartInfo("docker", "--version") + { + RedirectStandardOutput = true, + RedirectStandardError = true + }; + using (Process.Start(startInfo)) + { + return true; + } + } + catch(Win32Exception) + { + return false; + } + } } } diff --git a/test/KoreBuild.FunctionalTests/Utilities/TestApp.cs b/test/KoreBuild.FunctionalTests/Utilities/TestApp.cs index 6df3d8d..7211280 100644 --- a/test/KoreBuild.FunctionalTests/Utilities/TestApp.cs +++ b/test/KoreBuild.FunctionalTests/Utilities/TestApp.cs @@ -27,7 +27,17 @@ namespace KoreBuild.FunctionalTests public string WorkingDirectory { get; } - public async Task ExecuteBuild(ITestOutputHelper output, params string[] args) + public async Task ExecuteRun(ITestOutputHelper output, string[] koreBuildArgs, params string[] commandArgs) + { + return await ExecuteScript(output, "run", koreBuildArgs, commandArgs); + } + + public async Task ExecuteBuild(ITestOutputHelper output, params string[] commandArgs) + { + return await ExecuteScript(output, "build", new string[0], commandArgs); + } + + private async Task ExecuteScript(ITestOutputHelper output, string script, string[] koreBuildArgs, params string[] commandArgs) { output.WriteLine("Starting in " + WorkingDirectory); void Write(object sender, DataReceivedEventArgs e) @@ -41,22 +51,25 @@ namespace KoreBuild.FunctionalTests { cmd = "cmd.exe"; arguments.Add("/C"); - arguments.Add(@".\build.cmd"); + arguments.Add($@".\{script}.cmd"); } else { cmd = "bash"; - arguments.Add("./build.sh"); + arguments.Add($"./{script}.sh"); } + arguments.AddRange(koreBuildArgs); + arguments.AddRange(new[] { "-ToolsSource", _toolsSource, "-Update" }); + arguments.AddRange(commandArgs); + arguments.Add("/v:n"); - arguments.AddRange(args); var process = new Process { diff --git a/tools/KoreBuild.Console/Commands/DockerBuildCommand.cs b/tools/KoreBuild.Console/Commands/DockerBuildCommand.cs new file mode 100644 index 0000000..b725fbc --- /dev/null +++ b/tools/KoreBuild.Console/Commands/DockerBuildCommand.cs @@ -0,0 +1,182 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Reflection; +using Microsoft.Extensions.CommandLineUtils; + +namespace KoreBuild.Console.Commands +{ + internal class DockerBuildCommand : SubCommandBase + { + private const string DockerIgnore = ".dockerignore"; + private const string DockerfileExtension = ".dockerfile"; + private const string Owner = "aspnetbuild"; + private const string ImageName = "korebuild"; + + public CommandArgument ImageVariant { get; set; } + + public List Arguments {get; set; } + + public string Tag => $@"{Owner}/{ImageName}:{ImageVariant.Value}"; + + public override void Configure(CommandLineApplication application) + { + ImageVariant = application.Argument("image", "The docker image to run on."); + Arguments = application.RemainingArguments; + + base.Configure(application); + } + + protected override bool IsValid() + { + if(string.IsNullOrEmpty(ImageVariant?.Value)) + { + Reporter.Error("Image is a required argument."); + return false; + } + + return true; + } + + protected override int Execute() + { + var dockerFileName = GetDockerFileName(ImageVariant.Value); + var dockerFileSource = GetDockerFileSource(dockerFileName); + var dockerFileDestination = Path.Combine(RepoPath, GetDockerFileName(ImageVariant.Value)); + + File.Copy(dockerFileSource, dockerFileDestination, overwrite: true); + + var dockerIgnoreSource = GetDockerFileSource(DockerIgnore); + var dockerIgnoreDestination = Path.Combine(RepoPath, DockerIgnore); + + File.Copy(dockerIgnoreSource, dockerIgnoreDestination, overwrite: true); + + // If our ToolSource isn't http copy it to the docker context + var dockerToolsSource = ToolsSource; + string toolsSourceDestination = null; + if (!ToolsSource.StartsWith("http")) + { + dockerToolsSource = "ToolsSource"; + toolsSourceDestination = Path.Combine(RepoPath, dockerToolsSource); + DirectoryCopy(ToolsSource, toolsSourceDestination); + } + + try + { + var buildArgs = new List { "build" }; + + buildArgs.AddRange(new string[] { "-t", Tag, "-f", dockerFileDestination, RepoPath }); + var buildResult = RunDockerCommand(buildArgs); + + if (buildResult != 0) + { + return buildResult; + } + + var containerName = $"{Owner}_{DateTime.Now.ToString("yyyyMMddHHmmss")}"; + + var runArgs = new List { "run", "--rm", "-it", "--name", containerName, Tag }; + + runArgs.AddRange(new[] { "-ToolsSource", dockerToolsSource }); + + if (Arguments?.Count > 0) + { + runArgs.AddRange(Arguments); + } + + Reporter.Verbose($"Running in container '{containerName}'"); + return RunDockerCommand(runArgs); + } + finally{ + // Clean up the stuff we dumped there in order to get it in the docker context. + File.Delete(dockerFileDestination); + File.Delete(dockerIgnoreDestination); + if(toolsSourceDestination != null) + { + Directory.Delete(toolsSourceDestination, recursive: true); + } + } + } + + private string GetDockerFileSource(string fileName) + { + var executingDir = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location); + var source = Path.Combine(executingDir, "Commands", "DockerFiles", fileName); + + if(!File.Exists(source)) + { + Reporter.Error($"DockerFile '{source}' doesn't exist."); + throw new FileNotFoundException(); + } + + return source; + } + + private string GetDockerFileName(string platform) + { + return $"{platform}{DockerfileExtension}"; + } + + private int RunDockerCommand(List arguments) + { + var args = ArgumentEscaper.EscapeAndConcatenate(arguments.ToArray()); + Reporter.Verbose($"Running 'docker {args}'"); + + var psi = new ProcessStartInfo + { + FileName = "docker", + Arguments = args, + RedirectStandardError = true + }; + + var process = Process.Start(psi); + process.WaitForExit(); + + if(process.ExitCode != 0) + { + Reporter.Error(process.StandardError.ReadToEnd()); + } + + return process.ExitCode; + } + + private static void DirectoryCopy(string sourceDirName, string destDirName) + { + // Get the subdirectories for the specified directory. + DirectoryInfo dir = new DirectoryInfo(sourceDirName); + + if (!dir.Exists) + { + throw new DirectoryNotFoundException( + "Source directory does not exist or could not be found: " + + sourceDirName); + } + + DirectoryInfo[] dirs = dir.GetDirectories(); + // If the destination directory doesn't exist, create it. + if (!Directory.Exists(destDirName)) + { + Directory.CreateDirectory(destDirName); + } + + // Get the files in the directory and copy them to the new location. + FileInfo[] files = dir.GetFiles(); + foreach (FileInfo file in files) + { + string temppath = Path.Combine(destDirName, file.Name); + file.CopyTo(temppath, overwrite: true); + } + + // Copy subdirectories and their contents to the new location. + foreach (DirectoryInfo subdir in dirs) + { + string temppath = Path.Combine(destDirName, subdir.Name); + DirectoryCopy(subdir.FullName, temppath); + } + } + } +} diff --git a/tools/KoreBuild.Console/Commands/DockerFiles/.dockerignore b/tools/KoreBuild.Console/Commands/DockerFiles/.dockerignore new file mode 100644 index 0000000..26f97c8 --- /dev/null +++ b/tools/KoreBuild.Console/Commands/DockerFiles/.dockerignore @@ -0,0 +1,4 @@ +korebuild-lock.txt +**/bin +**/obj +artifacts diff --git a/tools/KoreBuild.Console/Commands/DockerFiles/jessie.dockerfile b/tools/KoreBuild.Console/Commands/DockerFiles/jessie.dockerfile new file mode 100644 index 0000000..dc4f027 --- /dev/null +++ b/tools/KoreBuild.Console/Commands/DockerFiles/jessie.dockerfile @@ -0,0 +1,16 @@ +FROM microsoft/dotnet:2.0-runtime-deps-jessie + +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + git \ +# KoreBuild dependencies + curl \ + unzip \ + apt-transport-https \ + && rm -rf /var/lib/apt/lists/* + +ADD . . + +ENV DOTNET_SKIP_FIRST_TIME_EXPERIENCE=true + +ENTRYPOINT ["./build.sh"] diff --git a/tools/KoreBuild.Console/Commands/DockerFiles/winservercore.dockerfile b/tools/KoreBuild.Console/Commands/DockerFiles/winservercore.dockerfile new file mode 100644 index 0000000..213d774 --- /dev/null +++ b/tools/KoreBuild.Console/Commands/DockerFiles/winservercore.dockerfile @@ -0,0 +1,18 @@ +FROM microsoft/aspnet:4.6.2 + + +# DevPack returns exit 0 immediately, but it's not done, so we wait. +# A more correct thing would be to block on a registry key existing or similar. +RUN \ + Invoke-WebRequest https://download.microsoft.com/download/F/1/D/F1DEB8DB-D277-4EF9-9F48-3A65D4D8F965/NDP461-DevPack-KB3105179-ENU.exe -OutFile ~\\net461dev.exe ; \ + ~\\net461dev.exe /Passive /NoRestart ; \ + Start-Sleep -s 10; \ + Remove-Item ~\\net461dev.exe -Force ; + +WORKDIR c:\\repo + +ADD . . + +ENV DOTNET_SKIP_FIRST_TIME_EXPERIENCE=true + +ENTRYPOINT ["build.cmd"] diff --git a/tools/KoreBuild.Console/Commands/RootCommand.cs b/tools/KoreBuild.Console/Commands/RootCommand.cs index 196dd0b..a825253 100644 --- a/tools/KoreBuild.Console/Commands/RootCommand.cs +++ b/tools/KoreBuild.Console/Commands/RootCommand.cs @@ -14,6 +14,7 @@ namespace KoreBuild.Console.Commands application.Command("install-tools", new InstallToolsCommand().Configure, throwOnUnexpectedArg:false); application.Command("msbuild", new MSBuildCommand().Configure, throwOnUnexpectedArg:false); + application.Command("docker-build", new DockerBuildCommand().Configure, throwOnUnexpectedArg: false); application.VersionOption("--version", GetVersion); diff --git a/tools/KoreBuild.Console/KoreBuild.Console.csproj b/tools/KoreBuild.Console/KoreBuild.Console.csproj index 508bd50..52a3f56 100644 --- a/tools/KoreBuild.Console/KoreBuild.Console.csproj +++ b/tools/KoreBuild.Console/KoreBuild.Console.csproj @@ -10,6 +10,7 @@ +