diff --git a/.gitignore b/.gitignore index 93024ca27..b4ece4ba8 100644 --- a/.gitignore +++ b/.gitignore @@ -18,7 +18,6 @@ x64/ x86/ bld/ -[Bb]in/ [Oo]bj/ [Ll]og/ @@ -272,11 +271,10 @@ images/runtime/dotnetcore/*.*/ !images/runtime/dotnetcore/2.2/ images/runtime/python/*.*/ -# Golang packages, if oryx is added to GOPATH (which it should) +# Go packages, if Oryx is added to GOPATH (which it should) pkg/ -bin/ -# Go packages from github +# Go packages from GitHub src/github.com/ src/startupscriptgenerator/vendor/ src/golang.org/ \ No newline at end of file diff --git a/build/__variables.sh b/build/__variables.sh index 0b2e2993b..95b34d2bf 100755 --- a/build/__variables.sh +++ b/build/__variables.sh @@ -12,6 +12,8 @@ declare -r BUILD_RUNTIMEIMAGES_USING_NOCACHE="$BUILD_RUNTIMEIMAGES_USING_NOCACHE declare -r BUILD_IMAGES_BUILD_CONTEXT_DIR="$__REPO_DIR/" declare -r BUILD_IMAGES_DOCKERFILE="$__REPO_DIR/images/build/Dockerfile" +declare -r BUILDER_BASE_IMAGE_DOCKERFILE="$__REPO_DIR/images/pack-builder/builder-base.Dockerfile" +declare -r PACK_IMAGE_DOCKERFILE="$__REPO_DIR/images/pack-builder/pack-runner.Dockerfile" declare -r ORYXTESTS_BUILDIMAGE_DOCKERFILE="$__REPO_DIR/tests/images/build/Dockerfile" declare -r RUNTIME_IMAGES_SRC_DIR="$__REPO_DIR/images/runtime" declare -r SOURCES_SRC_DIR="$__REPO_DIR/src" @@ -24,9 +26,12 @@ declare -r RUNTIME_IMAGES_ARTIFACTS_FILE="$ARTIFACTS_DIR/images/runtime-images.t declare -r ACR_BUILD_IMAGES_ARTIFACTS_FILE="$ARTIFACTS_DIR/images/build-images-acr.txt" declare -r ACR_RUNTIME_IMAGES_ARTIFACTS_FILE="$ARTIFACTS_DIR/images/runtime-images-acr.txt" -declare -r DOCKER_BUILD_IMAGES_REPO="oryxdevms/build" +declare -r DOCKER_DEV_REPO_BASE='oryxdevms' +declare -r DOCKER_BUILD_IMAGES_REPO="$DOCKER_DEV_REPO_BASE/build" +declare -r DOCKER_PACK_IMAGE_REPO="$DOCKER_DEV_REPO_BASE/pack" +declare -r DOCKER_BUILDER_BASE_IMAGE_REPO="$DOCKER_DEV_REPO_BASE/builder-base" declare -r ORYXTESTS_BUILDIMAGE_REPO="oryxtests/build" -declare -r DOCKER_RUNTIME_IMAGES_REPO="oryxdevms" +declare -r DOCKER_RUNTIME_IMAGES_REPO=$DOCKER_DEV_REPO_BASE declare -r ACR_DEV_NAME="oryxdevmcr.azurecr.io" declare -r ACR_BUILD_IMAGES_REPO="$ACR_DEV_NAME/public/oryx/build" declare -r ACR_RUNTIME_IMAGES_REPO="$ACR_DEV_NAME/public/oryx" diff --git a/build/build-buildimage-bases.sh b/build/build-buildimage-bases.sh old mode 100644 new mode 100755 diff --git a/build/build-buildimages.sh b/build/build-buildimages.sh index ad706928d..686681770 100755 --- a/build/build-buildimages.sh +++ b/build/build-buildimages.sh @@ -31,7 +31,7 @@ fi if [ "$EMBED_BUILDCONTEXT_IN_IMAGES" == "true" ] then ctxArgs="--build-arg GIT_COMMIT=$GIT_COMMIT --build-arg BUILD_NUMBER=$BUILD_NUMBER" - echo "build context: "$ctxArgs + echo "Build context args: $ctxArgs" fi function BuildAndTagStage() diff --git a/build/build-buildpacks-images.sh b/build/build-buildpacks-images.sh new file mode 100755 index 000000000..396d9d4a5 --- /dev/null +++ b/build/build-buildpacks-images.sh @@ -0,0 +1,36 @@ +#!/bin/bash +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT license. +# -------------------------------------------------------------------------------------------- + +set -e + +declare -r REPO_DIR=$( cd $( dirname "$0" ) && cd .. && pwd ) +source $REPO_DIR/build/__variables.sh + +# Build base image for builder +cd "$BUILD_IMAGES_BUILD_CONTEXT_DIR" +docker build -f "$BUILDER_BASE_IMAGE_DOCKERFILE" -t $DOCKER_BUILDER_BASE_IMAGE_REPO:latest . + +cd /tmp + +$REPO_DIR/images/pack-builder/install-pack.sh + +# Create builder +builderName="$DOCKER_DEV_REPO_BASE/pack-builder" +./pack create-builder $builderName \ + --builder-config $REPO_DIR/images/pack-builder/builder.toml \ + --no-pull + +# Remove pack & everything that was added by it +rm -f ./pack +rm -rf ~/.pack + +# Build an image that runs `pack` +cd "$BUILD_IMAGES_BUILD_CONTEXT_DIR" +docker build -f "$PACK_IMAGE_DOCKERFILE" \ + --build-arg BUILD_NUMBER='0.2.0' \ + --build-arg BUILDPACK_BUILDER_NAME="$builderName" \ + -t $DOCKER_PACK_IMAGE_REPO:latest \ + . diff --git a/doc/buildpack.md b/doc/buildpack.md new file mode 100644 index 000000000..7ff66fba0 --- /dev/null +++ b/doc/buildpack.md @@ -0,0 +1,9 @@ +# Oryx Buildpack + +Oryx provides a buildpack that runs it, so that Oryx can also be used via the [pack][] tool. + +## Related images + +WIP + +[pack]: https://github.com/buildpack/pack diff --git a/images/build/Dockerfile b/images/build/Dockerfile index e5cc40546..a68a89bf3 100644 --- a/images/build/Dockerfile +++ b/images/build/Dockerfile @@ -243,7 +243,9 @@ ARG AGENTBUILD=${AGENTBUILD} ARG BUILD_NUMBER=unspecified ENV GIT_COMMIT=${GIT_COMMIT} ENV BUILD_NUMBER=${BUILD_NUMBER} -RUN if [ -z "$AGENTBUILD" ]; then dotnet publish -r linux-x64 -o /opt/buildscriptgen/ -c Release BuildScriptGeneratorCli/BuildScriptGeneratorCli.csproj; fi +RUN if [ -z "$AGENTBUILD" ]; then \ + dotnet publish -r linux-x64 -o /opt/buildscriptgen/ -c Release BuildScriptGeneratorCli/BuildScriptGeneratorCli.csproj; \ + fi # This stage is only when building in devops agents FROM main AS copybuildscriptbinaries diff --git a/images/pack-builder/builder-base.Dockerfile b/images/pack-builder/builder-base.Dockerfile new file mode 100644 index 000000000..7dffc2b43 --- /dev/null +++ b/images/pack-builder/builder-base.Dockerfile @@ -0,0 +1,14 @@ +FROM oryxdevms/build:latest + +LABEL io.buildpacks.stack.id=com.microsoft.oryx.stack + +# Configure non-root user +RUN groupadd -g 1002 oryx_group && \ + useradd -u 1001 -g oryx_group oryx_user && \ + chown -R oryx_user:oryx_group /tmp && \ + mkdir -p /home/oryx_user && \ + chmod -R 777 /home/oryx_user + +ENV CNB_USER_ID=1001 CNB_GROUP_ID=1002 + +COPY --from=packs/samples:rc /lifecycle /lifecycle diff --git a/images/pack-builder/builder.toml b/images/pack-builder/builder.toml new file mode 100644 index 000000000..6569308ee --- /dev/null +++ b/images/pack-builder/builder.toml @@ -0,0 +1,13 @@ +[stack] + id = "com.microsoft.oryx.stack" + build-image = "oryxdevms/builder-base" + run-image = "oryxdevms/builder-base" + +[[buildpacks]] + id = "com.microsoft.oryx.buildpack" + uri = "oryx-buildpack" + +[[groups]] + [[groups.buildpacks]] + id = "com.microsoft.oryx.buildpack" + version = "0.0.1" diff --git a/images/pack-builder/install-pack.sh b/images/pack-builder/install-pack.sh new file mode 100755 index 000000000..b0687ff99 --- /dev/null +++ b/images/pack-builder/install-pack.sh @@ -0,0 +1,23 @@ +#!/bin/bash +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT license. +# -------------------------------------------------------------------------------------------- + +declare -r PACK_VERSION='0.1.0' + +if [[ "$OSTYPE" == "linux-gnu" ]]; then + packPlatform='linux'; +elif [[ "$OSTYPE" == "darwin"* ]]; then + packPlatform='macos'; +elif [[ "$OSTYPE" == "cygwin" || "$OSTYPE" == "msys" ]]; then + echo 'ERROR: `pack create-builder` is not implemented on Windows.' + exit 1 +else + echo 'ERROR: Could not detect compatible pack binary platform.' + exit 1 +fi + +wget -nv "https://github.com/buildpack/pack/releases/download/v$PACK_VERSION/pack-v$PACK_VERSION-$packPlatform.tgz" +tar -xvf "pack-v$PACK_VERSION-$packPlatform.tgz" +# `./pack` is now available for use diff --git a/images/pack-builder/oryx-buildpack/bin/build b/images/pack-builder/oryx-buildpack/bin/build new file mode 100755 index 000000000..da1e3894c --- /dev/null +++ b/images/pack-builder/oryx-buildpack/bin/build @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT license. +# -------------------------------------------------------------------------------------------- +# usage: bin/build +set -eo pipefail + +echo "# Running 'oryx build .' in '`pwd`'..." + +oryx build . +oryxExitStatus=$? + +echo "# Oryx exited with $oryxExitStatus..." +exit $oryxExitStatus diff --git a/images/pack-builder/oryx-buildpack/bin/detect b/images/pack-builder/oryx-buildpack/bin/detect new file mode 100755 index 000000000..ede1cde46 --- /dev/null +++ b/images/pack-builder/oryx-buildpack/bin/detect @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT license. +# -------------------------------------------------------------------------------------------- +# usage: bin/detect +set -eo pipefail + +sourceDir=`pwd` + +echo "# Running 'oryx buildpack-detect $sourceDir --platform-dir $1 --plan-path $2'..." + +exec oryx buildpack-detect $sourceDir --platform-dir $1 --plan-path $2 diff --git a/images/pack-builder/oryx-buildpack/buildpack.toml b/images/pack-builder/oryx-buildpack/buildpack.toml new file mode 100644 index 000000000..53232545a --- /dev/null +++ b/images/pack-builder/oryx-buildpack/buildpack.toml @@ -0,0 +1,7 @@ +[buildpack] +id = "com.microsoft.oryx.buildpack" +version = "0.0.1" +name = "Microsoft Oryx Buildpack" + +[[stacks]] +id = ["com.microsoft.oryx.stack"] diff --git a/images/pack-builder/pack-runner.Dockerfile b/images/pack-builder/pack-runner.Dockerfile new file mode 100644 index 000000000..d1f57b814 --- /dev/null +++ b/images/pack-builder/pack-runner.Dockerfile @@ -0,0 +1,11 @@ +FROM buildpack-deps:stable + +WORKDIR /tmp + +COPY images/buildpack-builder/install-pack.sh install-pack.sh +RUN ./install-pack.sh && mv pack /usr/local/bin + +ARG BUILDPACK_BUILDER_NAME +RUN pack set-default-builder $BUILDPACK_BUILDER_NAME + +ENTRYPOINT ["/usr/local/bin/pack"] diff --git a/src/BuildScriptGenerator/DefaultBuildScriptGenerator.cs b/src/BuildScriptGenerator/DefaultBuildScriptGenerator.cs index 5bdf1574a..b7a63ac39 100644 --- a/src/BuildScriptGenerator/DefaultBuildScriptGenerator.cs +++ b/src/BuildScriptGenerator/DefaultBuildScriptGenerator.cs @@ -36,7 +36,7 @@ namespace Microsoft.Oryx.BuildScriptGenerator script = null; var toolsToVersion = new Dictionary(); - List snippets; + IList snippets; var directoriesToExcludeFromCopyToIntermediateDir = new List(); var directoriesToExcludeFromCopyToBuildOutputDir = new List(); @@ -70,42 +70,31 @@ namespace Microsoft.Oryx.BuildScriptGenerator } } - private static string GetBenvArgs(Dictionary benvArgsMap) - { - var listOfBenvArgs = benvArgsMap.Select(t => $"{t.Key}={t.Value}"); - var benvArgs = string.Join(' ', listOfBenvArgs); - return benvArgs; - } - - private List GetBuildSnippets( - BuildScriptGeneratorContext context, - Dictionary toolsToVersion, - List directoriesToExcludeFromCopyToIntermediateDir, - List directoriesToExlcudeFromCopyToBuildOutputDir) + public IList> GetCompatiblePlatforms(BuildScriptGeneratorContext ctx) { bool providedLanguageFound = false; - var snippets = new List(); + var resultPlatforms = new List>(); foreach (var platform in _programmingPlatforms) { - if (!platform.IsEnabled(context)) + if (!platform.IsEnabled(ctx)) { _logger.LogDebug("{platformName} has been disabled", platform.Name); continue; } bool usePlatform = false; - var currPlatformMatchesProvided = !string.IsNullOrEmpty(context.Language) && - string.Equals(context.Language, platform.Name, StringComparison.OrdinalIgnoreCase); + var currPlatformMatchesProvided = !string.IsNullOrEmpty(ctx.Language) && + string.Equals(ctx.Language, platform.Name, StringComparison.OrdinalIgnoreCase); string targetVersionSpec = null; if (currPlatformMatchesProvided) { providedLanguageFound = true; - targetVersionSpec = context.LanguageVersion; + targetVersionSpec = ctx.LanguageVersion; usePlatform = true; } - else if (context.DisableMultiPlatformBuild && !string.IsNullOrEmpty(context.Language)) + else if (ctx.DisableMultiPlatformBuild && !string.IsNullOrEmpty(ctx.Language)) { _logger.LogDebug( "Multi platform build is disabled and platform was specified. " + @@ -117,7 +106,7 @@ namespace Microsoft.Oryx.BuildScriptGenerator if (!currPlatformMatchesProvided || string.IsNullOrEmpty(targetVersionSpec)) { _logger.LogDebug("Detecting platform using {platformName}", platform.Name); - var detectionResult = platform.Detect(context.SourceRepo); + var detectionResult = platform.Detect(ctx.SourceRepo); if (detectionResult != null) { _logger.LogDebug( @@ -136,43 +125,62 @@ namespace Microsoft.Oryx.BuildScriptGenerator if (usePlatform) { - var excludedDirs = platform.GetDirectoriesToExcludeFromCopyToIntermediateDir(context); - if (excludedDirs.Any()) - { - directoriesToExcludeFromCopyToIntermediateDir.AddRange(excludedDirs); - } - - excludedDirs = platform.GetDirectoriesToExcludeFromCopyToBuildOutputDir(context); - if (excludedDirs.Any()) - { - directoriesToExlcudeFromCopyToBuildOutputDir.AddRange(excludedDirs); - } - - string targetVersion = GetMatchingTargetVersion(platform, targetVersionSpec); - platform.SetVersion(context, targetVersion); - - string cleanOrNot = platform.IsCleanRepo(context.SourceRepo) ? "clean" : "not clean"; - _logger.LogDebug($"Repo is {cleanOrNot} for {platform.Name}"); - - var snippet = platform.GenerateBashBuildScriptSnippet(context); - if (snippet != null) - { - _logger.LogDebug("Script generator {scriptGenType} was used", platform.GetType()); - snippets.Add(snippet); - platform.SetRequiredTools(context.SourceRepo, targetVersion, toolsToVersion); - } - else - { - _logger.LogDebug("Script generator {scriptGenType} cannot be used", platform.GetType()); - } + resultPlatforms.Add(Tuple.Create(platform, targetVersionSpec)); } } // Even if a language was detected, we throw an error if the user provided // an unsupported language as target. - if (!string.IsNullOrEmpty(context.Language) && !providedLanguageFound) + if (!string.IsNullOrEmpty(ctx.Language) && !providedLanguageFound) { - ThrowInvalidLanguageProvided(context); + ThrowInvalidLanguageProvided(ctx); + } + + return resultPlatforms; + } + + private IList GetBuildSnippets( + BuildScriptGeneratorContext context, + Dictionary toolsToVersion, + List directoriesToExcludeFromCopyToIntermediateDir, + List directoriesToExlcudeFromCopyToBuildOutputDir) + { + var snippets = new List(); + + var platformsToUse = GetCompatiblePlatforms(context); + foreach (Tuple platformAndVersion in platformsToUse) + { + var (platform, targetVersionSpec) = platformAndVersion; + + var excludedDirs = platform.GetDirectoriesToExcludeFromCopyToIntermediateDir(context); + if (excludedDirs.Any()) + { + directoriesToExcludeFromCopyToIntermediateDir.AddRange(excludedDirs); + } + + excludedDirs = platform.GetDirectoriesToExcludeFromCopyToBuildOutputDir(context); + if (excludedDirs.Any()) + { + directoriesToExlcudeFromCopyToBuildOutputDir.AddRange(excludedDirs); + } + + string targetVersion = GetMatchingTargetVersion(platform, targetVersionSpec); + platform.SetVersion(context, targetVersion); + + string cleanOrNot = platform.IsCleanRepo(context.SourceRepo) ? "clean" : "not clean"; + _logger.LogDebug($"Repo is {cleanOrNot} for {platform.Name}"); + + var snippet = platform.GenerateBashBuildScriptSnippet(context); + if (snippet != null) + { + _logger.LogDebug("Script generator {scriptGenType} was used", platform.GetType()); + snippets.Add(snippet); + platform.SetRequiredTools(context.SourceRepo, targetVersion, toolsToVersion); + } + else + { + _logger.LogDebug("Script generator {scriptGenType} cannot be used", platform.GetType()); + } } return snippets; @@ -195,12 +203,19 @@ namespace Microsoft.Oryx.BuildScriptGenerator } } + private static string GetBenvArgs(Dictionary benvArgsMap) + { + var listOfBenvArgs = benvArgsMap.Select(t => $"{t.Key}={t.Value}"); + var benvArgs = string.Join(' ', listOfBenvArgs); + return benvArgs; + } + /// /// Builds the full build script from the list of snippets for each platform. /// /// Finalized build script as a string. private string BuildScriptFromSnippets( - List snippets, + IList snippets, Dictionary toolsToVersion, List directoriesToExcludeFromCopyToIntermediateDir, List directoriesToExcludeFromCopyToBuildOutputDir) diff --git a/src/BuildScriptGenerator/DefaultEnvironment.cs b/src/BuildScriptGenerator/DefaultEnvironment.cs index 75f9513b2..bdc663989 100644 --- a/src/BuildScriptGenerator/DefaultEnvironment.cs +++ b/src/BuildScriptGenerator/DefaultEnvironment.cs @@ -30,15 +30,15 @@ namespace Microsoft.Oryx.BuildScriptGenerator return null; } - public string GetEnvironmentVariable(string name) + public string GetEnvironmentVariable(string name, string defaultValue = null) { - return Environment.GetEnvironmentVariable(name); + return Environment.GetEnvironmentVariable(name) ?? defaultValue; } public IList GetEnvironmentVariableAsList(string name) { IList ret = null; - var values = Environment.GetEnvironmentVariable(name); + var values = GetEnvironmentVariable(name); if (!string.IsNullOrWhiteSpace(values)) { ret = values.Split(","); diff --git a/src/BuildScriptGenerator/DotNetCore/DotnetCoreScriptGeneratorOptionsSetup.cs b/src/BuildScriptGenerator/DotNetCore/DotnetCoreScriptGeneratorOptionsSetup.cs index 42955a834..b09d475c1 100644 --- a/src/BuildScriptGenerator/DotNetCore/DotnetCoreScriptGeneratorOptionsSetup.cs +++ b/src/BuildScriptGenerator/DotNetCore/DotnetCoreScriptGeneratorOptionsSetup.cs @@ -3,6 +3,7 @@ // Licensed under the MIT license. // -------------------------------------------------------------------------------------------- +using System.IO; using Microsoft.Extensions.Options; namespace Microsoft.Oryx.BuildScriptGenerator.DotNetCore @@ -10,7 +11,7 @@ namespace Microsoft.Oryx.BuildScriptGenerator.DotNetCore internal class DotnetCoreScriptGeneratorOptionsSetup : IConfigureOptions { internal const string DefaultVersion = DotNetCoreVersions.DotNetCore21Version; - internal const string InstalledVersionsDir = "/opt/dotnet/"; + internal const string InstalledVersionsDir = "/opt/dotnet/"; // TODO: remove hard-coded path private readonly IEnvironment _environment; diff --git a/src/BuildScriptGenerator/IBuildScriptGenerator.cs b/src/BuildScriptGenerator/IBuildScriptGenerator.cs index c5395545e..bbe6ff437 100644 --- a/src/BuildScriptGenerator/IBuildScriptGenerator.cs +++ b/src/BuildScriptGenerator/IBuildScriptGenerator.cs @@ -3,6 +3,9 @@ // Licensed under the MIT license. // -------------------------------------------------------------------------------------------- +using System; +using System.Collections.Generic; + namespace Microsoft.Oryx.BuildScriptGenerator { public interface IBuildScriptGenerator @@ -10,9 +13,20 @@ namespace Microsoft.Oryx.BuildScriptGenerator /// /// Tries to generate a bash script to build an application. /// - /// The with parameters for the script. + /// + /// The with parameters for the script. + /// /// The generated script if the operation was successful. /// true if the operation was successful, false otherwise. bool TryGenerateBashScript(BuildScriptGeneratorContext scriptGeneratorContext, out string script); + + /// + /// Determines which platforms can be used to build the given application. + /// + /// + /// The with parameters for check. + /// + /// a list of platform and version pairs. + IList> GetCompatiblePlatforms(BuildScriptGeneratorContext ctx); } } \ No newline at end of file diff --git a/src/BuildScriptGenerator/IEnvironment.cs b/src/BuildScriptGenerator/IEnvironment.cs index eca0830a0..8425ae62f 100644 --- a/src/BuildScriptGenerator/IEnvironment.cs +++ b/src/BuildScriptGenerator/IEnvironment.cs @@ -14,14 +14,13 @@ namespace Microsoft.Oryx.BuildScriptGenerator public interface IEnvironment { /// - /// Gets values of an environment variable. + /// Gets the value of an environment variable. /// - /// Name of the environment variable. Is case-sensitive. + /// Name of the environment variable. Case-sensitive. /// - /// The value of the environment variable specified by variable, - /// or null if the environment variable is not found. + /// The value of the given environment variable, or null if the environment variable isn't found. /// - string GetEnvironmentVariable(string name); + string GetEnvironmentVariable(string name, string defaultValue = null); /// /// Gets the value of an environment variable as a boolean, if found. The check diff --git a/src/BuildScriptGenerator/Node/NodeScriptGeneratorOptionsSetup.cs b/src/BuildScriptGenerator/Node/NodeScriptGeneratorOptionsSetup.cs index 16a20be41..3ae5c074f 100644 --- a/src/BuildScriptGenerator/Node/NodeScriptGeneratorOptionsSetup.cs +++ b/src/BuildScriptGenerator/Node/NodeScriptGeneratorOptionsSetup.cs @@ -3,6 +3,7 @@ // Licensed under the MIT license. // -------------------------------------------------------------------------------------------- +using System.IO; using Microsoft.Extensions.Options; namespace Microsoft.Oryx.BuildScriptGenerator.Node @@ -15,7 +16,7 @@ namespace Microsoft.Oryx.BuildScriptGenerator.Node internal const string NpmSupportedVersionsEnvVariable = "NPM_SUPPORTED_VERSIONS"; internal const string LegacyZipNodeModules = "ENABLE_NODE_MODULES_ZIP"; internal const string NodeLtsVersion = "8.11.2"; - internal const string InstalledNodeVersionsDir = "/opt/nodejs/"; + internal const string InstalledNodeVersionsDir = "/opt/nodejs/"; // TODO: remove hard-coded paths internal const string InstalledNpmVersionsDir = "/opt/npm/"; private readonly IEnvironment _environment; @@ -35,8 +36,10 @@ namespace Microsoft.Oryx.BuildScriptGenerator.Node options.NodeJsDefaultVersion = defaultVersion; options.NpmDefaultVersion = _environment.GetEnvironmentVariable(NpmDefaultVersion); + options.InstalledNodeVersionsDir = InstalledNodeVersionsDir; options.InstalledNpmVersionsDir = InstalledNpmVersionsDir; + options.SupportedNodeVersions = _environment.GetEnvironmentVariableAsList( NodeSupportedVersionsEnvVariable); options.SupportedNpmVersions = _environment.GetEnvironmentVariableAsList(NpmSupportedVersionsEnvVariable); diff --git a/src/BuildScriptGenerator/Python/PythonScriptGeneratorOptionsSetup.cs b/src/BuildScriptGenerator/Python/PythonScriptGeneratorOptionsSetup.cs index ef6ab18e6..b948fe41a 100644 --- a/src/BuildScriptGenerator/Python/PythonScriptGeneratorOptionsSetup.cs +++ b/src/BuildScriptGenerator/Python/PythonScriptGeneratorOptionsSetup.cs @@ -3,6 +3,7 @@ // Licensed under the MIT license. // -------------------------------------------------------------------------------------------- +using System.IO; using Microsoft.Extensions.Options; namespace Microsoft.Oryx.BuildScriptGenerator.Python @@ -28,7 +29,6 @@ namespace Microsoft.Oryx.BuildScriptGenerator.Python options.PythonDefaultVersion = defaultVersion; options.InstalledPythonVersionsDir = PythonConstants.InstalledPythonVersionsDir; - // Providing the supported versions through an environment variable allows us to use the tool in // other environments, e.g. our local machines for debugging. options.SupportedPythonVersions = _environment.GetEnvironmentVariableAsList( diff --git a/src/BuildScriptGeneratorCli/BuildScriptGenerator.cs b/src/BuildScriptGeneratorCli/BuildScriptGenerator.cs index 78120022b..315811e3c 100644 --- a/src/BuildScriptGeneratorCli/BuildScriptGenerator.cs +++ b/src/BuildScriptGeneratorCli/BuildScriptGenerator.cs @@ -26,6 +26,25 @@ namespace Microsoft.Oryx.BuildScriptGeneratorCli _logger = _serviceProvider.GetRequiredService>(); } + public static BuildScriptGeneratorContext CreateContext( + BuildScriptGeneratorOptions options, + CliEnvironmentSettings envSettings, + ISourceRepo sourceRepo) + { + return new BuildScriptGeneratorContext + { + SourceRepo = sourceRepo, + Language = options.Language, + LanguageVersion = options.LanguageVersion, + Properties = options.Properties, + EnableDotNetCore = !envSettings.DisableDotNetCore, + EnableNodeJs = !envSettings.DisableNodeJs, + EnablePython = !envSettings.DisablePython, + EnablePhp = !envSettings.DisablePhp, + DisableMultiPlatformBuild = envSettings.DisableMultiPlatformBuild + }; + } + public bool TryGenerateScript(out string generatedScript) { generatedScript = null; @@ -37,18 +56,7 @@ namespace Microsoft.Oryx.BuildScriptGeneratorCli var sourceRepoProvider = _serviceProvider.GetRequiredService(); var environment = _serviceProvider.GetRequiredService(); var sourceRepo = sourceRepoProvider.GetSourceRepo(); - var scriptGeneratorContext = new BuildScriptGeneratorContext - { - SourceRepo = sourceRepo, - Language = options.Language, - LanguageVersion = options.LanguageVersion, - Properties = options.Properties, - EnableDotNetCore = !environment.DisableDotNetCore, - EnableNodeJs = !environment.DisableNodeJs, - EnablePython = !environment.DisablePython, - EnablePhp = !environment.DisablePhp, - DisableMultiPlatformBuild = environment.DisableMultiPlatformBuild - }; + var scriptGeneratorContext = CreateContext(options, environment, sourceRepo); // Try generating a script if (!scriptGenerator.TryGenerateBashScript(scriptGeneratorContext, out generatedScript)) diff --git a/src/BuildScriptGeneratorCli/Commands/BuildCommand.cs b/src/BuildScriptGeneratorCli/Commands/BuildCommand.cs index 227f1d84d..5f1a935dc 100644 --- a/src/BuildScriptGeneratorCli/Commands/BuildCommand.cs +++ b/src/BuildScriptGeneratorCli/Commands/BuildCommand.cs @@ -20,7 +20,7 @@ using Microsoft.Oryx.Common; namespace Microsoft.Oryx.BuildScriptGeneratorCli { [Command("build", Description = "Generate and run build scripts.")] - internal class BuildCommand : BaseCommand + internal class BuildCommand : CommandBase { // Beginning and ending markers for build script output spans that should be time measured private readonly TextSpan[] _measurableStdOutSpans = diff --git a/src/BuildScriptGeneratorCli/Commands/BuildScriptCommand.cs b/src/BuildScriptGeneratorCli/Commands/BuildScriptCommand.cs index 443be23cc..b55bb1b80 100644 --- a/src/BuildScriptGeneratorCli/Commands/BuildScriptCommand.cs +++ b/src/BuildScriptGeneratorCli/Commands/BuildScriptCommand.cs @@ -14,7 +14,7 @@ using Microsoft.Oryx.Common; namespace Microsoft.Oryx.BuildScriptGeneratorCli { [Command("build-script", Description = "Generate build script to standard output.")] - internal class BuildScriptCommand : BaseCommand + internal class BuildScriptCommand : CommandBase { [Argument(0, Description = "The source directory.")] public string SourceDir { get; set; } diff --git a/src/BuildScriptGeneratorCli/Commands/BuildpackDetectCommand.cs b/src/BuildScriptGeneratorCli/Commands/BuildpackDetectCommand.cs new file mode 100644 index 000000000..31652e3ae --- /dev/null +++ b/src/BuildScriptGeneratorCli/Commands/BuildpackDetectCommand.cs @@ -0,0 +1,107 @@ +// -------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. +// -------------------------------------------------------------------------------------------- + +using System; +using System.IO; +using System.Linq; +using McMaster.Extensions.CommandLineUtils; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Microsoft.Oryx.BuildScriptGenerator; +using Microsoft.Oryx.Common; + +namespace Microsoft.Oryx.BuildScriptGeneratorCli +{ + [Command("buildpack-detect", Description = "Determines whether Oryx can be applied as a buildpack to " + + "an app in the current working directory.")] + internal class BuildpackDetectCommand : CommandBase + { + // CodeDetectFail @ https://github.com/buildpack/lifecycle/blob/master/detector.go + public const int DetectorFailCode = 100; + + [Argument(0, Description = "The source directory.")] + public string SourceDir { get; set; } + + [Option("--platform-dir ", CommandOptionType.SingleValue, Description = "Platform directory path.")] + public string PlatformDir { get; set; } + + [Option("--plan-path ", CommandOptionType.SingleValue, Description = "Build plan TOML path.")] + public string PlanPath { get; set; } + + internal override bool IsValidInput(IServiceProvider serviceProvider, IConsole console) + { + var result = true; + var options = serviceProvider.GetRequiredService>().Value; + var logger = serviceProvider.GetService>(); + + // Set from ConfigureBuildScriptGeneratorOptions + if (!Directory.Exists(options.SourceDir)) + { + logger.LogError("Could not find the source directory {srcDir}", options.SourceDir); + console.Error.WriteLine($"Error: Could not find the source directory '{options.SourceDir}'."); + result = false; + } + + if (!string.IsNullOrWhiteSpace(PlanPath)) + { + PlanPath = Path.GetFullPath(PlanPath); + if (!File.Exists(PlanPath)) + { + logger?.LogError("Could not find build plan file {planPath}", PlanPath); + console.Error.WriteLine($"Error: Could not find build plan file '{PlanPath}'."); + result = false; + } + } + + if (!string.IsNullOrWhiteSpace(PlatformDir)) + { + PlatformDir = Path.GetFullPath(PlatformDir); + if (!Directory.Exists(PlatformDir)) + { + logger?.LogError("Could not find platform directory {platformDir}", PlatformDir); + console.Error.WriteLine($"Error: Could not find platform directory '{PlatformDir}'."); + result = false; + } + } + + return result; + } + + internal override void ConfigureBuildScriptGeneratorOptions(BuildScriptGeneratorOptions options) + { + BuildScriptGeneratorOptionsHelper.ConfigureBuildScriptGeneratorOptions( + options, + SourceDir, + destinationDir: null, + intermediateDir: null, + language: null, + languageVersion: null, + scriptOnly: false, + properties: null); + } + + internal override int Execute(IServiceProvider serviceProvider, IConsole console) + { + var generator = serviceProvider.GetRequiredService(); + + var options = serviceProvider.GetRequiredService>().Value; + var env = serviceProvider.GetRequiredService(); + var repo = serviceProvider.GetRequiredService().GetSourceRepo(); + + var ctx = BuildScriptGenerator.CreateContext(options, env, repo); + var compatPlats = generator.GetCompatiblePlatforms(ctx); + + if (compatPlats != null && compatPlats.Any()) + { + console.WriteLine("# Detected platforms:"); + console.WriteLine(string.Join(' ', compatPlats.Select(pair => $"{pair.Item1.Name}=\"{pair.Item2}\""))); + return ProcessConstants.ExitSuccess; + } + + return DetectorFailCode; + } + } +} diff --git a/src/BuildScriptGeneratorCli/Commands/BaseCommand.cs b/src/BuildScriptGeneratorCli/Commands/CommandBase.cs similarity index 88% rename from src/BuildScriptGeneratorCli/Commands/BaseCommand.cs rename to src/BuildScriptGeneratorCli/Commands/CommandBase.cs index 5add66cd4..7e8a05936 100644 --- a/src/BuildScriptGeneratorCli/Commands/BaseCommand.cs +++ b/src/BuildScriptGeneratorCli/Commands/CommandBase.cs @@ -13,7 +13,7 @@ using Microsoft.Oryx.Common; namespace Microsoft.Oryx.BuildScriptGeneratorCli { - internal abstract class BaseCommand + internal abstract class CommandBase { private IServiceProvider _serviceProvider = null; @@ -33,6 +33,8 @@ namespace Microsoft.Oryx.BuildScriptGeneratorCli try { _serviceProvider = GetServiceProvider(); + _serviceProvider?.GetRequiredService>()?.LogInformation( + "Oryx command line: {cmdLine}", Environment.CommandLine); if (!IsValidInput(_serviceProvider, console)) { return ProcessConstants.ExitFailure; @@ -52,7 +54,7 @@ namespace Microsoft.Oryx.BuildScriptGeneratorCli } catch (Exception exc) { - _serviceProvider?.GetRequiredService>()?.LogError(exc, "Exception caught"); + _serviceProvider?.GetRequiredService>()?.LogError(exc, "Exception caught"); console.Error.WriteLine(Constants.GenericErrorMessage); if (ShowStackTrace) @@ -83,10 +85,7 @@ namespace Microsoft.Oryx.BuildScriptGeneratorCli internal virtual IServiceProvider GetServiceProvider() { var serviceProviderBuilder = new ServiceProviderBuilder(LogFilePath) - .ConfigureScriptGenerationOptions(o => - { - ConfigureBuildScriptGeneratorOptions(o); - }); + .ConfigureScriptGenerationOptions(opts => ConfigureBuildScriptGeneratorOptions(opts)); return serviceProviderBuilder.Build(); } diff --git a/src/BuildScriptGeneratorCli/Commands/LanguagesCommand.cs b/src/BuildScriptGeneratorCli/Commands/LanguagesCommand.cs index 45514230d..f42a8435b 100644 --- a/src/BuildScriptGeneratorCli/Commands/LanguagesCommand.cs +++ b/src/BuildScriptGeneratorCli/Commands/LanguagesCommand.cs @@ -19,7 +19,7 @@ namespace Microsoft.Oryx.BuildScriptGeneratorCli [Command( "languages", Description = "Show the list of supported platforms and other information like versions, properties etc.")] - internal class LanguagesCommand : BaseCommand + internal class LanguagesCommand : CommandBase { [Option("--json", Description = "Output the supported platform data in JSON format.")] public bool OutputJson { get; set; } @@ -73,10 +73,9 @@ namespace Microsoft.Oryx.BuildScriptGeneratorCli var defs = new DefinitionListFormatter(); defs.AddDefinition("Platform", platform.Name); - if (platform.Versions != null && platform.Versions.Any()) - { - defs.AddDefinition("Versions", string.Join(Environment.NewLine, platform.Versions)); - } + defs.AddDefinition("Versions", + (platform.Versions != null && platform.Versions.Any()) ? + string.Join(Environment.NewLine, platform.Versions) : "N/A"); if (platform.Properties != null && platform.Properties.Any()) { diff --git a/src/BuildScriptGeneratorCli/Commands/RunScriptCommand.cs b/src/BuildScriptGeneratorCli/Commands/RunScriptCommand.cs index d04615a9f..5a17b0aaa 100644 --- a/src/BuildScriptGeneratorCli/Commands/RunScriptCommand.cs +++ b/src/BuildScriptGeneratorCli/Commands/RunScriptCommand.cs @@ -14,7 +14,7 @@ using Microsoft.Oryx.Common; namespace Microsoft.Oryx.BuildScriptGeneratorCli { [Command("run-script", Description = "Generate startup script.")] - internal class RunScriptCommand : BaseCommand + internal class RunScriptCommand : CommandBase { [Option( "--platform", diff --git a/src/BuildScriptGeneratorCli/Program.cs b/src/BuildScriptGeneratorCli/Program.cs index 15173d0ee..9d0643474 100644 --- a/src/BuildScriptGeneratorCli/Program.cs +++ b/src/BuildScriptGeneratorCli/Program.cs @@ -15,6 +15,7 @@ namespace Microsoft.Oryx.BuildScriptGeneratorCli [Subcommand("languages", typeof(LanguagesCommand))] [Subcommand("build-script", typeof(BuildScriptCommand))] [Subcommand("run-script", typeof(RunScriptCommand))] + [Subcommand("buildpack-detect", typeof(BuildpackDetectCommand))] internal class Program { [Option(CommandOptionType.NoValue, Description = "Print version information.")] diff --git a/src/BuildScriptGeneratorCli/ServiceProviderBuilder.cs b/src/BuildScriptGeneratorCli/ServiceProviderBuilder.cs index 8adc76db7..88c439692 100644 --- a/src/BuildScriptGeneratorCli/ServiceProviderBuilder.cs +++ b/src/BuildScriptGeneratorCli/ServiceProviderBuilder.cs @@ -5,6 +5,7 @@ using System; using System.IO; +using JetBrains.Annotations; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using Microsoft.Oryx.BuildScriptGenerator; @@ -50,10 +51,7 @@ namespace Microsoft.Oryx.BuildScriptGeneratorCli public ServiceProviderBuilder ConfigureScriptGenerationOptions(Action configure) { - _serviceCollection.Configure(options => - { - configure(options); - }); + _serviceCollection.Configure(opts => configure(opts)); return this; } @@ -62,7 +60,7 @@ namespace Microsoft.Oryx.BuildScriptGeneratorCli return _serviceCollection.BuildServiceProvider(); } - private LoggingConfiguration BuildNLogConfiguration(string logPath) + private LoggingConfiguration BuildNLogConfiguration([CanBeNull] string logPath) { var config = new LoggingConfiguration(); diff --git a/tests/BuildScriptGenerator.Tests/Php/PhpLanguageDetectorTest.cs b/tests/BuildScriptGenerator.Tests/Php/PhpLanguageDetectorTest.cs index 2d9357738..4b3da9cf4 100644 --- a/tests/BuildScriptGenerator.Tests/Php/PhpLanguageDetectorTest.cs +++ b/tests/BuildScriptGenerator.Tests/Php/PhpLanguageDetectorTest.cs @@ -3,9 +3,7 @@ // Licensed under the MIT license. // -------------------------------------------------------------------------------------------- -using System; using System.Collections.Generic; -using System.IO; using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Options; using Microsoft.Oryx.BuildScriptGenerator.Exceptions; @@ -99,7 +97,10 @@ namespace Microsoft.Oryx.BuildScriptGenerator.Tests.Php var options = new PhpScriptGeneratorOptions(); optionsSetup.Configure(options); - return new PhpLanguageDetector(Options.Create(options), new TestPhpVersionProvider(supportedPhpVersions), NullLogger.Instance); + return new PhpLanguageDetector( + Options.Create(options), + new TestPhpVersionProvider(supportedPhpVersions), + NullLogger.Instance); } private class TestPhpVersionProvider : IPhpVersionProvider diff --git a/tests/BuildScriptGeneratorCli.Tests/BuildCommandTest.cs b/tests/BuildScriptGeneratorCli.Tests/BuildCommandTest.cs index 2afdf12b7..8cdbe9e15 100644 --- a/tests/BuildScriptGeneratorCli.Tests/BuildCommandTest.cs +++ b/tests/BuildScriptGeneratorCli.Tests/BuildCommandTest.cs @@ -239,8 +239,7 @@ namespace BuildScriptGeneratorCli.Tests [Theory] [MemberData(nameof(DestinationDirectoryPathData))] - public void IsValidInput_IsTrue_IfDestinationDirIsNotEmpty( - string destinationDir) + public void IsValidInput_IsTrue_IfDestinationDirIsNotEmpty(string destinationDir) { // Arrange var serviceProvider = new ServiceProviderBuilder() diff --git a/tests/BuildScriptGeneratorCli.Tests/BuildpackDetectCommandTest.cs b/tests/BuildScriptGeneratorCli.Tests/BuildpackDetectCommandTest.cs new file mode 100644 index 000000000..e1c33a721 --- /dev/null +++ b/tests/BuildScriptGeneratorCli.Tests/BuildpackDetectCommandTest.cs @@ -0,0 +1,76 @@ +// -------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. +// -------------------------------------------------------------------------------------------- + +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using System; +using System.IO; +using Xunit; +using Microsoft.Oryx.Tests.Common; +using Microsoft.Oryx.BuildScriptGenerator.Node; +using Microsoft.Oryx.BuildScriptGenerator; +using Microsoft.Oryx.BuildScriptGeneratorCli; +using Microsoft.Oryx.Common; + +namespace BuildScriptGeneratorCli.Tests +{ + public class BuildpackDetectCommandTest : IClassFixture + { + private readonly string _testDirPath; + + public BuildpackDetectCommandTest(TestTempDirTestFixture testFixture) + { + _testDirPath = testFixture.RootDirPath; + } + + [Fact] + public void Execute_Returns100_WhenSourceDirIsEmpty() + { + // Arrange + var srcDir = Path.Combine(_testDirPath, "emptydir"); + Directory.CreateDirectory(srcDir); + + var cmd = new BuildpackDetectCommand { SourceDir = srcDir }; + + // Act & Assert + Assert.Equal( + BuildpackDetectCommand.DetectorFailCode, + cmd.Execute(GetServiceProvider(cmd), new TestConsole())); + } + + [Fact] + public void Execute_OutputsNode_WhenPackageJsonExists() + { + // Arrange + var srcDir = Path.Combine(_testDirPath, "nodeappdir"); + Directory.CreateDirectory(srcDir); + File.WriteAllText(Path.Combine(srcDir, NodeConstants.PackageJsonFileName), "\n"); + + var cmd = new BuildpackDetectCommand { SourceDir = srcDir }; + var console = new TestConsole(); + + // Act + int exitCode = cmd.Execute(GetServiceProvider(cmd), console); + + // Assert + Assert.Equal(ProcessConstants.ExitSuccess, exitCode); + Assert.Contains( + $"{NodeConstants.NodeJsName}=\"{NodeScriptGeneratorOptionsSetup.NodeLtsVersion}\"", + console.StdOutput); + } + + private static IServiceProvider GetServiceProvider(BuildpackDetectCommand cmd) + { + var env = new TestEnvironment(); + env.SetEnvironmentVariable("NODE_SUPPORTED_VERSIONS", NodeScriptGeneratorOptionsSetup.NodeLtsVersion); + + var svcProvider = new ServiceProviderBuilder() + .ConfigureServices(svcs => svcs.Replace(ServiceDescriptor.Singleton(typeof(IEnvironment), env))) + .ConfigureScriptGenerationOptions(opts => cmd.ConfigureBuildScriptGeneratorOptions(opts)) + .Build(); + return svcProvider; + } + } +} diff --git a/tests/Oryx.RuntimeImage.Tests/DotnetCoreStartupScriptGenerationTest.cs b/tests/Oryx.RuntimeImage.Tests/DotnetCoreStartupScriptGenerationTest.cs index c3e2277b1..d6c3f50d4 100644 --- a/tests/Oryx.RuntimeImage.Tests/DotnetCoreStartupScriptGenerationTest.cs +++ b/tests/Oryx.RuntimeImage.Tests/DotnetCoreStartupScriptGenerationTest.cs @@ -453,7 +453,6 @@ namespace Microsoft.Oryx.RuntimeImage.Tests { // Arrange var appDir = "/app"; - var scriptLocation = "/tmp/run.sh"; var script = new ShellScriptBuilder() .AddCommand($"mkdir -p {appDir}") // no .csproj file .AddCommand($"oryx -sourcePath {appDir} -defaultAppFilePath /tmp/doesnotexist.dll") @@ -569,7 +568,6 @@ namespace Microsoft.Oryx.RuntimeImage.Tests var webApp2Dir = $"{repoDir}/src/Apps/WebApp2"; var defaultWebAppFile = "/tmp/defaultwebapp.dll"; var expectedStartupCommand = $"dotnet \"{defaultWebAppFile}\""; - var scriptLocation = "/tmp/run.sh"; var script = new ShellScriptBuilder() .AddCommand($"mkdir -p {webApp1Dir}") .AddCommand($"mkdir -p {webApp2Dir}") diff --git a/tests/Oryx.Tests.Common/TestEnvironment.cs b/tests/Oryx.Tests.Common/TestEnvironment.cs index 5e42d508f..20a51d84e 100644 --- a/tests/Oryx.Tests.Common/TestEnvironment.cs +++ b/tests/Oryx.Tests.Common/TestEnvironment.cs @@ -6,6 +6,7 @@ using System; using System.Collections; using System.Collections.Generic; +using System.Linq; using Microsoft.Oryx.BuildScriptGenerator; namespace Microsoft.Oryx.Tests.Common @@ -28,18 +29,26 @@ namespace Microsoft.Oryx.Tests.Common return null; } - public string GetEnvironmentVariable(string name) + public string GetEnvironmentVariable(string name, string defaultValue = null) { if (Variables.TryGetValue(name, out var value)) { return value; } - return null; + return defaultValue; } public IList GetEnvironmentVariableAsList(string name) { - return null; + IList ret = null; + var values = GetEnvironmentVariable(name); + if (!string.IsNullOrWhiteSpace(values)) + { + ret = values.Split(","); + ret = ret.Select(s => s.Trim()).ToList(); + } + + return ret; } public IDictionary GetEnvironmentVariables()