Enable supplying commands directly to the pre and post build script envrionment variables (#124)

This commit is contained in:
Kiran Challa 2019-05-08 13:47:12 -07:00 коммит произвёл GitHub
Родитель d9b0c53e88
Коммит 7869c96d80
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: 4AEE18F83AFDEB23
16 изменённых файлов: 354 добавлений и 70 удалений

Просмотреть файл

@ -67,12 +67,12 @@ fi
export SOURCE_DIR
export DESTINATION_DIR
{{ if PreBuildScriptPath | IsNotBlank }}
{{ if PreBuildCommand | IsNotBlank }}
# Make sure to cd to the source directory so that the pre-build script runs from there
cd "$SOURCE_DIR"
echo "{{ PreBuildScriptPrologue }}"
"{{ PreBuildScriptPath }}"
echo "{{ PreBuildScriptEpilogue }}"
echo "{{ PreBuildCommandPrologue }}"
{{ PreBuildCommand }}
echo "{{ PreBuildCommandEpilogue }}"
{{ end }}
{{ for Snippet in BuildScriptSnippets }}
@ -81,12 +81,12 @@ cd "$SOURCE_DIR"
{{~ Snippet }}
{{ end }}
{{ if PostBuildScriptPath | IsNotBlank }}
{{ if PostBuildCommand | IsNotBlank }}
# Make sure to cd to the source directory so that the post-build script runs from there
cd $SOURCE_DIR
echo "{{ PostBuildScriptPrologue }}"
"{{ PostBuildScriptPath }}"
echo "{{ PostBuildScriptEpilogue }}"
echo "{{ PostBuildCommandPrologue }}"
{{ PostBuildCommand }}
echo "{{ PostBuildCommandEpilogue }}"
{{ end }}
if [ "$SOURCE_DIR" != "$DESTINATION_DIR" ]

Просмотреть файл

@ -9,11 +9,11 @@ namespace Microsoft.Oryx.BuildScriptGenerator
{
public class BaseBashBuildScriptProperties
{
public const string PreBuildScriptPrologue = "Executing pre-build script...";
public const string PreBuildScriptEpilogue = "Finished executing pre-build script.";
public const string PreBuildCommandPrologue = "Executing pre-build command...";
public const string PreBuildCommandEpilogue = "Finished executing pre-build command.";
public const string PostBuildScriptPrologue = "Executing post-build script...";
public const string PostBuildScriptEpilogue = "Finished executing post-build script.";
public const string PostBuildCommandPrologue = "Executing post-build command...";
public const string PostBuildCommandEpilogue = "Finished executing post-build command.";
/// <summary>
/// Gets or sets the collection of build script snippets.
@ -21,9 +21,9 @@ namespace Microsoft.Oryx.BuildScriptGenerator
public IEnumerable<string> BuildScriptSnippets { get; set; }
/// <summary>
/// Gets or sets the path to the pre build script.
/// Gets or sets the the pre build script content
/// </summary>
public string PreBuildScriptPath { get; set; }
public string PreBuildCommand { get; set; }
/// <summary>
/// Gets or sets the argument to the benv command.
@ -31,9 +31,9 @@ namespace Microsoft.Oryx.BuildScriptGenerator
public string BenvArgs { get; set; }
/// <summary>
/// Gets or sets the path to the post build script.
/// Gets or sets the path to the post build script content.
/// </summary>
public string PostBuildScriptPath { get; set; }
public string PostBuildCommand { get; set; }
public IEnumerable<string> DirectoriesToExcludeFromCopyToBuildOutputDir { get; set; }

Просмотреть файл

@ -5,6 +5,7 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using Microsoft.Extensions.Logging;
using Microsoft.Oryx.BuildScriptGenerator.Exceptions;
@ -69,6 +70,7 @@ namespace Microsoft.Oryx.BuildScriptGenerator
directoriesToExcludeFromCopyToBuildOutputDir.Add(".git");
script = BuildScriptFromSnippets(
context.SourceRepo,
snippets,
toolsToVersion,
directoriesToExcludeFromCopyToIntermediateDir,
@ -249,6 +251,7 @@ namespace Microsoft.Oryx.BuildScriptGenerator
/// </summary>
/// <returns>Finalized build script as a string.</returns>
private string BuildScriptFromSnippets(
ISourceRepo sourceRepo,
IList<BuildScriptSnippet> snippets,
Dictionary<string, string> toolsToVersion,
List<string> directoriesToExcludeFromCopyToIntermediateDir,
@ -263,20 +266,24 @@ namespace Microsoft.Oryx.BuildScriptGenerator
.SelectMany(s => s.BuildProperties)
.ToDictionary(p => p.Key, p => p.Value);
(var preBuildCommand, var postBuildCommand) = PreAndPostBuildCommandHelper.GetPreAndPostBuildCommands(
sourceRepo,
environmentSettings);
var buildScriptProps = new BaseBashBuildScriptProperties()
{
BuildScriptSnippets = snippets.Select(s => s.BashBuildScriptSnippet),
BenvArgs = benvArgs,
PreBuildScriptPath = environmentSettings?.PreBuildScriptPath,
PostBuildScriptPath = environmentSettings?.PostBuildScriptPath,
PreBuildCommand = preBuildCommand,
PostBuildCommand = postBuildCommand,
DirectoriesToExcludeFromCopyToIntermediateDir = directoriesToExcludeFromCopyToIntermediateDir,
DirectoriesToExcludeFromCopyToBuildOutputDir = directoriesToExcludeFromCopyToBuildOutputDir,
ManifestFileName = Constants.ManifestFileName,
BuildProperties = buildProperties
};
LogScriptIfGiven("pre-build", buildScriptProps.PreBuildScriptPath);
LogScriptIfGiven("post-build", buildScriptProps.PostBuildScriptPath);
LogScriptIfGiven("pre-build", buildScriptProps.PreBuildCommand);
LogScriptIfGiven("post-build", buildScriptProps.PostBuildCommand);
script = TemplateHelpers.Render(
TemplateHelpers.TemplateResource.BaseBashScript,

Просмотреть файл

@ -139,39 +139,36 @@ namespace Microsoft.Oryx.BuildScriptGenerator
{
var environmentSettings = new EnvironmentSettings
{
PreBuildScriptPath = GetPath(EnvironmentSettingsKeys.PreBuildScriptPath),
PostBuildScriptPath = GetPath(EnvironmentSettingsKeys.PostBuildScriptPath)
PreBuildScriptPath = GetScriptAbsolutePath(TrimValue(EnvironmentSettingsKeys.PreBuildScriptPath)),
PostBuildScriptPath = GetScriptAbsolutePath(TrimValue(EnvironmentSettingsKeys.PostBuildScriptPath))
};
if (!string.IsNullOrEmpty(environmentSettings.PreBuildScriptPath))
{
environmentSettings.PreBuildScriptPath = GetScriptAbsolutePath(environmentSettings.PreBuildScriptPath);
}
if (!string.IsNullOrEmpty(environmentSettings.PostBuildScriptPath))
{
environmentSettings.PostBuildScriptPath = GetScriptAbsolutePath(
environmentSettings.PostBuildScriptPath);
}
environmentSettings.PreBuildCommand = TrimValue(EnvironmentSettingsKeys.PreBuildCommand);
environmentSettings.PostBuildCommand = TrimValue(EnvironmentSettingsKeys.PostBuildCommand);
return environmentSettings;
string GetPath(string name)
string TrimValue(string key)
{
var path = GetValue(name);
if (string.IsNullOrEmpty(path))
var value = GetValue(key);
if (string.IsNullOrEmpty(value))
{
return null;
}
path = path.Trim();
value = value.Trim();
var quote = '"';
if (path.StartsWith(quote) && path.EndsWith(quote))
if (value.Length > 1 && value.StartsWith(quote) && value.EndsWith(quote))
{
return path.Trim(quote);
value = value.Remove(0, 1);
if (value.Length > 0)
{
value = value.Remove(value.Length - 1);
}
}
return path;
return value;
}
string GetValue(string name)
@ -194,6 +191,11 @@ namespace Microsoft.Oryx.BuildScriptGenerator
string GetScriptAbsolutePath(string path)
{
if (string.IsNullOrEmpty(path))
{
return null;
}
if (!Path.IsPathFullyQualified(path))
{
path = Path.Combine(_sourceRepo.RootPath, path);

Просмотреть файл

@ -92,10 +92,10 @@ fi
# Make sure to create the destination dir so that pre-build script has access to it
mkdir -p "$DESTINATION_DIR"
{{ if PreBuildScriptPath | IsNotBlank }}
{{ if PreBuildCommand | IsNotBlank }}
# Make sure to cd to the source directory so that the pre-build script runs from there
cd "$SOURCE_DIR"
"{{ PreBuildScriptPath }}"
{{ PreBuildCommand }}
{{ end }}
echo
@ -112,10 +112,10 @@ dotnet restore "{{ ProjectFile }}"
{{ if ZipAllOutput }}
publishToDirectory "$tmpDestinationPublishDir"
{{ if PostBuildScriptPath | IsNotBlank }}
{{ if PostBuildCommand | IsNotBlank }}
# Make sure to cd to the source directory so that the post-build script runs from there
cd "$SOURCE_DIR"
"{{ PostBuildScriptPath }}"
{{ PostBuildCommand }}
{{ end }}
# Zip only the contents and not the parent directory
@ -129,10 +129,10 @@ dotnet restore "{{ ProjectFile }}"
{{ else }}
publishToDirectory "$DESTINATION_DIR"
{{ if PostBuildScriptPath | IsNotBlank }}
{{ if PostBuildCommand | IsNotBlank }}
# Make sure to cd to the source directory so that the post-build script runs from there
cd $SOURCE_DIR
"{{ PostBuildScriptPath }}"
{{ PostBuildCommand }}
{{ end }}
{{ end }}

Просмотреть файл

@ -20,9 +20,9 @@ namespace Microsoft.Oryx.BuildScriptGenerator.DotNetCore
public string BenvArgs { get; set; }
public string PreBuildScriptPath { get; set; }
public string PreBuildCommand { get; set; }
public string PostBuildScriptPath { get; set; }
public string PostBuildCommand { get; set; }
public bool ZipAllOutput { get; set; }

Просмотреть файл

@ -76,6 +76,10 @@ namespace Microsoft.Oryx.BuildScriptGenerator.DotNetCore
_environmentSettingsProvider.TryGetAndLoadSettings(out var environmentSettings);
(var preBuildCommand, var postBuildCommand) = PreAndPostBuildCommandHelper.GetPreAndPostBuildCommands(
context.SourceRepo,
environmentSettings);
var templateProperties = new DotNetCoreBashBuildSnippetProperties
{
ProjectFile = projectFile,
@ -84,8 +88,8 @@ namespace Microsoft.Oryx.BuildScriptGenerator.DotNetCore
BenvArgs = $"dotnet={context.DotnetCoreVersion}",
DirectoriesToExcludeFromCopyToIntermediateDir = GetDirectoriesToExcludeFromCopyToIntermediateDir(
context),
PreBuildScriptPath = environmentSettings?.PreBuildScriptPath,
PostBuildScriptPath = environmentSettings?.PostBuildScriptPath,
PreBuildCommand = preBuildCommand,
PostBuildCommand = postBuildCommand,
ManifestFileName = Constants.ManifestFileName,
ZipAllOutput = zipAllOutput,
};
@ -177,5 +181,18 @@ namespace Microsoft.Oryx.BuildScriptGenerator.DotNetCore
DotnetCoreConstants.OryxOutputPublishDirectory);
return (projectFile, publishDir);
}
private string GetCommandOrScript(string commandOrScript)
{
if (!string.IsNullOrEmpty(commandOrScript))
{
if (File.Exists(commandOrScript))
{
return $"\"{commandOrScript}\"";
}
}
return commandOrScript;
}
}
}

Просмотреть файл

@ -7,8 +7,14 @@ namespace Microsoft.Oryx.BuildScriptGenerator
{
public class EnvironmentSettings
{
// Note: The following two properties exist so that we do not break
// existing users who might still be using them
public string PreBuildScriptPath { get; set; }
public string PostBuildScriptPath { get; set; }
public string PreBuildCommand { get; set; }
public string PostBuildCommand { get; set; }
}
}

Просмотреть файл

@ -7,10 +7,21 @@ namespace Microsoft.Oryx.BuildScriptGenerator
{
public static class EnvironmentSettingsKeys
{
// Note: The following two constants exist so that we do not break
// existing users who might still be using them
public const string PreBuildScriptPath = "PRE_BUILD_SCRIPT_PATH";
public const string PostBuildScriptPath = "POST_BUILD_SCRIPT_PATH";
/// <summary>
/// Represents an line script or a path to a file
/// </summary>
public const string PreBuildCommand = "PRE_BUILD_COMMAND";
/// <summary>
/// Represents an line script or a path to a file
/// </summary>
public const string PostBuildCommand = "POST_BUILD_COMMAND";
public const string DotnetCoreDefaultVersion = "ORYX_DOTNETCORE_DEFAULT_VERSION";
public const string DotnetCoreSupportedVersions = "DOTNETCORE_SUPPORTED_VERSIONS";

Просмотреть файл

@ -0,0 +1,74 @@
// --------------------------------------------------------------------------------------------
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT license.
// --------------------------------------------------------------------------------------------
using System.IO;
namespace Microsoft.Oryx.BuildScriptGenerator
{
internal static class PreAndPostBuildCommandHelper
{
public static (string preBuildCommand, string postBuildCommand) GetPreAndPostBuildCommands(
ISourceRepo sourceRepo,
EnvironmentSettings settings)
{
if (settings == null)
{
return (null, null);
}
string preBuildCommand = null;
string postBuildCommand = null;
if (!string.IsNullOrEmpty(settings.PreBuildScriptPath))
{
preBuildCommand = $"\"{settings.PreBuildScriptPath}\"";
}
else if (!string.IsNullOrEmpty(settings.PreBuildCommand))
{
preBuildCommand = GetCommandOrFilePath(sourceRepo, settings.PreBuildCommand);
}
if (!string.IsNullOrEmpty(settings.PostBuildScriptPath))
{
postBuildCommand = $"\"{settings.PostBuildScriptPath}\"";
}
else if (!string.IsNullOrEmpty(settings.PostBuildCommand))
{
postBuildCommand = GetCommandOrFilePath(sourceRepo, settings.PostBuildCommand);
}
return (preBuildCommand: preBuildCommand, postBuildCommand: postBuildCommand);
}
private static string GetCommandOrFilePath(ISourceRepo sourceRepo, string commandOrScriptPath)
{
if (string.IsNullOrEmpty(commandOrScriptPath))
{
return null;
}
string fullyQualifiedPath = null;
if (Path.IsPathFullyQualified(commandOrScriptPath))
{
fullyQualifiedPath = commandOrScriptPath;
}
else
{
fullyQualifiedPath = Path.Combine(sourceRepo.RootPath, commandOrScriptPath);
}
if (fullyQualifiedPath != null)
{
var fullPath = Path.GetFullPath(fullyQualifiedPath);
if (File.Exists(Path.GetFullPath(fullPath)))
{
return $"\"{fullPath}\"";
}
}
return commandOrScriptPath;
}
}
}

Просмотреть файл

@ -27,12 +27,12 @@ namespace Microsoft.Oryx.BuildScriptGeneratorCli
{
new TextSpan(
"RunPreBuildScript",
BaseBashBuildScriptProperties.PreBuildScriptPrologue,
BaseBashBuildScriptProperties.PreBuildScriptEpilogue),
BaseBashBuildScriptProperties.PreBuildCommandPrologue,
BaseBashBuildScriptProperties.PreBuildCommandEpilogue),
new TextSpan(
"RunPostBuildScript",
BaseBashBuildScriptProperties.PostBuildScriptPrologue,
BaseBashBuildScriptProperties.PostBuildScriptEpilogue)
BaseBashBuildScriptProperties.PostBuildCommandPrologue,
BaseBashBuildScriptProperties.PostBuildCommandEpilogue)
};
[Argument(0, Description = "The source directory.")]

Просмотреть файл

@ -42,17 +42,17 @@ namespace Microsoft.Oryx.BuildScriptGenerator.Tests
const string script2 = "hijklmn";
var scriptProps = new BaseBashBuildScriptProperties()
{
PreBuildScriptPath = script1,
PostBuildScriptPath = script2
PreBuildCommand = script1,
PostBuildCommand = script2
};
// Act
var script = TemplateHelpers.Render(TemplateHelpers.TemplateResource.BaseBashScript, scriptProps);
// Assert
Assert.Contains("Executing pre-build script", script);
Assert.Contains("Executing pre-build command", script);
Assert.Contains(script1, script);
Assert.Contains("Executing post-build script", script);
Assert.Contains("Executing post-build command", script);
Assert.Contains(script2, script);
}
}

Просмотреть файл

@ -22,6 +22,53 @@ namespace Microsoft.Oryx.BuildScriptGenerator.Tests
_tempDirRoot = tempDirFixture.RootDirPath;
}
[Fact]
public void TryGetAndLoadSettings_TrimsQuotesAndWhitespace()
{
// Arrange
var sourceDir = CreateNewDir();
var scriptFile = Path.Combine(sourceDir, "a b.sh");
File.Create(scriptFile);
var testEnvironment = new TestEnvironment();
testEnvironment.SetEnvironmentVariable(EnvironmentSettingsKeys.PreBuildCommand, " \" a b c \" ");
testEnvironment.SetEnvironmentVariable(
EnvironmentSettingsKeys.PreBuildScriptPath,
$" \"{scriptFile}\" ");
testEnvironment.SetEnvironmentVariable(EnvironmentSettingsKeys.PostBuildCommand, " \" a b c \" ");
testEnvironment.SetEnvironmentVariable(
EnvironmentSettingsKeys.PostBuildScriptPath,
$" \"{scriptFile}\" ");
var provider = CreateProvider(sourceDir, testEnvironment);
// Act
provider.TryGetAndLoadSettings(out var settings);
// Assert
Assert.Equal(" a b c ", settings.PreBuildCommand);
Assert.Equal(scriptFile, settings.PreBuildScriptPath);
Assert.Equal(" a b c ", settings.PostBuildCommand);
Assert.Equal(scriptFile, settings.PostBuildScriptPath);
}
[Theory]
[InlineData("\"")]
[InlineData("a\"")]
[InlineData("a\"\"")]
public void TryGetAndLoadSettings_TrimsOnlyWhenMatchingQuotesAreFound(string value)
{
// Arrange
var sourceDir = CreateNewDir();
var testEnvironment = new TestEnvironment();
testEnvironment.SetEnvironmentVariable(EnvironmentSettingsKeys.PreBuildCommand, value);
var provider = CreateProvider(sourceDir, testEnvironment);
// Act
provider.TryGetAndLoadSettings(out var settings);
// Assert
Assert.Equal(value, settings.PreBuildCommand);
}
[Fact]
public void TryGetAndLoadSettings_PrefersPrefixedName_IfPresent()
{

Просмотреть файл

@ -472,6 +472,49 @@ namespace Microsoft.Oryx.BuildImage.Tests
result.GetDebugInfo());
}
[Fact]
public void Build_Executes_InlinePreAndPostBuildCommands()
{
// Arrange
var appName = "NetCoreApp21WebApp";
var volume = CreateSampleAppVolume(appName);
using (var sw = File.AppendText(Path.Combine(volume.MountedHostDir, "build.env")))
{
sw.NewLine = "\n";
sw.WriteLine("PRE_BUILD_COMMAND=\"echo from pre-build command\"");
sw.WriteLine("POST_BUILD_COMMAND=\"echo from post-build command\"");
}
var appDir = volume.ContainerDir;
var tempOutputDir = "/tmp/output";
var script = new ShellScriptBuilder()
.AddBuildCommand($"{appDir} -o {tempOutputDir} -l dotnet --language-version 2.1")
.ToString();
// Act
var result = _dockerCli.Run(
Settings.BuildImageName,
SampleAppsTestBase.CreateAppNameEnvVar(appName),
volume,
commandToExecuteOnRun: "/bin/bash",
commandArguments:
new[]
{
"-c",
script
});
// Assert
RunAsserts(
() =>
{
Assert.True(result.IsSuccess);
Assert.Contains("from pre-build command", result.StdOut);
Assert.Contains("from post-build command", result.StdOut);
},
result.GetDebugInfo());
}
[Fact]
public void Build_CopiesContentCreatedByPostBuildScript_ToExplicitOutputDirectory_AndOutpuIsZipped()
{

Просмотреть файл

@ -1072,6 +1072,48 @@ namespace Microsoft.Oryx.BuildImage.Tests
result.GetDebugInfo());
}
[Fact]
public void Build_Executes_InlinePreAndPostBuildCommands()
{
// Arrange
var appName = "flask-app";
var volume = CreateSampleAppVolume("flask-app");
using (var sw = File.AppendText(Path.Combine(volume.MountedHostDir, "build.env")))
{
sw.NewLine = "\n";
sw.WriteLine("PRE_BUILD_COMMAND=\"echo from pre-build command\"");
sw.WriteLine("POST_BUILD_COMMAND=\"echo from post-build command\"");
}
var appDir = volume.ContainerDir;
var script = new ShellScriptBuilder()
.AddBuildCommand($"{appDir} -o /tmp/output")
.ToString();
// Act
var result = _dockerCli.Run(
Settings.BuildImageName,
SampleAppsTestBase.CreateAppNameEnvVar(appName),
volume,
commandToExecuteOnRun: "/bin/bash",
commandArguments:
new[]
{
"-c",
script
});
// Assert
RunAsserts(
() =>
{
Assert.True(result.IsSuccess);
Assert.Contains("from pre-build command", result.StdOut);
Assert.Contains("from post-build command", result.StdOut);
},
result.GetDebugInfo());
}
[Fact]
public void Django_CollectStaticFailure_DoesNotFailBuild()
{

Просмотреть файл

@ -58,7 +58,9 @@ namespace Microsoft.Oryx.Tests.Common
string[] runArgs,
Func<Task> assertAction)
{
var AppNameEnvVariable = new EnvironmentVariable(LoggingConstants.AppServiceAppNameEnvironmentVariableName, appName);
var AppNameEnvVariable = new EnvironmentVariable(
LoggingConstants.AppServiceAppNameEnvironmentVariableName,
appName);
environmentVariables.Add(AppNameEnvVariable);
return BuildRunAndAssertAppAsync(
output,
@ -68,7 +70,7 @@ namespace Microsoft.Oryx.Tests.Common
runtimeImageName,
environmentVariables,
portMapping,
link:null,
link: null,
runCmd,
runArgs,
assertAction);
@ -146,19 +148,29 @@ namespace Microsoft.Oryx.Tests.Common
output);
// Run
await RunAndAssertAppAsync(runtimeImageName, output, volumes, environmentVariables, portMapping, link, runCmd, runArgs, assertAction, dockerCli);
await RunAndAssertAppAsync(
runtimeImageName,
output,
volumes,
environmentVariables,
portMapping,
link,
runCmd,
runArgs,
assertAction,
dockerCli);
}
public static async Task RunAndAssertAppAsync(
string imageName,
ITestOutputHelper output,
List<DockerVolume> volumes,
List<EnvironmentVariable> environmentVariables,
string portMapping,
string link,
string runCmd,
string[] runArgs,
Func<Task> assertAction,
List<DockerVolume> volumes,
List<EnvironmentVariable> environmentVariables,
string portMapping,
string link,
string runCmd,
string[] runArgs,
Func<Task> assertAction,
DockerCli dockerCli)
{
DockerRunCommandProcessResult runResult = null;
@ -204,11 +216,34 @@ namespace Microsoft.Oryx.Tests.Common
break;
}
catch (Exception ex) when (ex.InnerException is IOException || ex.InnerException is SocketException)
catch (Exception ex) when (ex.InnerException is IOException ||
ex.InnerException is SocketException)
{
if (i == MaxRetryCount - 1)
{
output.WriteLine(runResult.GetDebugInfo());
string debugInfo = string.Empty;
// ToString() on StringBuilder is throwing an exception probably because of a
// multithreading issue where the container is still writing data into it and we are trying
// to retrieve the content out of it.
try
{
// TO i
// System.ArgumentOutOfRangeException : Index was out of range. Must be non-negative
// and less than the size of the collection.
// Parameter name: chunkLength
// Stack Trace:
// at System.Text.StringBuilder.ToString()
debugInfo = runResult.GetDebugInfo();
output.WriteLine(debugInfo);
}
catch (Exception debugInfoException)
{
output.WriteLine(
"An error occurred while trying to get data from the output and error " +
"streams of the container. Exception: " + debugInfoException.ToString());
}
throw;
}
}