microsoft
/
Oryx
зеркало из https://github.com/microsoft/Oryx.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

Added a sentinel file to indicate earlier download was successful (#539)

This commit is contained in:
Kiran 2020-03-27 09:40:33 -07:00 коммит произвёл GitHub
Родитель e107d283c7
Коммит fe43df757f
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: 4AEE18F83AFDEB23
11 изменённых файлов: 219 добавлений и 60 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

1
build/__sdkStorageConstants.sh
Просмотреть файл

@ -7,3 +7,4 @@ PROD_SDK_STORAGE_BASE_URL='https://oryxsdk-cdn.azureedge.net'
DEFAULT_VERSION_FILE_NAME='defaultVersion.txt'
VERSIONS_TO_BUILD_FILE_NAME='versionsToBuild.txt'
CONTAINER_METADATA_URL_FORMAT='{0}/{1}?restype=container&comp=list&include=metadata'
SDK_DOWNLOAD_SENTINEL_FILE_NAME='.oryx-sdkdownload-sentinel'

1
build/constants.yaml
Просмотреть файл

@ -132,6 +132,7 @@
default-version-file-name: defaultVersion.txt
versions-to-build-file-name: versionsToBuild.txt
container-metadata-url-format: '{0}/{1}?restype=container&comp=list&include=metadata'
sdk-download-sentinel-file-name: .oryx-sdkdownload-sentinel
outputs:
- type: shell
directory: images

15
src/BuildScriptGenerator/DotNetCore/DotNetCorePlatformInstaller.cs
Просмотреть файл

@ -5,7 +5,7 @@
using System.Text;
using Microsoft.Extensions.Options;
using Microsoft.Oryx.BuildScriptGenerator.DotNetCore;
using Microsoft.Oryx.Common;
namespace Microsoft.Oryx.BuildScriptGenerator.DotNetCore
{
@ -28,6 +28,8 @@ namespace Microsoft.Oryx.BuildScriptGenerator.DotNetCore
var sdkVersion = versionMap[runtimeVersion];
var dirToInstall =
$"{Constants.TemporaryInstallationDirectoryRoot}/{DotNetCoreConstants.LanguageName}/sdks/{sdkVersion}";
var sentinelFileDir =
$"{Constants.TemporaryInstallationDirectoryRoot}/{DotNetCoreConstants.LanguageName}/runtimes/{runtimeVersion}";
var sdkInstallerScript = GetInstallerScriptSnippet(
DotNetCoreConstants.LanguageName,
sdkVersion,
@ -39,7 +41,9 @@ namespace Microsoft.Oryx.BuildScriptGenerator.DotNetCore
scriptBuilder
.AppendLine(sdkInstallerScript)
.AppendLine($"mkdir -p {dotnetDir}/runtimes/{runtimeVersion}")
.AppendLine($"echo '{sdkVersion}' > {dotnetDir}/runtimes/{runtimeVersion}/sdkVersion.txt");
.AppendLine($"echo '{sdkVersion}' > {dotnetDir}/runtimes/{runtimeVersion}/sdkVersion.txt")
// Write out a sentinel file to indicate downlaod and extraction was successful
.AppendLine($"echo > {sentinelFileDir}/{SdkStorageConstants.SdkDownloadSentinelFileName}");
return scriptBuilder.ToString();
}
@ -47,11 +51,8 @@ namespace Microsoft.Oryx.BuildScriptGenerator.DotNetCore
{
return IsVersionInstalled(
version,
installationDirs: new[]
{
DotNetCoreConstants.InstalledDotNetCoreRuntimeVersionsDir,
$"{Constants.TemporaryInstallationDirectoryRoot}/dotnet/runtimes"
});
builtInDir: DotNetCoreConstants.InstalledDotNetCoreRuntimeVersionsDir,
dynamicInstallDir: $"{Constants.TemporaryInstallationDirectoryRoot}/dotnet/runtimes");
}
}
}

7
src/BuildScriptGenerator/Node/NodePlatformInstaller.cs
Просмотреть файл

@ -25,11 +25,8 @@ namespace Microsoft.Oryx.BuildScriptGenerator.Node
{
return IsVersionInstalled(
version,
installationDirs: new[]
{
NodeConstants.InstalledNodeVersionsDir,
$"{Constants.TemporaryInstallationDirectoryRoot}/nodejs"
});
builtInDir: NodeConstants.InstalledNodeVersionsDir,
dynamicInstallDir: $"{Constants.TemporaryInstallationDirectoryRoot}/nodejs");
}
}
}

35
src/BuildScriptGenerator/PlatformInstallerBase.cs
Просмотреть файл

@ -4,6 +4,8 @@
// --------------------------------------------------------------------------------------------
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using Microsoft.Extensions.Options;
@ -70,23 +72,44 @@ namespace Microsoft.Oryx.BuildScriptGenerator
.AppendLine($"rm -f {tarFile}")
.AppendLine("PLATFORM_SETUP_ELAPSED_TIME=$(($SECONDS - $PLATFORM_SETUP_START))")
.AppendLine("echo \"Done in $PLATFORM_SETUP_ELAPSED_TIME sec(s).\"")
.AppendLine("echo");
.AppendLine("echo")
// Write out a sentinel file to indicate downlaod and extraction was successful
.AppendLine($"echo > {versionDirInTemp}/{SdkStorageConstants.SdkDownloadSentinelFileName}");
return snippet.ToString();
}
protected bool IsVersionInstalled(string lookupVersion, string[] installationDirs)
protected bool IsVersionInstalled(string lookupVersion, string builtInDir, string dynamicInstallDir)
{
foreach (var installationDir in installationDirs)
var versionsFromDisk = VersionProviderHelper.GetVersionsFromDirectory(builtInDir);
if (HasVersion(versionsFromDisk))
{
var versionsFromDisk = VersionProviderHelper.GetVersionsFromDirectory(installationDir);
if (versionsFromDisk.Any(onDiskVersion
=> string.Equals(lookupVersion, onDiskVersion, StringComparison.OrdinalIgnoreCase)))
return true;
}
versionsFromDisk = VersionProviderHelper.GetVersionsFromDirectory(dynamicInstallDir);
if (HasVersion(versionsFromDisk))
{
// Only if there is a sentinel file we want to indicate that a version exists.
// This is because a user could kill a build midway which might leave the download of an SDK
// in a corrupt state.
var sentinelFile = Path.Combine(
dynamicInstallDir,
lookupVersion,
SdkStorageConstants.SdkDownloadSentinelFileName);
if (File.Exists(sentinelFile))
{
return true;
}
}
return false;
bool HasVersion(IEnumerable<string> versionsOnDisk)
{
return versionsOnDisk.Any(onDiskVersion
=> string.Equals(lookupVersion, onDiskVersion, StringComparison.OrdinalIgnoreCase));
}
}
private string GetPlatformBinariesStorageBaseUrl()

7
src/BuildScriptGenerator/Python/PythonPlatformInstaller.cs
Просмотреть файл

@ -25,11 +25,8 @@ namespace Microsoft.Oryx.BuildScriptGenerator.Python
{
return IsVersionInstalled(
version,
installationDirs: new[]
{
PythonConstants.InstalledPythonVersionsDir,
$"{Constants.TemporaryInstallationDirectoryRoot}/python"
});
builtInDir: PythonConstants.InstalledPythonVersionsDir,
dynamicInstallDir: $"{Constants.TemporaryInstallationDirectoryRoot}/python");
}
}
}

1
src/Common/SdkStorageConstants.cs
Просмотреть файл

@ -11,5 +11,6 @@ namespace Microsoft.Oryx.Common
public const string DefaultVersionFileName = "defaultVersion.txt";
public const string VersionsToBuildFileName = "versionsToBuild.txt";
public const string ContainerMetadataUrlFormat = "{0}/{1}?restype=container&comp=list&include=metadata";
public const string SdkDownloadSentinelFileName = ".oryx-sdkdownload-sentinel";
}
}

1
src/startupscriptgenerator/src/common/consts/sdk_storage_constants.go
Просмотреть файл

@ -9,3 +9,4 @@ const ProdSdkStorageBaseUrl string = "https://oryxsdk-cdn.azureedge.net"
const DefaultVersionFileName string = "defaultVersion.txt"
const VersionsToBuildFileName string = "versionsToBuild.txt"
const ContainerMetadataUrlFormat string = "{0}/{1}?restype=container&comp=list&include=metadata"
const SdkDownloadSentinelFileName string = ".oryx-sdkdownload-sentinel"

50
tests/Oryx.BuildImage.Tests/DotNetCore/DotNetCoreDynamicInstallTest.cs
Просмотреть файл

@ -5,7 +5,7 @@
using System.Collections.Generic;
using System.IO;
using Microsoft.Oryx.BuildScriptGeneratorCli;
using Microsoft.Oryx.BuildScriptGenerator.DotNetCore;
using Microsoft.Oryx.Common;
using Microsoft.Oryx.Tests.Common;
using Xunit;
@ -41,7 +41,8 @@ namespace Microsoft.Oryx.BuildImage.Tests
SdkStorageConstants.SdkStorageBaseUrlKeyName,
SdkStorageConstants.DevSdkStorageBaseUrl)
.AddBuildCommand(
$"{appDir} -i /tmp/int -o {appOutputDir} --platform dotnet --platform-version {runtimeVersion}")
$"{appDir} -i /tmp/int -o {appOutputDir} " +
$"--platform {DotNetCoreConstants.LanguageName} --platform-version {runtimeVersion}")
.AddFileExistsCheck($"{appOutputDir}/{appName}.dll")
.ToString();
@ -64,5 +65,50 @@ namespace Microsoft.Oryx.BuildImage.Tests
},
result.GetDebugInfo());
}
[Fact]
public void DynamicInstall_ReInstallsSdk_IfSentinelFileIsNotPresent()
{
// Arrange
var appName = "NetCoreApp31.MvcApp";
var runtimeVersion = "3.1.2"; //NOTE: use the full version so that we know the install directory path
var installationDir = $"{BuildScriptGenerator.Constants.TemporaryInstallationDirectoryRoot}/dotnet/runtimes/{runtimeVersion}";
var sentinelFile = $"{installationDir}/{SdkStorageConstants.SdkDownloadSentinelFileName}";
var volume = CreateSampleAppVolume(appName);
var appDir = volume.ContainerDir;
var appOutputDir = "/tmp/output";
var buildCmd = $"{appDir} -i /tmp/int -o {appOutputDir} " +
$"--platform {DotNetCoreConstants.LanguageName} --platform-version {runtimeVersion}";
var script = new ShellScriptBuilder()
.SetEnvironmentVariable(
SdkStorageConstants.SdkStorageBaseUrlKeyName,
SdkStorageConstants.DevSdkStorageBaseUrl)
.AddBuildCommand(buildCmd)
.AddFileExistsCheck($"{appOutputDir}/{appName}.dll")
.AddFileExistsCheck(sentinelFile)
.AddCommand($"rm -f {sentinelFile}")
.AddBuildCommand(buildCmd)
.AddFileExistsCheck(sentinelFile)
.ToString();
// Act
var result = _dockerCli.Run(new DockerRunArguments
{
ImageId = _imageHelper.GetGitHubActionsBuildImage(),
EnvironmentVariables = new List<EnvironmentVariable> { CreateAppNameEnvVar(appName) },
Volumes = new List<DockerVolume> { volume },
CommandToExecuteOnRun = "/bin/bash",
CommandArguments = new[] { "-c", script }
});
// Assert
RunAsserts(
() =>
{
Assert.True(result.IsSuccess);
Assert.Contains(string.Format(SdkVersionMessageFormat, runtimeVersion), result.StdOut);
},
result.GetDebugInfo());
}
}
}

46
tests/Oryx.BuildImage.Tests/Node/NodeDynamicInstallationTest.cs
Просмотреть файл

@ -4,6 +4,7 @@
// --------------------------------------------------------------------------------------------
using System.Collections.Generic;
using Microsoft.Oryx.BuildScriptGenerator.Node;
using Microsoft.Oryx.BuildScriptGeneratorCli;
using Microsoft.Oryx.Common;
using Microsoft.Oryx.Tests.Common;
@ -60,6 +61,51 @@ namespace Microsoft.Oryx.BuildImage.Tests
result.GetDebugInfo());
}
[Fact]
public void DynamicInstall_ReInstallsSdk_IfSentinelFileIsNotPresent()
{
// Arrange
var version = "12.16.1"; //NOTE: use the full version so that we know the install directory path
var installationDir = $"{BuildScriptGenerator.Constants.TemporaryInstallationDirectoryRoot}/nodejs/{version}";
var sentinelFile = $"{installationDir}/{SdkStorageConstants.SdkDownloadSentinelFileName}";
var volume = CreateWebFrontEndVolume();
var appDir = volume.ContainerDir;
var appOutputDir = "/tmp/webfrontend-output";
var buildCmd = $"{appDir} -i /tmp/int -o {appOutputDir} " +
$"--platform {NodeConstants.NodeJsName} --platform-version {version}";
var script = new ShellScriptBuilder()
.AddCommand(GetSnippetToCleanUpExistingInstallation())
.SetEnvironmentVariable(
SdkStorageConstants.SdkStorageBaseUrlKeyName,
SdkStorageConstants.DevSdkStorageBaseUrl)
.AddBuildCommand(buildCmd)
.AddDirectoryExistsCheck($"{appOutputDir}/node_modules")
.AddFileExistsCheck(sentinelFile)
.AddCommand($"rm -f {sentinelFile}")
.AddBuildCommand(buildCmd)
.AddDirectoryExistsCheck($"{appOutputDir}/node_modules")
.AddFileExistsCheck(sentinelFile)
.AddBuildCommand(buildCmd)
.ToString();
// Act
var result = _dockerCli.Run(new DockerRunArguments
{
ImageId = _imageHelper.GetGitHubActionsBuildImage(),
Volumes = new List<DockerVolume> { volume },
CommandToExecuteOnRun = "/bin/bash",
CommandArguments = new[] { "-c", script }
});
// Assert
RunAsserts(
() =>
{
Assert.True(result.IsSuccess);
},
result.GetDebugInfo());
}
private string GetSnippetToCleanUpExistingInstallation()
{
return $"rm -rf {DefaultInstallationRootDir}; mkdir -p {DefaultInstallationRootDir}";

115
tests/Oryx.BuildImage.Tests/Python/PythonDynamicInstallationTest.cs
Просмотреть файл

@ -4,7 +4,7 @@
// --------------------------------------------------------------------------------------------
using System.Collections.Generic;
using Microsoft.Oryx.BuildScriptGenerator;
using Microsoft.Oryx.BuildScriptGenerator.Python;
using Microsoft.Oryx.BuildScriptGeneratorCli;
using Microsoft.Oryx.Common;
using Microsoft.Oryx.Tests.Common;
@ -37,42 +37,87 @@ namespace Microsoft.Oryx.BuildImage.Tests
[MemberData(nameof(ImageNameData))]
public void GeneratesScript_AndBuilds(string imageName)
{
// Arrange
var version = "3.8.1";
var appName = "flask-app";
var volume = CreateSampleAppVolume(appName);
var appDir = volume.ContainerDir;
var appOutputDir = "/tmp/app-output";
var script = new ShellScriptBuilder()
.AddCommand(GetSnippetToCleanUpExistingInstallation())
.SetEnvironmentVariable(SettingsKeys.EnableDynamicInstall, true.ToString())
.SetEnvironmentVariable(
SdkStorageConstants.SdkStorageBaseUrlKeyName,
SdkStorageConstants.DevSdkStorageBaseUrl)
.AddBuildCommand(
$"{appDir} --platform python --platform-version {version} -o {appOutputDir}")
.ToString();
// Arrange
var version = "3.8.1";
var installationDir = $"{BuildScriptGenerator.Constants.TemporaryInstallationDirectoryRoot}/python/{version}";
var appName = "flask-app";
var volume = CreateSampleAppVolume(appName);
var appDir = volume.ContainerDir;
var appOutputDir = "/tmp/app-output";
var script = new ShellScriptBuilder()
.AddCommand(GetSnippetToCleanUpExistingInstallation())
.SetEnvironmentVariable(SettingsKeys.EnableDynamicInstall, true.ToString())
.SetEnvironmentVariable(
SdkStorageConstants.SdkStorageBaseUrlKeyName,
SdkStorageConstants.DevSdkStorageBaseUrl)
.AddBuildCommand(
$"{appDir} --platform {PythonConstants.PythonName} --platform-version {version} -o {appOutputDir}")
.ToString();
// Act
var result = _dockerCli.Run(new DockerRunArguments
{
ImageId = imageName,
EnvironmentVariables = new List<EnvironmentVariable> { CreateAppNameEnvVar(appName) },
Volumes = new List<DockerVolume> { volume },
CommandToExecuteOnRun = "/bin/bash",
CommandArguments = new[] { "-c", script }
});
// Act
var result = _dockerCli.Run(new DockerRunArguments
{
ImageId = imageName,
EnvironmentVariables = new List<EnvironmentVariable> { CreateAppNameEnvVar(appName) },
Volumes = new List<DockerVolume> { volume },
CommandToExecuteOnRun = "/bin/bash",
CommandArguments = new[] { "-c", script }
});
// Assert
RunAsserts(
() =>
{
Assert.True(result.IsSuccess);
Assert.Contains(
$"Python Version: {BuildScriptGenerator.Constants.TemporaryInstallationDirectoryRoot}/python/{version}/bin/python3",
result.StdOut);
},
result.GetDebugInfo());
// Assert
RunAsserts(
() =>
{
Assert.True(result.IsSuccess);
Assert.Contains(
$"Python Version: {installationDir}/bin/python3",
result.StdOut);
},
result.GetDebugInfo());
}
[Fact]
public void DynamicInstall_ReInstallsSdk_IfSentinelFileIsNotPresent()
{
// Arrange
var version = "3.8.1"; //NOTE: use the full version so that we know the install directory path
var installationDir = $"{BuildScriptGenerator.Constants.TemporaryInstallationDirectoryRoot}/python/{version}";
var sentinelFile = $"{installationDir}/{SdkStorageConstants.SdkDownloadSentinelFileName}";
var appName = "flask-app";
var volume = CreateSampleAppVolume(appName);
var appDir = volume.ContainerDir;
var appOutputDir = "/tmp/app-output";
var buildCmd = $"{appDir} --platform {PythonConstants.PythonName} --platform-version {version} -o {appOutputDir}";
var script = new ShellScriptBuilder()
.AddCommand(GetSnippetToCleanUpExistingInstallation())
.SetEnvironmentVariable(SettingsKeys.EnableDynamicInstall, true.ToString())
.SetEnvironmentVariable(
SdkStorageConstants.SdkStorageBaseUrlKeyName,
SdkStorageConstants.DevSdkStorageBaseUrl)
.AddBuildCommand(buildCmd)
.AddFileExistsCheck(sentinelFile)
.AddCommand($"rm -f {sentinelFile}")
.AddBuildCommand(buildCmd)
.AddFileExistsCheck(sentinelFile)
.ToString();
// Act
var result = _dockerCli.Run(new DockerRunArguments
{
ImageId = _imageHelper.GetGitHubActionsBuildImage(),
EnvironmentVariables = new List<EnvironmentVariable> { CreateAppNameEnvVar(appName) },
Volumes = new List<DockerVolume> { volume },
CommandToExecuteOnRun = "/bin/bash",
CommandArguments = new[] { "-c", script }
});
// Assert
RunAsserts(
() =>
{
Assert.True(result.IsSuccess);
},
result.GetDebugInfo());
}
private string GetSnippetToCleanUpExistingInstallation()