зеркало из https://github.com/microsoft/BuildXL.git
Merged PR 669803: [VM] Introduce temp folder shared by pips executed in VM
When pips are executed in VM, some tools cannot be executed with `net use`, and thus those tools need to be copied to a temporary location in the VM. The pip can use its specified temp folder as this temporary location. But since pips's temp folder is usually unique, it means that the same tool will be copied multiple times to the VM if there are multiple pips needing that tool. Not only that it will degrades performance, the VM will run out of space quickly because it has many pips running there at the same time, and all of them need a large tool. With this shared temp folder, one pip can copy the tool once, and other pips can simply reuse it.
This commit is contained in:
Iman Narasamdya
Родитель
ed4dd50e67
Коммит
4679e7cd81
|
|
@ -2995,6 +2995,11 @@ namespace BuildXL.Processes
|
|||
|
||||
environmentVariables = environmentVariables.Override(overridenEnvVars);
|
||||
}
|
||||
|
||||
environmentVariables = environmentVariables.Override(new[]
|
||||
{
|
||||
new KeyValuePair<string, string>(VmSpecialEnvironmentVariables.VmSharedTemp, PrepareSharedTempDirectoryForVm().ToString(m_pathTable))
|
||||
});
|
||||
}
|
||||
|
||||
return true;
|
||||
|
|
@ -3037,6 +3042,17 @@ namespace BuildXL.Processes
|
|||
return true;
|
||||
}
|
||||
|
||||
private AbsolutePath PrepareSharedTempDirectoryForVm()
|
||||
{
|
||||
AbsolutePath vmTempRoot = m_sandboxConfig.RedirectedTempFolderRootForVmExecution.IsValid
|
||||
? m_sandboxConfig.RedirectedTempFolderRootForVmExecution
|
||||
: AbsolutePath.Create(m_pathTable, VmConstants.Temp.Root);
|
||||
|
||||
AbsolutePath vmSharedTemp = vmTempRoot.Combine(m_pathTable, VmSpecialFilesAndDirectories.SharedTempFolder);
|
||||
AddUntrackedScopeToManifest(vmSharedTemp);
|
||||
return vmSharedTemp;
|
||||
}
|
||||
|
||||
private bool PrepareTempDirectoryForVm(AbsolutePath tempDirectoryPath)
|
||||
{
|
||||
// Suppose that the temp directory is D:\Bxl\Out\Object\Pip123\Temp\t_0.
|
||||
|
|
@ -3119,7 +3135,7 @@ namespace BuildXL.Processes
|
|||
Contract.Assert(m_fileAccessManifest != null);
|
||||
|
||||
// Ensure that T:\BxlTemp\D__\Bxl\Out\Object\Pip123\Temp\t_0 is untracked. Thus, there is no need for a directory translation.
|
||||
m_fileAccessManifest.AddScope(tempRedirectedPath, mask: m_excludeReportAccessMask, values: FileAccessPolicy.AllowAll | FileAccessPolicy.AllowRealInputTimestamps);
|
||||
AddUntrackedScopeToManifest(tempRedirectedPath);
|
||||
|
||||
Tracing.Logger.Log.PipTempSymlinkRedirection(
|
||||
m_loggingContext,
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@ using BuildXL.SandboxedProcessExecutor.Tracing;
|
|||
using BuildXL.Utilities;
|
||||
using BuildXL.Utilities.Instrumentation.Common;
|
||||
using BuildXL.Utilities.Tracing;
|
||||
using BuildXL.Utilities.VmCommandProxy;
|
||||
|
||||
#nullable enable
|
||||
|
||||
|
|
@ -500,6 +501,18 @@ namespace BuildXL.SandboxedProcessExecutor
|
|||
}
|
||||
}
|
||||
|
||||
string vmSharedTemp = info.EnvironmentVariables.TryGetValue(VmSpecialEnvironmentVariables.VmSharedTemp, null);
|
||||
|
||||
if (!string.IsNullOrEmpty(vmSharedTemp))
|
||||
{
|
||||
var result = EnsureDirectoryExists(vmSharedTemp);
|
||||
if (!result.Succeeded)
|
||||
{
|
||||
m_logger.LogError($"Failed to prepare VM shared temporary directory '{vmSharedTemp}': {result.Failure.DescribeIncludingInnerFailures()}");
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
if (remoteData == null)
|
||||
{
|
||||
return true;
|
||||
|
|
|
|||
|
|
@ -121,6 +121,19 @@ namespace Test.BuildXL.Utilities
|
|||
});
|
||||
}
|
||||
|
||||
[Fact(Skip = "Waiting for a new BuildXL LKG")]
|
||||
public void VmSharedTempFolderExists()
|
||||
{
|
||||
EnsureRunInVm(() =>
|
||||
{
|
||||
string vmSharedTemp = Environment.GetEnvironmentVariable(VmSpecialEnvironmentVariables.VmSharedTemp);
|
||||
XAssert.IsNotNull(vmSharedTemp);
|
||||
string expectedVmSharedTemp = Path.Combine(VmConstants.Temp.Root, VmSpecialFilesAndDirectories.SharedTempFolder);
|
||||
XAssert.AreEqual(expectedVmSharedTemp.ToUpperInvariant(), vmSharedTemp.ToUpperInvariant());
|
||||
XAssert.IsTrue(Directory.Exists(vmSharedTemp));
|
||||
});
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void VmPipCurrentlyDoesNotHaveUserProfileEnvironmentVariablesSpecified()
|
||||
{
|
||||
|
|
|
|||
|
|
@ -60,9 +60,13 @@ namespace BuildXL.Utilities.VmCommandProxy
|
|||
/// that do not work well when executed from the network mapped D: drive. It will copy the original .exe and its entire folder into the VM
|
||||
/// and re-execute the original command-line from there.
|
||||
/// CODESYNC (CB codebase): private\Common\VmUtils\VmCommandProxy\VmCommandProxy.cs
|
||||
///
|
||||
/// </summary>
|
||||
public const string CopyLocalShimDirectory = @"C:\VmAgent\Dependencies\CopyLocalShim";
|
||||
|
||||
/// <summary>
|
||||
/// Named of shared temp folder that all pips executing in the VM will have untracked accesses.
|
||||
/// </summary>
|
||||
public const string SharedTempFolder = "BuildXLVmSharedTemp";
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
|
@ -86,6 +90,16 @@ namespace BuildXL.Utilities.VmCommandProxy
|
|||
/// </summary>
|
||||
public const string VmOriginalTemp = "[BUILDXL]VM_ORIGINAL_TEMP";
|
||||
|
||||
/// <summary>
|
||||
/// Environment variable whose value is a path to a temp folder shared by all pips executing in the VM.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Shared temp folder is needed particularly in the case of pips needing to use <see cref="VmSpecialFilesAndDirectories.CopyLocalShimDirectory"/>.
|
||||
/// One pip will copy the tool into a folder in this shared temp folder once, and other pips will simply use the copied tool in that folder. Without
|
||||
/// this folder, each pip will copy the tool into its own temp folder. Not only that this has performance impact, the VM will run out of space quickly.
|
||||
/// </remarks>
|
||||
public const string VmSharedTemp = "[BUILDXL]VM_SHARED_TEMP";
|
||||
|
||||
/// <summary>
|
||||
/// Property indicating if a process is running in VM.
|
||||
/// </summary>
|
||||
|
|
|
|||
|
|
@ -61,6 +61,9 @@ param(
|
|||
[Parameter(HelpMessage = "Build queue")]
|
||||
[string]$Queue = "BuildXL_Internal_PR",
|
||||
|
||||
[Parameter(HelpMessage = "Build engine drop")]
|
||||
[string]$BxlDrop = "",
|
||||
|
||||
[Parameter(ValueFromRemainingArguments=$true)]
|
||||
[string[]]$BuildXLArguments = ""
|
||||
)
|
||||
|
|
@ -134,12 +137,17 @@ if ($beforeCommit -ne $afterCommit)
|
|||
# 2- Generate the cb.exe arguments and send the build to Cloudbuild
|
||||
Write-Host ">>> Sending the build to CloudBuild"
|
||||
|
||||
# BuildXL version in your repo needs to be used to get cache hits in your local builds.
|
||||
# To this end, we infer BuildXL version from BuildXLLkgVersion.cmd. If we use the version
|
||||
# in CloudBuild, we may not get cache hits from the remote cache.
|
||||
$bxlVersionLine = Get-Content -Path "$PSScriptRoot\BuildXLLkgVersion.cmd" | Select-String "BUILDXL_LKG_VERSION" | select-object -First 1
|
||||
$bxlVersion = $bxlVersionLine.Line.Split("=")[1];
|
||||
$bxlDrop = 'https://cloudbuild.artifacts.visualstudio.com/DefaultCollection/_apis/drop/drops/buildxl.dogfood.' + $bxlVersion + '?root=release/win-x64';
|
||||
if (![string]::IsNullOrEmpty($BxlDrop)) {
|
||||
$bxlEngine = $BxlDrop + '?root=release/win-x64';
|
||||
}
|
||||
else {
|
||||
# BuildXL version in your repo needs to be used to get cache hits in your local builds.
|
||||
# To this end, we infer BuildXL version from BuildXLLkgVersion.cmd. If we use the version
|
||||
# in CloudBuild, we may not get cache hits from the remote cache.
|
||||
$bxlVersionLine = Get-Content -Path "$PSScriptRoot\BuildXLLkgVersion.cmd" | Select-String "BUILDXL_LKG_VERSION" | select-object -First 1
|
||||
$bxlVersion = $bxlVersionLine.Line.Split("=")[1];
|
||||
$bxlEngine = 'https://cloudbuild.artifacts.visualstudio.com/DefaultCollection/_apis/drop/drops/buildxl.dogfood.' + $bxlVersion + '?root=release/win-x64';
|
||||
}
|
||||
|
||||
$disableCache = ($Cache -eq "Disable");
|
||||
$consumeCache = ($Cache -eq "Consume");
|
||||
|
|
@ -172,7 +180,7 @@ $requestBody = "
|
|||
'MaxBuilders': '$maxBuilders',
|
||||
'Description': 'BuildXL buddy build $([DateTime]::UtcNow.ToLocalTIme().ToString())',
|
||||
'ToolPaths': {
|
||||
'DominoEngine': '$bxlDrop'
|
||||
'DominoEngine': '$bxlEngine'
|
||||
},
|
||||
'GenericRunnerOptions': {
|
||||
'CacheVstsAccountName': '$CacheAccount',
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче