microsoft
/
BuildXL
зеркало из https://github.com/microsoft/BuildXL.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

Merged PR 558613: Remove symlink definition files and lazy symlink materialization 

Remove symlink definition files and lazy symlink materialization, non of the features are used at all. Accompanying flags have been removed.
This commit is contained in:
Kristijan Simic 2020-06-18 15:39:45 +00:00
Родитель b94c096014
Коммит ad322c5736
29 изменённых файлов: 34 добавлений и 998 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

14
Public/Src/App/Bxl/Args.cs
Просмотреть файл

@ -810,9 +810,6 @@ namespace BuildXL
"pipWarningTimeoutMultiplier",
opt =>
sandboxConfiguration.WarningTimeoutMultiplier = (int)CommandLineUtilities.ParseDoubleOption(opt, 0.000001, 1000000)),
OptionHandlerFactory.CreateOption(
"populateSymlinkDirectory",
opt => engineConfiguration.PopulateSymlinkDirectories.Add(CommandLineUtilities.ParsePathOption(opt, pathTable))),
OptionHandlerFactory.CreateOption(
"posixDeleteMode",
opt => FileUtilities.PosixDeleteMode = CommandLineUtilities.ParseEnumOption<PosixDeleteMode>(opt)),
@ -975,9 +972,6 @@ namespace BuildXL
OptionHandlerFactory.CreateOption(
"substTarget",
opt => loggingConfiguration.SubstTarget = CommandLineUtilities.ParsePathOption(opt, pathTable)),
OptionHandlerFactory.CreateOption(
"symlinkDefinitionFile",
opt => layoutConfiguration.SymlinkDefinitionFile = CommandLineUtilities.ParsePathOption(opt, pathTable)),
OptionHandlerFactory.CreateOption(
"telemetryTagPrefix",
opt => schedulingConfiguration.TelemetryTagPrefix = CommandLineUtilities.ParseStringOption(opt)),
@ -1010,10 +1004,6 @@ namespace BuildXL
"typeCheck",
sign => { /* Do nothing Office still passes this flag even though it is deprecated. */ }),
OptionHandlerFactory.CreateOption(
"unexpectedSymlinkAccessReportingMode",
opt => schedulingConfiguration.UnexpectedSymlinkAccessReportingMode = CommandLineUtilities.ParseEnumOption<UnexpectedSymlinkAccessReportingMode>(opt)),
// <Begin unsafe arguments>
// Unsafe options should follow the pattern that enabling them (i.e. "/unsafe_option" or "/unsafe_option+") should lead to an unsafe configuration
// Unsafe options must pass the optional parameter isUnsafe as true
@ -1163,10 +1153,6 @@ namespace BuildXL
"unsafe_IgnoreZwRenameFileInformation",
sign => sandboxConfiguration.UnsafeSandboxConfigurationMutable.IgnoreZwRenameFileInformation = sign,
isUnsafe: true),
OptionHandlerFactory.CreateBoolOption(
"unsafe_LazySymlinkCreation",
sign => schedulingConfiguration.UnsafeLazySymlinkCreation = sign,
isUnsafe: true),
OptionHandlerFactory.CreateBoolOption(
"unsafe_MonitorFileAccesses",
sign =>

2
Public/Src/App/Bxl/BuildXLApp.cs
Просмотреть файл

@ -312,8 +312,6 @@ namespace BuildXL
(int)EngineLogEventId.HistoricMetadataCacheSaved,
(int)EngineLogEventId.HistoricPerfDataLoaded,
(int)EngineLogEventId.HistoricPerfDataSaved,
(int)SchedulerLogEventId.CreateSymlinkFromSymlinkMap,
(int)SchedulerLogEventId.SymlinkFileTraceMessage,
(int)SharedLogEventId.StartEngineRun,
(int)EngineLogEventId.StartCheckingForPipGraphReuse,
(int)EngineLogEventId.EndCheckingForPipGraphReuse,

20
Public/Src/App/Bxl/HelpText.cs
Просмотреть файл

@ -1162,10 +1162,6 @@ namespace BuildXL
Strings.HelpText_DisplayHelp_VsOutputSrc,
HelpLevel.Verbose);
hw.WriteOption(
"/symlinkDefinitionFile:<file>",
Strings.HelpText_DisplayHelp_SymlinkDefinitionFile);
hw.WriteOption(
"/inputChanges:<file>",
Strings.HelpText_DisplayHelp_InputChanges);
@ -1174,22 +1170,6 @@ namespace BuildXL
"/telemetryTagPrefix:<string>",
Strings.HelpText_DisplayHelp_TelemetryTagPrefix);
hw.WriteOption(
"/populateSymlinkDirectory:<file>",
Strings.HelpText_DisplayHelp_PopulateSymlinkDirectory);
hw.WriteOption(
"/unsafe_LazySymlinkCreation[+|-]",
Strings.HelpText_DisplayHelp_LazySymlinkCreation);
hw.WriteOption(
"/unsafe_LazySymlinkCreation[+|-]",
Strings.HelpText_DisplayHelp_LazySymlinkCreation);
hw.WriteOption(
"/unexpectedSymlinkAccessReportingMode:<mode>",
Strings.HelpText_DisplayHelp_UnexpectedSymlinkAccessReportingMode);
hw.WriteOption(
"/unsafe_DisableGraphPostValidation[+|-]",
Strings.HelpText_DisplayHelp_DisableGraphPostValidation);

12
Public/Src/App/Bxl/Strings.resx
Просмотреть файл

@ -886,12 +886,6 @@ Example: ad2d42d2ec5d2ca0c0b7ad65402d07c7ef40b91e</value>
<data name="HelpText_DisplayHelp_AllowFetchingCachedGraphFromContentCache" xml:space="preserve">
<value>Allow fetching cached graph from content cache. Defaults to on.</value>
</data>
<data name="HelpText_DisplayHelp_SymlinkDefinitionFile" xml:space="preserve">
<value>Specifies the file containing mappings from symlinks to their targets, and those symlinks will be created, if necessary, before pip executions</value>
</data>
<data name="HelpText_DisplayHelp_LazySymlinkCreation" xml:space="preserve">
<value>Creates symlink from the symlink definition file lazily. Defaults to off.</value>
</data>
<data name="HelpText_DisplayHelp_CompressGraphFiles" xml:space="preserve">
<value>When enabled, graph files are compressed.</value>
</data>
@ -907,12 +901,6 @@ Example: ad2d42d2ec5d2ca0c0b7ad65402d07c7ef40b91e</value>
<data name="HelpText_DisplayHelp_UseFileContentTablePathMappings" xml:space="preserve">
<value>Use file content table path mappings to avoid opening handles for hashing files already in the table. Defaults to off.</value>
</data>
<data name="HelpText_DisplayHelp_PopulateSymlinkDirectory" xml:space="preserve">
<value>Specifies a directory to eagerly populate with symlinks when symlink definition file and lazy symlink creation are enabled.</value>
</data>
<data name="HelpText_DisplayHelp_UnexpectedSymlinkAccessReportingMode" xml:space="preserve">
<value>Specifies the when unexpected symlink file/directory accesses are reported as warnings: None, ExecutionOnly (only for executed processes), All (executed and cached process observations report unexpected accesses). Default: All. NOTE: This is only enabled when a symlink definition file is provided.</value>
</data>
<data name="HelpText_DisplayHelp_DisableGraphPostValidation" xml:space="preserve">
<value>Disables post validation of graph construction. Defaults to on.</value>
</data>

19
Public/Src/Engine/Dll/Engine.FrontEnd.cs
Просмотреть файл

@ -357,22 +357,6 @@ namespace BuildXL.Engine
return cacheGraphStats;
}
AsyncOut<AbsolutePath> symlinkFileLocation = new AsyncOut<AbsolutePath>();
if (!SymlinkDefinitionFileProvider.TryFetchWorkerSymlinkFileAsync(
outerLoggingContext,
Context.PathTable,
cacheForWorker,
Configuration.Layout,
m_workerService,
symlinkFileLocation).Result)
{
cacheGraphStats.CacheMissReason = GraphCacheMissReason.NoFingerprintFromMaster;
cacheGraphStats.MissReason = cacheGraphStats.CacheMissReason;
return cacheGraphStats;
}
m_workerSymlinkDefinitionFile = symlinkFileLocation.Value;
// Success. Populate the stats
cacheGraphStats.WasHit = true;
cacheGraphStats.WorkerHit = true;
@ -594,9 +578,6 @@ namespace BuildXL.Engine
m_collector,
m_directoryTranslator,
engineState,
symlinkDefinitionFile: IsDistributedWorker ?
m_workerSymlinkDefinitionFile.Value :
Configuration.Layout.SymlinkDefinitionFile,
tempCleaner: m_tempCleaner,
buildEngineFingerprint).GetAwaiter().GetResult();
}

23
Public/Src/Engine/Dll/Engine.cs
Просмотреть файл

@ -246,8 +246,6 @@ namespace BuildXL.Engine
[CanBeNull]
private readonly string m_buildVersion;
private AbsolutePath? m_workerSymlinkDefinitionFile = null;
private bool IsDistributedMaster => Configuration.Distribution.BuildRole == DistributedBuildRoles.Master;
private bool IsDistributedWorker => Configuration.Distribution.BuildRole == DistributedBuildRoles.Worker;
@ -2344,7 +2342,6 @@ namespace BuildXL.Engine
{ "unsafe_IgnoreZwCreateOpenQueryFamily", Logger.Log.ConfigUnsafeMonitorZwCreateOpenQueryFileOff },
{ "unsafe_IgnoreZwOtherFileInformation", Logger.Log.ConfigIgnoreZwOtherFileInformation },
{ "unsafe_IgnoreZwRenameFileInformation", Logger.Log.ConfigIgnoreZwRenameFileInformation },
{ "unsafe_LazySymlinkCreation", Logger.Log.ConfigUnsafeLazySymlinkCreation },
{ "unsafe_MonitorFileAccesses", Logger.Log.ConfigUnsafeDisabledFileAccessMonitoring },
{ "unsafe_PreserveOutputs", Logger.Log.ConfigPreserveOutputs },
{ "unsafe_PreserveOutputsTrustLevel", loggingContext => { } /* Special case: unsafe option we do not want logged */ },
@ -2768,16 +2765,6 @@ namespace BuildXL.Engine
m_enginePerformanceInfo.CacheInitializationDurationMs = (long)cacheInitializationTask.InitializationTime.TotalMilliseconds;
}
Task<Possible<SymlinkDefinitions>> symlinkDefinitionsTask =
SymlinkDefinitionFileProvider.TryPrepareSymlinkDefinitionsAsync(
loggingContext,
reuseResult,
Configuration,
m_masterService,
cacheInitializationTask,
Context,
m_tempCleaner);
m_buildViewModel.SetContext(Context);
var phase = Configuration.Engine.Phase;
@ -2918,15 +2905,6 @@ namespace BuildXL.Engine
return ConstructScheduleResult.Failure;
}
var maybeSymlinkDefinitions = symlinkDefinitionsTask.GetAwaiter().GetResult();
if (!maybeSymlinkDefinitions.Succeeded)
{
Contract.Assert(
loggingContext.ErrorWasLogged,
"Failed to load symlink definitions file, but no error was logged.");
return ConstructScheduleResult.Failure;
}
CacheInitializer cacheInitializerForGraphConstruction = possibleCacheInitializer.Result;
engineSchedule = EngineSchedule.Create(
@ -2942,7 +2920,6 @@ namespace BuildXL.Engine
performanceCollector: m_collector,
directoryTranslator: m_directoryTranslator,
maxDegreeOfParallelism: Configuration.FrontEnd.MaxFrontEndConcurrency(),
symlinkDefinitions: maybeSymlinkDefinitions.Result,
tempCleaner: m_tempCleaner,
buildEngineFingerprint: graphFingerprint?.ExactFingerprint.BuildEngineHash.ToString(),
detoursListener: TestHooks?.DetoursListener);

29
Public/Src/Engine/Dll/EngineSchedule.cs
Просмотреть файл

@ -187,7 +187,6 @@ namespace BuildXL.Engine
PerformanceCollector performanceCollector,
DirectoryTranslator directoryTranslator,
int maxDegreeOfParallelism,
SymlinkDefinitions symlinkDefinitions,
TempCleaner tempCleaner,
string buildEngineFingerprint,
IDetoursEventListener detoursListener = null)
@ -288,7 +287,6 @@ namespace BuildXL.Engine
previousInputsSalt: previousOutputsSalt.Value,
directoryTranslator: directoryTranslator,
pipTwoPhaseCache: twoPhaseCache,
symlinkDefinitions: symlinkDefinitions,
buildEngineFingerprint: buildEngineFingerprint,
vmInitializer: VmInitializer.CreateFromEngine(
configuration.Layout.BuildEngineDirectory.ToString(context.PathTable),
@ -1532,7 +1530,6 @@ namespace BuildXL.Engine
PerformanceCollector performanceCollector,
DirectoryTranslator directoryTranslator,
EngineState engineState,
AbsolutePath symlinkDefinitionFile,
TempCleaner tempCleaner,
string buildEngineFingerprint)
{
@ -1561,31 +1558,6 @@ namespace BuildXL.Engine
GraphCacheFile.ConfigState,
reader => ConfigFileState.DeserializeAsync(reader, loggingContext, pipExecutionContextTask));
Task<SymlinkDefinitions> symlinkDefinitionsTask = Task.Run(
async () =>
{
if (symlinkDefinitionFile.IsValid)
{
var pathTable = await pathTableTask;
var symlinkFilePath = symlinkDefinitionFile.ToString(oldContext.PathTable);
SchedulerLogger.Log.SymlinkFileTraceMessage(loggingContext, I($"Loading symlink file from location '{symlinkFilePath}' with reused pip graph."));
// Scheduler needs symlink map to create symlinks lazily.
var symlinkDefinitionsResult = await SymlinkDefinitions.TryLoadAsync(loggingContext, pathTable, symlinkFilePath,
symlinksDebugPath: configuration.Logging.LogsDirectory.Combine(oldContext.PathTable, "DebugSymlinksDefinitions.log").ToString(oldContext.PathTable),
tempDirectoryCleaner: tempCleaner);
if (!symlinkDefinitionsResult.Succeeded)
{
symlinkDefinitionsResult.Failure.Throw();
}
return symlinkDefinitionsResult.Result;
}
return null;
});
EngineContext newContext;
if (await pipExecutionContextTask != null)
@ -1707,7 +1679,6 @@ namespace BuildXL.Engine
fingerprintSalt: configFileState.CacheSalt,
directoryTranslator: directoryTranslator,
pipTwoPhaseCache: pipTwoPhaseCache,
symlinkDefinitions: await symlinkDefinitionsTask,
buildEngineFingerprint: buildEngineFingerprint,
vmInitializer: VmInitializer.CreateFromEngine(
newConfiguration.Layout.BuildEngineDirectory.ToString(newContext.PathTable),

182
Public/Src/Engine/Dll/SymlinkDefinitionFileProvider.cs
Просмотреть файл

@ -1,182 +0,0 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.
using System.Diagnostics.ContractsLight;
using System.Threading.Tasks;
using BuildXL.Cache.ContentStore.Hashing;
using BuildXL.Engine.Cache;
using BuildXL.Engine.Cache.Artifacts;
using BuildXL.Engine.Cache.Fingerprints;
using BuildXL.Engine.Distribution;
using BuildXL.Native.IO;
using BuildXL.Scheduler.Artifacts;
using BuildXL.Scheduler.Tracing;
using BuildXL.Storage.Fingerprints;
using BuildXL.Utilities;
using BuildXL.Utilities.Configuration;
using BuildXL.Utilities.Instrumentation.Common;
using static BuildXL.Utilities.FormattableStringEx;
namespace BuildXL.Engine
{
/// <summary>
/// Provider for symlink file.
/// </summary>
internal static class SymlinkDefinitionFileProvider
{
/// <summary>
/// Loads configured symlink definitions (if not already loaded)
/// Stores to cache for use by workers in distributed build
/// Eagerly creates symlinks if lazy symlink creation is disabled
/// </summary>
public static async Task<Possible<SymlinkDefinitions>> TryPrepareSymlinkDefinitionsAsync(
LoggingContext loggingContext,
GraphReuseResult reuseResult,
IConfiguration configuration,
MasterService masterService,
CacheInitializationTask cacheInitializerTask,
PipExecutionContext context,
ITempCleaner tempDirectoryCleaner = null)
{
var pathTable = context.PathTable;
bool isDistributedMaster = configuration.Distribution.BuildRole == DistributedBuildRoles.Master;
Possible<SymlinkDefinitions> maybeSymlinkDefinitions = new Possible<SymlinkDefinitions>((SymlinkDefinitions)null);
if (reuseResult?.IsFullReuse == true)
{
maybeSymlinkDefinitions = reuseResult.EngineSchedule.Scheduler.SymlinkDefinitions;
}
else if (configuration.Layout.SymlinkDefinitionFile.IsValid)
{
var symlinkFilePath = configuration.Layout.SymlinkDefinitionFile.ToString(pathTable);
Logger.Log.SymlinkFileTraceMessage(loggingContext, I($"Loading symlink file from location '{symlinkFilePath}'."));
maybeSymlinkDefinitions = await SymlinkDefinitions.TryLoadAsync(
loggingContext,
pathTable,
symlinkFilePath,
symlinksDebugPath: configuration.Logging.LogsDirectory.Combine(pathTable, "DebugSymlinksDefinitions.log").ToString(pathTable),
tempDirectoryCleaner: tempDirectoryCleaner);
}
if (!maybeSymlinkDefinitions.Succeeded || maybeSymlinkDefinitions.Result == null)
{
return maybeSymlinkDefinitions;
}
// Need to store symlinks to cache for workers
if (configuration.Distribution.BuildRole == DistributedBuildRoles.Master)
{
var possibleCacheInitializer = await cacheInitializerTask;
if (!possibleCacheInitializer.Succeeded)
{
return possibleCacheInitializer.Failure;
}
Logger.Log.SymlinkFileTraceMessage(loggingContext, I($"Storing symlink file for use by workers."));
var symlinkFile = configuration.Layout.SymlinkDefinitionFile.Expand(pathTable);
var possibleStore = await TryStoreToCacheAsync(
loggingContext,
cache: possibleCacheInitializer.Result.CreateCacheForContext().ArtifactContentCache,
symlinkFile: symlinkFile);
if (!possibleStore.Succeeded)
{
return possibleStore.Failure;
}
masterService.SymlinkFileContentHash = possibleStore.Result;
Logger.Log.SymlinkFileTraceMessage(loggingContext, I($"Stored symlink file for use by workers."));
}
if (!configuration.Schedule.UnsafeLazySymlinkCreation || configuration.Engine.PopulateSymlinkDirectories.Count != 0)
{
// Symlink definition file is defined, and BuildXL intends to create it eagerly.
// At this point master and worker should have had its symlink definition file, if specified.
if (!FileContentManager.CreateSymlinkEagerly(loggingContext, configuration, pathTable, maybeSymlinkDefinitions.Result, context.CancellationToken))
{
return new Failure<string>("Failed eagerly creating symlinks");
}
}
return maybeSymlinkDefinitions;
}
/// <summary>
/// Tries to store symlink file to cache.
/// </summary>
public static async Task<Possible<ContentHash>> TryStoreToCacheAsync(
LoggingContext loggingContext,
IArtifactContentCache cache,
ExpandedAbsolutePath symlinkFile)
{
var possibleStore = await cache.TryStoreAsync(FileRealizationMode.HardLinkOrCopy, symlinkFile);
if (!possibleStore.Succeeded)
{
Tracing.Logger.Log.FailedStoreSymlinkFileToCache(loggingContext, symlinkFile.ExpandedPath, possibleStore.Failure.DescribeIncludingInnerFailures());
return possibleStore.Failure;
}
Logger.Log.SymlinkFileTraceMessage(loggingContext, I($"Stored symlink file '{symlinkFile}' with hash '{possibleStore.Result}'."));
return possibleStore.Result;
}
private const string SymlinkFileName = "SymlinkDefinitions";
/// <summary>
/// Tries to fetch symlink file from the cache.
/// </summary>
public static async Task<bool> TryFetchWorkerSymlinkFileAsync(
LoggingContext loggingContext,
PathTable pathTable,
EngineCache cache,
ILayoutConfiguration layoutConfiguration,
WorkerService workerService,
AsyncOut<AbsolutePath> symlinkPathAsyncOut)
{
Contract.Requires(loggingContext != null);
Contract.Requires(pathTable != null);
Contract.Requires(cache != null);
Contract.Requires(workerService != null);
Contract.Requires(symlinkPathAsyncOut != null);
symlinkPathAsyncOut.Value = AbsolutePath.Invalid;
var symlinkFileContentHash = workerService.BuildStartData.SymlinkFileContentHash.ToContentHash();
if (symlinkFileContentHash == WellKnownContentHashes.AbsentFile)
{
// Absent file meaning there is no symlink file to use
return true;
}
Logger.Log.SymlinkFileTraceMessage(loggingContext, I($"Attempting to retrieve symlink file with hash '{symlinkFileContentHash}'."));
var maybeLoaded = await cache.ArtifactContentCache.TryLoadAvailableContentAsync(new[] { symlinkFileContentHash });
if (!maybeLoaded.Succeeded)
{
Tracing.Logger.Log.FailedLoadSymlinkFileFromCache(loggingContext, maybeLoaded.Failure.DescribeIncludingInnerFailures());
return false;
}
var destinationPath = layoutConfiguration.EngineCacheDirectory.Combine(pathTable, SymlinkFileName).Expand(pathTable);
var materializedFile =
await
cache.ArtifactContentCache.TryMaterializeAsync(
FileRealizationMode.HardLinkOrCopy,
destinationPath,
symlinkFileContentHash);
if (!materializedFile.Succeeded)
{
Tracing.Logger.Log.FailedMaterializeSymlinkFileFromCache(
loggingContext,
destinationPath.ExpandedPath,
materializedFile.Failure.DescribeIncludingInnerFailures());
return false;
}
Logger.Log.SymlinkFileTraceMessage(loggingContext, I($"Symlink file with hash '{symlinkFileContentHash}' materialized to location '{destinationPath}'."));
symlinkPathAsyncOut.Value = destinationPath.Path;
return true;
}
}
}

9
Public/Src/Engine/Dll/Tracing/Log.cs
Просмотреть файл

@ -1132,15 +1132,6 @@ namespace BuildXL.Engine.Tracing
Message = "/unsafe_OptimizedAstConversion enabled: Some analyses during AST conversions are disabled and some AST constructs, like types, are not converted.")]
public abstract void ConfigUnsafeOptimizedAstConversion(LoggingContext context);
[GeneratedEvent(
(ushort)LogEventId.ConfigUnsafeLazySymlinkCreation,
EventGenerators = EventGenerators.LocalOnly,
EventLevel = Level.Warning,
Keywords = (int)Keywords.UserMessage,
EventTask = (int)Tasks.Engine,
Message = "/unsafe_LazySymlinkCreation enabled: {ShortProductName} is configured to create symlinks from symlink definition manifest lazily. This might lead to incorrect builds because symlinks may have not been created when a pip consumes (read or probe) it or enumerates its parent.")]
public abstract void ConfigUnsafeLazySymlinkCreation(LoggingContext context);
[GeneratedEvent(
(ushort)LogEventId.ConfigDebuggingAndProfilingCannotBeSpecifiedSimultaneously,
EventGenerators = EventGenerators.LocalOnly,

2
Public/Src/Engine/Dll/Tracing/LogEventId.cs
Просмотреть файл

@ -89,7 +89,7 @@ namespace BuildXL.Engine.Tracing
ConfigIgnoreDynamicWritesOnAbsentProbes = 916,
ConfigIgnoreSetFileInformationByHandle = 917,
ConfigPreserveOutputs = 918,
ConfigUnsafeLazySymlinkCreation = 919,
// was ConfigUnsafeLazySymlinkCreation = 919,
ConfigDisableDetours = 920,
ConfigDebuggingAndProfilingCannotBeSpecifiedSimultaneously = 921,
ConfigIgnoreGetFinalPathNameByHandle = 922,

243
Public/Src/Engine/Scheduler/Artifacts/FileContentManager.cs
Просмотреть файл

@ -70,7 +70,6 @@ namespace BuildXL.Scheduler.Artifacts
[SuppressMessage("Microsoft.Design", "CA1001:TypesThatOwnDisposableFieldsShouldBeDisposable")]
public sealed class FileContentManager : IQueryableFileContentManager
{
private const int MAX_SYMLINK_TRAVERSALS = 100;
private readonly ITempCleaner m_tempDirectoryCleaner;
#region Internal State
@ -186,15 +185,6 @@ namespace BuildXL.Scheduler.Artifacts
private readonly ConcurrentBigMap<FileArtifact, DirectoryArtifact> m_dynamicOutputFileDirectories =
new ConcurrentBigMap<FileArtifact, DirectoryArtifact>();
/// <summary>
/// Symlink definitions.
/// </summary>
private readonly SymlinkDefinitions m_symlinkDefinitions;
/// <summary>
/// Flag indicating if symlink should be created lazily.
/// </summary>
private bool LazySymlinkCreation => m_host.Configuration.Schedule.UnsafeLazySymlinkCreation && m_symlinkDefinitions != null;
#endregion
#region External State (i.e. passed into constructor)
@ -254,13 +244,11 @@ namespace BuildXL.Scheduler.Artifacts
public FileContentManager(
IFileContentManagerHost host,
IOperationTracker operationTracker,
SymlinkDefinitions symlinkDefinitions = null,
ITempCleaner tempDirectoryCleaner = null)
{
m_host = host;
ArtifactContentCache = new ElidingArtifactContentCacheWrapper(host.ArtifactContentCache);
OperationTracker = operationTracker;
m_symlinkDefinitions = symlinkDefinitions;
m_tempDirectoryCleaner = tempDirectoryCleaner;
m_outputMaterializationExclusionMap = new FlaggedHierarchicalNameDictionary<Unit>(host.Context.PathTable, HierarchicalNameTable.NameFlags.Root);
@ -830,114 +818,11 @@ namespace BuildXL.Scheduler.Artifacts
}
}
/// <summary>
/// Attempts to the symlink target if registered
/// </summary>
public AbsolutePath TryGetRegisteredSymlinkFinalTarget(AbsolutePath symlink)
{
if (m_symlinkDefinitions == null)
{
return AbsolutePath.Invalid;
}
AbsolutePath symlinkTarget = symlink;
for (int i = 0; i < MAX_SYMLINK_TRAVERSALS; i++)
{
var next = m_symlinkDefinitions.TryGetSymlinkTarget(symlinkTarget);
if (next.IsValid)
{
symlinkTarget = next;
}
else
{
return symlinkTarget == symlink ? AbsolutePath.Invalid : symlinkTarget;
}
}
// Symlink chain is too long
List<AbsolutePath> symlinkChain = new List<AbsolutePath>();
symlinkTarget = symlink;
for (int i = 0; i < MAX_SYMLINK_TRAVERSALS; i++)
{
symlinkChain.Add(symlinkTarget);
symlinkTarget = m_symlinkDefinitions.TryGetSymlinkTarget(symlinkTarget);
}
throw new BuildXLException(I(
$"Registered symlink chain exceeds max length of {MAX_SYMLINK_TRAVERSALS}: {string.Join("->" + Environment.NewLine, symlinkChain)}"));
}
/// <summary>
/// Reports an unexpected access which the file content manager can check to verify that access is safe with respect to lazy
/// symlink creation
/// </summary>
internal void ReportUnexpectedSymlinkAccess(LoggingContext loggingContext, string pipDescription, AbsolutePath path, ObservedInputType observedInputType, CompactSet<ReportedFileAccess> reportedAccesses)
{
if (TryGetSymlinkPathKind(path, out var symlinkPathKind))
{
using (var toolPathSetWrapper = Pools.StringSetPool.GetInstance())
using (var toolFileNameSetWrapper = Pools.StringSetPool.GetInstance())
{
var toolPathSet = toolPathSetWrapper.Instance;
var toolFileNameSet = toolFileNameSetWrapper.Instance;
foreach (var reportedAccess in reportedAccesses)
{
if (!string.IsNullOrEmpty(reportedAccess.Process.Path))
{
if (toolPathSet.Add(reportedAccess.Process.Path))
{
AbsolutePath toolPath;
if (AbsolutePath.TryCreate(Context.PathTable, reportedAccess.Process.Path, out toolPath))
{
toolFileNameSet.Add(toolPath.GetName(Context.PathTable).ToString(Context.StringTable));
}
}
}
}
Logger.Log.UnexpectedAccessOnSymlinkPath(
pipDescription: pipDescription,
context: loggingContext,
path: path.ToString(Context.PathTable),
pathKind: symlinkPathKind,
inputType: observedInputType.ToString(),
tools: string.Join(", ", toolFileNameSet));
}
}
}
internal bool TryGetSymlinkPathKind(AbsolutePath path, out string kind)
{
kind = null;
if (m_symlinkDefinitions == null)
{
return false;
}
if (m_symlinkDefinitions.IsSymlink(path))
{
kind = "file";
}
else if (m_symlinkDefinitions.HasNestedSymlinks(path))
{
kind = "directory";
}
return kind != null;
}
/// <summary>
/// Gets the updated semantic path information for the given path with data from the file content manager
/// </summary>
internal SemanticPathInfo GetUpdatedSemanticPathInfo(in SemanticPathInfo mountInfo)
{
if (mountInfo.IsValid && LazySymlinkCreation && m_symlinkDefinitions.HasNestedSymlinks(mountInfo.Root))
{
// Rewrite the semantic path info to indicate that the mount has potential build outputs
return new SemanticPathInfo(mountInfo.RootName, mountInfo.Root, mountInfo.Flags | SemanticPathFlags.HasPotentialBuildOutputs);
}
return mountInfo;
}
@ -946,9 +831,7 @@ namespace BuildXL.Scheduler.Artifacts
/// </summary>
public bool HasPotentialBuildOutputs(AbsolutePath directoryPath, in SemanticPathInfo mountInfo, bool isReadOnlyDirectory)
{
// If (1) the directory is writeable, or (2) the directory contains symlinks, that may have not been created, then use the graph enumeration.
return (mountInfo.IsWritable && !isReadOnlyDirectory) ||
(LazySymlinkCreation && m_symlinkDefinitions.DirectoryContainsSymlink(directoryPath));
return (mountInfo.IsWritable && !isReadOnlyDirectory);
}
/// <summary>
@ -1495,98 +1378,6 @@ namespace BuildXL.Scheduler.Artifacts
return fileContentInfo;
}
/// <summary>
/// Creates all symlink files in the symlink definitions
/// </summary>
public static bool CreateSymlinkEagerly(LoggingContext loggingContext, IConfiguration configuration, PathTable pathTable, SymlinkDefinitions symlinkDefinitions, CancellationToken cancellationToken)
{
Contract.Requires(loggingContext != null);
Contract.Requires(symlinkDefinitions != null);
Contract.Requires(!configuration.Schedule.UnsafeLazySymlinkCreation || configuration.Engine.PopulateSymlinkDirectories.Count != 0);
Logger.Log.SymlinkFileTraceMessage(loggingContext, I($"Eagerly creating symlinks found in symlink file."));
int createdSymlinkCount = 0;
int reuseExistingSymlinkCount = 0;
int failedSymlinkCount = 0;
var startTime = TimestampUtilities.Timestamp;
var symlinkDirectories = symlinkDefinitions.DirectorySymlinkContents.Keys.ToList();
var populateSymlinkDirectories = new HashSet<HierarchicalNameId>(configuration.Engine.PopulateSymlinkDirectories.Select(p => p.Value));
Parallel.ForEach(
symlinkDirectories,
new ParallelOptions
{
MaxDegreeOfParallelism = configuration.Schedule.MaxProcesses,
},
symlinkDirectory =>
{
bool populateSymlinks = !configuration.Schedule.UnsafeLazySymlinkCreation;
if (!populateSymlinks)
{
// If populating symlinks lazily, check if the directory is under the explicitly specified symlink directories
// to populate
foreach (var parent in pathTable.EnumerateHierarchyBottomUp(symlinkDirectory.Value))
{
if (populateSymlinkDirectories.Contains(parent))
{
populateSymlinks = true;
break;
}
}
}
if (!populateSymlinks)
{
return;
}
var directorySymlinks = symlinkDefinitions.DirectorySymlinkContents[symlinkDirectory];
foreach (var symlinkPath in directorySymlinks)
{
var symlink = symlinkPath.ToString(pathTable);
var symlinkTarget = symlinkDefinitions[symlinkPath].ToString(pathTable);
if (cancellationToken.IsCancellationRequested)
{
Interlocked.Increment(ref failedSymlinkCount);
break;
}
bool created;
var maybeSymlink = FileUtilities.TryCreateSymlinkIfNotExistsOrTargetsDoNotMatch(symlink, symlinkTarget, true, out created);
if (maybeSymlink.Succeeded)
{
if (created)
{
Interlocked.Increment(ref createdSymlinkCount);
}
else
{
Interlocked.Increment(ref reuseExistingSymlinkCount);
}
}
else
{
Interlocked.Increment(ref failedSymlinkCount);
Logger.Log.FailedToCreateSymlinkFromSymlinkMap(loggingContext, symlink, symlinkTarget, maybeSymlink.Failure.DescribeIncludingInnerFailures());
}
}
});
Logger.Log.CreateSymlinkFromSymlinkMap(
loggingContext,
createdSymlinkCount,
reuseExistingSymlinkCount,
failedSymlinkCount,
(int)(TimestampUtilities.Timestamp - startTime).TotalMilliseconds);
return failedSymlinkCount == 0;
}
private async Task<ArtifactMaterializationResult> TryMaterializeArtifactsCore(
PipInfo pipInfo,
OperationContext operationContext,
@ -1690,36 +1481,10 @@ namespace BuildXL.Scheduler.Artifacts
}
else if (sealDirectoryKind == SealDirectoryKind.SourceTopDirectoryOnly)
{
IReadOnlyList<AbsolutePath> paths;
if (LazySymlinkCreation && m_symlinkDefinitions.TryGetSymlinksInDirectory(directory.Path, out paths))
{
foreach (var path in paths)
{
AddFileMaterialization(
state,
FileArtifact.CreateSourceFile(path),
directoryAllowReadOnlyOverride,
m_symlinkDefinitions[path]);
}
}
continue;
}
else if (sealDirectoryKind == SealDirectoryKind.SourceAllDirectories)
{
if (LazySymlinkCreation && m_symlinkDefinitions.HasNestedSymlinks(directory))
{
foreach (var node in Context.PathTable.EnumerateHierarchyTopDown(directory.Path.Value))
{
var path = new AbsolutePath(node);
AddFileMaterialization(
state,
FileArtifact.CreateSourceFile(path),
directoryAllowReadOnlyOverride,
m_symlinkDefinitions.TryGetSymlinkTarget(path));
}
}
continue;
}
@ -1747,13 +1512,13 @@ namespace BuildXL.Scheduler.Artifacts
private AbsolutePath TryGetSymlinkTarget(FileArtifact file)
{
if (file.IsOutputFile || !LazySymlinkCreation)
if (file.IsOutputFile)
{
// Only source files can be declared as symlinks
return AbsolutePath.Invalid;
}
return m_symlinkDefinitions.TryGetSymlinkTarget(file);
return AbsolutePath.Invalid;
}
private void MarkDirectoryMaterializations(PipArtifactsState state)
@ -2787,7 +2552,7 @@ namespace BuildXL.Scheduler.Artifacts
}
else
{
// Just store placeholder task for output files/lazy symlinks since they are not verified
// Just store placeholder task for output files/symlinks since they are not verified
state.HashTasks.Add(s_placeHolderFileHashTask);
}
}

212
Public/Src/Engine/Scheduler/Artifacts/SymlinkDefinitions.cs
Просмотреть файл

@ -1,212 +0,0 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.
using System;
using System.Collections.Generic;
using System.Diagnostics.ContractsLight;
using System.IO;
using System.Threading.Tasks;
using BuildXL.Native.IO;
using BuildXL.Scheduler.Tracing;
using BuildXL.Utilities;
using BuildXL.Utilities.Collections;
using BuildXL.Utilities.Configuration;
using BuildXL.Utilities.Instrumentation.Common;
using static BuildXL.Utilities.FormattableStringEx;
namespace BuildXL.Scheduler.Artifacts
{
/// <summary>
/// Symlink definitions containing mappings from symlinks to their targets.
/// </summary>
public sealed class SymlinkDefinitions
{
private readonly PathTable m_pathTable;
private readonly Dictionary<AbsolutePath, List<AbsolutePath>> m_directorySymlinkContents;
private readonly ConcurrentBigMap<AbsolutePath, AbsolutePath> m_symlinkDefinitionMap;
private readonly HashSet<HierarchicalNameId> m_directoriesContainingSymlinks;
/// <summary>
/// Mappings from symlinks to their targets.
/// </summary>
public IReadOnlyDictionary<AbsolutePath, List<AbsolutePath>> DirectorySymlinkContents => m_directorySymlinkContents;
/// <summary>
/// Creates an instance of <see cref="SymlinkDefinitions" />.
/// </summary>
public SymlinkDefinitions(PathTable pathTable, ConcurrentBigMap<AbsolutePath, AbsolutePath> symlinkDefinitionMap)
{
Contract.Requires(pathTable != null);
Contract.Requires(symlinkDefinitionMap != null);
Contract.Requires(pathTable != null);
m_pathTable = pathTable;
m_symlinkDefinitionMap = symlinkDefinitionMap;
m_directorySymlinkContents = new Dictionary<AbsolutePath, List<AbsolutePath>>();
m_directoriesContainingSymlinks = new HashSet<HierarchicalNameId>();
foreach (var symlink in m_symlinkDefinitionMap.Keys)
{
var directory = symlink.GetParent(m_pathTable);
foreach (var pathId in m_pathTable.EnumerateHierarchyBottomUp(directory.Value))
{
if (!m_directoriesContainingSymlinks.Add(pathId))
{
break;
}
}
List<AbsolutePath> paths;
if (!m_directorySymlinkContents.TryGetValue(directory, out paths))
{
paths = new List<AbsolutePath>();
m_directorySymlinkContents.Add(directory, paths);
}
paths.Add(symlink);
}
}
/// <summary>
/// Load symlink definitions serialized using <see cref="PathMapSerializer"/>
/// </summary>
public static async Task<Possible<SymlinkDefinitions>> TryLoadAsync(
LoggingContext loggingContext,
PathTable pathTable,
string filePath,
string symlinksDebugPath,
ITempCleaner tempDirectoryCleaner = null)
{
try
{
var pathMap = await PathMapSerializer.LoadAsync(filePath, pathTable);
var definitions = new SymlinkDefinitions(pathTable, pathMap);
Logger.Log.SymlinkFileTraceMessage(
loggingContext,
I($"Loaded symlink definitions with {definitions.m_symlinkDefinitionMap.Count} entries and {definitions.m_directorySymlinkContents.Count} directories."));
if (EngineEnvironmentSettings.DebugSymlinkDefinitions && symlinksDebugPath != null)
{
FileUtilities.DeleteFile(symlinksDebugPath, tempDirectoryCleaner: tempDirectoryCleaner);
using (var writer = new StreamWriter(symlinksDebugPath))
{
foreach (var entry in pathMap)
{
writer.WriteLine("Source: {0}", entry.Key.ToString(pathTable));
writer.WriteLine("Target: {0}", entry.Value.ToString(pathTable));
}
}
}
return definitions;
}
catch (Exception ex)
{
Logger.Log.FailedLoadSymlinkFile(loggingContext, ex.GetLogEventMessage());
return new Failure<string>("Failed loading symlink definition file");
}
}
/// <summary>
/// Deserialize symlink definitions serialized using <see cref="Serialize(BuildXLWriter, SymlinkDefinitions)"/>
/// </summary>
public static Possible<SymlinkDefinitions> Deserialize(LoggingContext loggingContext, PathTable pathTable, BuildXLReader reader)
{
try
{
bool isNull = reader.ReadBoolean();
if (isNull)
{
return (SymlinkDefinitions)null;
}
var pathMap = ConcurrentBigMap<AbsolutePath, AbsolutePath>.Deserialize(
reader,
() => new KeyValuePair<AbsolutePath, AbsolutePath>(
key: reader.ReadAbsolutePath(),
value: reader.ReadAbsolutePath()));
return new SymlinkDefinitions(pathTable, pathMap);
}
catch (Exception ex)
{
Logger.Log.FailedLoadSymlinkFile(loggingContext, ex.GetLogEventMessage());
return new Failure<string>("Failed loading symlink definition file");
}
}
/// <summary>
/// Serialize the symlink definitions for load using <see cref="Deserialize"/>
/// </summary>
/// <param name="writer">the writer</param>
/// <param name="symlinkDefinitions">the symlink definitions (may be null)</param>
public static void Serialize(BuildXLWriter writer, SymlinkDefinitions symlinkDefinitions)
{
writer.Write(symlinkDefinitions == null);
symlinkDefinitions?.m_symlinkDefinitionMap.Serialize(
writer,
kvp =>
{
writer.Write(kvp.Key);
writer.Write(kvp.Value);
});
}
/// <summary>
/// Checks if symlink definition contains the given path as symlink.
/// </summary>
public bool IsSymlink(AbsolutePath path) => m_symlinkDefinitionMap.ContainsKey(path);
/// <summary>
/// Gets the symlink target or <see cref="AbsolutePath.Invalid"/> if the path is not a registered symlink
/// </summary>
public AbsolutePath TryGetSymlinkTarget(AbsolutePath symlink) => m_symlinkDefinitionMap.TryGet(symlink).Item.Value;
/// <summary>
/// Indexer.
/// </summary>
public AbsolutePath this[AbsolutePath path] => m_symlinkDefinitionMap[path];
/// <summary>
/// Tries to get symlink target.
/// </summary>
public bool TryGetSymlinkTarget(AbsolutePath symlink, out AbsolutePath symlinkTarget)
=> m_symlinkDefinitionMap.TryGetValue(symlink, out symlinkTarget);
/// <summary>
/// Checks if a directory contains symlinks.
/// </summary>
public bool DirectoryContainsSymlink(AbsolutePath directory) => m_directorySymlinkContents.ContainsKey(directory);
/// <summary>
/// Tries to get all symlinks inside a given directory.
/// </summary>
public bool TryGetSymlinksInDirectory(AbsolutePath directory, out IReadOnlyList<AbsolutePath> paths)
{
paths = null;
List<AbsolutePath> tempPaths;
if (!m_directorySymlinkContents.TryGetValue(directory, out tempPaths))
{
return false;
}
paths = tempPaths;
return true;
}
/// <summary>
/// Checks whether the directory or any of its children contain symlinks
/// </summary>
public bool HasNestedSymlinks(AbsolutePath directory)
{
return m_directoriesContainingSymlinks.Contains(directory.Value);
}
/// <summary>
/// Number of mappings in this instance of <see cref="SymlinkDefinitions"/>.
/// </summary>
public int Count => m_symlinkDefinitionMap.Count;
}
}

1
Public/Src/Engine/Scheduler/Fingerprints/ObservedInputProcessor.cs
Просмотреть файл

@ -1666,7 +1666,6 @@ namespace BuildXL.Scheduler.Fingerprints
Possible<PathExistence> existence;
// Check if path will be eventually produced as a file/directory by querying 'output file system'
// TODO: Should the file content manager be queried about eventual production (i.e. lazy symlink creation)?
existence = FileSystemView.GetExistence(path, FileSystemViewMode.Output);
// NOTE: We don't check success of existence from produced file system as it should never return failure

50
Public/Src/Engine/Scheduler/PipExecutor.cs
Просмотреть файл

@ -159,28 +159,21 @@ namespace BuildXL.Scheduler
FileMaterializationInfo sourceMaterializationInfo = environment.State.FileContentManager.GetInputContent(pip.Source);
FileContentInfo sourceContentInfo = sourceMaterializationInfo.FileContentInfo;
var symlinkTarget = environment.State.FileContentManager.TryGetRegisteredSymlinkFinalTarget(pip.Source.Path);
ReadOnlyArray<AbsolutePath> symlinkChain;
bool isSymLink = symlinkTarget.IsValid;
if (isSymLink)
var symlinkTarget = AbsolutePath.Invalid;
bool isSymLink = false;
var possibleSymlinkChain = CheckValidSymlinkChainAsync(pip.Source, environment);
if (!possibleSymlinkChain.Succeeded)
{
symlinkChain = ReadOnlyArray<AbsolutePath>.FromWithoutCopy(new[] { symlinkTarget });
possibleSymlinkChain.Failure.Throw();
}
else
{
// pip.Source is not a registered symlink - check if this file forms a proper symlink chain
var possibleSymlinkChain = CheckValidSymlinkChainAsync(pip.Source, environment);
if (!possibleSymlinkChain.Succeeded)
{
possibleSymlinkChain.Failure.Throw();
}
symlinkChain = possibleSymlinkChain.Result;
if (symlinkChain.Length > 0)
{
symlinkTarget = symlinkChain[symlinkChain.Length - 1];
isSymLink = true;
}
symlinkChain = possibleSymlinkChain.Result;
if (symlinkChain.Length > 0)
{
symlinkTarget = symlinkChain[symlinkChain.Length - 1];
isSymLink = true;
}
if (isSymLink && !environment.Configuration.Schedule.AllowCopySymlink)
@ -3629,16 +3622,12 @@ namespace BuildXL.Scheduler
public void ReportUnexpectedAccess(ObservedPathEntry assertion, ObservedInputType observedInputType)
{
if (m_environment.Configuration.Schedule.UnexpectedSymlinkAccessReportingMode == UnexpectedSymlinkAccessReportingMode.All)
{
m_environment.State.FileContentManager.ReportUnexpectedSymlinkAccess(m_operationContext, m_pipDescription, assertion.Path, observedInputType, reportedAccesses: default(CompactSet<ReportedFileAccess>));
}
// noop
}
public bool IsReportableUnexpectedAccess(AbsolutePath path)
{
return m_environment.Configuration.Schedule.UnexpectedSymlinkAccessReportingMode == UnexpectedSymlinkAccessReportingMode.All &&
m_environment.State.FileContentManager.TryGetSymlinkPathKind(path, out var kind);
return false;
}
}
@ -3683,21 +3672,12 @@ namespace BuildXL.Scheduler
public void ReportUnexpectedAccess(ObservedFileAccess observation, ObservedInputType observedInputType)
{
if (m_environment.Configuration.Schedule.UnexpectedSymlinkAccessReportingMode != UnexpectedSymlinkAccessReportingMode.None)
{
m_environment.State.FileContentManager.ReportUnexpectedSymlinkAccess(
m_operationContext,
m_processDescription,
observation.Path,
observedInputType,
observation.Accesses);
}
// noop
}
public bool IsReportableUnexpectedAccess(AbsolutePath path)
{
return m_environment.Configuration.Schedule.UnexpectedSymlinkAccessReportingMode != UnexpectedSymlinkAccessReportingMode.None &&
m_environment.State.FileContentManager.TryGetSymlinkPathKind(path, out var kind);
return false;
}
public ObservedInputAccessCheckFailureAction OnAccessCheckFailure(ObservedFileAccess observation, bool fromTopLevelDirectory)

10
Public/Src/Engine/Scheduler/Scheduler.cs
Просмотреть файл

@ -186,11 +186,6 @@ namespace BuildXL.Scheduler
/// </summary>
private readonly FileContentManager m_fileContentManager;
/// <summary>
/// Symlink definitions
/// </summary>
public readonly SymlinkDefinitions SymlinkDefinitions;
/// <summary>
/// Tracker of output materializations.
/// </summary>
@ -826,7 +821,6 @@ namespace BuildXL.Scheduler
private bool InputsLazilyMaterialized =>
m_scheduleConfiguration.EnableLazyOutputMaterialization
|| IsDistributedBuild
|| m_scheduleConfiguration.UnsafeLazySymlinkCreation
|| m_scheduleConfiguration.OutputMaterializationExclusionRoots.Count != 0;
/// <summary>
@ -1080,7 +1074,6 @@ namespace BuildXL.Scheduler
DirectoryTranslator directoryTranslator = null,
IIpcProvider ipcProvider = null,
PipTwoPhaseCache pipTwoPhaseCache = null,
SymlinkDefinitions symlinkDefinitions = null,
JournalState journalState = null,
VmInitializer vmInitializer = null,
SchedulerTestHooks testHooks = null)
@ -1190,8 +1183,7 @@ namespace BuildXL.Scheduler
m_dropPipTracker = new DropPipTracker(Context);
OperationTracker = new OperationTracker(loggingContext, this);
SymlinkDefinitions = symlinkDefinitions;
m_fileContentManager = new FileContentManager(this, OperationTracker, symlinkDefinitions);
m_fileContentManager = new FileContentManager(this, OperationTracker);
m_apiServer = null;
m_writableDrives = new HashSet<string>(StringComparer.OrdinalIgnoreCase);

50
Public/Src/Engine/Scheduler/Tracing/Log.cs
Просмотреть файл

@ -3056,56 +3056,6 @@ namespace BuildXL.Scheduler.Tracing
Message = "{0}")]
public abstract void CriticalPathChain(LoggingContext context, string criticalPathMessage);
#region Symlink file
[GeneratedEvent(
(int)LogEventId.FailedLoadSymlinkFile,
EventGenerators = EventGenerators.LocalOnly,
EventLevel = Level.Error,
Keywords = (int)Keywords.UserMessage,
EventTask = (int)Tasks.Engine,
Message = "Failed to load symlink file: {message}.")]
public abstract void FailedLoadSymlinkFile(LoggingContext context, string message);
[GeneratedEvent(
(ushort)LogEventId.FailedToCreateSymlinkFromSymlinkMap,
EventGenerators = EventGenerators.LocalOnly,
EventLevel = Level.Error,
Keywords = (int)Keywords.UserMessage,
EventTask = (ushort)Tasks.Storage,
Message = "Failed to create symlink from '{source}' to '{target}': {message}")]
public abstract void FailedToCreateSymlinkFromSymlinkMap(LoggingContext loggingContext, string source, string target, string message);
[GeneratedEvent(
(ushort)LogEventId.CreateSymlinkFromSymlinkMap,
EventGenerators = EventGenerators.LocalOnly | Generators.Statistics,
EventLevel = Level.Verbose,
Keywords = (int)Keywords.UserMessage,
EventTask = (ushort)Tasks.Storage,
Message = "Symlink creations: Created symlinks: {createdSymlinkCount} | Reuse symlinks: {reuseSymlinkCount} | Failed creations: {failedSymlinkCount} | Elapsed time: {createSymlinkDurationMs}ms")]
public abstract void CreateSymlinkFromSymlinkMap(LoggingContext loggingContext, int createdSymlinkCount, int reuseSymlinkCount, int failedSymlinkCount, int createSymlinkDurationMs);
[GeneratedEvent(
(int)LogEventId.SymlinkFileTraceMessage,
EventGenerators = EventGenerators.LocalOnly,
EventLevel = Level.Informational,
Keywords = (int)Keywords.UserMessage,
EventTask = (int)Tasks.Engine,
Message = "{message}")]
public abstract void SymlinkFileTraceMessage(LoggingContext context, string message);
[GeneratedEvent(
(int)LogEventId.UnexpectedAccessOnSymlinkPath,
EventGenerators = EventGenerators.LocalOnly,
// TODO: Should this be informational?
EventLevel = Level.Warning,
Keywords = (int)Keywords.UserMessage,
EventTask = (int)Tasks.Engine,
Message = "[{pipDescription}] Unexpected access on symlink {pathKind} path '{path}': {inputType} (Tools: {tools}).")]
public abstract void UnexpectedAccessOnSymlinkPath(LoggingContext context, string pipDescription, string path, string pathKind, string inputType, string tools);
#endregion
#region Preserved output tracker
[GeneratedEvent(

10
Public/Src/Engine/Scheduler/Tracing/LogEventId.cs
Просмотреть файл

@ -233,11 +233,11 @@ namespace BuildXL.Scheduler.Tracing
PipIsIncrementallySkippedDueToCleanMaterialized = 3631,
// Symlink file.
FailedToCreateSymlinkFromSymlinkMap = 3632,
FailedLoadSymlinkFile = 3633,
CreateSymlinkFromSymlinkMap = 3634,
SymlinkFileTraceMessage = 3635,
UnexpectedAccessOnSymlinkPath = 3636,
// was FailedToCreateSymlinkFromSymlinkMap = 3632,
// was FailedLoadSymlinkFile = 3633,
// was CreateSymlinkFromSymlinkMap = 3634,
// was SymlinkFileTraceMessage = 3635,
// was UnexpectedAccessOnSymlinkPath = 3636,
// Preserved outputs tracker.
// Reserved = 3640,

55
Public/Src/Engine/UnitTests/Engine/MiniBuild.cs
Просмотреть файл

@ -492,61 +492,6 @@ namespace Test.BuildXL.EngineTests
AssertInformationalEventLogged(LogEventId.FetchedSerializedGraphFromCache, count: 0);
}
[TheoryIfSupported(requiresSymlinkPermission: true)]
[InlineData(true)]
[InlineData(false)]
public void MiniBuildCopySymlink(bool lazySymlinkCreation)
{
using (var tempFiles = new TempFileStorage(canGetFileNames: true, rootPath: TestOutputDirectory))
{
const string SymlinkSource1 = "symlink1.lnk";
const string SymlinkSource2 = "symlink2.lnk";
const string CopiedSymlink1 = "copied-" + SymlinkSource1;
const string CopiedSymlink2 = "copied-" + SymlinkSource2;
const string SymlinkTarget1 = "target1.txt";
const string SymlinkTarget2 = "target2.txt";
SetupCopySymlink(SymlinkSource1, CopiedSymlink1, SymlinkSource2, CopiedSymlink2);
Configuration.Schedule.UnsafeLazySymlinkCreation = lazySymlinkCreation;
Configuration.Cache.CacheGraph = true;
var sourceDirectory = Configuration.Layout.SourceDirectory.ToString(Context.PathTable);
// Write symlink targets.
File.WriteAllText(Path.Combine(sourceDirectory, SymlinkTarget1), "1");
File.WriteAllText(Path.Combine(sourceDirectory, SymlinkTarget2), "2");
// Write symlink definition file.
var symlinkDefinitionFile = Path.Combine(sourceDirectory, "SymlinkDefinition");
var pathMapSerializer = new PathMapSerializer(symlinkDefinitionFile);
pathMapSerializer.OnNext(
new KeyValuePair<string, string>(Path.Combine(sourceDirectory, SymlinkSource1), Path.Combine(sourceDirectory, SymlinkTarget1)));
pathMapSerializer.OnNext(
new KeyValuePair<string, string>(Path.Combine(sourceDirectory, SymlinkSource2), Path.Combine(sourceDirectory, SymlinkTarget2)));
((IObserver<KeyValuePair<string, string>>)pathMapSerializer).OnCompleted();
// Allow copying symlink for testing, and set the symlink definition file.
Configuration.Schedule.AllowCopySymlink = true;
Configuration.Layout.SymlinkDefinitionFile = AbsolutePath.Create(Context.PathTable, symlinkDefinitionFile);
IgnoreWarnings();
RunEngine();
var objectDirectoryPath = Configuration.Layout.ObjectDirectory.ToString(Context.PathTable);
XAssert.IsTrue(File.Exists(Path.Combine(objectDirectoryPath, CopiedSymlink1)));
XAssert.IsTrue(File.Exists(Path.Combine(objectDirectoryPath, CopiedSymlink2)));
string content1 = File.ReadAllText(Path.Combine(objectDirectoryPath, CopiedSymlink1));
string content2 = File.ReadAllText(Path.Combine(objectDirectoryPath, CopiedSymlink2));
XAssert.AreEqual("1", content1);
XAssert.AreEqual("2", content2);
}
}
[Fact]
public void MiniBuildCachedGraphWithAndWithoutEnvironmentAccess()
{

5
Public/Src/Utilities/Configuration/EngineEnvironmentSettings.cs
Просмотреть файл

@ -76,11 +76,6 @@ namespace BuildXL.Utilities.Configuration
/// </summary>
public static readonly Setting<bool> BypassNugetDownload = CreateSetting("BuildXLBypassNugetDownload", value => value == "1");
/// <summary>
/// Emit file with all symlink definitions
/// </summary>
public static readonly Setting<bool> DebugSymlinkDefinitions = CreateSetting("DebugSymlinkDefinitions", value => value == "1");
/// <summary>
/// Allows optionally specifying an alternative timeout fo connect between IDE service and BuildXL task
/// </summary>

5
Public/Src/Utilities/Configuration/IEngineConfiguration.cs
Просмотреть файл

@ -91,11 +91,6 @@ namespace BuildXL.Utilities.Configuration
[CanBeNull]
IReadOnlyList<AbsolutePath> ScrubDirectories { get; }
/// <summary>
/// Directories to eagerly populate with symlinks
/// </summary>
IReadOnlyList<AbsolutePath> PopulateSymlinkDirectories { get; }
/// <summary>
/// Directories under the object directory root will get shortened to avoid too long path names. Defaults to 64 characters for relative output directories.
/// </summary>

5
Public/Src/Utilities/Configuration/ILayoutConfiguration.cs
Просмотреть файл

@ -67,11 +67,6 @@ namespace BuildXL.Utilities.Configuration
/// </summary>
AbsolutePath FileContentTableFile { get; }
/// <summary>
/// Path to file defining symlinks
/// </summary>
AbsolutePath SymlinkDefinitionFile { get; }
/// <summary>
/// File change tracker file.
/// </summary>

10
Public/Src/Utilities/Configuration/IScheduleConfiguration.cs
Просмотреть файл

@ -194,11 +194,6 @@ namespace BuildXL.Utilities.Configuration
/// </remarks>
int ProcessRetries { get; }
/// <summary>
/// Create symlink lazily from the symlink definition manifest.
/// </summary>
bool UnsafeLazySymlinkCreation { get; }
/// <summary>
/// Enables lazy materialization of write file outputs. Defaults to off (on for CloudBuild)
/// </summary>
@ -212,11 +207,6 @@ namespace BuildXL.Utilities.Configuration
/// </summary>
bool WriteIpcOutput { get; }
/// <summary>
/// Gets the mode for reporting unexpected symlink accesses which defines when unexpected accesses are reported
/// </summary>
UnexpectedSymlinkAccessReportingMode UnexpectedSymlinkAccessReportingMode { get; }
/// <summary>
/// Stores pip outputs to cache.
/// </summary>

9
Public/Src/Utilities/Configuration/Mutable/EngineConfiguration.cs
Просмотреть файл

@ -28,7 +28,6 @@ namespace BuildXL.Utilities.Configuration.Mutable
BuildLockWaitTimeoutMins = 0;
DirectoriesToTranslate = new List<TranslateDirectoryData>();
ScrubDirectories = new List<AbsolutePath>();
PopulateSymlinkDirectories = new List<AbsolutePath>();
CompressGraphFiles = false;
FileChangeTrackerInitializationMode = FileChangeTrackerInitializationMode.ResumeExisting;
LogStatistics = true;
@ -68,7 +67,6 @@ namespace BuildXL.Utilities.Configuration.Mutable
template.DirectoriesToTranslate.Select(
d => new TranslateDirectoryData(d.RawUserOption, pathRemapper.Remap(d.FromPath), pathRemapper.Remap(d.ToPath))).ToList();
ScrubDirectories = pathRemapper.Remap(template.ScrubDirectories);
PopulateSymlinkDirectories = pathRemapper.Remap(template.PopulateSymlinkDirectories);
CompressGraphFiles = template.CompressGraphFiles;
FileChangeTrackerInitializationMode = template.FileChangeTrackerInitializationMode;
LogStatistics = template.LogStatistics;
@ -144,13 +142,6 @@ namespace BuildXL.Utilities.Configuration.Mutable
/// <inheritdoc />
IReadOnlyList<TranslateDirectoryData> IEngineConfiguration.DirectoriesToTranslate => DirectoriesToTranslate;
/// <nodoc />
[SuppressMessage("Microsoft.Usage", "CA2227:CollectionPropertiesShouldBeReadOnly")]
public List<AbsolutePath> PopulateSymlinkDirectories { get; set; }
/// <inheritdoc />
IReadOnlyList<AbsolutePath> IEngineConfiguration.PopulateSymlinkDirectories => PopulateSymlinkDirectories;
/// <inheritdoc />
public bool CompressGraphFiles { get; set; }

4
Public/Src/Utilities/Configuration/Mutable/LayoutConfiguration.cs
Просмотреть файл

@ -30,7 +30,6 @@ namespace BuildXL.Utilities.Configuration.Mutable
TempDirectory = pathRemapper.Remap(template.TempDirectory);
BuildEngineDirectory = pathRemapper.Remap(template.BuildEngineDirectory);
FileContentTableFile = pathRemapper.Remap(template.FileContentTableFile);
SymlinkDefinitionFile = pathRemapper.Remap(template.SymlinkDefinitionFile);
SchedulerFileChangeTrackerFile = pathRemapper.Remap(template.SchedulerFileChangeTrackerFile);
IncrementalSchedulingStateFile = pathRemapper.Remap(template.IncrementalSchedulingStateFile);
FingerprintStoreDirectory = pathRemapper.Remap(template.FingerprintStoreDirectory);
@ -73,9 +72,6 @@ namespace BuildXL.Utilities.Configuration.Mutable
/// <inheritdoc />
public AbsolutePath FileContentTableFile { get; set; }
/// <inheritdoc />
public AbsolutePath SymlinkDefinitionFile { get; set; }
/// <inheritdoc />
public AbsolutePath SchedulerFileChangeTrackerFile { get; set; }

10
Public/Src/Utilities/Configuration/Mutable/ScheduleConfiguration.cs
Просмотреть файл

@ -49,8 +49,6 @@ namespace BuildXL.Utilities.Configuration.Mutable
ProcessRetries = 0;
UnsafeLazySymlinkCreation = false;
UnexpectedSymlinkAccessReportingMode = UnexpectedSymlinkAccessReportingMode.All;
StoreOutputsToCache = true;
// TODO: Fix me.
@ -121,8 +119,6 @@ namespace BuildXL.Utilities.Configuration.Mutable
UnsafeDisableGraphPostValidation = template.UnsafeDisableGraphPostValidation;
ProcessRetries = template.ProcessRetries;
UnsafeLazySymlinkCreation = template.UnsafeLazySymlinkCreation;
UnexpectedSymlinkAccessReportingMode = template.UnexpectedSymlinkAccessReportingMode;
StoreOutputsToCache = template.StoreOutputsToCache;
EnableLazyWriteFileMaterialization = template.EnableLazyWriteFileMaterialization;
@ -286,18 +282,12 @@ namespace BuildXL.Utilities.Configuration.Mutable
/// <inheritdoc />
public int ProcessRetries { get; set; }
/// <inheritdoc />
public bool UnsafeLazySymlinkCreation { get; set; }
/// <inheritdoc />
public bool EnableLazyWriteFileMaterialization { get; set; }
/// <inheritdoc />
public bool WriteIpcOutput { get; set; }
/// <inheritdoc />
public UnexpectedSymlinkAccessReportingMode UnexpectedSymlinkAccessReportingMode { get; set; }
/// <inheritdoc />
public bool StoreOutputsToCache { get; set; }

26
Public/Src/Utilities/Configuration/UnexpectedSymlinkAccessReportingMode.cs
Просмотреть файл

@ -1,26 +0,0 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.
namespace BuildXL.Utilities.Configuration
{
/// <summary>
/// Specifies when unexpected file accesses for symlink paths are reported
/// </summary>
public enum UnexpectedSymlinkAccessReportingMode : byte
{
/// <summary>
/// All outputs are required
/// </summary>
None,
/// <summary>
/// Report unexpected symlink accesses for executed processes only (i.e. not cached)
/// </summary>
ExecutionOnly,
/// <summary>
/// Report unexpected symlink accesses for executed and cached processes
/// </summary>
All,
}
}

11
Public/Src/Utilities/UnitTests/Utilities/PathMapSerializerTests.cs
Просмотреть файл

@ -19,11 +19,12 @@ namespace Test.BuildXL.Utilities
public void BasicTest()
{
var symlinkMap = new Dictionary<string, string>
{
[A("X","symlink1.lnk")] = A("X","target1"),
[A("X","symlink2.lnk")] = A("X","target2")
};
var file = GetFullPath("SymlinkDefinition");
{
[A("X","file1_cpy.txt")] = A("X","file1.txt"),
[A("X","file2_cpy.txt")] = A("X","file2.txt")
};
var file = GetFullPath("SomeFileMappingDefinition");
var pathMapSerializer = new PathMapSerializer(file);
foreach (var mapping in symlinkMap)

2
RunCheckInTests.cmd
Просмотреть файл

@ -218,7 +218,7 @@ endlocal && exit /b 0
set start=!time!
set stepName=Running SymLink Tests
call :StatusMessage !stepName!
call :RunBxl -Use RunCheckinTests %BUILDXL_ARGS% /unsafe_IgnoreProducingSymlinks+ /c:%ENLISTMENTROOT%\Public\Src\Sandbox\Windows\DetoursTests\SymLink1\config.dsc /TraceInfo:RunCheckinTests=Symlink /logsDirectory:%~dp0out\Logs\SymLinkTest\
call :RunBxl -Use RunCheckinTests %BUILDXL_ARGS% /c:%ENLISTMENTROOT%\Public\Src\Sandbox\Windows\DetoursTests\SymLink1\config.dsc /TraceInfo:RunCheckinTests=Symlink /logsDirectory:%~dp0out\Logs\SymLinkTest\
rmdir /s /q %ENLISTMENTROOT%\Public\Src\Sandbox\Windows\DetoursTests\SymLink1\Out
if !ERRORLEVEL! NEQ 0 (exit /b 1)
call :RecordStep "!stepName!" !start!

2
Shared/Scripts/PR/TestWithDeployment.cmd
Просмотреть файл

@ -71,7 +71,7 @@ echo Success: Administrative permissions confirmed. Running symlink tests...
set start=%time%
set stepName=Running SymLink Tests
call %PRROOT%\Utilities\StatusMessage.cmd %stepName%
call %PRROOT%\Utilities\RunBxl.cmd -Use RunCheckinTests %BUILDXL_ARGS% /unsafe_IgnoreProducingSymlinks+ /c:%ENLISTMENTROOT%\Public\Src\Sandbox\Windows\DetoursTests\SymLink1\config.dsc /viewer:disable /TraceInfo:RunCheckinTests=Symlink /logsDirectory:%~dp0out\Logs\SymLinkTest\
call %PRROOT%\Utilities\RunBxl.cmd -Use RunCheckinTests %BUILDXL_ARGS% /c:%ENLISTMENTROOT%\Public\Src\Sandbox\Windows\DetoursTests\SymLink1\config.dsc /viewer:disable /TraceInfo:RunCheckinTests=Symlink /logsDirectory:%~dp0out\Logs\SymLinkTest\
rmdir /s /q %ENLISTMENTROOT%\Public\Src\Sandbox\Windows\DetoursTests\SymLink1\Out
if %ERRORLEVEL% NEQ 0 (exit /b 1)
call :RecordStep "%stepName%" %start%