microsoft
/
BuildXL
зеркало из https://github.com/microsoft/BuildXL.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

Merged PR 644709: Explicitly report directory probes in detours 

Related work items: #1816341
This commit is contained in:
Pasindu Gunasekara 🍣 2022-02-05 23:57:23 +00:00
Родитель 6d8db05662
Коммит b565f0b5ee
26 изменённых файлов: 231 добавлений и 112 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

1
Private/Wdg/ExecutionLogSdk/ReportedFileAccessDescriptor.cs
Просмотреть файл

@ -48,6 +48,7 @@ namespace Tool.ExecutionLogSdk
shareMode: reportedFileAccess.ShareMode,
creationDisposition: reportedFileAccess.CreationDisposition,
flagsAndAttributes: reportedFileAccess.FlagsAndAttributes,
openedFileOrDirectoryAttribute: reportedFileAccess.OpenedFileOrDirectoryAttributes,
manifestPath: reportedFileAccess.ManifestPath,
path: reportedFileAccess.GetPath(pathTable),
enumeratePattern: reportedFileAccess.EnumeratePattern);

4
Public/Src/App/Bxl/Args.cs
Просмотреть файл

@ -471,6 +471,10 @@ namespace BuildXL
loggingConfiguration,
ideConfiguration,
frontEndConfiguration)),
OptionHandlerFactory.CreateBoolOption(
"explicitlyReportDirectoryProbes",
sign => sandboxConfiguration.ExplicitlyReportDirectoryProbes = sign
),
OptionHandlerFactory.CreateOption(
"exportGraph",
opt =>

6
Public/Src/App/Bxl/HelpText.cs
Просмотреть файл

@ -857,6 +857,12 @@ namespace BuildXL
Strings.HelpText_DisplayHelp_TreatAbsentDirectoryAsExistentUnderOpaque,
HelpLevel.Verbose);
hw.WriteOption(
"/explicitlyReportDirectoryProbes[+|-]",
Strings.HelpText_DisplayHelp_ExplicitlyReportDirectoryProbes,
HelpLevel.Verbose
);
#endregion
hw.WriteBanner(

3
Public/Src/App/Bxl/Strings.resx
Просмотреть файл

@ -1111,4 +1111,7 @@ Example: ad2d42d2ec5d2ca0c0b7ad65402d07c7ef40b91e</value>
<data name="HelpText_DisplayHelp_RunInSubst" xml:space="preserve">
<value>Improves path stability across potentially heterogeneous machines by internally mapping a source path (typically the source of the repo to build) into a drive letter. If the source path is not explicitly provided with /substSource, the location of the main config file is used. Only effective on Windows, in other platforms the option is ignored. Useful for dev cache.</value>
</data>
<data name="HelpText_DisplayHelp_ExplicitlyReportDirectoryProbes" xml:space="preserve">
<value>When enabled, detours will explicitly report directory probes. Note that this may result in an increased amount of DFAs.</value>
</data>
</root>

1
Public/Src/Engine/Dll/Tracing/Log.cs
Просмотреть файл

@ -2919,7 +2919,6 @@ If you can't update and need this feature after July 2018 please reach out to th
EventTask = (int)Tasks.Engine,
Message = "Pip {pipId} timed out remotely on step {step} on worker {worker}.")]
public abstract void PipTimedOutRemotely(LoggingContext context, string pipId, string step, string worker);
}
/// <summary>

51
Public/Src/Engine/Processes/FileAccessManifest.cs
Просмотреть файл

@ -35,6 +35,11 @@ namespace BuildXL.Processes
/// </summary>
private FileAccessManifestFlag m_fileAccessManifestFlag;
/// <summary>
/// Extra file access manifest flags.
/// </summary>
private FileAccessManifestExtraFlag m_fileAccessManifestExtraFlag;
/// <summary>
/// Name of semaphore for message count.
/// </summary>
@ -88,15 +93,20 @@ namespace BuildXL.Processes
EnforceAccessPoliciesOnDirectoryCreation = false;
IgnoreCreateProcessReport = false;
ProbeDirectorySymlinkAsDirectory = false;
ExplicitlyReportDirectoryProbes = false;
}
private bool GetFlag(FileAccessManifestFlag flag) => (m_fileAccessManifestFlag & flag) != 0;
private bool GetExtraFlag(FileAccessManifestExtraFlag flag) => (m_fileAccessManifestExtraFlag & flag) != 0;
/// <summary>
/// Gets file access manifest flag.
/// </summary>
internal FileAccessManifestFlag Flag => m_fileAccessManifestFlag;
internal FileAccessManifestExtraFlag ExtraFlag => m_fileAccessManifestExtraFlag;
private void SetFlag(FileAccessManifestFlag flag, bool value)
{
if (value)
@ -109,6 +119,18 @@ namespace BuildXL.Processes
}
}
private void SetExtraFlag(FileAccessManifestExtraFlag flag, bool value)
{
if (value)
{
m_fileAccessManifestExtraFlag |= flag;
}
else
{
m_fileAccessManifestExtraFlag &= ~flag;
}
}
/// <summary>
/// Flag indicating if the manifest tree block is sealed.
/// </summary>
@ -416,6 +438,15 @@ namespace BuildXL.Processes
set => SetFlag(FileAccessManifestFlag.QBuildIntegrated, value);
}
/// <summary>
/// When enabled, directory accesses will be explcitily reported by detours.
/// </summary>
public bool ExplicitlyReportDirectoryProbes
{
get => GetExtraFlag(FileAccessManifestExtraFlag.ExplicitlyReportDirectoryProbes);
set => SetExtraFlag(FileAccessManifestExtraFlag.ExplicitlyReportDirectoryProbes, value);
}
/// <summary>
/// A location for a file where Detours to log failure messages.
/// </summary>
@ -787,6 +818,16 @@ namespace BuildXL.Processes
writer.Write((uint)extraFlags);
}
private static FileAccessManifestExtraFlag ReadExtraFlagsBlock(BinaryReader reader)
{
#if DEBUG
CheckedCode.EnsureRead(reader, CheckedCode.ExtraFlags);
#endif
return (FileAccessManifestExtraFlag)reader.ReadUInt32();
}
private static void WritePipId(BinaryWriter writer, long pipId)
{
#if DEBUG
@ -961,7 +1002,7 @@ namespace BuildXL.Processes
WriteTranslationPathStrings(writer, DirectoryTranslator);
WriteErrorDumpLocation(writer, InternalDetoursErrorNotificationFile);
WriteFlagsBlock(writer, m_fileAccessManifestFlag);
WriteExtraFlagsBlock(writer, FileAccessManifestExtraFlag.None);
WriteExtraFlagsBlock(writer, m_fileAccessManifestExtraFlag);
WritePipId(writer, PipId);
WriteReportBlock(writer, setup);
WriteDllBlock(writer, setup);
@ -1011,6 +1052,7 @@ namespace BuildXL.Processes
WriteTranslationPathStrings(writer, DirectoryTranslator);
WriteErrorDumpLocation(writer, InternalDetoursErrorNotificationFile);
WriteFlagsBlock(writer, m_fileAccessManifestFlag);
WriteExtraFlagsBlock(writer, m_fileAccessManifestExtraFlag);
WritePipId(writer, PipId);
WriteChars(writer, m_messageCountSemaphoreName);
@ -1036,6 +1078,7 @@ namespace BuildXL.Processes
DirectoryTranslator directoryTranslator = ReadTranslationPathStrings(reader);
string internalDetoursErrorNotificationFile = ReadErrorDumpLocation(reader);
FileAccessManifestFlag fileAccessManifestFlag = ReadFlagsBlock(reader);
FileAccessManifestExtraFlag fileAccessManifestExtraFlag = ReadExtraFlagsBlock(reader);
long pipId = ReadPipId(reader);
string messageCountSemaphoreName = ReadChars(reader);
@ -1053,6 +1096,7 @@ namespace BuildXL.Processes
InternalDetoursErrorNotificationFile = internalDetoursErrorNotificationFile,
PipId = pipId,
m_fileAccessManifestFlag = fileAccessManifestFlag,
m_fileAccessManifestExtraFlag = fileAccessManifestExtraFlag,
m_sealedManifestTreeBlock = sealedManifestTreeBlock,
m_messageCountSemaphoreName = messageCountSemaphoreName
};
@ -1150,9 +1194,10 @@ namespace BuildXL.Processes
// CODESYNC: DataTypes.h
[Flags]
private enum FileAccessManifestExtraFlag
internal enum FileAccessManifestExtraFlag
{
None = 0,
NoneExtra = 0,
ExplicitlyReportDirectoryProbes = 0x1
}
private readonly struct FileAccessScope

17
Public/Src/Engine/Processes/ReportedFileAccess.cs
Просмотреть файл

@ -71,6 +71,10 @@ namespace BuildXL.Processes
/// <summary>
/// Computed attributes for this file or directory access.
/// </summary>
/// <remarks>
/// This is separate from <see cref="FlagsAndAttributes"/>. This represents the attributes artifact that is opened/created by the operation.
/// <see cref="FlagsAndAttributes"/> represents the attributes applied to this operation.
/// </remarks>
public readonly FlagsAndAttributes OpenedFileOrDirectoryAttributes;
/// <summary>
@ -226,7 +230,8 @@ namespace BuildXL.Processes
DesiredAccess,
ShareMode,
CreationDisposition,
FlagsAndAttributes,
FlagsAndAttributes,
OpenedFileOrDirectoryAttributes,
ManifestPath,
Path,
EnumeratePattern,
@ -248,6 +253,7 @@ namespace BuildXL.Processes
ShareMode,
CreationDisposition,
flagsAndAttributes,
OpenedFileOrDirectoryAttributes,
manifestPath,
path,
EnumeratePattern,
@ -289,6 +295,7 @@ namespace BuildXL.Processes
ShareMode == other.ShareMode &&
CreationDisposition == other.CreationDisposition &&
FlagsAndAttributes == other.FlagsAndAttributes &&
OpenedFileOrDirectoryAttributes == other.OpenedFileOrDirectoryAttributes &&
string.Equals(EnumeratePattern, other.EnumeratePattern, OperatingSystemHelper.PathComparison) &&
Method == other.Method;
}
@ -845,6 +852,7 @@ namespace BuildXL.Processes
writer.Write((uint)ShareMode);
writer.Write((uint)CreationDisposition);
writer.Write((uint)FlagsAndAttributes);
writer.Write((uint)OpenedFileOrDirectoryAttributes);
if (writePath != null)
{
@ -881,6 +889,7 @@ namespace BuildXL.Processes
shareMode: (ShareMode)reader.ReadUInt32(),
creationDisposition: (CreationDisposition)reader.ReadUInt32(),
flagsAndAttributes: (FlagsAndAttributes)reader.ReadUInt32(),
openedFileOrDirectoryAttribute: (FlagsAndAttributes)reader.ReadUInt32(),
manifestPath: readPath != null ? readPath(reader) : reader.ReadAbsolutePath(),
path: reader.ReadNullableString(),
enumeratePattern: reader.ReadNullableString(),
@ -898,7 +907,7 @@ namespace BuildXL.Processes
{
unchecked
{
return HashCodeHelper.Combine(
return HashCodeHelper.Combine(new int[] {
string.IsNullOrEmpty(Path) ? ManifestPath.GetHashCode() : OperatingSystemHelper.PathComparer.GetHashCode(Path),
string.IsNullOrEmpty(EnumeratePattern) ? 0 : OperatingSystemHelper.PathComparer.GetHashCode(EnumeratePattern),
Process != null ? (int)Process.ProcessId : 0,
@ -909,7 +918,9 @@ namespace BuildXL.Processes
(int)DesiredAccess,
(int)ShareMode,
(int)CreationDisposition,
(int)FlagsAndAttributes);
(int)FlagsAndAttributes,
(int)OpenedFileOrDirectoryAttributes
});
}
}

35
Public/Src/Engine/Processes/SandboxedProcessPipExecutor.cs
Просмотреть файл

@ -320,6 +320,7 @@ namespace BuildXL.Processes
IgnoreCreateProcessReport = m_sandboxConfig.UnsafeSandboxConfiguration.IgnoreCreateProcessReport,
ProbeDirectorySymlinkAsDirectory = m_sandboxConfig.UnsafeSandboxConfiguration.ProbeDirectorySymlinkAsDirectory,
SubstituteProcessExecutionInfo = shimInfo,
ExplicitlyReportDirectoryProbes = m_sandboxConfig.ExplicitlyReportDirectoryProbes,
};
if (!m_sandboxConfig.UnsafeSandboxConfiguration.MonitorFileAccesses)
@ -1457,6 +1458,7 @@ namespace BuildXL.Processes
ShareMode.FILE_SHARE_NONE,
isRead ? CreationDisposition.OPEN_ALWAYS : CreationDisposition.CREATE_ALWAYS,
FlagsAndAttributes.FILE_ATTRIBUTE_NORMAL,
openedFileOrDirectoryAttribute: FlagsAndAttributes.FILE_ATTRIBUTE_NORMAL,
manifestPath,
path: (path == manifestPath) ? null : path.ToString(m_pathTable),
enumeratePattern: null,
@ -2122,10 +2124,10 @@ namespace BuildXL.Processes
else
{
// The default policy is to only allow absent file probes. But if undeclared source reads are enabled, we allow any kind of read
var rootFileAccessPolicy = m_pip.AllowUndeclaredSourceReads ?
(FileAccessPolicy.AllowReadAlways | FileAccessPolicy.ReportAccess) :
// All directory enumerations are reported unless explicitly excluded
(FileAccessPolicy.AllowReadIfNonexistent | FileAccessPolicy.ReportDirectoryEnumerationAccess | FileAccessPolicy.ReportAccessIfNonexistent);
// All directory enumerations are reported unless explicitly excluded
var rootFileAccessPolicy = m_pip.AllowUndeclaredSourceReads
? (FileAccessPolicy.AllowReadAlways | FileAccessPolicy.ReportAccess)
: (FileAccessPolicy.AllowReadIfNonexistent | FileAccessPolicy.ReportDirectoryEnumerationAccess | FileAccessPolicy.ReportAccessIfNonexistent);
m_fileAccessManifest.AddScope(
AbsolutePath.Invalid,
@ -3915,7 +3917,7 @@ namespace BuildXL.Processes
var accessesUnsorted = accessesUnsortedWrapper.Instance;
foreach (KeyValuePair<AbsolutePath, CompactSet<ReportedFileAccess>> entry in accessesByPath)
{
bool? isDirectoryLocation = null;
bool isDirectoryLocation = false;
bool hasEnumeration = false;
bool isProbe = true;
@ -3939,18 +3941,15 @@ namespace BuildXL.Processes
bool isPathCandidateToBeOwnedByASharedOpaque = false;
foreach (var access in entry.Value)
{
// Detours reports a directory probe with a trailing backslash.
isDirectoryLocation =
// If the path is available and ends with a trailing backlash, we know that represents
// a directory
((isDirectoryLocation == null || isDirectoryLocation.Value) &&
access.Path != null && access.Path.EndsWith("\\", StringComparison.OrdinalIgnoreCase))
||
// If FILE_ATTRIBUTE_DIRECTORY flag is present, that means detours understood the operation
// as happening on a directory.
// TODO: this flag is not properly propagated for all detours operations.
access.FlagsAndAttributes.HasFlag(FlagsAndAttributes.FILE_ATTRIBUTE_DIRECTORY);
// If isDirectoryLocation was not already set, try one of the methods below
isDirectoryLocation |=
// If the path is available and ends with a trailing backlash, we know that represents a directory
(access.Path != null && access.Path.EndsWith("\\", StringComparison.OrdinalIgnoreCase))
// If FILE_ATTRIBUTE_DIRECTORY flag is present, that means detours understood the operation as happening on a directory. TODO: this flag is not properly propagated for all detours operations.
|| access.FlagsAndAttributes.HasFlag(FlagsAndAttributes.FILE_ATTRIBUTE_DIRECTORY)
// The IsOpenedHandleDirectory function uses attributes that report symlink/junction for directories as directories rather than files. This one is the most reliable.
|| (m_sandboxConfig.ExplicitlyReportDirectoryProbes && access.IsOpenedHandleDirectory());
// To treat the paths as file probes, all accesses to the path must be the probe access.
isProbe &= access.RequestedAccess == RequestedAccess.Probe;
@ -4033,7 +4032,7 @@ namespace BuildXL.Processes
observationFlags |= ObservationFlags.FileProbe;
}
if (isDirectoryLocation != null && isDirectoryLocation.Value)
if (isDirectoryLocation)
{
observationFlags |= ObservationFlags.DirectoryLocation;
}

4
Public/Src/Engine/Scheduler/Tracing/DumpPipLiteAnalysisUtilities.cs
Просмотреть файл

@ -468,7 +468,9 @@ namespace BuildXL.Scheduler.Tracing
ManifestPath = CreateString(data.ManifestPath, pathTable),
Process = data.Process.ProcessId,
ExplicitlyReported = data.ExplicitlyReported,
EnumeratePattern = data.EnumeratePattern
EnumeratePattern = data.EnumeratePattern,
IsOpenedHandleDirectory = data.IsOpenedHandleDirectory(),
OpenedFileOrDirectoryAttributes = CreateString(data.OpenedFileOrDirectoryAttributes)
};
}

12
Public/Src/Engine/Scheduler/Tracing/ExecutionLog.Events.cs
Просмотреть файл

@ -483,6 +483,12 @@ namespace BuildXL.Scheduler.Tracing
/// </summary>
public string RequiredKextVersionNumber;
/// <summary>
/// Whether /unsafe_explicitlyReportDirectoryProbes flag was passed to BuildXL. (disabled by default)
/// </summary>
public bool ExplicitlyReportDirectoryProbes;
/// <inheritdoc />
public ExecutionLogEventMetadata<BuildSessionConfigurationEventData> Metadata => ExecutionLogMetadata.BuildSessionConfiguration;
@ -514,6 +520,7 @@ namespace BuildXL.Scheduler.Tracing
PipWarningsPromotedToErrors = salts.PipWarningsPromotedToErrors;
ValidateDistribution = salts.ValidateDistribution;
RequiredKextVersionNumber = salts.RequiredKextVersionNumber;
ExplicitlyReportDirectoryProbes = salts.ExplicitlyReportDirectoryProbes;
}
/// <summary>
@ -546,7 +553,8 @@ namespace BuildXL.Scheduler.Tracing
normalizeReadTimestamps: NormalizeReadTimestamps,
validateDistribution: ValidateDistribution,
pipWarningsPromotedToErrors: PipWarningsPromotedToErrors,
requiredKextVersionNumber: RequiredKextVersionNumber
requiredKextVersionNumber: RequiredKextVersionNumber,
explicitlyReportDirectoryProbes: ExplicitlyReportDirectoryProbes
)
{
// Constructor appends EngineEnvironmentSettings.FingerprintSalt
@ -582,6 +590,7 @@ namespace BuildXL.Scheduler.Tracing
writer.Write(PipWarningsPromotedToErrors);
writer.Write(RequiredKextVersionNumber);
writer.Write(IgnoreFullReparsePointResolving);
writer.Write(ExplicitlyReportDirectoryProbes);
}
/// <inheritdoc />
@ -610,6 +619,7 @@ namespace BuildXL.Scheduler.Tracing
PipWarningsPromotedToErrors = reader.ReadBoolean();
RequiredKextVersionNumber = reader.ReadString();
IgnoreFullReparsePointResolving = reader.ReadBoolean();
ExplicitlyReportDirectoryProbes = reader.ReadBoolean();
}
}

2
Public/Src/Engine/Scheduler/Tracing/SerializedPip.cs
Просмотреть файл

@ -306,6 +306,8 @@ namespace BuildXL.Scheduler.Tracing
public uint Process { get; set; }
public bool ExplicitlyReported { get; set; }
public string EnumeratePattern { get; set; }
public bool IsOpenedHandleDirectory { get; set; }
public string OpenedFileOrDirectoryAttributes { get; set; }
}
public class ProcessDetouringStatusDataJson

33
Public/Src/Engine/UnitTests/Scheduler.IntegrationTest/BaselineTests.cs
Просмотреть файл

@ -6,7 +6,6 @@ using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using BuildXL.Pips;
using BuildXL.Pips.Builders;
using BuildXL.Pips.Operations;
@ -750,26 +749,40 @@ namespace IntegrationTest.BuildXL.Scheduler
/// </summary>
/// <remarks>See bug 1816341 for more context on what is being tested here.</remarks>
[Feature(Features.DirectoryProbe)]
[Fact]
public void ValidatePipUnderbuildWithDirectoryProbes()
[Theory]
[MemberData(nameof(TruthTable.GetTable), 1, MemberType = typeof(TruthTable))]
public void ValidatePipUnderbuildWithDirectoryProbes(bool explicitlyReport)
{
DirectoryArtifact directoryToProbe = CreateOutputDirectoryArtifact();
Directory.CreateDirectory(ArtifactToString(directoryToProbe));
Configuration.Sandbox.ExplicitlyReportDirectoryProbes = explicitlyReport;
Process pip = CreateAndSchedulePipBuilder(new Operation[]
DirectoryArtifact directoryToProbe = CreateUniqueDirectoryArtifact(root: Path.Combine(SourceRoot, "Foo"));
DirectoryArtifact dir = SealDirectory(directoryToProbe.Path.GetParent(Context.PathTable), SealDirectoryKind.SourceAllDirectories);
var builder = CreatePipBuilder(new[]
{
Operation.DirProbe(directoryToProbe),
Operation.WriteFile(CreateOutputFileArtifact())
}).Process;
});
builder.AddInputDirectory(dir);
var pip = SchedulePipBuilder(builder).Process;
RunScheduler().AssertCacheMiss(pip.PipId);
RunScheduler().AssertCacheHit(pip.PipId);
Directory.Delete(ArtifactToString(directoryToProbe));
// Since detours does not report directory probes, we see a cache hit here even though we expect a cache miss (causing an underbuild)
// See where the explicitReport variable is being calcuated in PolicyResult::CheckReadAccess to see why these are not reported
RunScheduler().AssertCacheHit(pip.PipId);
if (explicitlyReport)
{
RunScheduler().AssertCacheMiss(pip.PipId);
RunScheduler().AssertCacheHit(pip.PipId);
}
else
{
// Since detours does not report directory probes, we see a cache hit here even though we expect a cache miss (causing an underbuild)
// See where the explicitReport variable is being calcuated in PolicyResult::CheckReadAccess to see why these are not reported
RunScheduler().AssertCacheHit(pip.PipId);
}
}
[Feature(Features.DirectoryProbe)]

25
Public/Src/Pips/Dll/Graph/ExtraFingerprintSalts.cs
Просмотреть файл

@ -46,7 +46,8 @@ namespace BuildXL.Pips.Graph
normalizeReadTimestamps: true,
pipWarningsPromotedToErrors: false,
validateDistribution: false,
requiredKextVersionNumber: s_requiredKextVersionNumber
requiredKextVersionNumber: s_requiredKextVersionNumber,
explicitlyReportDirectoryProbes: false
);
/// <summary>
@ -90,7 +91,8 @@ namespace BuildXL.Pips.Graph
PipFingerprintingVersion.TwoPhaseV2,
fingerprintSalt,
searchPathToolsHash,
requiredKextVersionNumber: s_requiredKextVersionNumber
requiredKextVersionNumber: s_requiredKextVersionNumber,
config.Sandbox.ExplicitlyReportDirectoryProbes
)
{
}
@ -132,6 +134,7 @@ namespace BuildXL.Pips.Graph
/// <param name="fingerprintSalt">The extra, optional fingerprint salt.</param>
/// <param name="searchPathToolsHash">The extra, optional salt of path fragments of tool locations for tools using search path enumeration.</param>
/// <param name="requiredKextVersionNumber">The required kernel extension version number.</param>
/// <param name="explicitlyReportDirectoryProbes">Whether /unsafe_explicitlyReportDirectoryProbes was passed to BuildXL.</param>
public ExtraFingerprintSalts(
bool ignoreSetFileInformationByHandle,
bool ignoreZwRenameFileInformation,
@ -154,7 +157,8 @@ namespace BuildXL.Pips.Graph
PipFingerprintingVersion fingerprintVersion,
string fingerprintSalt,
ContentHash? searchPathToolsHash,
string requiredKextVersionNumber
string requiredKextVersionNumber,
bool explicitlyReportDirectoryProbes
)
{
IgnoreSetFileInformationByHandle = ignoreSetFileInformationByHandle;
@ -180,6 +184,7 @@ namespace BuildXL.Pips.Graph
PipWarningsPromotedToErrors = pipWarningsPromotedToErrors;
RequiredKextVersionNumber = requiredKextVersionNumber;
m_calculatedSaltsFingerprint = null;
ExplicitlyReportDirectoryProbes = explicitlyReportDirectoryProbes;
}
#pragma warning restore CS1572
@ -302,6 +307,11 @@ namespace BuildXL.Pips.Graph
/// </summary>
public string RequiredKextVersionNumber { get; set; }
/// <summary>
/// Whether /unsafe_explicitlyReportDirectoryProbes flag was passed to BuildXL. (disabled by default)
/// </summary>
public bool ExplicitlyReportDirectoryProbes { get; set; }
/// <nodoc />
public static bool operator ==(ExtraFingerprintSalts left, ExtraFingerprintSalts right)
{
@ -343,7 +353,8 @@ namespace BuildXL.Pips.Graph
&& SearchPathToolsHash.Value.Equals(SearchPathToolsHash.Value)
&& other.PipWarningsPromotedToErrors == PipWarningsPromotedToErrors
&& other.ValidateDistribution == ValidateDistribution
&& other.RequiredKextVersionNumber.Equals(RequiredKextVersionNumber);
&& other.RequiredKextVersionNumber.Equals(RequiredKextVersionNumber)
&& other.ExplicitlyReportDirectoryProbes.Equals(ExplicitlyReportDirectoryProbes);
}
/// <inheritdoc />
@ -379,6 +390,7 @@ namespace BuildXL.Pips.Graph
hashCode = (hashCode * 397) ^ ValidateDistribution.GetHashCode();
hashCode = (hashCode * 397) ^ (RequiredKextVersionNumber?.GetHashCode() ?? 0);
hashCode = (hashCode * 397) ^ IgnoreFullReparsePointResolving.GetHashCode();
hashCode = (hashCode * 397) ^ ExplicitlyReportDirectoryProbes.GetHashCode();
return hashCode;
}
@ -438,6 +450,11 @@ namespace BuildXL.Pips.Graph
fingerprinter.Add(nameof(RequiredKextVersionNumber), RequiredKextVersionNumber);
}
if (ExplicitlyReportDirectoryProbes)
{
fingerprinter.Add(nameof(ExplicitlyReportDirectoryProbes), 1);
}
fingerprinter.Add(nameof(FingerprintVersion), (int)FingerprintVersion);
}

19
Public/Src/Sandbox/MacOs/Interop/Sandbox/Data/SandboxedPip.hpp
Просмотреть файл

@ -42,29 +42,32 @@ public:
~SandboxedPip();
/*! Process id of the root process of this pip. */
inline const pid_t GetProcessId() const { return processId_; }
inline const pid_t GetProcessId() const { return processId_; }
/*! A unique identifier of this pip. */
inline const pipid_t GetPipId() const { return fam_.GetPipId()->PipId; }
inline const pipid_t GetPipId() const { return fam_.GetPipId()->PipId; }
/*! File access manifest record for this pip (to be used for checking file accesses) */
inline const PCManifestRecord GetManifestRecord() const { return fam_.GetUnixRootNode(); }
inline const PCManifestRecord GetManifestRecord() const { return fam_.GetUnixRootNode(); }
/*! File access manifest flags */
inline const FileAccessManifestFlag GetFamFlags() const { return fam_.GetFamFlags(); }
inline const FileAccessManifestFlag GetFamFlags() const { return fam_.GetFamFlags(); }
/*! File access manifest extra flags */
inline const FileAccessManifestExtraFlag GetFamExtraFlags() const { return fam_.GetFamExtraFlags(); }
/*!
* Returns the full path of the root process of this pip.
* The lenght of the path is stored in the 'length' argument because the path is not necessarily 0-terminated.
*/
inline const char* GetProcessPath(int *length) const { return fam_.GetProcessPath(length); }
inline const char* GetReportsPath(int *length) const { return fam_.GetReportsPath(length); }
inline const char* GetProcessPath(int *length) const { return fam_.GetProcessPath(length); }
inline const char* GetReportsPath(int *length) const { return fam_.GetReportsPath(length); }
/*! Number of currently active processes in this pip's process tree */
inline const int GetTreeSize() const { return processTreeCount_; }
inline const int GetTreeSize() const { return processTreeCount_; }
/*! When this returns true, child processes should not be tracked. */
bool AllowChildProcessesToBreakAway() const { return fam_.AllowChildProcessesToBreakAway(); }
bool AllowChildProcessesToBreakAway() const { return fam_.AllowChildProcessesToBreakAway(); }
#pragma mark Process Tree Tracking

2
Public/Src/Sandbox/MacOs/Interop/Sandbox/Handlers/AccessHandler.cpp
Просмотреть файл

@ -130,7 +130,7 @@ PolicyResult AccessHandler::PolicyForPath(const char *absolutePath)
log_error("Invalid policy cursor for path '%s'", absolutePath);
}
return PolicyResult(GetPip()->GetFamFlags(), absolutePath, cursor);
return PolicyResult(GetPip()->GetFamFlags(), GetPip()->GetFamExtraFlags(), absolutePath, cursor);
}
static bool is_prefix(const char *s1, const char *s2)

17
Public/Src/Sandbox/MacOs/Sandbox/Src/FileAccessManifest/FileAccessManifestParser.hpp
Просмотреть файл

@ -44,14 +44,15 @@ public:
bool init(const BYTE *payload, size_t payloadSize);
inline bool IsValid() const { return error_ == nullptr; }
inline bool HasErrors() const { return !IsValid(); }
inline const char* Error() const { return error_; }
inline PCManifestRecord GetManifestRootNode() const { return root_; }
inline PCManifestRecord GetUnixRootNode() const { return root_->BucketCount > 0 ? root_->GetChildRecord(0) : root_; }
inline PCManifestPipId GetPipId() const { return pipId_; }
inline FileAccessManifestFlag GetFamFlags() const { return static_cast<FileAccessManifestFlag>(flags_->Flags); }
inline bool AllowChildProcessesToBreakAway() const { return manifestChildProcessesToBreakAwayFromJob_->Count > 0; }
inline bool IsValid() const { return error_ == nullptr; }
inline bool HasErrors() const { return !IsValid(); }
inline const char* Error() const { return error_; }
inline PCManifestRecord GetManifestRootNode() const { return root_; }
inline PCManifestRecord GetUnixRootNode() const { return root_->BucketCount > 0 ? root_->GetChildRecord(0) : root_; }
inline PCManifestPipId GetPipId() const { return pipId_; }
inline FileAccessManifestFlag GetFamFlags() const { return static_cast<FileAccessManifestFlag>(flags_->Flags); }
inline FileAccessManifestExtraFlag GetFamExtraFlags() const { return static_cast<FileAccessManifestExtraFlag>(extraFlags_->ExtraFlags); }
inline bool AllowChildProcessesToBreakAway() const { return manifestChildProcessesToBreakAwayFromJob_->Count > 0; }
inline const char* GetReportsPath(int *length) const
{
*length = report_->Size;

2
Public/Src/Sandbox/MacOs/Sandbox/Src/Kauth/AccessHandler.cpp
Просмотреть файл

@ -158,7 +158,7 @@ PolicyResult AccessHandler::PolicyForPath(const char *absolutePath)
log_error("Invalid policy cursor for path '%s'", absolutePath);
}
return PolicyResult(GetPip()->getFamFlags(), absolutePath, cursor);
return PolicyResult(GetPip()->getFamFlags(), GetPip()->getFamExtraFlags(), absolutePath, cursor);
}
#define VATTR_GET(vp, ctx, vap, attr, errno, result) \

11
Public/Src/Sandbox/MacOs/Sandbox/Src/SandboxedPip.hpp
Просмотреть файл

@ -99,19 +99,22 @@ public:
pipid_t getPipId() const { return fam_.GetPipId()->PipId; }
/*! File access manifest record for this pip (to be used for checking file accesses) */
PCManifestRecord getManifestRecord() const { return fam_.GetUnixRootNode(); }
PCManifestRecord getManifestRecord() const { return fam_.GetUnixRootNode(); }
/*! File access manifest flags */
FileAccessManifestFlag getFamFlags() const { return fam_.GetFamFlags(); }
FileAccessManifestFlag getFamFlags() const { return fam_.GetFamFlags(); }
/*! File access manifest extra flags */
FileAccessManifestExtraFlag getFamExtraFlags() const { return fam_.GetFamExtraFlags(); }
/*! When this returns true, child processes should not be tracked. */
bool AllowChildProcessesToBreakAway() const { return fam_.AllowChildProcessesToBreakAway(); }
bool AllowChildProcessesToBreakAway() const { return fam_.AllowChildProcessesToBreakAway(); }
/*!
* Returns the full path of the root process of this pip.
* The lenght of the path is stored in the 'length' argument because the path is not necessarily 0-terminated.
*/
const char* getProcessPath(int *length) const { return fam_.GetProcessPath(length); }
const char* getProcessPath(int *length) const { return fam_.GetProcessPath(length); }
/*! Various counters. */
AllCounters* Counters() { return &counters_; }

13
Public/Src/Sandbox/Windows/DetoursServices/DataTypes.h
Просмотреть файл

@ -90,10 +90,21 @@ inline bool CheckReportAnyAccess(FileAccessManifestFlag flags, bool accessDenied
//
// Keep this in sync with the C# version declared in FileAccessManifest.cs
//
#define FOR_ALL_FAM_EXTRA_FLAGS(m) \
m(NoneExtra, 0x0) \
m(ExplicitlyReportDirectoryProbes, 0x1)
enum class FileAccessManifestExtraFlag {
None = 0x0,
FOR_ALL_FAM_EXTRA_FLAGS(GEN_FAM_FLAG_ENUM_NAME_VALUE)
};
DEFINE_ENUM_FLAG_OPERATORS(FileAccessManifestExtraFlag)
// Checker function for FileAccessManifestFlagExtra enum.
#define GEN_FAM_EXTRA_FLAG_CHECKER(flag_name, flag_value) \
inline bool Check##flag_name(FileAccessManifestExtraFlag flags) { return (flags & FileAccessManifestExtraFlag::flag_name) != FileAccessManifestExtraFlag::NoneExtra; }
FOR_ALL_FAM_EXTRA_FLAGS(GEN_FAM_EXTRA_FLAG_CHECKER)
//
// CODESYNC: Keep this in sync with the C# version declared in Public\Src\Engine\Processes\FileAccessPolicy.cs
//

44
Public/Src/Sandbox/Windows/DetoursServices/DetouredFunctions.cpp
Просмотреть файл

@ -910,42 +910,6 @@ static bool IsHandleOfDirectoryAndGetAttributes(_In_ HANDLE hFile, _In_ bool tre
return isDirectory;
}
/// <summary>
/// Checks if a handle or a path points to a directory.
/// </summary>
/// <remarks>
/// This function first tries to get attributes via the given handle, and, if failed (e.g., the handle has
/// missing permisisons or is <code>INVALID_HANDLE_VALUE</code>), the function calls <code>GetFileAttributes</code> on the path.
/// </remarks>
static bool IsHandleOrPathToDirectory(_In_ HANDLE hFile, _In_ LPCWSTR lpFileName, bool treatReparsePointAsFile)
{
bool isHandleOfDirectory;
return hFile == INVALID_HANDLE_VALUE || !TryCheckHandleOfDirectory(hFile, treatReparsePointAsFile, isHandleOfDirectory)
? IsPathToDirectory(lpFileName, treatReparsePointAsFile)
: isHandleOfDirectory;
}
/// <summary>
/// Checks if a handle or a path points to a directory but treat reparse point according to the desired accesses and flags.
/// </summary>
static bool IsHandleOrPathToDirectory(
_In_ HANDLE hFile,
_In_ LPCWSTR lpFileName,
_In_ DWORD dwDesiredAccess,
_In_ DWORD dwFlagsAndAttributes,
_In_ PolicyResult* policyResult)
{
bool treatReparsePointAsFile =
!ProbeDirectorySymlinkAsDirectory() // It is set globally that directory symlink probe should not be treated as directory.
&& WantsProbeOnlyAccess(dwDesiredAccess) // Probe-only access.
&& FlagsAndAttributesContainReparsePointFlag(dwFlagsAndAttributes) // Open attribute contains reparse point flag.
&& (policyResult == nullptr // No policy is specified,
|| !policyResult->TreatDirectorySymlinkAsDirectory()); // or policy does not mandate directory symlink to be treated as directory.
return IsHandleOrPathToDirectory(hFile, lpFileName, treatReparsePointAsFile);
}
/// <summary>
/// Gets the file attributes for a given path. Returns false if no valid attributes were found or if a NULL path is provided.
/// </summary>
@ -1034,7 +998,7 @@ static bool IsHandleOrPathToDirectory(
GetFileAttributesByPath(lpFileName, /*ref*/ fileOrDirectoryAttribute);
}
return IsDirectoryFromAttributes(/*ref*/ fileOrDirectoryAttribute, treatReparsePointAsFile);
return IsDirectoryFromAttributes(fileOrDirectoryAttribute, treatReparsePointAsFile);
}
/// <summary>
@ -6344,7 +6308,7 @@ NTSTATUS NTAPI Detoured_ZwCreateFile(
CheckIfNtCreateDispositionImpliesWriteOrDelete(CreateDisposition) ||
CheckIfNtCreateMayDeleteFile(CreateOptions, DesiredAccess)) &&
// Force directory checking using path, instead of handle, because the value of *FileHandle is still undefined, i.e., neither valid nor not valid.
!IsHandleOrPathToDirectory(INVALID_HANDLE_VALUE, path.GetPathString(), opContext.DesiredAccess, CreateOptions, &policyResult))
!IsHandleOrPathToDirectory(INVALID_HANDLE_VALUE, path.GetPathString(), opContext.DesiredAccess, CreateOptions, &policyResult, /*ref*/opContext.OpenedFileOrDirectoryAttributes))
{
error = GetLastError();
accessCheck = policyResult.CheckWriteAccess();
@ -6649,7 +6613,7 @@ NTSTATUS NTAPI Detoured_NtCreateFile(
CheckIfNtCreateDispositionImpliesWriteOrDelete(CreateDisposition) ||
CheckIfNtCreateMayDeleteFile(CreateOptions, DesiredAccess)) &&
// Force directory checking using path, instead of handle, because the value of *FileHandle is still undefined, i.e., neither valid nor not valid.
!IsHandleOrPathToDirectory(INVALID_HANDLE_VALUE, path.GetPathString(), opContext.DesiredAccess, CreateOptions, &policyResult))
!IsHandleOrPathToDirectory(INVALID_HANDLE_VALUE, path.GetPathString(), opContext.DesiredAccess, CreateOptions, &policyResult, /*ref*/opContext.OpenedFileOrDirectoryAttributes))
{
error = GetLastError();
accessCheck = policyResult.CheckWriteAccess();
@ -6937,7 +6901,7 @@ NTSTATUS NTAPI Detoured_ZwOpenFile(
CheckIfNtCreateDispositionImpliesWriteOrDelete(FILE_OPEN) ||
CheckIfNtCreateMayDeleteFile(OpenOptions, DesiredAccess)) &&
// Force directory checking using path, instead of handle, because the value of *FileHandle is still undefined, i.e., neither valid nor not valid.
!IsHandleOrPathToDirectory(INVALID_HANDLE_VALUE, path.GetPathString(), opContext.DesiredAccess, OpenOptions, &policyResult))
!IsHandleOrPathToDirectory(INVALID_HANDLE_VALUE, path.GetPathString(), opContext.DesiredAccess, OpenOptions, &policyResult, /*ref*/opContext.OpenedFileOrDirectoryAttributes))
{
accessCheck = policyResult.CheckWriteAccess();

6
Public/Src/Sandbox/Windows/DetoursServices/FileAccessHelpers.h
Просмотреть файл

@ -338,6 +338,12 @@ inline bool Should##flag_name() { return Check##flag_name(g_fileAccessManifestFl
FOR_ALL_FAM_FLAGS(GEN_CHECK_GLOBAL_FAM_FLAG)
inline bool ReportAnyAccess(bool accessDenied) { return CheckReportAnyAccess(g_fileAccessManifestFlags, accessDenied); }
#define GEN_CHECK_GLOBAL_FAM_EXTRA_FLAG(flag_name, flag_value) \
inline bool flag_name() { return Check##flag_name(g_fileAccessManifestExtraFlags); } \
inline bool Should##flag_name() { return Check##flag_name(g_fileAccessManifestExtraFlags); }
FOR_ALL_FAM_EXTRA_FLAGS(GEN_CHECK_GLOBAL_FAM_EXTRA_FLAG)
inline LPCTSTR InternalDetoursErrorNotificationFile()
{
return g_internalDetoursErrorNotificationFile;

12
Public/Src/Sandbox/Windows/DetoursServices/PolicyResult.h
Просмотреть файл

@ -111,15 +111,16 @@ public:
private:
FileAccessManifestFlag m_famFlag;
FileAccessManifestExtraFlag m_famExtraFlag;
public:
PolicyResult(FileAccessManifestFlag famFlag)
: m_famFlag(famFlag), m_isIndeterminate(true)
PolicyResult(FileAccessManifestFlag famFlag, FileAccessManifestExtraFlag famExtraFlag)
: m_famFlag(famFlag), m_isIndeterminate(true), m_famExtraFlag(famExtraFlag)
{
}
PolicyResult(FileAccessManifestFlag famFlag, CanonicalizedPathType path, PolicySearchCursor cursor)
: PolicyResult(famFlag)
PolicyResult(FileAccessManifestFlag famFlag, FileAccessManifestExtraFlag famExtraFlag, CanonicalizedPathType path, PolicySearchCursor cursor)
: PolicyResult(famFlag, famExtraFlag)
{
Initialize(path, cursor);
}
@ -128,6 +129,9 @@ public:
FOR_ALL_FAM_FLAGS(GEN_CHECK_FAM_FLAG_FUNC)
inline bool ReportAnyAccess(bool accessDenied) const { return CheckReportAnyAccess(m_famFlag, accessDenied); }
#define GEN_CHECK_FAM_EXTRA_FLAG_FUNC(flag_name, flag_value) inline bool flag_name() const { return Check##flag_name(m_famExtraFlag); }
FOR_ALL_FAM_EXTRA_FLAGS(GEN_CHECK_FAM_EXTRA_FLAG_FUNC)
#endif // _WIN32
// Performs an access check for a read-access, based on dynamically-observed read context (existence, etc.)

4
Public/Src/Sandbox/Windows/DetoursServices/PolicyResult_common.cpp
Просмотреть файл

@ -98,7 +98,9 @@ AccessCheckResult PolicyResult::CheckReadAccess(RequestedReadAccess readAccessRe
? ResultAction::Allow
: (FailUnexpectedFileAccesses() ? ResultAction::Deny : ResultAction::Warn);
bool explicitReport = !context.OpenedDirectory &&
// When ExplicitlyReportDirectoryProbes is set, if the request access is a probe then explicitly report it
// When ExplicitlyReportDirectoryProbes is not set, do not explicitly report any operations on directories (context.OpenedDirectory)
bool explicitReport = ((ExplicitlyReportDirectoryProbes() && accessRequested == RequestedAccess::Probe) || !context.OpenedDirectory) &&
((exists && ((m_policy & FileAccessPolicy::FileAccessPolicy_ReportAccessIfExistent) != 0)) ||
(!exists && ((m_policy & FileAccessPolicy::FileAccessPolicy_ReportAccessIfNonExistent) != 0)));

8
Public/Src/Utilities/Configuration/ISandboxConfiguration.cs
Просмотреть файл

@ -277,5 +277,13 @@ namespace BuildXL.Utilities.Configuration
/// This list is only considered when <see cref="IUnsafeSandboxConfiguration.IgnoreFullReparsePointResolving"/> is set to true and<see cref="IUnsafeSandboxConfiguration.EnableFullReparsePointResolving"/> is set to false.
/// </summary>
IReadOnlyList<AbsolutePath> DirectoriesToEnableFullReparsePointParsing { get; }
/// <summary>
/// Enable explicitly reporting directory probes from detours to help avoid underbuilds caused by unreported directory probes.
/// </summary>
/// <remarks>
/// This is an experimental feature, enabling this option may result in more DFAs on a build.
/// </remarks>
public bool ExplicitlyReportDirectoryProbes { get; }
}
}

2
Public/Src/Utilities/Configuration/IUnsafeSandboxConfiguration.cs
Просмотреть файл

@ -293,7 +293,7 @@ namespace BuildXL.Utilities.Configuration
ProbeDirectorySymlinkAsDirectory = reader.ReadBoolean(),
IgnoreFullReparsePointResolving = reader.ReadBoolean(),
SkipFlaggingSharedOpaqueOutputs = reader.ReadBoolean() ? (bool?)reader.ReadBoolean() : null,
EnableFullReparsePointResolving = reader.ReadBoolean() ? (bool?) reader.ReadBoolean() : null,
EnableFullReparsePointResolving = reader.ReadBoolean() ? (bool?)reader.ReadBoolean() : null,
};
}

9
Public/Src/Utilities/Configuration/Mutable/SandboxConfiguration.cs
Просмотреть файл

@ -52,7 +52,8 @@ namespace BuildXL.Utilities.Configuration.Mutable
PreserveOutputsForIncrementalTool = false;
GlobalUnsafePassthroughEnvironmentVariables = new List<string>();
VmConcurrencyLimit = 0;
DirectoriesToEnableFullReparsePointParsing = new List<AbsolutePath>();
DirectoriesToEnableFullReparsePointParsing = new List<AbsolutePath>();
ExplicitlyReportDirectoryProbes = false;
}
/// <nodoc />
@ -102,7 +103,8 @@ namespace BuildXL.Utilities.Configuration.Mutable
PreserveOutputsForIncrementalTool = template.PreserveOutputsForIncrementalTool;
GlobalUnsafePassthroughEnvironmentVariables = new List<string>(template.GlobalUnsafePassthroughEnvironmentVariables);
VmConcurrencyLimit = template.VmConcurrencyLimit;
DirectoriesToEnableFullReparsePointParsing = pathRemapper.Remap(template.DirectoriesToEnableFullReparsePointParsing);
DirectoriesToEnableFullReparsePointParsing = pathRemapper.Remap(template.DirectoriesToEnableFullReparsePointParsing);
ExplicitlyReportDirectoryProbes = template.ExplicitlyReportDirectoryProbes;
}
/// <inheritdoc />
@ -270,5 +272,8 @@ namespace BuildXL.Utilities.Configuration.Mutable
/// <inheritdoc />
IReadOnlyList<AbsolutePath> ISandboxConfiguration.DirectoriesToEnableFullReparsePointParsing => DirectoriesToEnableFullReparsePointParsing;
/// <inheritdoc />
public bool ExplicitlyReportDirectoryProbes { get; set; }
}
}