Populate RocksDB with all XLG events (#671)

AB#1574994
AB#1574153
AB#1574993

Populated the rest of the XLG events, and updated the protobuf file accordingly. Now the DB is written to immediately rather than let the events buffer up in memory first.

Public/Src/Tools/Execution.Analyzer/Analyzers.Core/XLGPlusPlus/BXLInvocationAnalyzer.cs

@@ -3,12 +3,8 @@
 
 using System;
 using System.IO;
-using System.Text;
 using BuildXL.Analyzers.Core.XLGPlusPlus;
-using BuildXL.Scheduler.Tracing;
 using BuildXL.ToolSupport;
-using BuildXL.Utilities;
-using Google.Protobuf;
 
 namespace BuildXL.Execution.Analyzer
 {
@@ -62,7 +58,7 @@ namespace BuildXL.Execution.Analyzer
         private static void WriteDominoInvocationHelp(HelpWriter writer)
         {
             writer.WriteBanner("BXL Invocation \"Analyzer\"");
-            writer.WriteModeOption(nameof(AnalysisMode.BXLInvocationXLG), "Gets and outputs information related to BXL invocation events from the database.");
+            writer.WriteModeOption(nameof(AnalysisMode.BXLInvocationXLG), "Gets and outputs information related to BXL invocation events from RocksDB.");
             writer.WriteOption("inputDir", "Required. The directory to read the RocksDB database from", shortName: "i");
             writer.WriteOption("outputFile", "Required. The file where to write the results", shortName: "o");
         }
@@ -81,9 +77,9 @@ namespace BuildXL.Execution.Analyzer
         /// <inheritdoc/>
         public override int Analyze()
         {
-            var dataStore = new XLGppDataStore(storeDirectory: InputDirPath);
+            var dataStore = new XldbDataStore(storeDirectory: InputDirPath);
 
-            File.WriteAllLines(OutputFilePath, dataStore.GetEventsByType((int)ExecutionEventId.DominoInvocation));
+            File.WriteAllLines(OutputFilePath, dataStore.GetEventsByType(Xldb.ExecutionEventId.DominoInvocation));
 
             return 0;
         }

Public/Src/Tools/Execution.Analyzer/Analyzers.Core/XLGPlusPlus/Events.proto

@@ -3,15 +3,26 @@
 
 syntax = "proto3";
 
+import "tools/google/protobuf/timestamp.proto";
+import "tools/google/protobuf/duration.proto";
+
 package BuildXL.Execution.Analyzer;
 
-option csharp_namespace = "BuildXL.Execution.Analyzer";
+option csharp_namespace = "BuildXL.Execution.Analyzer.Xldb";
+
+//
+//  Generic Event Info (key, count, etc)
+//
+
+// Message that keeps track of total number of events processed
+message EventCount{
+    uint32 Value = 1;
+}
 
 // Event query should be of this format. It also serves as the key to the DB
 message EventTypeQuery{
-    
     // ID of the Event - required
-    int32 EventTypeID = 1; 
+    ExecutionEventId EventTypeID = 1; 
     
     // Other ideas to keep in EventTypeQuery were as follows:
     // string (or int64) Path (or hash of path) = 2;
@@ -22,31 +33,837 @@ message EventTypeQuery{
     string UUID = 99;   
 }
 
-// The BXL Invocation Event message
+//
+// Enums and Helper Structs
+//
+
+// Note that enum values use C++ scoping rules, meaning that enum values are siblings of their type, 
+// not children of it. Therefore, enum values must be unique within "BuildXL.Execution.Analyzer", not 
+// just within a particular enum. If there are conflicting enum values, append the name of the enum in front.
+// ie: ChooseWorkerCpu ---> PipExecutionStep_ChooseWorkerCpu
+
+enum ExecutionEventId{
+    FileArtifactContentDecided = 0;
+    WorkerList = 1;
+    ExecutionEventId_PipExecutionPerformance = 2;
+    DirectoryMembershipHashed = 3;
+
+    // Deprecated in favor of ProcessFingerprintComputation (enum 10)
+    ObservedInputs = 4; 
+
+    ProcessExecutionMonitoringReported = 5;
+    ExecutionEventId_ExtraEventDataReported = 6;
+    DependencyViolationReported = 7;
+    PipExecutionStepPerformanceReported = 8;
+    ResourceUsageReported = 9;
+    ProcessFingerprintComputation = 10;
+    PipCacheMiss = 11;
+    PipExecutionDirectoryOutputs = 12;
+    DominoInvocation = 13;
+}
+
+enum PipOutputOrigin{
+    Produced = 0;
+    UpToDate = 1;
+    DeployedFromCache = 2;
+    NotMaterialized = 3;
+}
+
+enum PipExecutionStep{
+    PipExecutionStep_None = 0;
+    Start = 1;
+    Cancel = 2;
+    SkipDueToFailedDependencies = 3;
+    CheckIncrementalSkip = 4;
+    MaterializeInputs = 5;
+    MaterializeOutputs = 6;
+    ExecuteNonProcessPip = 7;
+    PipExecutionStep_CacheLookup = 8;
+    RunFromCache = 9;
+    ExecuteProcess = 10;
+    PostProcess = 11;
+    HandleResult = 12;
+    PipExecutionStep_ChooseWorkerCpu = 13;
+    PipExecutionStep_ChooseWorkerCacheLookup = 14;
+    Done = 15;
+}
+
+enum WorkDispatcher_DispatcherKind{
+    WorkDispatcherKind_None = 0;
+    IO = 1;
+    WorkDispatcher_DispatcherKind_CPU = 2;
+    Light = 3;
+    WorkDispatcher_DispatcherKind_CacheLookup = 4;
+    WorkDispatcher_DispatcherKind_ChooseWorkerCpu = 5;
+    WorkDispatcher_DispatcherKind_ChooseWorkerCacheLookup = 6;
+    Materialize = 7;
+}
+
+enum FileMonitoringViolationAnalyzer_DependencyViolationType{
+    DoubleWrite = 0;
+    ReadRace = 1;
+    UndeclaredOrderedRead = 2;
+    MissingSourceDependency = 3;
+    UndeclaredReadCycle = 4;
+    UndeclaredOutput = 5;
+    ReadUndeclaredOutput = 6;
+    WriteInSourceSealDirectory = 7;
+    WriteInUndeclaredSourceRead = 8;
+    WriteInExistingFile = 9;
+    WriteOnAbsentPathProbe = 10;
+    AbsentPathProbeUnderUndeclaredOpaque = 11;
+    WriteToTempPathInsideSharedOpaque = 12;
+}
+
+enum FileMonitoringViolationAnalyzer_AccessLevel{
+    FileMonitoringViolationAnalyzer_AccessLevel_Read = 0;
+    FileMonitoringViolationAnalyzer_AccessLevel_Write = 1;
+}
+
+enum PipCacheMissType{
+    Invalid = 0;
+    MissForDescriptorsDueToStrongFingerprints = 53;
+    MissForDescriptorsDueToWeakFingerprints = 54;
+    MissForDescriptorsDueToArtificialMissOptions = 55;
+    MissForCacheEntry = 56;
+    MissDueToInvalidDescriptors = 57;
+    MissForProcessMetadata = 58;
+    MissForProcessMetadataFromHistoricMetadata = 59;
+    MissForProcessOutputContent = 60;
+    Hit = 51;
+    MissForProcessConfiguredUncacheable = 61;
+}
+
+enum ExecutionSampler_LimitingResource{
+    GraphShape = 0;
+    ExecutionSampler_LimitingResource_CPU = 1;
+    Disk = 2;
+    Memory = 3;
+    ConcurrencyLimit = 4;
+    ProjectedMemory = 5;
+    Semaphore = 6;
+    Other = 7;
+}
+
+enum CreationDisposition{
+    // First enum value in proto must be 0
+    CreationDisposition_DEFAULT_ZERO = 0;
+    CREATE_NEW = 1;
+    CREATE_ALWAYS = 2;
+    OPEN_EXISTING = 3;
+    OPEN_ALWAYS = 4;
+    TRUNCATE_EXISTING = 5;
+}
+
+enum DesiredAccess{
+    // First enum value in proto must be 0
+    DesiredAccess_DEFAULT_ZERO = 0;
+    DELETE = 0x00010000;
+    SYNCHRONIZE = 0x00100000;
+    FILE_APPEND_DATA = 0x00000004;
+    FILE_WRITE_EA = 0x00000010;
+    FILE_WRITE_ATTRIBUTES = 0x00000100;
+    FILE_WRITE_DATA = 0x00000002;
+    GENERIC_ALL = 0x10000000;
+    GENERIC_EXECUTE = 0x20000000;
+    GENERIC_WRITE = 0x40000000;
+    GENERIC_READ = -2147483648;
+}
+
+enum FlagsAndAttributes{
+    // First enum value in proto must be 0
+    FlagsAndAttributes_DEFAULT_ZERO = 0;
+    FILE_ATTRIBUTE_READONLY = 0x00000001;
+    FILE_ATTRIBUTE_HIDDEN = 0x00000002;
+    FILE_ATTRIBUTE_SYSTEM = 0x00000004;
+    FILE_ATTRIBUTE_ARCHIVE = 0x00000020;
+    FILE_ATTRIBUTE_NORMAL = 0x00000080;
+    FILE_ATTRIBUTE_TEMPORARY = 0x00000100;
+    FILE_ATTRIBUTE_OFFLINE = 0x00001000;
+    FILE_ATTRIBUTE_ENCRYPED = 0x00004000;
+    FILE_FLAG_OPEN_NO_RECALL = 0x00100000;
+    FILE_FLAG_OPEN_REPARSE_POINT = 0x00200000;
+    FILE_FLAG_SESSION_AWARE = 0x00800000;
+    FILE_FLAG_POSIX_SEMANTICS = 0x01000000;
+    FILE_FLAG_BACKUP_SEMANTICS = 0x02000000;
+    FILE_FLAG_DELETE_ON_CLOSE = 0x04000000;
+    FILE_FLAG_SEQUENTIAL_SCAN = 0x08000000;
+    FILE_FLAG_RANDOM_ACCESS = 0x10000000;
+    FILE_FLAG_NO_BUFFERING = 0x20000000;
+    FILE_FLAG_OVERLAPPED = 0x40000000;
+    FILE_FLAG_WRITE_THROUGH = -2147483648;
+}
+
+enum ShareMode{
+    FILE_SHARE_NONE = 0x0;
+    FILE_SHARE_READ = 0x1;
+    FILE_SHARE_WRITE = 0x2;
+    FILE_SHARE_DELETE = 0x4;
+}
+
+enum FileAccessStatus{
+    FileAccessStatus_None = 0;
+    Allowed = 1;
+    Denied = 2;
+    CannotDetermineByPolicy = 3;
+}
+
+enum FileAccessStatusMethod{
+    PolicyBased = 0;
+    FileExistenceBased = 1;
+}
+
+enum RequestedAccess{
+    RequestedAccess_None = 0;
+    RequestedAccess_Read = 1;
+    RequestedAccess_Write = 2;
+    Probe = 4;
+    Enumerate = 8;
+    EnumerationProbe = 16;
+    // Read | Write = 1 | 2 = 3
+    ReadWrite = 3; 
+    // Read | Write | Probe | Enumerate | EnumerationProbe
+    All = 31;
+}
+
+enum ReportedFileOperation{
+    Unknown = 0;
+    CreateFile = 1;
+    GetFileAttributes = 2;
+    GetFileAttributesEx = 3;
+    Process = 4;
+    FindFirstFileEx = 5;
+    FindNextFile = 6;
+    CreateDirectory = 7;
+    DeleteFile = 8;
+    MoveFileSource = 9;
+    MoveFileDestination = 10;
+    SetFileInformationByHandleSource = 11;
+    SetFileInformationByHandleDest = 12;
+    ZwSetRenameInformationFileSource = 13;
+    ZwSetRenameInformationFileDest = 14;
+    ZwSetLinkInformationFile = 15;
+    ZwSetDispositionInformationFile = 16;
+    ZwSetModeInformationFile = 17;
+    ZwSetFileNameInformationFileSource = 18;
+    ZwSetFileNameInformationFileDest = 19;
+    CopyFileSource = 20;
+    CopyFileDestination = 21;
+    CreateHardLinkSource = 22;
+    CreateHardLinkDestination = 23;
+    RemoveDirectory = 24;
+    NtQueryDirectoryFile = 25;
+    ZwQueryDirectoryFile = 26;
+    NtCreateFile = 27;
+    ZwCreateFile = 28;
+    ZwOpenFile = 29;
+    ChangedReadWriteToReadAccess = 30;
+    FirstAllowWriteCheckInProcess = 31;
+    ReparsePointTarget = 32;
+    CreateSymbolicLinkSource = 33;
+    MoveFileWithProgressSource = 34;
+    MoveFileWithProgressDest = 35;
+    MultipleOperations = 36;
+    ProcessExit = 37;
+    MacLookup = 38;
+    MacReadlink = 39;
+    MacVNodeCreate = 40;
+    KAuthMoveSource = 41;
+    KAuthMoveDest = 42;
+    KAuthCreateHardlinkSource = 43;
+    KAuthCreateHardlinkDest = 44;
+    KAuthCopySource = 45;
+    KAuthCopyDest = 46;
+    KAuthDeleteDir = 47;
+    KAuthDeleteFile = 48;
+    KAuthOpenDir = 49;
+    KAuthReadFile = 50;
+    KAuthCreateDir = 51;
+    KAuthWriteFile = 52;
+    KAuthClose = 53;
+    KAuthCloseModified = 54;
+    KAuthGetAttributes = 55;
+    KAuthVNodeExecute = 56;
+    KAuthVNodeWrite = 57;
+    KAuthVNodeRead = 58;
+    KAuthVNodeProb = 59;
+}
+
+enum FingerprintComputationKind{
+    Execution = 0;
+    CacheCheck = 1;
+}
+
+enum ObservedInputType{
+    AbsentPathProbe = 0;
+    FileContentRead = 1;
+    DirectoryEnumeration = 2;
+    ExistingDirectoryProbe = 3;
+    ExistingFileProbe = 4;
+}
+
+enum PreserveOutputsMode{
+    Disabled = 0;
+    Enabled = 1;
+    Reset = 2;
+}
+
+enum SandboxKind{
+    SandboxKind_None = 0;
+    Default = 1;
+    WinDetours = 2;
+    MaxOsKext = 3;
+    MacOsKextIgnoreFileAccesses = 4;
+}
+
+enum DoubleWritePolicy{
+    DoubleWritesAreErrors = 0;
+    AllowSameContentDoubleWrites = 1;
+    UnsafeFirstDoubleWriteWins = 2;
+}
+
+message AbsolutePath{
+    string Value = 1;
+}
+
+message ContentHash{
+    string Value = 1;
+}
+
+message FileArtifact{
+    AbsolutePath path = 1;
+
+    int32 RewriteCount = 2;
+}
+
+message FileContentInfo{
+    // [IsKnownExistenceFlag | Existence | IsKnownLengthFlag | Length]; 
+    int64 LengthAndExistence = 1;
+
+    ContentHash Hash = 2;
+}
+
+message PipExecutionPerformance{
+    int32 PipExecutionLevel = 1;
+
+    google.protobuf.Timestamp ExecutionStart = 2;
+    
+    google.protobuf.Timestamp ExecutionStop = 3;
+}
+
+message ProcessPipExecutionPerformance{
+    google.protobuf.Duration ProcessExecutionTime = 1;
+    
+    IOTypeCounters ReadCounters = 2;
+
+    IOTypeCounters WriteCounters = 3;
+
+    IOTypeCounters OtherCounters = 4;
+    
+    google.protobuf.Duration UserTime = 5;
+
+    google.protobuf.Duration KernelTime = 6;
+
+    uint64 PeakMemoryUsage = 7;
+
+    int32 PeakMemoryUsageMb = 8;
+
+    uint32 NumberOfProcesses = 9;
+
+    FileMonitoringViolationCounters FileMonitoringViolationCounters = 10;
+
+    Fingerprint Fingerprint = 11;
+
+    uint64 CacheDescriptorId = 12;
+}
+
+message FileMonitoringViolationCounters{
+    int32 NumFileAccessesWhitelistedButNotCacheable = 1;
+
+    int32 NumFileAccessesWhitelistedAndCacheable = 2;
+
+    int32 NumFileAccessViolationsNotWhitelisted = 3;
+}
+
+message Fingerprint{
+    int32 Length = 1;
+
+    bytes Bytes = 2;
+}
+
+message IOTypeCounters{
+    uint64 OperationCount = 1;
+
+    uint64 TransferCOunt = 2;
+}
+
+message ReportedProcess{
+    string Path = 1;
+
+    uint32 ProcessId = 2;
+
+    string ProcessArgs = 3;
+
+    IOTypeCounters ReadCounters = 4;
+
+    IOTypeCounters WriteCounters = 5;
+
+    IOTypeCounters OtherCounters = 6;
+
+    google.protobuf.Timestamp CreationTime = 7;
+
+    google.protobuf.Timestamp ExitTime = 8;
+
+    google.protobuf.Duration KernelTime = 9;
+
+    google.protobuf.Duration UserTime = 10;
+
+    uint32 ExitCode = 11;
+
+    uint32 ParentProcessId = 12;
+}
+
+message ReportedFileAccess{
+    CreationDisposition CreationDisposition = 1;
+
+    DesiredAccess DesiredAccess = 2;
+
+    uint32 Error = 3;
+
+    uint64 Usn = 4;
+
+    FlagsAndAttributes FlagsAndAttributes = 5;
+
+    string Path = 6;
+
+    string ManifestPath = 7;
+
+    ReportedProcess Process = 8;
+
+    ShareMode ShareMode = 9;
+
+    FileAccessStatus Status = 10;
+
+    FileAccessStatusMethod Method = 11;
+
+    RequestedAccess RequestedAccess = 12;
+
+    ReportedFileOperation Operation = 13;
+
+    bool ExplicitlyReported = 14;
+
+    string EnumeratePattern = 15;
+}
+
+message ProcessDetouringStatusData{
+    uint64 ProcessID = 1;
+
+    uint32 ReportStatus = 2;
+
+    string ProcessName = 3;
+
+    string StartApplicationName = 4;
+
+    string StartCommandLine = 5;
+
+    bool NeedsInjection = 6;
+
+    uint64 Job = 7;
+
+    bool DisableDetours = 8;
+
+    uint32 CreationFlags = 9;
+
+    bool Detoured = 10;
+
+    uint32 Error = 11;
+}
+
+message WeakContentFingerPrint{
+    Fingerprint Hash = 1;
+}
+
+message StrongContentFingerPrint{
+    Fingerprint Hash = 1;
+}
+
+message ObservedPathEntry{
+    AbsolutePath Path = 1;
+
+    string EnumeratePatternRegex = 2;
+}
+
+message UnsafeSandboxConfiguration{
+    PreserveOutputsMode PreserveOutputs = 1;
+
+    bool MonitorFileAccesses = 2;
+
+    bool IgnoreZwRenameFileInformation = 3;
+
+    bool IgnoreZwOtherFileInformation = 4;
+
+    bool IgnoreNonCreateFileReparsePoints = 5;
+
+    bool IgnoreSetFileInformationByHandle = 6;
+
+    bool IgnoreReparsePoints = 7;
+
+    bool IgnorePreloadedDlls = 8;
+
+    bool ExistingDirectoryProbesAsEnumerations = 9;
+
+    bool MonitorNtCreateFile = 10;
+
+    bool MonitorZwCreateOpenQueryFile = 11;
+
+    SandboxKind SandboxKind = 12;
+
+    bool UnexpectedFileAccessesAreErrors = 13;
+
+    bool IgnoreGetFinalPathNameByHandle = 14;
+
+    bool IgnoreDynamicWritesOnAbsentProbes = 15;
+
+    DoubleWritePolicy DoubleWritePolicy = 16;
+
+    bool IgnoreUndeclaredAccessesUnderSharedOpaques = 17;
+}
+
+message UnsafeOptions{
+    ContentHash PreserveOutputsSalt = 1;
+
+    UnsafeSandboxConfiguration UnsafeConfiguration = 2;
+}
+
+message StringId{
+    int32 Value = 1;
+}
+
+message ObservedInput{
+    ObservedInputType Type = 1;
+
+    ContentHash Hash = 2;
+
+    ObservedPathEntry PathEntry = 3;
+
+    AbsolutePath Path = 4;
+
+    bool IsSearchPath = 5;
+
+    bool IsDirectoryPath = 6;
+
+    bool DirectoryEnumeration = 7;
+}
+
+message ObservedPathSet{
+    repeated ObservedPathEntry Paths = 1;
+
+    repeated StringId ObservedAccessedFileNames = 2;
+
+    UnsafeOptions UnsafeOptions = 3;
+}
+
+message ProcessStrongFingerprintComputationData{
+    ObservedPathSet PathSet = 1;
+
+    ContentHash PathSetHash = 2;
+
+    repeated ObservedPathEntry PathEntries = 3;
+
+    UnsafeOptions UnsafeOptions = 4;
+
+    repeated StringId ObservedAccessedFileNames = 5;
+
+    repeated StrongContentFingerPrint PriorStrongFingerprints = 6;
+
+    bool Succeeded = 7;
+
+    bool IsStrongFingerprintHit = 8;
+
+    StrongContentFingerPrint ComputedStrongFingerprint = 9;
+
+    repeated ObservedInput ObservedInputs = 10;
+}
+
+message DirectoryFingerprint{
+    ContentHash Hash = 1;
+}
+
+message DirectoryOutput{
+    DirectoryArtifact DirectoryArtifact = 1;
+    
+    repeated FileArtifact fileArtifactArray = 2;
+}
+
+message DirectoryArtifact{
+    bool IsValid = 1;
+
+    AbsolutePath Path = 2;
+
+    uint32 PartialSealID = 3;
+
+    bool IsSharedOpaque = 4;
+}
+
+//
+//  Event Data
+//
+
+// The FileArtifactContentDecided Event message
+message FileArtifactContentDecidedEvent{
+    string UUID = 99;
+
+    uint32 WorkerID = 1;
+
+    FileArtifact FileArtifact = 2;
+
+    FileContentInfo FileContentInfo = 3;
+
+    PipOutputOrigin OutputOrigin = 4;
+}
+
+// The WorkerList Event message
+message WorkerListEvent{
+    string UUID = 99;
+
+    uint32 WorkerID = 1;
+
+    repeated string Workers = 2;
+}
+
+// The PipExecutionPerformance Event message
+message PipExecutionPerformanceEvent{
+    string UUID = 99;
+
+    uint32 WorkerID = 1;
+
+    uint32 PipID = 2;
+
+    PipExecutionPerformance PipExecutionPerformance = 3;
+
+    ProcessPipExecutionPerformance ProcessPipExecutionPerformance = 4;
+}
+
+// The DirectoryMembershipHashed Event message
+message DirectoryMembershipHashedEvent{
+    string UUID = 99;
+
+    uint32 WorkerID = 1;
+
+    DirectoryFingerprint DirectoryFingerprint = 3;
+
+    AbsolutePath Directory = 4;
+
+    bool IsStatic = 5;
+
+    bool IsSearchPath = 6;
+
+    uint32 PipID = 7;
+
+    repeated AbsolutePath Members = 8;
+
+    string EnumeratePatternRegex = 9;
+}
+
+// The ProcessExecutionMonitoringReported Event message
+message ProcessExecutionMonitoringReportedEvent{
+    string UUID = 99;
+
+    uint32 WorkerID = 1;
+
+    uint32 PipID = 2;
+
+    repeated ReportedProcess ReportedProcesses = 3;
+
+    repeated ReportedFileAccess ReportedFileAccesses = 4;
+    
+    repeated ReportedFileAccess WhitelistedReportedFileAccesses = 5;
+
+    repeated ProcessDetouringStatusData ProcessDetouringStatuses = 6;
+}
+
+// The ExtraEventDataReported Event message
+message ExtraEventDataReported{
+    string UUID = 99;
+
+    uint32 WorkerID = 1;
+
+    bool DisableDetours = 2;
+
+    bool IgnoreReparsePoints = 3;
+
+    bool IgnorePreloadedDlls = 4;
+
+    bool ExistingDirectoryProbesAsEnumerations = 5;
+
+    bool NtFileCreateMonitored = 6;
+
+    bool ZwFileCreateOpenMonitored = 7;
+
+    bool IgnoreZwRenameFileInformation = 8;
+
+    bool IgnoreZwOtherFileInformation = 9;
+
+    bool IgnoreNonCreateFileReparsePoints = 10;
+
+    bool IgnoreSetFileInformationByHandle = 11;
+
+    bool IgnoreGetFinalPathNameByHandle = 12;
+
+    int32 FingerprintVersion = 13;
+
+    string FingerprintSalt = 14;
+
+    ContentHash SearchPathToolsHash = 15;
+
+    bool UnexpectedFileAccessesAreErrors = 16;
+
+    bool MonitorFileAccesses = 17;
+
+    bool MaskUntrackedAccesses = 18;
+
+    bool NormalizeReadTimestamps = 19;
+
+    bool PipWarningsPromotedToErrors = 20;
+
+    bool ValidateDistribution = 21;
+
+    string RequiredKextVersionNumber = 22;
+}
+
+// The DependencyViolationReported Event message
+message DependencyViolationReportedEvent{
+    string UUID = 99;
+
+    uint32 WorkerID = 1;
+
+    uint32 ViolatorPipID = 2;
+
+    uint32 RelatedPipID = 3;
+
+    FileMonitoringViolationAnalyzer_DependencyViolationType ViolationType = 4;
+
+    FileMonitoringViolationAnalyzer_AccessLevel AccessLevel = 5;
+
+    AbsolutePath Path = 6;
+}
+
+// The PipExecutionStepPerformanceReported Event message
+message PipExecutionStepPerformanceReportedEvent{
+    string UUID = 99;
+
+    uint32 WorkerID = 1;
+
+    uint32 PipID = 2;
+
+    google.protobuf.Timestamp StartTime = 3;
+
+    google.protobuf.Duration Duration = 4;
+
+    PipExecutionStep Step = 5;
+
+    WorkDispatcher_DispatcherKind Dispatcher = 6;
+}
+
+// The StatusReported Event message
+message StatusReportedEvent{
+    string UUID = 99;
+
+    uint32 WorkerID = 1;
+
+    google.protobuf.Timestamp Time = 2;
+
+    int32 CpuPercent = 3;
+
+    repeated int32 DiskPercents = 4;
+    
+    repeated int32 DiskQueueDepths = 5;
+
+    int32 RamPercent = 6;
+
+    int32 MachineRamUtilizationMB = 7;
+
+    int32 CommitPercent = 8;
+
+    int32 CommitTotalMB = 9;
+
+    int32 ProcessCpuPercent = 10;
+
+    int32 ProcessWorkingSetMB = 11;
+
+    int32 CpuWaiting = 12;
+    
+    int32 CpuRunning = 13;
+
+    int32 IoCurrentMax = 14;
+
+    int32 IoWaiting = 15;
+
+    int32 IoRunning = 16;
+
+    int32 LookupWaiting = 17;
+
+    int32 LookupRunning = 18;
+
+    int32 ExternalProcesses = 19;
+
+    repeated int64 PipsSucceededAllTypes = 20;
+
+    ExecutionSampler_LimitingResource LimitingResource = 21;
+
+    int32 UnresponsivenessFactor = 22;
+
+    int64 ProcessPipsPending = 23;
+
+    int64 ProcessPipsAllocatedSlots = 24;
+}
+
+// The ProcessFingerprintComputation Event message
+message ProcessFingerprintComputationEvent{
+    string UUID = 99;
+
+    uint32 WorkerID = 1;
+    
+    FingerprintComputationKind Kind = 2;
+
+    uint32 PipID = 3;
+    
+    WeakContentFingerPrint WeakFingerprint = 4;
+
+    repeated ProcessStrongFingerprintComputationData StrongFingerprintComputations = 5;
+}
+
+// The PipCacheMiss Event message
+message PipCacheMissEvent{
+    string UUID = 99;
+
+    uint32 WorkerID = 1;
+
+    uint32 PipID = 2;
+
+    PipCacheMissType CacheMissType = 3;
+}
+
+// The PipExecutionDirectoryOutputs Event message
+message PipExecutionDirectoryOutputsEvent{
+    string UUID = 99;
+
+    uint32 WorkerID = 1;
+
+    repeated DirectoryOutput DirectoryOutput = 2;
+}
+
+// The BXLInvocation Event message
 message BXLInvocationEvent{
+    string UUID = 99;
 
-    // Unique identifier for the event - required, TODO: Remove as it may not be needed.
-    string UUID = 1;
+    uint32 WorkerID = 1;
 
-    // Identifies the worker which invoked this event
-    uint32 WorkerID = 2;
-
-    // Identifies the source that called this event
-    string SubstSource = 3;
+    AbsolutePath SubstSource = 2;
     
-    // Identifies the target of this event
-    string SubstTarget = 4;
+    AbsolutePath SubstTarget = 3;
 
-    // Whether the source is valid
-    bool IsSubstSourceValid = 5;
+    bool IsSubstSourceValid = 4;
     
-    // Whether the target is valid
-    bool IsSubstTargetValid = 6; 
-}
-
-// Contains a list of BXL Invocation Events
-message BXLInvocationEventList{
-
-    // List of different BXL Invocation events
-    repeated BXLInvocationEvent BXLInvEventList = 1;
-}
+    bool IsSubstTargetValid = 5; 
+}

Public/Src/Tools/Execution.Analyzer/Analyzers.Core/XLGPlusPlus/XLGToDBAnalyzer.cs

@@ -2,18 +2,17 @@
 // Licensed under the MIT license. See LICENSE file in the project root for full license information.
 
 using System;
+using System.Diagnostics;
 using System.IO;
+using System.Linq;
+using System.Text;
+using System.Threading;
+using BuildXL.Engine.Cache.KeyValueStores;
 using BuildXL.Scheduler.Tracing;
 using BuildXL.ToolSupport;
 using BuildXL.Utilities;
+using BuildXL.Utilities.ParallelAlgorithms;
 using Google.Protobuf;
-using BuildXL.Engine.Cache;
-using BuildXL.Engine.Cache.KeyValueStores;
-using BuildXL.Engine.Cache.Serialization;
-using BuildXL.Native.IO;
-using System.Diagnostics;
-using BuildXL.Execution.Analyzer.Model;
-using BuildXL.Analyzers.Core.XLGPlusPlus;
 
 namespace BuildXL.Execution.Analyzer
 {
@@ -35,12 +34,15 @@ namespace BuildXL.Execution.Analyzer
                 }
             }
 
-
             if (string.IsNullOrEmpty(outputDirPath))
             {
                 throw Error("/outputDir parameter is required");
             }
 
+            if (Directory.Exists(outputDirPath) && Directory.EnumerateFileSystemEntries(outputDirPath).Any())
+            {
+                throw Error("Directory provided exists and is non-empty. Aborting analyzer.");
+            }
 
             return new XLGToDBAnalyzer(GetAnalysisInput())
             {
@@ -55,7 +57,60 @@ namespace BuildXL.Execution.Analyzer
         {
             writer.WriteBanner("XLG to DB \"Analyzer\"");
             writer.WriteModeOption(nameof(AnalysisMode.XlgToDb), "Dumps event data from the xlg into a database.");
-            writer.WriteOption("outputDir", "Required. The directory to write out the RocksDB database", shortName: "o");
+            writer.WriteOption("outputDir", "Required. The new (or existing, but empty) directory to write out the RocksDB database", shortName: "o");
+        }
+    }
+
+    internal sealed class XLGToDBAnalyzer : Analyzer
+    {
+        private XLGToDBAnalyzerInner m_inner;
+        private ActionBlockSlim<Action> m_actionBlock;
+        public string OutputDirPath
+        {
+            set => m_inner.OutputDirPath = value;
+        }
+
+        public XLGToDBAnalyzer(AnalysisInput input)
+            : base(input)
+        {
+            m_inner = new XLGToDBAnalyzerInner(input);
+            m_actionBlock = new ActionBlockSlim<Action>(12, action => action());
+        }
+
+        /// <inheritdoc/>
+        public override bool CanHandleEvent(ExecutionEventId eventId, uint workerId, long timestamp, int eventPayloadSize)
+        {
+            return m_inner.CanHandleEvent(eventId, workerId, timestamp, eventPayloadSize);
+        }
+
+        /// <inheritdoc/>
+        protected override void ReportUnhandledEvent<TEventData>(TEventData data)
+        {
+            var workerId = CurrentEventWorkerId;
+            m_actionBlock.Post(() =>
+            {
+                m_inner.ProcessEvent(data, workerId);
+            });
+        }
+
+        /// <inheritdoc/>
+        public override void Prepare()
+        {
+            m_inner.Prepare();
+        }
+
+        /// <inheritdoc/>
+        public override int Analyze()
+        {
+            m_actionBlock.Complete();
+            m_actionBlock.CompletionAsync().GetAwaiter().GetResult();
+            return m_inner.Analyze();
+        }
+
+        /// <inheritdoc/>
+        public override void Dispose()
+        {
+            m_inner.Dispose();
         }
     }
 
@@ -63,99 +118,296 @@ namespace BuildXL.Execution.Analyzer
     /// <summary>
     /// Analyzer to dump xlg events and other data into RocksDB
     /// </summary>
-    internal sealed class XLGToDBAnalyzer : Analyzer
+    internal sealed class XLGToDBAnalyzerInner : Analyzer
     {
+        /// <summary>
+        /// Output directory path
+        /// </summary>
         public string OutputDirPath;
+
+        /// <summary>
+        /// Store WorkerID.Value to pass into protobuf object to identify this event
+        /// </summary>
+        public ThreadLocal<uint> WorkerID = new ThreadLocal<uint>();
+
         private bool m_accessorSucceeded;
-        private BXLInvocationEventList m_invocationEventList = new BXLInvocationEventList();
-        private KeyValueStoreAccessor Accessor { get; set; }
-        private uint WorkerID { get; set; }
 
+        private KeyValueStoreAccessor m_accessor;
 
-        public XLGToDBAnalyzer(AnalysisInput input) : base(input) { }
+        private Stopwatch m_stopWatch;
+
+        private int m_eventCount;
+
+        public XLGToDBAnalyzerInner(AnalysisInput input) : base(input)
+        {
+            m_stopWatch = new Stopwatch();
+            m_stopWatch.Start();
+        }
+
+        internal void ProcessEvent<TEventData>(TEventData data, uint workerId) where TEventData : struct, IExecutionLogEventData<TEventData>
+        {
+            var eventCount = Interlocked.Increment(ref m_eventCount);
+
+            if (eventCount % 10000 == 0)
+            {
+                Console.WriteLine("Processed {0} events so far.", eventCount);
+                Console.WriteLine("Total time elapsed: {0} seconds", m_stopWatch.ElapsedMilliseconds / 1000.0);
+            }
+
+            WorkerID.Value = workerId;
+            data.Metadata.LogToTarget(data, this);
+        }
 
         /// <inheritdoc/>
         public override void Prepare()
         {
-            try
-            {
-                Directory.Delete(path: OutputDirPath, recursive: true);
-            }
-            catch (Exception e)
-            {
-                Console.WriteLine("No such dir or could not delete dir with exception {0}.\nIf dir still exists, this analyzer will append data to existing DB.", e);
-            }
-
             var accessor = KeyValueStoreAccessor.Open(storeDirectory: OutputDirPath);
 
             if (accessor.Succeeded)
             {
-                Accessor = accessor.Result;
+                m_accessor = accessor.Result;
                 m_accessorSucceeded = true;
             }
             else
             {
                 Console.Error.WriteLine("Could not access RocksDB datastore. Exiting analyzer.");
             }
+
+            m_eventCount = 0;
         }
 
         /// <inheritdoc/>
         public override int Analyze()
         {
-            if (!m_accessorSucceeded)
+            Console.WriteLine("Num events ingested = {0}", m_eventCount);
+            Console.WriteLine("Total time elapsed: {0} seconds", m_stopWatch.ElapsedMilliseconds / 1000.0);
+            var ec = new Xldb.EventCount
             {
-                return 0;
-            }
+                Value = (uint)m_eventCount
+            };
 
-            Analysis.IgnoreResult(
-              Accessor.Use(database =>
-              {
-                  foreach (var invEvent in m_invocationEventList.BXLInvEventList)
-                  {
-                      var eq = new EventTypeQuery
-                      {
-                          EventTypeID = (int)ExecutionEventId.DominoInvocation,
-                          UUID = invEvent.UUID
-                      };
-
-                      database.Put(eq.ToByteArray(), invEvent.ToByteArray());
-                  }
-              })
-            );
-
-            Accessor.Dispose();
+            WriteToDb(Encoding.ASCII.GetBytes("EventCount"), ec.ToByteArray());
             return 0;
         }
 
+        /// <inheritdoc/>
+        public override void Dispose()
+        {
+            m_accessor.Dispose();
+        }
+
         /// <inheritdoc/>
         public override bool CanHandleEvent(ExecutionEventId eventId, uint workerId, long timestamp, int eventPayloadSize)
         {
-            if (eventId.Equals(ExecutionEventId.DominoInvocation))
-            {
-                WorkerID = workerId; // store workerID to pass into protobuf object to identify this event
-                return true;
-            }
-            return false; // return false to keep the event from being parsed
+            return (m_accessorSucceeded && eventId != ExecutionEventId.ObservedInputs);
         }
 
         /// <summary>
-        /// Override the DominoInvocationEvent to capture its data and store it in the protobuf 
+        /// Override event to capture its data and store it in the protobuf 
+        /// </summary>
+        public override void FileArtifactContentDecided(FileArtifactContentDecidedEventData data)
+        {
+            var fileArtifactContentDecidedEvent = data.ToFileArtifactContentDecidedEvent(WorkerID.Value, PathTable);
+            var eq = new Xldb.EventTypeQuery
+            {
+                EventTypeID = Xldb.ExecutionEventId.FileArtifactContentDecided,
+                UUID = fileArtifactContentDecidedEvent.UUID
+            };
+
+            WriteToDb(eq.ToByteArray(), fileArtifactContentDecidedEvent.ToByteArray());
+        }
+
+        /// <summary>
+        /// Override event to capture its data and store it in the protobuf 
+        /// </summary>
+        public override void WorkerList(WorkerListEventData data)
+        {
+            var workerListEvent = data.ToWorkerListEvent(WorkerID.Value);
+            var eq = new Xldb.EventTypeQuery
+            {
+                EventTypeID = Xldb.ExecutionEventId.WorkerList,
+                UUID = workerListEvent.UUID
+            };
+
+            WriteToDb(eq.ToByteArray(), workerListEvent.ToByteArray());
+        }
+
+        /// <summary>
+        /// Override event to capture its data and store it in the protobuf 
+        /// </summary>
+        public override void PipExecutionPerformance(PipExecutionPerformanceEventData data)
+        {
+            var pipExecPerfEvent = data.ToPipExecutionPerformanceEvent();
+            var eq = new Xldb.EventTypeQuery
+            {
+                EventTypeID = Xldb.ExecutionEventId.PipExecutionPerformance,
+                UUID = pipExecPerfEvent.UUID
+            };
+
+            WriteToDb(eq.ToByteArray(), pipExecPerfEvent.ToByteArray());
+        }
+
+        /// <summary>
+        /// Override event to capture its data and store it in the protobuf 
+        /// </summary>
+        public override void DirectoryMembershipHashed(DirectoryMembershipHashedEventData data)
+        {
+            var directoryMembershipEvent = data.ToDirectoryMembershipHashedEvent(WorkerID.Value, PathTable);
+            var eq = new Xldb.EventTypeQuery
+            {
+                EventTypeID = Xldb.ExecutionEventId.DirectoryMembershipHashed,
+                UUID = directoryMembershipEvent.UUID
+            };
+
+            WriteToDb(eq.ToByteArray(), directoryMembershipEvent.ToByteArray());
+        }
+
+        /// <summary>
+        /// Override event to capture its data and store it in the protobuf 
+        /// </summary>
+        public override void ProcessExecutionMonitoringReported(ProcessExecutionMonitoringReportedEventData data)
+        {
+            var processExecMonitoringReportedEvent = data.ToProcessExecutionMonitoringReportedEvent(WorkerID.Value, PathTable);
+            var eq = new Xldb.EventTypeQuery
+            {
+                EventTypeID = Xldb.ExecutionEventId.ProcessExecutionMonitoringReported,
+                UUID = processExecMonitoringReportedEvent.UUID
+            };
+
+            WriteToDb(eq.ToByteArray(), processExecMonitoringReportedEvent.ToByteArray());
+        }
+
+        /// <summary>
+        /// Override event to capture its data and store it in the protobuf 
+        /// </summary>
+        public override void ProcessFingerprintComputed(ProcessFingerprintComputationEventData data)
+        {
+            var processFingerprintComputedEvent = data.ToProcessFingerprintComputationEvent(WorkerID.Value, PathTable);
+            var eq = new Xldb.EventTypeQuery
+            {
+                EventTypeID = Xldb.ExecutionEventId.ProcessFingerprintComputation,
+                UUID = processFingerprintComputedEvent.UUID
+            };
+
+            WriteToDb(eq.ToByteArray(), processFingerprintComputedEvent.ToByteArray());
+        }
+
+        /// <summary>
+        /// Override event to capture its data and store it in the protobuf 
+        /// </summary>
+        public override void ExtraEventDataReported(ExtraEventData data)
+        {
+            var extraEvent = data.ToExtraEventDataReported(WorkerID.Value);
+            var eq = new Xldb.EventTypeQuery
+            {
+                EventTypeID = Xldb.ExecutionEventId.ExtraEventDataReported,
+                UUID = extraEvent.UUID
+            };
+
+            WriteToDb(eq.ToByteArray(), extraEvent.ToByteArray());
+        }
+
+        /// <summary>
+        /// Override event to capture its data and store it in the protobuf 
+        /// </summary>
+        public override void DependencyViolationReported(DependencyViolationEventData data)
+        {
+            var dependencyViolationEvent = data.ToDependencyViolationReportedEvent(WorkerID.Value, PathTable);
+            var eq = new Xldb.EventTypeQuery
+            {
+                EventTypeID = Xldb.ExecutionEventId.DependencyViolationReported,
+                UUID = dependencyViolationEvent.UUID
+            };
+
+            WriteToDb(eq.ToByteArray(), dependencyViolationEvent.ToByteArray());
+        }
+
+        /// <summary>
+        /// Override event to capture its data and store it in the protobuf 
+        /// </summary>
+        public override void PipExecutionStepPerformanceReported(PipExecutionStepPerformanceEventData data)
+        {
+            var pipExecStepPerformanceEvent = data.ToPipExecutionStepPerformanceReportedEvent(WorkerID.Value);
+            var eq = new Xldb.EventTypeQuery
+            {
+                EventTypeID = Xldb.ExecutionEventId.PipExecutionStepPerformanceReported,
+                UUID = pipExecStepPerformanceEvent.UUID
+            };
+
+            WriteToDb(eq.ToByteArray(), pipExecStepPerformanceEvent.ToByteArray());
+        }
+
+        /// <summary>
+        /// Override event to capture its data and store it in the protobuf 
+        /// </summary>
+        public override void PipCacheMiss(PipCacheMissEventData data)
+        {
+            var pipCacheMissEvent = data.ToPipCacheMissEvent(WorkerID.Value);
+            var eq = new Xldb.EventTypeQuery
+            {
+                EventTypeID = Xldb.ExecutionEventId.PipCacheMiss,
+                UUID = pipCacheMissEvent.UUID
+            };
+
+            WriteToDb(eq.ToByteArray(), pipCacheMissEvent.ToByteArray());
+        }
+
+        /// <summary>
+        /// Override event to capture its data and store it in the protobuf 
+        /// </summary>
+        public override void StatusReported(StatusEventData data)
+        {
+            var statusReportedEvent = data.ToResourceUsageReportedEvent(WorkerID.Value);
+            var eq = new Xldb.EventTypeQuery
+            {
+                EventTypeID = Xldb.ExecutionEventId.ResourceUsageReported,
+                UUID = statusReportedEvent.UUID
+            };
+
+            WriteToDb(eq.ToByteArray(), statusReportedEvent.ToByteArray());
+        }
+
+        /// <summary>
+        /// Override event to capture its data and store it in the protobuf 
         /// </summary>
         public override void DominoInvocation(DominoInvocationEventData data)
         {
-            var domInvEvent = new BXLInvocationEvent();
-            var loggingConfig = data.Configuration.Logging;
+            var bxlInvEvent = data.ToBXLInvocationEvent(WorkerID.Value, PathTable);
+            var eq = new Xldb.EventTypeQuery
+            {
+                EventTypeID = Xldb.ExecutionEventId.DominoInvocation,
+                UUID = bxlInvEvent.UUID
+            };
 
-            var uuid = Guid.NewGuid().ToString();
+            WriteToDb(eq.ToByteArray(), bxlInvEvent.ToByteArray());
+        }
 
-            domInvEvent.UUID = uuid;
-            domInvEvent.WorkerID = WorkerID;
-            domInvEvent.SubstSource = loggingConfig.SubstSource.ToString(PathTable, PathFormat.HostOs);
-            domInvEvent.SubstTarget = loggingConfig.SubstTarget.ToString(PathTable, PathFormat.HostOs);
-            domInvEvent.IsSubstSourceValid = loggingConfig.SubstSource.IsValid;
-            domInvEvent.IsSubstTargetValid = loggingConfig.SubstTarget.IsValid;
+        /// <summary>
+        /// Override event to capture its data and store it in the protobuf 
+        /// </summary>
+        public override void PipExecutionDirectoryOutputs(PipExecutionDirectoryOutputs data)
+        {
+            var pipExecDirectoryOutputEvent = data.ToPipExecutionDirectoryOutputsEvent(WorkerID.Value, PathTable);
+            var eq = new Xldb.EventTypeQuery
+            {
+                EventTypeID = Xldb.ExecutionEventId.PipExecutionDirectoryOutputs,
+                UUID = pipExecDirectoryOutputEvent.UUID
+            };
 
-            m_invocationEventList.BXLInvEventList.Add(domInvEvent);
+            WriteToDb(eq.ToByteArray(), pipExecDirectoryOutputEvent.ToByteArray());
+        }
+
+        /// <summary>
+        /// Write a key/value pair to the db
+        /// </summary>
+        public void WriteToDb(byte[] key, byte[] value)
+        {
+            Analysis.IgnoreResult(
+                m_accessor.Use(database =>
+                {
+                    database.Put(key, value);
+                })
+            );
         }
     }
 }

Public/Src/Tools/Execution.Analyzer/Analyzers.Core/XLGPlusPlus/XldbDataStore.cs

@@ -4,16 +4,14 @@
 using System;
 using System.Collections.Generic;
 using System.Diagnostics.ContractsLight;
-using System.Linq;
 using BuildXL.Engine.Cache.KeyValueStores;
-using BuildXL.Execution.Analyzer;
-using BuildXL.Scheduler.Tracing;
+using BuildXL.Execution.Analyzer.Xldb;
 using BuildXL.Utilities;
 using Google.Protobuf;
 
 namespace BuildXL.Analyzers.Core.XLGPlusPlus
 {
-    public class XLGppDataStore: IDisposable
+    public sealed class XldbDataStore : IDisposable
     {
         /// <summary>
         /// Rocks DB Accessor for XLG++ data
@@ -23,7 +21,7 @@ namespace BuildXL.Analyzers.Core.XLGPlusPlus
         /// <summary>
         /// Open the datastore and populate the KeyValueStoreAccessor for the XLG++ DB
         /// </summary>
-        public XLGppDataStore(string storeDirectory,
+        public XldbDataStore(string storeDirectory,
             bool defaultColumnKeyTracked = false,
             IEnumerable<string> additionalColumns = null,
             IEnumerable<string> additionalKeyTrackedColumns = null,
@@ -56,9 +54,9 @@ namespace BuildXL.Analyzers.Core.XLGPlusPlus
         /// Gets all the events of a certain type from the DB
         /// </summary>
         /// <returns>List of event objects recovered from DB </returns>
-        public IEnumerable<string> GetEventsByType(int eventTypeID)
+        public IEnumerable<string> GetEventsByType(ExecutionEventId eventTypeID)
         {
-            Contract.Assert(Accessor != null, "XLGppStore must be initialized via OpenDatastore first");
+            Contract.Assert(Accessor != null, "XldbStore must be initialized via OpenDatastore first");
 
             var storedEvents = new List<string>();
             var eventQuery = new EventTypeQuery
@@ -70,42 +68,16 @@ namespace BuildXL.Analyzers.Core.XLGPlusPlus
                 {
                     foreach (var kvp in database.PrefixSearch(eventQuery.ToByteArray()))
                     {
-                        Console.WriteLine(BXLInvocationEvent.Parser.ParseFrom(kvp.Value));
                         storedEvents.Add(BXLInvocationEvent.Parser.ParseFrom(kvp.Value).ToString());
                     }
                 })
             );
-            
+
             return storedEvents;
         }
 
-        /// <summary>
-        /// Method to test if the appropriate things have been stored in DB.
-        /// NOTE: For internal testing/Debugging only!
-        /// </summary>
-        /// <returns>Stored data in string format</returns>
-        public string GetStoredData()
-        {
-            Contract.Assert(Accessor != null, "XLGppStore must be initialized via OpenDatastore first");
-
-            string value = null;
-            Analysis.IgnoreResult(
-                Accessor.Use(database =>
-                {
-                    database.TryGetValue("foo", out value);
-                    foreach (var kvp in database.PrefixSearch("b"))
-                    {
-                        Console.WriteLine("The key is {0}, and the value is {1}", kvp.Key, kvp.Value);
-                    }
-                })
-            );
-            return value;
-        }
-
         public void Dispose()
         {
-            Contract.Assert(Accessor != null, "XLGppStore must be initialized via OpenDatastore first");
-
             Accessor.Dispose();
         }
     }

Public/Src/Tools/Execution.Analyzer/Analyzers.Core/XLGPlusPlus/XldbProtobufExtensions.cs

@@ -0,0 +1,563 @@
+// Copyright (c) Microsoft. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+using System;
+using System.Linq;
+using BuildXL.Cache.MemoizationStore.Interfaces.Sessions;
+using BuildXL.Pips;
+using BuildXL.Processes;
+using BuildXL.Scheduler.Fingerprints;
+using BuildXL.Scheduler.Tracing;
+using BuildXL.Utilities;
+
+namespace BuildXL.Execution.Analyzer
+{
+    /// <summary>
+    /// Extension methods for Xldb ProtoBuf conversions.
+    /// </summary>
+    public static class XldbProtobufExtensions
+    {
+        /// <nodoc />
+        public static Xldb.FileArtifactContentDecidedEvent ToFileArtifactContentDecidedEvent(this FileArtifactContentDecidedEventData data, uint workerID, PathTable pathTable)
+        {
+            var Uuid = Guid.NewGuid().ToString();
+
+            return new Xldb.FileArtifactContentDecidedEvent()
+            {
+                UUID = Uuid,
+                WorkerID = workerID,
+                FileArtifact = data.FileArtifact.ToFileArtifact(pathTable),
+                FileContentInfo = new Xldb.FileContentInfo
+                {
+                    LengthAndExistence = data.FileContentInfo.SerializedLengthAndExistence,
+                    Hash = new Xldb.ContentHash() { Value = data.FileContentInfo.Hash.ToString() }
+                },
+                OutputOrigin = (Xldb.PipOutputOrigin)data.OutputOrigin
+            };
+        }
+
+        /// <nodoc />
+        public static Xldb.WorkerListEvent ToWorkerListEvent(this WorkerListEventData data, uint workerID)
+        {
+            var Uuid = Guid.NewGuid().ToString();
+
+            var workerListEvent = new Xldb.WorkerListEvent
+            {
+                UUID = Uuid
+            };
+
+            workerListEvent.Workers.AddRange(data.Workers.Select(worker => worker));
+            return workerListEvent;
+        }
+
+        /// <nodoc />
+        public static Xldb.PipExecutionPerformanceEvent ToPipExecutionPerformanceEvent(this PipExecutionPerformanceEventData data)
+        {
+            var pipExecPerfEvent = new Xldb.PipExecutionPerformanceEvent();
+            var Uuid = Guid.NewGuid().ToString();
+
+            var pipExecPerformance = new Xldb.PipExecutionPerformance();
+            pipExecPerformance.PipExecutionLevel = (int)data.ExecutionPerformance.ExecutionLevel;
+            pipExecPerformance.ExecutionStart = Google.Protobuf.WellKnownTypes.Timestamp.FromDateTime(data.ExecutionPerformance.ExecutionStart);
+            pipExecPerformance.ExecutionStop = Google.Protobuf.WellKnownTypes.Timestamp.FromDateTime(data.ExecutionPerformance.ExecutionStop);
+
+            var processPipExecPerformance = new Xldb.ProcessPipExecutionPerformance();
+            var performance = data.ExecutionPerformance as ProcessPipExecutionPerformance;
+            if (performance != null)
+            {
+                processPipExecPerformance.ProcessExecutionTime = Google.Protobuf.WellKnownTypes.Duration.FromTimeSpan(performance.ProcessExecutionTime);
+                processPipExecPerformance.ReadCounters = new Xldb.IOTypeCounters()
+                {
+                    OperationCount = performance.IO.ReadCounters.OperationCount,
+                    TransferCOunt = performance.IO.ReadCounters.TransferCount
+                };
+
+                processPipExecPerformance.WriteCounters = new Xldb.IOTypeCounters()
+                {
+                    OperationCount = performance.IO.WriteCounters.OperationCount,
+                    TransferCOunt = performance.IO.WriteCounters.TransferCount
+                };
+
+                processPipExecPerformance.OtherCounters = new Xldb.IOTypeCounters()
+                {
+                    OperationCount = performance.IO.OtherCounters.OperationCount,
+                    TransferCOunt = performance.IO.OtherCounters.TransferCount
+                };
+
+                processPipExecPerformance.UserTime = Google.Protobuf.WellKnownTypes.Duration.FromTimeSpan(performance.UserTime);
+                processPipExecPerformance.KernelTime = Google.Protobuf.WellKnownTypes.Duration.FromTimeSpan(performance.KernelTime);
+                processPipExecPerformance.PeakMemoryUsage = performance.PeakMemoryUsage;
+                processPipExecPerformance.PeakMemoryUsageMb = performance.PeakMemoryUsageMb;
+                processPipExecPerformance.NumberOfProcesses = performance.NumberOfProcesses;
+
+                processPipExecPerformance.FileMonitoringViolationCounters = new Xldb.FileMonitoringViolationCounters()
+                {
+                    NumFileAccessesWhitelistedAndCacheable = performance.FileMonitoringViolations.NumFileAccessesWhitelistedAndCacheable,
+                    NumFileAccessesWhitelistedButNotCacheable = performance.FileMonitoringViolations.NumFileAccessesWhitelistedButNotCacheable,
+                    NumFileAccessViolationsNotWhitelisted = performance.FileMonitoringViolations.NumFileAccessViolationsNotWhitelisted
+                };
+
+                processPipExecPerformance.Fingerprint = performance.Fingerprint.ToFingerprint();
+
+                if (performance.CacheDescriptorId.HasValue)
+                {
+                    processPipExecPerformance.CacheDescriptorId = performance.CacheDescriptorId.Value;
+                }
+            }
+
+            pipExecPerfEvent.UUID = Uuid;
+            pipExecPerfEvent.WorkerID = data.ExecutionPerformance.WorkerId;
+            pipExecPerfEvent.PipID = data.PipId.Value;
+            pipExecPerfEvent.PipExecutionPerformance = pipExecPerformance;
+            pipExecPerfEvent.ProcessPipExecutionPerformance = processPipExecPerformance;
+            return pipExecPerfEvent;
+        }
+
+        /// <nodoc />
+        public static Xldb.DirectoryMembershipHashedEvent ToDirectoryMembershipHashedEvent(this DirectoryMembershipHashedEventData data, uint workerID, PathTable pathTable)
+        {
+            var Uuid = Guid.NewGuid().ToString();
+
+            var directoryMembershipEvent = new Xldb.DirectoryMembershipHashedEvent()
+            {
+                UUID = Uuid,
+                WorkerID = workerID,
+                DirectoryFingerprint = new Xldb.DirectoryFingerprint()
+                {
+                    Hash = new Xldb.ContentHash() { Value = data.DirectoryFingerprint.Hash.ToString() }
+                },
+                Directory = data.Directory.ToAbsolutePath(pathTable),
+                IsStatic = data.IsSearchPath,
+                IsSearchPath = data.IsSearchPath,
+                PipID = data.PipId.Value,
+                EnumeratePatternRegex = data.EnumeratePatternRegex ?? ""
+            };
+
+            directoryMembershipEvent.Members.AddRange(data.Members.Select(member => member.ToAbsolutePath(pathTable)));
+
+            return directoryMembershipEvent;
+        }
+
+        /// <nodoc />
+        public static Xldb.ProcessExecutionMonitoringReportedEvent ToProcessExecutionMonitoringReportedEvent(this ProcessExecutionMonitoringReportedEventData data, uint workerID, PathTable pathTable)
+        {
+            var Uuid = Guid.NewGuid().ToString();
+
+            var processExecutionMonitoringReportedEvent = new Xldb.ProcessExecutionMonitoringReportedEvent
+            {
+                UUID = Uuid,
+                WorkerID = workerID,
+                PipID = data.PipId.Value
+            };
+
+            processExecutionMonitoringReportedEvent.ReportedProcesses.AddRange(
+                data.ReportedProcesses.Select(rp => rp.ToReportedProcess()));
+            processExecutionMonitoringReportedEvent.ReportedFileAccesses.AddRange(
+                data.ReportedFileAccesses.Select(reportedFileAccess => reportedFileAccess.ToReportedFileAccess(pathTable)));
+            processExecutionMonitoringReportedEvent.WhitelistedReportedFileAccesses.AddRange(
+                data.WhitelistedReportedFileAccesses.Select(
+                    whiteListReportedFileAccess => whiteListReportedFileAccess.ToReportedFileAccess(pathTable)));
+
+            foreach (var processDetouringStatus in data.ProcessDetouringStatuses)
+            {
+                processExecutionMonitoringReportedEvent.ProcessDetouringStatuses.Add(new Xldb.ProcessDetouringStatusData()
+                {
+                    ProcessID = processDetouringStatus.ProcessId,
+                    ReportStatus = processDetouringStatus.ReportStatus,
+                    ProcessName = processDetouringStatus.ProcessName,
+                    StartApplicationName = processDetouringStatus.StartApplicationName,
+                    StartCommandLine = processDetouringStatus.StartCommandLine,
+                    NeedsInjection = processDetouringStatus.NeedsInjection,
+                    Job = processDetouringStatus.Job,
+                    DisableDetours = processDetouringStatus.DisableDetours,
+                    CreationFlags = processDetouringStatus.CreationFlags,
+                    Detoured = processDetouringStatus.Detoured,
+                    Error = processDetouringStatus.Error
+                });
+            }
+
+            return processExecutionMonitoringReportedEvent;
+        }
+
+        /// <nodoc />
+        public static Xldb.ProcessFingerprintComputationEvent ToProcessFingerprintComputationEvent(this ProcessFingerprintComputationEventData data, uint workerID, PathTable pathTable)
+        {
+            var Uuid = Guid.NewGuid().ToString();
+
+            var processFingerprintComputationEvent = new Xldb.ProcessFingerprintComputationEvent
+            {
+                UUID = Uuid,
+                WorkerID = workerID,
+                Kind = (Xldb.FingerprintComputationKind)data.Kind,
+                PipID = data.PipId.Value,
+                WeakFingerprint = new Xldb.WeakContentFingerPrint()
+                {
+                    Hash = data.WeakFingerprint.Hash.ToFingerprint()
+                },
+            };
+
+            foreach (var strongFingerprintComputation in data.StrongFingerprintComputations)
+            {
+                var processStrongFingerprintComputationData = new Xldb.ProcessStrongFingerprintComputationData()
+                {
+                    PathSet = strongFingerprintComputation.PathSet.ToObservedPathSet(pathTable),
+                    PathSetHash = new Xldb.ContentHash()
+                    {
+                        Value = strongFingerprintComputation.PathSetHash.ToString()
+                    },
+                    UnsafeOptions = strongFingerprintComputation.UnsafeOptions.ToUnsafeOptions(),
+                    Succeeded = strongFingerprintComputation.Succeeded,
+                    IsStrongFingerprintHit = strongFingerprintComputation.IsStrongFingerprintHit,
+                    ComputedStrongFingerprint = new Xldb.StrongContentFingerPrint()
+                    {
+                        Hash = strongFingerprintComputation.ComputedStrongFingerprint.Hash.ToFingerprint()
+                    }
+                };
+
+                processStrongFingerprintComputationData.PathEntries.AddRange(
+                    strongFingerprintComputation.PathEntries.Select(
+                        pathEntry => pathEntry.ToObservedPathEntry(pathTable)));
+                processStrongFingerprintComputationData.ObservedAccessedFileNames.AddRange(
+                    strongFingerprintComputation.ObservedAccessedFileNames.Select(
+                        observedAccessedFileName => new Xldb.StringId() { Value = observedAccessedFileName.Value }));
+                processStrongFingerprintComputationData.PriorStrongFingerprints.AddRange(
+                    strongFingerprintComputation.PriorStrongFingerprints.Select(
+                        priorStrongFingerprint => new Xldb.StrongContentFingerPrint() { Hash = priorStrongFingerprint.Hash.ToFingerprint() }));
+
+                foreach (var observedInput in strongFingerprintComputation.ObservedInputs)
+                {
+                    processStrongFingerprintComputationData.ObservedInputs.Add(new Xldb.ObservedInput()
+                    {
+                        Type = (Xldb.ObservedInputType)observedInput.Type,
+                        Hash = new Xldb.ContentHash()
+                        {
+                            Value = observedInput.Hash.ToString()
+                        },
+                        PathEntry = observedInput.PathEntry.ToObservedPathEntry(pathTable),
+                        Path = observedInput.Path.ToAbsolutePath(pathTable),
+                        IsSearchPath = observedInput.IsSearchPath,
+                        IsDirectoryPath = observedInput.IsDirectoryPath,
+                        DirectoryEnumeration = observedInput.DirectoryEnumeration
+                    });
+                }
+
+                processFingerprintComputationEvent.StrongFingerprintComputations.Add(processStrongFingerprintComputationData);
+            }
+
+            return processFingerprintComputationEvent;
+        }
+
+        /// <nodoc />
+        public static Xldb.ExtraEventDataReported ToExtraEventDataReported(this ExtraEventData data, uint workerID)
+        {
+            var Uuid = Guid.NewGuid().ToString();
+
+            return new Xldb.ExtraEventDataReported
+            {
+                UUID = Uuid,
+                WorkerID = workerID,
+                DisableDetours = data.DisableDetours,
+                IgnoreReparsePoints = data.IgnoreReparsePoints,
+                IgnorePreloadedDlls = data.IgnorePreloadedDlls,
+                ExistingDirectoryProbesAsEnumerations = data.ExistingDirectoryProbesAsEnumerations,
+                NtFileCreateMonitored = data.NtFileCreateMonitored,
+                ZwFileCreateOpenMonitored = data.ZwFileCreateOpenMonitored,
+                IgnoreZwRenameFileInformation = data.IgnoreZwRenameFileInformation,
+                IgnoreZwOtherFileInformation = data.IgnoreZwOtherFileInformation,
+                IgnoreNonCreateFileReparsePoints = data.IgnoreNonCreateFileReparsePoints,
+                IgnoreSetFileInformationByHandle = data.IgnoreSetFileInformationByHandle,
+                IgnoreGetFinalPathNameByHandle = data.IgnoreGetFinalPathNameByHandle,
+                FingerprintVersion = (int)data.FingerprintVersion,
+                FingerprintSalt = data.FingerprintSalt,
+                SearchPathToolsHash = new Xldb.ContentHash() { Value = data.SearchPathToolsHash.ToString() },
+                UnexpectedFileAccessesAreErrors = data.UnexpectedFileAccessesAreErrors,
+                MonitorFileAccesses = data.MonitorFileAccesses,
+                MaskUntrackedAccesses = data.MaskUntrackedAccesses,
+                NormalizeReadTimestamps = data.NormalizeReadTimestamps,
+                PipWarningsPromotedToErrors = data.PipWarningsPromotedToErrors,
+                ValidateDistribution = data.ValidateDistribution,
+                RequiredKextVersionNumber = data.RequiredKextVersionNumber
+            };
+        }
+
+        /// <nodoc />
+        public static Xldb.DependencyViolationReportedEvent ToDependencyViolationReportedEvent(this DependencyViolationEventData data, uint workerID, PathTable pathTable)
+        {
+            var Uuid = Guid.NewGuid().ToString();
+
+            return new Xldb.DependencyViolationReportedEvent()
+            {
+                UUID = Uuid,
+                WorkerID = workerID,
+                ViolatorPipID = data.ViolatorPipId.Value,
+                RelatedPipID = data.RelatedPipId.Value,
+                ViolationType = (Xldb.FileMonitoringViolationAnalyzer_DependencyViolationType)data.ViolationType,
+                AccessLevel = (Xldb.FileMonitoringViolationAnalyzer_AccessLevel)data.AccessLevel,
+                Path = data.Path.ToAbsolutePath(pathTable)
+            };
+        }
+
+        /// <nodoc />
+        public static Xldb.PipExecutionStepPerformanceReportedEvent ToPipExecutionStepPerformanceReportedEvent(this PipExecutionStepPerformanceEventData data, uint workerID)
+        {
+            var Uuid = Guid.NewGuid().ToString();
+
+            var pipExecStepPerformanceEvent = new Xldb.PipExecutionStepPerformanceReportedEvent
+            {
+                UUID = Uuid,
+                WorkerID = workerID,
+                PipID = data.PipId.Value,
+                StartTime = Google.Protobuf.WellKnownTypes.Timestamp.FromDateTime(data.StartTime),
+                Duration = Google.Protobuf.WellKnownTypes.Duration.FromTimeSpan(data.Duration),
+                Step = (Xldb.PipExecutionStep)data.Step,
+                Dispatcher = (Xldb.WorkDispatcher_DispatcherKind)data.Dispatcher
+            };
+
+            return pipExecStepPerformanceEvent;
+        }
+
+        /// <nodoc />
+        public static Xldb.PipCacheMissEvent ToPipCacheMissEvent(this PipCacheMissEventData data, uint workerID)
+        {
+            var Uuid = Guid.NewGuid().ToString();
+            return new Xldb.PipCacheMissEvent()
+            {
+                UUID = Uuid,
+                WorkerID = workerID,
+                PipID = data.PipId.Value,
+                CacheMissType = (Xldb.PipCacheMissType)data.CacheMissType
+            };
+        }
+
+        /// <nodoc />
+        public static Xldb.StatusReportedEvent ToResourceUsageReportedEvent(this StatusEventData data, uint workerID)
+        {
+            var Uuid = Guid.NewGuid().ToString();
+
+            var statusReportedEvent = new Xldb.StatusReportedEvent()
+            {
+                UUID = Uuid,
+                WorkerID = workerID,
+                Time = Google.Protobuf.WellKnownTypes.Timestamp.FromDateTime(data.Time),
+                CpuPercent = data.CpuPercent,
+                RamPercent = data.RamPercent,
+                MachineRamUtilizationMB = data.MachineRamUtilizationMB,
+                CommitPercent = data.CommitPercent,
+                CommitTotalMB = data.CommitTotalMB,
+                ProcessCpuPercent = data.ProcessCpuPercent,
+                ProcessWorkingSetMB = data.ProcessWorkingSetMB,
+                CpuWaiting = data.CpuWaiting,
+                CpuRunning = data.CpuRunning,
+                IoCurrentMax = data.IoCurrentMax,
+                IoWaiting = data.IoWaiting,
+                IoRunning = data.IoRunning,
+                LookupWaiting = data.LookupWaiting,
+                LookupRunning = data.LookupRunning,
+                ExternalProcesses = data.ExternalProcesses,
+                LimitingResource = (Xldb.ExecutionSampler_LimitingResource)data.LimitingResource,
+                UnresponsivenessFactor = data.UnresponsivenessFactor,
+                ProcessPipsPending = data.ProcessPipsPending,
+                ProcessPipsAllocatedSlots = data.ProcessPipsAllocatedSlots
+            };
+
+            statusReportedEvent.DiskPercents.AddRange(data.DiskPercents.Select(percent => percent));
+            statusReportedEvent.DiskQueueDepths.AddRange(data.DiskQueueDepths.Select(depth => depth));
+            statusReportedEvent.PipsSucceededAllTypes.AddRange(data.PipsSucceededAllTypes.Select(type => type));
+
+            return statusReportedEvent;
+        }
+
+        /// <nodoc />
+        public static Xldb.BXLInvocationEvent ToBXLInvocationEvent(this DominoInvocationEventData data, uint workerID, PathTable pathTable)
+        {
+            var loggingConfig = data.Configuration.Logging;
+            var Uuid = Guid.NewGuid().ToString();
+
+            var bxlInvEvent = new Xldb.BXLInvocationEvent
+            {
+                UUID = Uuid,
+                WorkerID = workerID,
+                SubstSource = loggingConfig.SubstSource.ToAbsolutePath(pathTable),
+                SubstTarget = loggingConfig.SubstTarget.ToAbsolutePath(pathTable),
+                IsSubstSourceValid = loggingConfig.SubstSource.IsValid,
+                IsSubstTargetValid = loggingConfig.SubstTarget.IsValid
+            };
+
+            return bxlInvEvent;
+        }
+
+        /// <nodoc />
+        public static Xldb.PipExecutionDirectoryOutputsEvent ToPipExecutionDirectoryOutputsEvent(this PipExecutionDirectoryOutputs data, uint workerID, PathTable pathTable)
+        {
+            var Uuid = Guid.NewGuid().ToString();
+
+            var pipExecDirectoryOutputEvent = new Xldb.PipExecutionDirectoryOutputsEvent();
+            pipExecDirectoryOutputEvent.UUID = Uuid;
+            pipExecDirectoryOutputEvent.WorkerID = workerID;
+
+            foreach (var (directoryArtifact, fileArtifactArray) in data.DirectoryOutputs)
+            {
+                var directoryOutput = new Xldb.DirectoryOutput()
+                {
+                    DirectoryArtifact = new Xldb.DirectoryArtifact()
+                    {
+                        IsValid = directoryArtifact.IsValid,
+                        Path = directoryArtifact.Path.ToAbsolutePath(pathTable),
+                        PartialSealID = directoryArtifact.PartialSealId,
+                        IsSharedOpaque = directoryArtifact.IsSharedOpaque
+                    }
+                };
+
+                directoryOutput.FileArtifactArray.AddRange(
+                    fileArtifactArray.Select(
+                        file => file.ToFileArtifact(pathTable)));
+                pipExecDirectoryOutputEvent.DirectoryOutput.Add(directoryOutput);
+            }
+            return pipExecDirectoryOutputEvent;
+        }
+
+        /// <nodoc />
+        public static Xldb.ReportedFileAccess ToReportedFileAccess(this ReportedFileAccess reportedFileAccess, PathTable pathTable)
+        {
+            return new Xldb.ReportedFileAccess()
+            {
+                CreationDisposition = (Xldb.CreationDisposition)reportedFileAccess.CreationDisposition,
+                DesiredAccess = (Xldb.DesiredAccess)reportedFileAccess.DesiredAccess,
+                Error = reportedFileAccess.Error,
+                Usn = reportedFileAccess.Usn.Value,
+                FlagsAndAttributes = (Xldb.FlagsAndAttributes)reportedFileAccess.FlagsAndAttributes,
+                Path = reportedFileAccess.Path,
+                ManifestPath = reportedFileAccess.ManifestPath.ToString(pathTable, PathFormat.HostOs),
+                Process = reportedFileAccess.Process.ToReportedProcess(),
+                ShareMode = (Xldb.ShareMode)reportedFileAccess.ShareMode,
+                Status = (Xldb.FileAccessStatus)reportedFileAccess.Status,
+                Method = (Xldb.FileAccessStatusMethod)reportedFileAccess.Method,
+                RequestedAccess = (Xldb.RequestedAccess)reportedFileAccess.RequestedAccess,
+                Operation = (Xldb.ReportedFileOperation)reportedFileAccess.Operation,
+                ExplicitlyReported = reportedFileAccess.ExplicitlyReported,
+                EnumeratePattern = reportedFileAccess.EnumeratePattern
+            };
+        }
+
+        /// <nodoc />
+        public static Xldb.ObservedPathSet ToObservedPathSet(this ObservedPathSet pathSet, PathTable pathTable)
+        {
+            var observedPathSet = new Xldb.ObservedPathSet();
+            observedPathSet.Paths.AddRange(pathSet.Paths.Select(pathEntry => pathEntry.ToObservedPathEntry(pathTable)));
+            observedPathSet.ObservedAccessedFileNames.AddRange(
+                pathSet.ObservedAccessedFileNames.Select(
+                    observedAccessedFileName => new Xldb.StringId() { Value = observedAccessedFileName.Value }));
+            observedPathSet.UnsafeOptions = pathSet.UnsafeOptions.ToUnsafeOptions();
+
+            return observedPathSet;
+        }
+
+        /// <nodoc />
+        public static Xldb.UnsafeOptions ToUnsafeOptions(this UnsafeOptions unsafeOption)
+        {
+            var unsafeOpt = new Xldb.UnsafeOptions()
+            {
+                PreserveOutputsSalt = new Xldb.ContentHash()
+                {
+                    Value = unsafeOption.PreserveOutputsSalt.ToString()
+                },
+                UnsafeConfiguration = new Xldb.UnsafeSandboxConfiguration()
+                {
+                    PreserveOutputs = (Xldb.PreserveOutputsMode)unsafeOption.UnsafeConfiguration.PreserveOutputs,
+                    MonitorFileAccesses = unsafeOption.UnsafeConfiguration.MonitorFileAccesses,
+                    IgnoreZwRenameFileInformation = unsafeOption.UnsafeConfiguration.IgnoreZwRenameFileInformation,
+                    IgnoreZwOtherFileInformation = unsafeOption.UnsafeConfiguration.IgnoreZwOtherFileInformation,
+                    IgnoreNonCreateFileReparsePoints = unsafeOption.UnsafeConfiguration.IgnoreNonCreateFileReparsePoints,
+                    IgnoreSetFileInformationByHandle = unsafeOption.UnsafeConfiguration.IgnoreSetFileInformationByHandle,
+                    IgnoreReparsePoints = unsafeOption.UnsafeConfiguration.IgnoreReparsePoints,
+                    IgnorePreloadedDlls = unsafeOption.UnsafeConfiguration.IgnorePreloadedDlls,
+                    ExistingDirectoryProbesAsEnumerations = unsafeOption.UnsafeConfiguration.ExistingDirectoryProbesAsEnumerations,
+                    MonitorNtCreateFile = unsafeOption.UnsafeConfiguration.MonitorNtCreateFile,
+                    MonitorZwCreateOpenQueryFile = unsafeOption.UnsafeConfiguration.MonitorZwCreateOpenQueryFile,
+                    SandboxKind = (Xldb.SandboxKind)unsafeOption.UnsafeConfiguration.SandboxKind,
+                    UnexpectedFileAccessesAreErrors = unsafeOption.UnsafeConfiguration.UnexpectedFileAccessesAreErrors,
+                    IgnoreGetFinalPathNameByHandle = unsafeOption.UnsafeConfiguration.IgnoreGetFinalPathNameByHandle,
+                    IgnoreDynamicWritesOnAbsentProbes = unsafeOption.UnsafeConfiguration.IgnoreDynamicWritesOnAbsentProbes,
+                    IgnoreUndeclaredAccessesUnderSharedOpaques = unsafeOption.UnsafeConfiguration.IgnoreUndeclaredAccessesUnderSharedOpaques,
+                }
+            };
+
+            if (unsafeOption.UnsafeConfiguration.DoubleWritePolicy != null)
+            {
+                unsafeOpt.UnsafeConfiguration.DoubleWritePolicy = (Xldb.DoubleWritePolicy)unsafeOption.UnsafeConfiguration.DoubleWritePolicy;
+            }
+            return unsafeOpt;
+        }
+
+        /// <nodoc />
+        public static Xldb.AbsolutePath ToAbsolutePath(this AbsolutePath path, PathTable pathTable)
+        {
+            return new Xldb.AbsolutePath()
+            {
+                Value = path.ToString(pathTable, PathFormat.HostOs)
+            };
+        }
+
+        /// <nodoc />
+        public static Xldb.FileArtifact ToFileArtifact(this FileArtifact fileArtifact, PathTable pathTable)
+        {
+            return new Xldb.FileArtifact
+            {
+                Path = fileArtifact.Path.ToAbsolutePath(pathTable),
+                RewriteCount = fileArtifact.RewriteCount,
+            };
+        }
+
+        /// <nodoc />
+        public static Xldb.ReportedProcess ToReportedProcess(this ReportedProcess reportedProcess)
+        {
+            return new Xldb.ReportedProcess()
+            {
+                Path = reportedProcess.Path,
+                ProcessId = reportedProcess.ProcessId,
+                ProcessArgs = reportedProcess.ProcessArgs,
+                ReadCounters = new Xldb.IOTypeCounters
+                {
+                    OperationCount = reportedProcess.IOCounters.ReadCounters.OperationCount,
+                    TransferCOunt = reportedProcess.IOCounters.ReadCounters.TransferCount
+                },
+                WriteCounters = new Xldb.IOTypeCounters
+                {
+                    OperationCount = reportedProcess.IOCounters.WriteCounters.OperationCount,
+                    TransferCOunt = reportedProcess.IOCounters.WriteCounters.TransferCount
+                },
+                OtherCounters = new Xldb.IOTypeCounters
+                {
+                    OperationCount = reportedProcess.IOCounters.OtherCounters.OperationCount,
+                    TransferCOunt = reportedProcess.IOCounters.OtherCounters.TransferCount
+                },
+                CreationTime = Google.Protobuf.WellKnownTypes.Timestamp.FromDateTime(reportedProcess.CreationTime),
+                ExitTime = Google.Protobuf.WellKnownTypes.Timestamp.FromDateTime(reportedProcess.ExitTime),
+                KernelTime = Google.Protobuf.WellKnownTypes.Duration.FromTimeSpan(reportedProcess.KernelTime),
+                UserTime = Google.Protobuf.WellKnownTypes.Duration.FromTimeSpan(reportedProcess.UserTime),
+                ExitCode = reportedProcess.ExitCode,
+                ParentProcessId = reportedProcess.ParentProcessId
+            };
+        }
+
+        /// <nodoc />
+        public static Xldb.ObservedPathEntry ToObservedPathEntry(this ObservedPathEntry pathEntry, PathTable pathTable)
+        {
+            return new Xldb.ObservedPathEntry()
+            {
+                Path = pathEntry.Path.ToAbsolutePath(pathTable),
+                EnumeratePatternRegex = pathEntry.EnumeratePatternRegex ?? ""
+            };
+        }
+
+        /// <nodoc />
+        public static Xldb.Fingerprint ToFingerprint(this Fingerprint fingerprint)
+        {
+            return new Xldb.Fingerprint()
+            {
+                Length = fingerprint.Length,
+                Bytes = Google.Protobuf.ByteString.CopyFrom(fingerprint.ToByteArray())
+            };
+        }
+    }
+}

Public/Src/Tools/Execution.Analyzer/BxlAnalyzer.dsc

@@ -1,13 +1,10 @@
 // Copyright (c) Microsoft. All rights reserved.
 // Licensed under the MIT license. See LICENSE file in the project root for full license information.
-
 import * as Managed from "Sdk.Managed";
 import * as GrpcSdk from "Sdk.Protocols.Grpc";
 
 namespace Execution.Analyzer {
-
     export declare const qualifier: BuildXLSdk.DefaultQualifier;
-
     @@public
     export const exe = BuildXLSdk.executable({
         assemblyName: "bxlanalyzer",
@@ -17,10 +14,14 @@ namespace Execution.Analyzer {
         skipDocumentationGeneration: true,
         sources: [
             ...globR(d`.`, "*.cs"),
-            ...GrpcSdk.generate({proto: [f`Analyzers.core\XLGPlusPlus\Events.proto`]}).sources,
+            ...GrpcSdk.generate({
+                proto: globR(d`.`, "*.proto"),
+                includes: [importFrom("Google.Protobuf.Tools").Contents.all],
+            }).sources,
         ],
         references: [
-            ...addIf(BuildXLSdk.isFullFramework,
+            ...addIf(
+                BuildXLSdk.isFullFramework,
                 NetFx.System.IO.dll,
                 NetFx.System.Web.dll,
                 NetFx.System.Xml.dll,
@@ -51,18 +52,14 @@ namespace Execution.Analyzer {
             importFrom("BuildXL.Utilities.Instrumentation").Tracing.dll,
             importFrom("BuildXL.Utilities").Collections.dll,
             importFrom("BuildXL.Utilities").Configuration.dll,
+            importFrom("Google.Protobuf").pkg,
             importFrom("Newtonsoft.Json").pkg,
             importFrom("Microsoft.IdentityModel.Clients.ActiveDirectory").pkg,
             importFrom("Microsoft.TeamFoundationServer.Client").pkg,
             importFrom("Microsoft.VisualStudio.Services.Client").pkg,
             importFrom("Microsoft.VisualStudio.Services.InteractiveClient").pkg,
-            importFrom("Google.Protobuf").pkg,
         ],
-        internalsVisibleTo: [
-            "Test.Tool.Analyzers",
-        ],
-        defineConstants: addIf(BuildXLSdk.Flags.isVstsArtifactsEnabled,
-            "FEATURE_VSTS_ARTIFACTSERVICES"
-        ),
+        internalsVisibleTo: ["Test.Tool.Analyzers"],
+        defineConstants: addIf(BuildXLSdk.Flags.isVstsArtifactsEnabled, "FEATURE_VSTS_ARTIFACTSERVICES"),
     });
-}
+}

Public/Src/Tools/UnitTests/Analyzers/XLGToDBAnalyzerTests.cs

@@ -19,19 +19,22 @@ namespace Test.Tool.Analyzers
     /// </summary>
     public class XLGToDBAnalyzerTests : AnalyzerTestBase
     {
-        private AbsolutePath DirPath { get; set; }
+        private AbsolutePath OutputDirPath { get; set; }
+        private AbsolutePath TestDirPath { get; set; }
+
         public XLGToDBAnalyzerTests(ITestOutputHelper output) : base(output)
         {
             AnalysisMode = AnalysisMode.XlgToDb;
 
-            DirPath = Combine(AbsolutePath.Create(Context.PathTable, TemporaryDirectory), "xlgtodb");
+            OutputDirPath = Combine(AbsolutePath.Create(Context.PathTable, TemporaryDirectory), "XlgToDb");
+            TestDirPath = Combine(AbsolutePath.Create(Context.PathTable, TemporaryDirectory), "XlgToDbTest");
 
             ModeSpecificDefaultArgs = new Option[]
             {
                 new Option
                 {
                     Name = "outputDir",
-                    Value = DirPath.ToString(Context.PathTable)
+                    Value = OutputDirPath.ToString(Context.PathTable)
                 }
             };
         }
@@ -39,7 +42,7 @@ namespace Test.Tool.Analyzers
         /// <summary>
         /// This test makes sure that a RocksDB database was created. 
         /// If a DB is created, and events have been populated, one or more sst 
-        /// files will also be created, and this test makes sure there is at least 
+        /// files will also be created, and this test makes sure there is at least one
         /// such sst file that is present.
         /// </summary>
         [Fact]
@@ -47,7 +50,7 @@ namespace Test.Tool.Analyzers
         {
             Configuration.Logging.LogExecution = true;
 
-            var file = FileArtifact.CreateOutputFile(Combine(DirPath, "blah.txt"));
+            var file = FileArtifact.CreateOutputFile(Combine(TestDirPath, "blah.txt"));
 
             var pipA = CreateAndSchedulePipBuilder(new Operation[]
             {
@@ -64,7 +67,7 @@ namespace Test.Tool.Analyzers
 
             var analyzerRes = RunAnalyzer(buildA).AssertSuccess();
 
-            XAssert.AreNotEqual(Directory.GetFiles(DirPath.ToString(Context.PathTable), "*.sst").Length, 0);
+            XAssert.AreNotEqual(Directory.GetFiles(OutputDirPath.ToString(Context.PathTable), "*.sst").Length, 0);
         }
     }
 }

Public/Src/Utilities/Utilities/Tracing/BinaryLogReader.cs

@@ -224,7 +224,12 @@ namespace BuildXL.Utilities.Tracing
                         Contract.Assert(LogStream.Position <= (position + header.EventPayloadSize), "Event handler read beyond the event payload");
                     }
 
-                    m_logStreamReader.ReadBytes((int)(m_nextReadPosition.Value - LogStream.Position));
+                    // Seek to the start of the next event as we may not have read the entire payload (i.e. EventStatsAnalyzer)
+                    if (LogStream.Position != m_nextReadPosition.Value)
+                    {
+                        LogStream.Seek(m_nextReadPosition.Value, SeekOrigin.Begin);
+                    }
+
                     return EventReadResult.Success;
                 }
             }

config.dsc

@@ -193,6 +193,7 @@ config({
                 { id: "Grpc.Core", version: "1.18.0" },
                 { id: "Grpc.Tools", version: "1.18.0" },
                 { id: "Google.Protobuf", version: "3.7.0" },
+                { id: "Google.Protobuf.Tools", version: "3.7.0" },
                 { id: "Redis-64", version: "3.0.503" },
 
                 // Testing