diff --git a/PerfettoCds/PerfettoCds.csproj b/PerfettoCds/PerfettoCds.csproj index 708dbaf..bd764ce 100644 --- a/PerfettoCds/PerfettoCds.csproj +++ b/PerfettoCds/PerfettoCds.csproj @@ -2,7 +2,7 @@ netstandard2.1 - 1.4.0 + 1.5.0 true Microsoft Microsoft Corp. diff --git a/PerfettoCds/Pipeline/Args.cs b/PerfettoCds/Pipeline/Args.cs new file mode 100644 index 0000000..94570aa --- /dev/null +++ b/PerfettoCds/Pipeline/Args.cs @@ -0,0 +1,53 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using Microsoft.Performance.SDK.Processing; +using PerfettoProcessor; +using Utilities; + +namespace PerfettoCds +{ + public class Args + { + public List ArgKeys { get; private set; } + public List Values { get; private set; } + + public static Args ParseArgs(IEnumerable perfettoArgEvents) + { + var args = new Args(); + args.ArgKeys = new List(); + args.Values = new List(); + + // Each event has multiple of these "debug annotations". They get stored in lists + foreach (var arg in perfettoArgEvents) + { + args.ArgKeys.Add(Common.StringIntern(arg.ArgKey)); + switch (arg.ValueType) + { + case "json": + case "string": + args.Values.Add(Common.StringIntern(arg.StringValue)); + break; + case "bool": + case "int": + args.Values.Add(arg.IntValue); + break; + case "uint": + case "pointer": + args.Values.Add((uint)arg.IntValue); + break; + case "real": + args.Values.Add(arg.RealValue); + break; + default: + throw new Exception("Unexpected Perfetto value type"); + } + } + + return args; + } + } +} diff --git a/PerfettoCds/Pipeline/CompositeDataCookers/PerfettoCpuSamplingEventCooker.cs b/PerfettoCds/Pipeline/CompositeDataCookers/PerfettoCpuSamplingEventCooker.cs index 86adafb..6f28a5b 100644 --- a/PerfettoCds/Pipeline/CompositeDataCookers/PerfettoCpuSamplingEventCooker.cs +++ b/PerfettoCds/Pipeline/CompositeDataCookers/PerfettoCpuSamplingEventCooker.cs @@ -32,7 +32,7 @@ namespace PerfettoCds.Pipeline.CompositeDataCookers PerfettoPluginConstants.StackProfileMappingCookerPath, PerfettoPluginConstants.StackProfileSymbolCookerPath, PerfettoPluginConstants.ThreadCookerPath, - PerfettoPluginConstants.ProcessCookerPath, + PerfettoPluginConstants.ProcessRawCookerPath, }; [DataOutput] @@ -47,7 +47,7 @@ namespace PerfettoCds.Pipeline.CompositeDataCookers { // Gather the data from all the SQL tables var threadData = requiredData.QueryOutput>(new DataOutputPath(PerfettoPluginConstants.ThreadCookerPath, nameof(PerfettoThreadCooker.ThreadEvents))); - var processData = requiredData.QueryOutput>(new DataOutputPath(PerfettoPluginConstants.ProcessCookerPath, nameof(PerfettoProcessCooker.ProcessEvents))); + var processData = requiredData.QueryOutput>(new DataOutputPath(PerfettoPluginConstants.ProcessRawCookerPath, nameof(PerfettoProcessRawCooker.ProcessEvents))); var perfSampleData = requiredData.QueryOutput>(new DataOutputPath(PerfettoPluginConstants.PerfSampleCookerPath, nameof(PerfettoPerfSampleCooker.PerfSampleEvents))); var stackProfileCallSiteData = requiredData.QueryOutput>(new DataOutputPath(PerfettoPluginConstants.StackProfileCallSiteCookerPath, nameof(PerfettoStackProfileCallSiteCooker.StackProfileCallSiteEvents))); diff --git a/PerfettoCds/Pipeline/CompositeDataCookers/PerfettoCpuSchedEventCooker.cs b/PerfettoCds/Pipeline/CompositeDataCookers/PerfettoCpuSchedEventCooker.cs index 4ac4329..01b8bc7 100644 --- a/PerfettoCds/Pipeline/CompositeDataCookers/PerfettoCpuSchedEventCooker.cs +++ b/PerfettoCds/Pipeline/CompositeDataCookers/PerfettoCpuSchedEventCooker.cs @@ -28,7 +28,7 @@ namespace PerfettoCds.Pipeline.CompositeDataCookers public IReadOnlyCollection RequiredDataCookers => new[] { PerfettoPluginConstants.ThreadCookerPath, - PerfettoPluginConstants.ProcessCookerPath, + PerfettoPluginConstants.ProcessRawCookerPath, PerfettoPluginConstants.SchedSliceCookerPath, PerfettoPluginConstants.FtraceEventCookerPath }; @@ -49,11 +49,11 @@ namespace PerfettoCds.Pipeline.CompositeDataCookers { // Gather the data from all the SQL tables var threadData = requiredData.QueryOutput>(new DataOutputPath(PerfettoPluginConstants.ThreadCookerPath, nameof(PerfettoThreadCooker.ThreadEvents))); - var processData = requiredData.QueryOutput>(new DataOutputPath(PerfettoPluginConstants.ProcessCookerPath, nameof(PerfettoProcessCooker.ProcessEvents))); + var processData = requiredData.QueryOutput>(new DataOutputPath(PerfettoPluginConstants.ProcessRawCookerPath, nameof(PerfettoProcessRawCooker.ProcessEvents))); PopulateCpuSchedulingEvents(requiredData, threadData, processData); } - void PopulateCpuSchedulingEvents(IDataExtensionRetrieval requiredData, ProcessedEventData threadData, ProcessedEventData processData) + void PopulateCpuSchedulingEvents(IDataExtensionRetrieval requiredData, ProcessedEventData threadData, ProcessedEventData processData) { var schedSliceData = requiredData.QueryOutput>(new DataOutputPath(PerfettoPluginConstants.SchedSliceCookerPath, nameof(PerfettoSchedSliceCooker.SchedSliceEvents))); @@ -138,31 +138,31 @@ namespace PerfettoCds.Pipeline.CompositeDataCookers } } - void PopulateCpuWakeEvents(IDataExtensionRetrieval requiredData, ProcessedEventData threadData, ProcessedEventData processData) + void PopulateCpuWakeEvents(IDataExtensionRetrieval requiredData, ProcessedEventData threadData, ProcessedEventData processData) { var schedWakeData = requiredData.QueryOutput>(new DataOutputPath(PerfettoPluginConstants.FtraceEventCookerPath, nameof(PerfettoFtraceEventCooker.FtraceEvents))) .Where(f => f.Name == "sched_wakeup"); - Dictionary tidToThreadMap = threadData + Dictionary tidToThreadMap = threadData .ToLookup(t => t.Tid) .ToDictionary(tg => tg.Key, tg => tg.Last()); - Dictionary upidToProcessMap = processData + Dictionary upidToProcessMap = processData .ToLookup(p => p.Upid) .ToDictionary(pg => pg.Key, pg => pg.Last()); // Create events out of the joined results foreach (var wake in schedWakeData) { - long wokenTid = long.Parse(wake.Values[1]); // This field name is pid but it is woken thread's Tid. + var wokenTid = uint.Parse(wake.Values[1]); // This field name is pid but it is woken thread's Tid. PerfettoThreadEvent wokenThread = tidToThreadMap[wokenTid]; string wokenThreadName = wokenThread.Name; - long? wokenPid = wokenThread.Upid; + var wokenPid = wokenThread.Upid; string wokenProcessName = wokenPid != null ? upidToProcessMap[wokenPid.Value].Name : wake.Values[0]; // This field name is comms but it is woken process name. string wakerThreadName = wake.ThreadName; - long wakerTid = wake.Tid; + var wakerTid = wake.Tid; PerfettoThreadEvent wakerThread = tidToThreadMap[wakerTid]; - long? wakerPid = wakerThread.Upid; + var wakerPid = wakerThread.Upid; string wakerProcessName = wakerPid != null ? upidToProcessMap[wakerPid.Value].Name : String.Empty; PerfettoCpuWakeEvent ev = new PerfettoCpuWakeEvent diff --git a/PerfettoCds/Pipeline/CompositeDataCookers/PerfettoFrameEventCooker.cs b/PerfettoCds/Pipeline/CompositeDataCookers/PerfettoFrameEventCooker.cs index 462b35d..2483d86 100644 --- a/PerfettoCds/Pipeline/CompositeDataCookers/PerfettoFrameEventCooker.cs +++ b/PerfettoCds/Pipeline/CompositeDataCookers/PerfettoFrameEventCooker.cs @@ -30,7 +30,7 @@ namespace PerfettoCds.Pipeline.CompositeDataCookers public IReadOnlyCollection RequiredDataCookers => new[] { PerfettoPluginConstants.ThreadCookerPath, - PerfettoPluginConstants.ProcessCookerPath, + PerfettoPluginConstants.ProcessRawCookerPath, PerfettoPluginConstants.ActualFrameCookerPath, PerfettoPluginConstants.ExpectedFrameCookerPath }; @@ -48,11 +48,11 @@ namespace PerfettoCds.Pipeline.CompositeDataCookers { // Gather the data from all the SQL tables var threadData = requiredData.QueryOutput>(new DataOutputPath(PerfettoPluginConstants.ThreadCookerPath, nameof(PerfettoThreadCooker.ThreadEvents))); - var processData = requiredData.QueryOutput>(new DataOutputPath(PerfettoPluginConstants.ProcessCookerPath, nameof(PerfettoProcessCooker.ProcessEvents))); + var processData = requiredData.QueryOutput>(new DataOutputPath(PerfettoPluginConstants.ProcessRawCookerPath, nameof(PerfettoProcessRawCooker.ProcessEvents))); PopulateFrameEvents(requiredData, threadData, processData); } - void PopulateFrameEvents(IDataExtensionRetrieval requiredData, ProcessedEventData threadData, ProcessedEventData processData) + void PopulateFrameEvents(IDataExtensionRetrieval requiredData, ProcessedEventData threadData, ProcessedEventData processData) { var actualFrameData = requiredData.QueryOutput>(new DataOutputPath(PerfettoPluginConstants.ActualFrameCookerPath, nameof(PerfettoActualFrameCooker.ActualFrameEvents))); var expectedFrameData = requiredData.QueryOutput>(new DataOutputPath(PerfettoPluginConstants.ExpectedFrameCookerPath, nameof(PerfettoExpectedFrameCooker.ExpectedFrameEvents))); diff --git a/PerfettoCds/Pipeline/CompositeDataCookers/PerfettoFtraceEventCooker.cs b/PerfettoCds/Pipeline/CompositeDataCookers/PerfettoFtraceEventCooker.cs index b4cbf4d..efc3959 100644 --- a/PerfettoCds/Pipeline/CompositeDataCookers/PerfettoFtraceEventCooker.cs +++ b/PerfettoCds/Pipeline/CompositeDataCookers/PerfettoFtraceEventCooker.cs @@ -30,7 +30,7 @@ namespace PerfettoCds.Pipeline.CompositeDataCookers { PerfettoPluginConstants.RawCookerPath, PerfettoPluginConstants.ThreadCookerPath, - PerfettoPluginConstants.ProcessCookerPath, + PerfettoPluginConstants.ProcessRawCookerPath, PerfettoPluginConstants.ArgCookerPath }; @@ -54,7 +54,7 @@ namespace PerfettoCds.Pipeline.CompositeDataCookers // Gather the data from all the SQL tables var rawData = requiredData.QueryOutput>(new DataOutputPath(PerfettoPluginConstants.RawCookerPath, nameof(PerfettoRawCooker.RawEvents))); var threadData = requiredData.QueryOutput>(new DataOutputPath(PerfettoPluginConstants.ThreadCookerPath, nameof(PerfettoThreadCooker.ThreadEvents))); - var processData = requiredData.QueryOutput>(new DataOutputPath(PerfettoPluginConstants.ProcessCookerPath, nameof(PerfettoProcessCooker.ProcessEvents))); + var processData = requiredData.QueryOutput>(new DataOutputPath(PerfettoPluginConstants.ProcessRawCookerPath, nameof(PerfettoProcessRawCooker.ProcessEvents))); var argsData = requiredData.QueryOutput>(new DataOutputPath(PerfettoPluginConstants.ArgCookerPath, nameof(PerfettoArgCooker.ArgEvents))); // Join them all together diff --git a/PerfettoCds/Pipeline/CompositeDataCookers/PerfettoGenericEventCooker.cs b/PerfettoCds/Pipeline/CompositeDataCookers/PerfettoGenericEventCooker.cs index 8e96880..1d2bac3 100644 --- a/PerfettoCds/Pipeline/CompositeDataCookers/PerfettoGenericEventCooker.cs +++ b/PerfettoCds/Pipeline/CompositeDataCookers/PerfettoGenericEventCooker.cs @@ -61,7 +61,7 @@ namespace PerfettoCds.Pipeline.CompositeDataCookers PerfettoPluginConstants.ArgCookerPath, PerfettoPluginConstants.ThreadTrackCookerPath, PerfettoPluginConstants.ThreadCookerPath, - PerfettoPluginConstants.ProcessCookerPath, + PerfettoPluginConstants.ProcessEventCookerPath, PerfettoPluginConstants.ProcessTrackCookerPath }; @@ -154,7 +154,7 @@ namespace PerfettoCds.Pipeline.CompositeDataCookers var argData = requiredData.QueryOutput>(new DataOutputPath(PerfettoPluginConstants.ArgCookerPath, nameof(PerfettoArgCooker.ArgEvents))); var threadTrackData = requiredData.QueryOutput>(new DataOutputPath(PerfettoPluginConstants.ThreadTrackCookerPath, nameof(PerfettoThreadTrackCooker.ThreadTrackEvents))); var threadData = requiredData.QueryOutput>(new DataOutputPath(PerfettoPluginConstants.ThreadCookerPath, nameof(PerfettoThreadCooker.ThreadEvents))); - var processData = requiredData.QueryOutput>(new DataOutputPath(PerfettoPluginConstants.ProcessCookerPath, nameof(PerfettoProcessCooker.ProcessEvents))); + var processData = requiredData.QueryOutput>(new DataOutputPath(PerfettoPluginConstants.ProcessEventCookerPath, nameof(PerfettoProcessEventCooker.ProcessEvents))); var processTrackData = requiredData.QueryOutput>(new DataOutputPath(PerfettoPluginConstants.ProcessTrackCookerPath, nameof(PerfettoProcessTrackCooker.ProcessTrackEvents))); // Join them all together @@ -173,11 +173,13 @@ namespace PerfettoCds.Pipeline.CompositeDataCookers from thread in td.DefaultIfEmpty() join threadProcess in processData on thread?.Upid equals threadProcess.Upid into pd from threadProcess in pd.DefaultIfEmpty() + join threadProcessProcess in processData on threadProcess?.Upid equals threadProcessProcess.Upid into pd1 + from threadProcessProcess in pd1.DefaultIfEmpty() join processTrack in processTrackData on slice.TrackId equals processTrack.Id into ptd from processTrack in ptd.DefaultIfEmpty() - join process in processData on processTrack?.Upid equals process.Upid into pd2 - from process in pd2.DefaultIfEmpty() - select new { slice, args, threadTrack, thread, threadProcess, process }; + join processTrackProcess in processData on processTrack?.Upid equals processTrackProcess.Upid into pd2 + from processTrackProcess in pd2.DefaultIfEmpty() + select new { slice, args, threadTrack, thread, threadProcess, threadProcessProcess, processTrackProcess }; var longestRelTS = joined.Max(f => f.slice?.RelativeTimestamp); var longestEndTs = longestRelTS.HasValue ? new Timestamp(longestRelTS.Value) : Timestamp.MaxValue; @@ -188,6 +190,8 @@ namespace PerfettoCds.Pipeline.CompositeDataCookers MaximumEventFieldCount = Math.Max(MaximumEventFieldCount, result.args.Count()); string provider = string.Empty; + + // TODO - Replace with Args.ParseArgs List argKeys = new List(); List values = new List(); // Each event has multiple of these "debug annotations". They get stored in lists @@ -230,6 +234,7 @@ namespace PerfettoCds.Pipeline.CompositeDataCookers } string processName = string.Empty; + string processLabel = string.Empty; string threadName = string.Empty; if (result.thread != null) { @@ -241,9 +246,18 @@ namespace PerfettoCds.Pipeline.CompositeDataCookers { processName = $"{result.threadProcess.Name} ({result.threadProcess.Pid})"; } - else if (result.process != null) + else if (result.processTrackProcess != null) { - processName = $"{result.process.Name} ({result.process.Pid})"; + processName = $"{result.processTrackProcess.Name} ({result.processTrackProcess.Pid})"; + } + + if (result.threadProcessProcess != null) + { + processLabel = result.threadProcessProcess.Label; + } + else if (result.processTrackProcess != null) + { + processLabel = result.processTrackProcess.Label; } int parentTreeDepthLevel = 0; @@ -293,6 +307,7 @@ namespace PerfettoCds.Pipeline.CompositeDataCookers values, argKeys, processName, + processLabel, threadName, provider, result.threadTrack, diff --git a/PerfettoCds/Pipeline/CompositeDataCookers/PerfettoLogcatEventCooker.cs b/PerfettoCds/Pipeline/CompositeDataCookers/PerfettoLogcatEventCooker.cs index 992231c..75c1720 100644 --- a/PerfettoCds/Pipeline/CompositeDataCookers/PerfettoLogcatEventCooker.cs +++ b/PerfettoCds/Pipeline/CompositeDataCookers/PerfettoLogcatEventCooker.cs @@ -28,7 +28,7 @@ namespace PerfettoCds.Pipeline.CompositeDataCookers { PerfettoPluginConstants.AndroidLogCookerPath, PerfettoPluginConstants.ThreadCookerPath, - PerfettoPluginConstants.ProcessCookerPath + PerfettoPluginConstants.ProcessRawCookerPath }; [DataOutput] @@ -50,7 +50,7 @@ namespace PerfettoCds.Pipeline.CompositeDataCookers { // Gather the data from all the SQL tables var threadData = requiredData.QueryOutput>(new DataOutputPath(PerfettoPluginConstants.ThreadCookerPath, nameof(PerfettoThreadCooker.ThreadEvents))); - var processData = requiredData.QueryOutput>(new DataOutputPath(PerfettoPluginConstants.ProcessCookerPath, nameof(PerfettoProcessCooker.ProcessEvents))); + var processData = requiredData.QueryOutput>(new DataOutputPath(PerfettoPluginConstants.ProcessRawCookerPath, nameof(PerfettoProcessRawCooker.ProcessEvents))); var androidLogData = requiredData.QueryOutput>(new DataOutputPath(PerfettoPluginConstants.AndroidLogCookerPath, nameof(PerfettoAndroidLogCooker.AndroidLogEvents))); // Join them all together diff --git a/PerfettoCds/Pipeline/CompositeDataCookers/PerfettoProcessEventCooker.cs b/PerfettoCds/Pipeline/CompositeDataCookers/PerfettoProcessEventCooker.cs new file mode 100644 index 0000000..f3c7652 --- /dev/null +++ b/PerfettoCds/Pipeline/CompositeDataCookers/PerfettoProcessEventCooker.cs @@ -0,0 +1,105 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +using System; +using System.Collections.Generic; +using System.Linq; +using Microsoft.Performance.SDK; +using Microsoft.Performance.SDK.Extensibility; +using Microsoft.Performance.SDK.Extensibility.DataCooking; +using Microsoft.Performance.SDK.Processing; +using PerfettoCds.Pipeline.DataOutput; +using PerfettoCds.Pipeline.SourceDataCookers; +using PerfettoProcessor; + +namespace PerfettoCds.Pipeline.CompositeDataCookers +{ + /// + /// Pulls data from multiple individual SQL tables and joins them to create events for process output + /// + public sealed class PerfettoProcessEventCooker : CookedDataReflector, ICompositeDataCookerDescriptor + { + public static readonly DataCookerPath DataCookerPath = PerfettoPluginConstants.ProcessEventCookerPath; + + public string Description => "Process event composite cooker"; + + public DataCookerPath Path => DataCookerPath; + + // Declare all of the cookers that are used by this CompositeCooker. + public IReadOnlyCollection RequiredDataCookers => new[] + { + PerfettoPluginConstants.ProcessRawCookerPath, + PerfettoPluginConstants.ArgCookerPath, + PerfettoPluginConstants.PackageListCookerPath, + }; + + [DataOutput] + public ProcessedEventData ProcessEvents { get; } + + /// + /// The highest number of arg fields found in any single event. + /// + [DataOutput] + public int MaximumArgsEventFieldCount { get; private set; } + + public PerfettoProcessEventCooker() : base(PerfettoPluginConstants.ProcessEventCookerPath) + { + this.ProcessEvents = + new ProcessedEventData(); + } + + public void OnDataAvailable(IDataExtensionRetrieval requiredData) + { + // Gather the data from all the SQL tables + var processData = requiredData.QueryOutput>(new DataOutputPath(PerfettoPluginConstants.ProcessRawCookerPath, nameof(PerfettoProcessRawCooker.ProcessEvents))); + var argData = requiredData.QueryOutput>(new DataOutputPath(PerfettoPluginConstants.ArgCookerPath, nameof(PerfettoArgCooker.ArgEvents))); + var packageListData = requiredData.QueryOutput>(new DataOutputPath(PerfettoPluginConstants.PackageListCookerPath, nameof(PerfettoPackageListCooker.PackageListEvents))); + + // Join them all together + + // Contains the information for each process entry with args + var joined = from process in processData + join arg in argData on process.ArgSetId equals arg.ArgSetId into args + join packageList in packageListData on process.Uid equals packageList.Uid into pld + from packageList in pld.DefaultIfEmpty() + join parentProcess in processData on process.ParentUpid equals parentProcess.Upid into pp + from parentProcess in pp.DefaultIfEmpty() + select new { process, args, packageList, parentProcess }; + + // Create events out of the joined results + foreach (var result in joined) + { + var args = Args.ParseArgs(result.args); + MaximumArgsEventFieldCount = Math.Max(MaximumArgsEventFieldCount, args.ArgKeys.Count()); + + const string ChromeProcessLabel = "chrome.process_label[0]"; + string processLabel = null; + if (args.ArgKeys.Contains(ChromeProcessLabel)) + { + processLabel = (string) args.Values[args.ArgKeys.IndexOf(ChromeProcessLabel)]; + } + + var ev = new PerfettoProcessEvent + ( + result.process.Id, + result.process.Type, + result.process.Upid, + result.process.Pid, + result.process.Name, + processLabel, + result.process.RelativeStartTimestamp.HasValue ? new Timestamp(result.process.RelativeStartTimestamp.Value) : Timestamp.Zero, + result.process.RelativeEndTimestamp.HasValue ? new Timestamp(result.process.RelativeEndTimestamp.Value) : Timestamp.MaxValue, + result.process.ParentUpid, + result.parentProcess, + result.process.Uid, + result.process.AndroidAppId, + result.process.CmdLine, + args.ArgKeys.ToArray(), + args.Values.ToArray(), + result.packageList + ); + this.ProcessEvents.AddEvent(ev); + } + this.ProcessEvents.FinalizeData(); + } + } +} diff --git a/PerfettoCds/Pipeline/CompositeDataCookers/PerfettoProcessMemoryEventCooker.cs b/PerfettoCds/Pipeline/CompositeDataCookers/PerfettoProcessMemoryEventCooker.cs index 907a29b..466dc24 100644 --- a/PerfettoCds/Pipeline/CompositeDataCookers/PerfettoProcessMemoryEventCooker.cs +++ b/PerfettoCds/Pipeline/CompositeDataCookers/PerfettoProcessMemoryEventCooker.cs @@ -29,7 +29,7 @@ namespace PerfettoCds.Pipeline.CompositeDataCookers { PerfettoPluginConstants.CounterCookerPath, PerfettoPluginConstants.ProcessCounterTrackCookerPath, - PerfettoPluginConstants.ProcessCookerPath + PerfettoPluginConstants.ProcessRawCookerPath }; [DataOutput] @@ -46,7 +46,7 @@ namespace PerfettoCds.Pipeline.CompositeDataCookers // Gather the data from all the SQL tables var counterData = requiredData.QueryOutput>(new DataOutputPath(PerfettoPluginConstants.CounterCookerPath, nameof(PerfettoCounterCooker.CounterEvents))); var processCounterTrackData = requiredData.QueryOutput>(new DataOutputPath(PerfettoPluginConstants.ProcessCounterTrackCookerPath, nameof(PerfettoProcessCounterTrackCooker.ProcessCounterTrackEvents))); - var processData = requiredData.QueryOutput>(new DataOutputPath(PerfettoPluginConstants.ProcessCookerPath, nameof(PerfettoProcessCooker.ProcessEvents))); + var processData = requiredData.QueryOutput>(new DataOutputPath(PerfettoPluginConstants.ProcessRawCookerPath, nameof(PerfettoProcessRawCooker.ProcessEvents))); // Join them all together // Counter table contains the memory count value, timestamp diff --git a/PerfettoCds/Pipeline/DataOutput/PerfettoFtraceEvent.cs.cs b/PerfettoCds/Pipeline/DataOutput/PerfettoFtraceEvent.cs.cs index dc62a5e..e78f37f 100644 --- a/PerfettoCds/Pipeline/DataOutput/PerfettoFtraceEvent.cs.cs +++ b/PerfettoCds/Pipeline/DataOutput/PerfettoFtraceEvent.cs.cs @@ -15,7 +15,7 @@ namespace PerfettoCds.Pipeline.DataOutput public string ProcessFormattedName { get; } public string ThreadFormattedName { get; } public string ThreadName { get; } - public long Tid { get; } + public uint Tid { get; } public uint Cpu { get; } // Name of the ftrace event public string Name { get; } @@ -28,7 +28,7 @@ namespace PerfettoCds.Pipeline.DataOutput string processFormattedName, string threadFormattedName, string threadName, - long tid, + uint tid, uint cpu, string name, List values, diff --git a/PerfettoCds/Pipeline/DataOutput/PerfettoGenericEvent.cs b/PerfettoCds/Pipeline/DataOutput/PerfettoGenericEvent.cs index 62b8143..e87bb0b 100644 --- a/PerfettoCds/Pipeline/DataOutput/PerfettoGenericEvent.cs +++ b/PerfettoCds/Pipeline/DataOutput/PerfettoGenericEvent.cs @@ -27,6 +27,7 @@ namespace PerfettoCds.Pipeline.DataOutput // From Process table public string Process { get; } + public string ProcessLabel { get; } // From Thread table public string Thread { get; } @@ -50,6 +51,7 @@ namespace PerfettoCds.Pipeline.DataOutput List values, List argKeys, string process, + string processLabel, string thread, string provider, PerfettoThreadTrackEvent threadTrack, @@ -66,6 +68,7 @@ namespace PerfettoCds.Pipeline.DataOutput Values = values.ToArray(); ArgKeys = argKeys.ToArray(); Process = Common.StringIntern(process); + ProcessLabel = Common.StringIntern(processLabel); Thread = Common.StringIntern(thread); Provider = Common.StringIntern(provider); ThreadTrack = threadTrack; diff --git a/PerfettoCds/Pipeline/DataOutput/PerfettoProcessEvent.cs b/PerfettoCds/Pipeline/DataOutput/PerfettoProcessEvent.cs new file mode 100644 index 0000000..6d27d58 --- /dev/null +++ b/PerfettoCds/Pipeline/DataOutput/PerfettoProcessEvent.cs @@ -0,0 +1,102 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +using Microsoft.Performance.SDK; +using PerfettoProcessor; +using System.Collections.Generic; +using Utilities; + +namespace PerfettoCds.Pipeline.DataOutput +{ + /// + /// Contains information of processes seen during the trace (composite) + /// + public class PerfettoProcessEvent + { + public long Id { get; } + public string Type { get; } + + /// + /// Unique process id. This is != the OS pid.This is a monotonic number associated to each process + /// The OS process id(pid) cannot be used as primary key because tids and pids are recycled by most kernels. + /// + public uint Upid { get; } + + /// + /// The OS id for this process. Note: this is notunique over the lifetime of the trace so cannot be used as a primary key. + /// + public uint Pid { get; } + + /// + /// The name of the process. Can be populated from manysources (e.g. ftrace, /proc scraping, track event etc). + /// + public string Name { get; } + + /// + /// The process name label populated from args or other means + /// + public string Label { get; } + + /// + /// The start timestamp of this process (if known). Isnull in most cases unless a process creation event is enabled (e.g. task_newtask ftrace event on Linux/Android). + /// + public Timestamp StartTimestamp { get; } + + /// + /// The end timestamp of this process (if known). Isnull in most cases unless a process destruction event is enabled (e.g. sched_process_free ftrace event on Linux/Android). + /// + public Timestamp EndTimestamp { get; } + + /// + /// The upid of the process which caused this process to be spawned + /// + public uint? ParentUpid { get; } + + /// + /// The parent process which caused this process to be spawned + /// + public PerfettoProcessRawEvent ParentProcess { get; } + + /// + /// The Unix user id of the process + /// + public uint? Uid { get; } + + /// + /// Android appid of this process. + /// + public uint? AndroidAppId { get; } + + /// + /// /proc/cmdline for this process. + /// + public string CmdLine { get; } + + // From Args table. The args for a process. Variable number per event + /// + /// Extra args for this process + /// + public string[] ArgKeys { get; } + public object[] Values { get; } + public PerfettoPackageListEvent PackageList { get; } + + public PerfettoProcessEvent(long id, string type, uint upid, uint pid, string name, string label, Timestamp startTimestamp, Timestamp endTimestamp, uint? parentUpid, PerfettoProcessRawEvent parentProcess, uint? uid, uint? androidAppId, string cmdLine, string[] argKeys, object[] values, PerfettoPackageListEvent packageList) + { + Id = id; + Type = type; + Upid = upid; + Pid = pid; + Name = Common.StringIntern(name); + Label = Common.StringIntern(label); + StartTimestamp = startTimestamp; + EndTimestamp = endTimestamp; + ParentUpid = parentUpid; + ParentProcess = parentProcess; + Uid = uid; + AndroidAppId = androidAppId; + CmdLine = cmdLine; + ArgKeys = argKeys; + Values = values; + PackageList = packageList; + } + } +} diff --git a/PerfettoCds/Pipeline/PerfettoPluginConstants.cs b/PerfettoCds/Pipeline/PerfettoPluginConstants.cs index 6c71e01..3e6e841 100644 --- a/PerfettoCds/Pipeline/PerfettoPluginConstants.cs +++ b/PerfettoCds/Pipeline/PerfettoPluginConstants.cs @@ -20,7 +20,7 @@ namespace PerfettoCds public const string ArgCookerId = nameof(PerfettoArgCooker); public const string ThreadCookerId = nameof(PerfettoThreadCooker); public const string ThreadTrackCookerId = nameof(PerfettoThreadTrackCooker); - public const string ProcessCookerId = nameof(PerfettoProcessCooker); + public const string ProcessRawCookerId = nameof(PerfettoProcessRawCooker); public const string SchedSliceCookerId = nameof(PerfettoSchedSliceCooker); public const string AndroidLogCookerId = nameof(PerfettoAndroidLogCooker); public const string RawCookerId = nameof(PerfettoRawCooker); @@ -37,9 +37,11 @@ namespace PerfettoCds public const string StackProfileSymbolCookerId = nameof(PerfettoStackProfileSymbolCooker); public const string ExpectedFrameCookerId = nameof(PerfettoExpectedFrameCooker); public const string ActualFrameCookerId = nameof(PerfettoActualFrameCooker); + public const string PackageListCookerId = nameof(PerfettoPackageListCooker); // ID for composite data cookers public const string GenericEventCookerId = nameof(PerfettoGenericEventCooker); + public const string ProcessEventCookerId = nameof(PerfettoProcessEventCooker); public const string CpuSchedEventCookerId = nameof(PerfettoCpuSchedEventCooker); public const string LogcatEventCookerId = nameof(PerfettoLogcatEventCooker); public const string FtraceEventCookerId = nameof(PerfettoFtraceEventCooker); @@ -56,7 +58,7 @@ namespace PerfettoCds public const string ArgEvent = PerfettoArgEvent.Key; public const string ThreadTrackEvent = PerfettoThreadTrackEvent.Key; public const string ThreadEvent = PerfettoThreadEvent.Key; - public const string ProcessEvent = PerfettoProcessEvent.Key; + public const string ProcessEventRaw = PerfettoProcessRawEvent.Key; public const string SchedSliceEvent = PerfettoSchedSliceEvent.Key; public const string AndroidLogEvent = PerfettoAndroidLogEvent.Key; public const string RawEvent = PerfettoRawEvent.Key; @@ -73,9 +75,11 @@ namespace PerfettoCds public const string StackProfileSymbolEvent = PerfettoStackProfileSymbolEvent.Key; public const string ExpectedFrameEvent = PerfettoExpectedFrameEvent.Key; public const string ActualFrameEvent = PerfettoActualFrameEvent.Key; + public const string PackageListEvent = PerfettoPackageListEvent.Key; // Output events for composite cookers - public const string GenericEvent = nameof(PerfettoGenericEvent); + public const string GenericEvent = nameof(PerfettoGenericEvent); + public const string ProcessEvent = nameof(PerfettoProcessEvent); public const string CpuSchedEvent = nameof(PerfettoCpuSchedEvent); public const string LogcatEvent = nameof(PerfettoLogcatEvent); public const string FtraceEvent = nameof(PerfettoFtraceEvent); @@ -96,8 +100,8 @@ namespace PerfettoCds DataCookerPath.ForSource(PerfettoPluginConstants.ParserId, PerfettoPluginConstants.ThreadTrackCookerId); public static readonly DataCookerPath ThreadCookerPath = DataCookerPath.ForSource(PerfettoPluginConstants.ParserId, PerfettoPluginConstants.ThreadCookerId); - public static readonly DataCookerPath ProcessCookerPath = - DataCookerPath.ForSource(PerfettoPluginConstants.ParserId, PerfettoPluginConstants.ProcessCookerId); + public static readonly DataCookerPath ProcessRawCookerPath = + DataCookerPath.ForSource(PerfettoPluginConstants.ParserId, PerfettoPluginConstants.ProcessRawCookerId); public static readonly DataCookerPath SchedSliceCookerPath = DataCookerPath.ForSource(PerfettoPluginConstants.ParserId, PerfettoPluginConstants.SchedSliceCookerId); public static readonly DataCookerPath AndroidLogCookerPath = @@ -130,10 +134,14 @@ namespace PerfettoCds DataCookerPath.ForSource(PerfettoPluginConstants.ParserId, PerfettoPluginConstants.ExpectedFrameCookerId); public static readonly DataCookerPath ActualFrameCookerPath = DataCookerPath.ForSource(PerfettoPluginConstants.ParserId, PerfettoPluginConstants.ActualFrameCookerId); + public static readonly DataCookerPath PackageListCookerPath = + DataCookerPath.ForSource(PerfettoPluginConstants.ParserId, PerfettoPluginConstants.PackageListCookerId); // Paths for composite cookers public static readonly DataCookerPath GenericEventCookerPath = DataCookerPath.ForComposite(PerfettoPluginConstants.GenericEventCookerId); + public static readonly DataCookerPath ProcessEventCookerPath = + DataCookerPath.ForComposite(PerfettoPluginConstants.ProcessEventCookerId); public static readonly DataCookerPath CpuSchedEventCookerPath = DataCookerPath.ForComposite(PerfettoPluginConstants.CpuSchedEventCookerId); public static readonly DataCookerPath LogcatEventCookerPath = diff --git a/PerfettoCds/Pipeline/PerfettoSourceParser.cs b/PerfettoCds/Pipeline/PerfettoSourceParser.cs index 3959263..8682305 100644 --- a/PerfettoCds/Pipeline/PerfettoSourceParser.cs +++ b/PerfettoCds/Pipeline/PerfettoSourceParser.cs @@ -34,7 +34,8 @@ namespace PerfettoCds private IProgress Progress; private double CurrentProgress; - public Timestamp FirstEventTimestamp { get; private set; } + public Timestamp FirstEventTimestamp { get; private set; } + public Timestamp LastEventTimestamp { get; private set; } /// /// Increase the progress percentage by a fixed percent @@ -163,7 +164,7 @@ namespace PerfettoCds new PerfettoArgEvent(), new PerfettoThreadTrackEvent(), new PerfettoThreadEvent(), - new PerfettoProcessEvent(), + new PerfettoProcessRawEvent(), new PerfettoSchedSliceEvent(), new PerfettoAndroidLogEvent(), new PerfettoRawEvent(), @@ -198,10 +199,10 @@ namespace PerfettoCds // Run the query and process the events. var dateTimeQueryStarted = DateTime.UtcNow; - traceProc.QueryTraceForEvents(query.GetSqlQuery(), query.GetEventKey(), EventCallback); + var rowCount = traceProc.QueryTraceForEvents(query.GetSqlQuery(), query.GetEventKey(), EventCallback); var dateTimeQueryFinished = DateTime.UtcNow; - logger.Verbose($"Query for {query.GetEventKey()} completed in {(dateTimeQueryFinished - dateTimeQueryStarted).TotalSeconds}s at {dateTimeQueryFinished.ToString("MM/dd/yyyy HH:mm:ss.fff")} UTC"); + logger.Verbose($"Query for {query.GetEventKey()} returning {rowCount} rows completed in {(dateTimeQueryFinished - dateTimeQueryStarted).TotalSeconds}s at {dateTimeQueryFinished.ToString("MM/dd/yyyy HH:mm:ss.fff")} UTC"); IncreaseProgress(queryProgressIncrease); @@ -222,6 +223,7 @@ namespace PerfettoCds // Get the delta between the first and last event var eventDelta = new Timestamp(lastEventTime.ToNanoseconds - firstEventTime.ToNanoseconds); this.FirstEventTimestamp = firstEventTime; + this.LastEventTimestamp = lastEventTime; // The starting UTC time is from the snapshot. We need to adjust it based on when the first event happened // The highest precision DateTime has is ticks (a tick is a group of 100 nanoseconds) diff --git a/PerfettoCds/Pipeline/SourceDataCookers/PerfettoPackageListCooker.cs b/PerfettoCds/Pipeline/SourceDataCookers/PerfettoPackageListCooker.cs new file mode 100644 index 0000000..3e4a508 --- /dev/null +++ b/PerfettoCds/Pipeline/SourceDataCookers/PerfettoPackageListCooker.cs @@ -0,0 +1,53 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +using System.Collections.Generic; +using System.Threading; +using Microsoft.Performance.SDK; +using Microsoft.Performance.SDK.Extensibility; +using Microsoft.Performance.SDK.Extensibility.DataCooking; +using Microsoft.Performance.SDK.Extensibility.DataCooking.SourceDataCooking; +using Microsoft.Performance.SDK.Processing; +using PerfettoCds.Pipeline.Events; +using PerfettoProcessor; + +namespace PerfettoCds.Pipeline.SourceDataCookers +{ + /// + /// Cooks the data from the package_list table in Perfetto traces + /// + public sealed class PerfettoPackageListCooker : SourceDataCooker + { + public override string Description => "Processes events from the package_list Perfetto SQL table"; + + // + // The data this cooker outputs. Tables or other cookers can query for this data + // via the SDK runtime + // + [DataOutput] + public ProcessedEventData PackageListEvents { get; } + + // Instructs runtime to only send events with the given keys this data cooker + public override ReadOnlyHashSet DataKeys => + new ReadOnlyHashSet(new HashSet { PerfettoPluginConstants.PackageListEvent }); + + + public PerfettoPackageListCooker() : base(PerfettoPluginConstants.PackageListCookerPath) + { + this.PackageListEvents = new ProcessedEventData(); + } + + public override DataProcessingResult CookDataElement(PerfettoSqlEventKeyed perfettoEvent, PerfettoSourceParser context, CancellationToken cancellationToken) + { + var newEvent = (PerfettoPackageListEvent)perfettoEvent.SqlEvent; + this.PackageListEvents.AddEvent(newEvent); + + return DataProcessingResult.Processed; + } + + public override void EndDataCooking(CancellationToken cancellationToken) + { + base.EndDataCooking(cancellationToken); + this.PackageListEvents.FinalizeData(); + } + } +} diff --git a/PerfettoCds/Pipeline/SourceDataCookers/PerfettoProcessCooker.cs b/PerfettoCds/Pipeline/SourceDataCookers/PerfettoProcessRawCooker.cs similarity index 72% rename from PerfettoCds/Pipeline/SourceDataCookers/PerfettoProcessCooker.cs rename to PerfettoCds/Pipeline/SourceDataCookers/PerfettoProcessRawCooker.cs index dc683c7..e93cc9f 100644 --- a/PerfettoCds/Pipeline/SourceDataCookers/PerfettoProcessCooker.cs +++ b/PerfettoCds/Pipeline/SourceDataCookers/PerfettoProcessRawCooker.cs @@ -15,7 +15,7 @@ namespace PerfettoCds.Pipeline.SourceDataCookers /// /// Cooks the data from the Process table in Perfetto traces /// - public sealed class PerfettoProcessCooker : SourceDataCooker + public sealed class PerfettoProcessRawCooker : SourceDataCooker { public override string Description => "Processes events from the process Perfetto SQL table"; @@ -24,23 +24,25 @@ namespace PerfettoCds.Pipeline.SourceDataCookers // via the SDK runtime // [DataOutput] - public ProcessedEventData ProcessEvents { get; } + public ProcessedEventData ProcessEvents { get; } // Instructs runtime to only send events with the given keys this data cooker public override ReadOnlyHashSet DataKeys => - new ReadOnlyHashSet(new HashSet { PerfettoPluginConstants.ProcessEvent }); + new ReadOnlyHashSet(new HashSet { PerfettoPluginConstants.ProcessEventRaw }); - public PerfettoProcessCooker() : base(PerfettoPluginConstants.ProcessCookerPath) + public PerfettoProcessRawCooker() : base(PerfettoPluginConstants.ProcessRawCookerPath) { - this.ProcessEvents = new ProcessedEventData(); + this.ProcessEvents = new ProcessedEventData(); } public override DataProcessingResult CookDataElement(PerfettoSqlEventKeyed perfettoEvent, PerfettoSourceParser context, CancellationToken cancellationToken) { - var newEvent = (PerfettoProcessEvent)perfettoEvent.SqlEvent; + var newEvent = (PerfettoProcessRawEvent)perfettoEvent.SqlEvent; newEvent.RelativeStartTimestamp = newEvent.StartTimestamp - context.FirstEventTimestamp.ToNanoseconds; - newEvent.RelativeEndTimestamp = newEvent.EndTimestamp - context.FirstEventTimestamp.ToNanoseconds; + newEvent.RelativeEndTimestamp = newEvent.EndTimestamp.HasValue ? + newEvent.EndTimestamp - context.FirstEventTimestamp.ToNanoseconds : + (context.LastEventTimestamp - context.FirstEventTimestamp).ToNanoseconds; this.ProcessEvents.AddEvent(newEvent); return DataProcessingResult.Processed; diff --git a/PerfettoCds/Pipeline/SourceDataCookers/PerfettoThreadCooker.cs b/PerfettoCds/Pipeline/SourceDataCookers/PerfettoThreadCooker.cs index 447b4b4..b58d570 100644 --- a/PerfettoCds/Pipeline/SourceDataCookers/PerfettoThreadCooker.cs +++ b/PerfettoCds/Pipeline/SourceDataCookers/PerfettoThreadCooker.cs @@ -39,8 +39,11 @@ namespace PerfettoCds.Pipeline.SourceDataCookers public override DataProcessingResult CookDataElement(PerfettoSqlEventKeyed perfettoEvent, PerfettoSourceParser context, CancellationToken cancellationToken) { var newEvent = (PerfettoThreadEvent)perfettoEvent.SqlEvent; - newEvent.RelativeStartTimestamp = newEvent.StartTimestamp - context.FirstEventTimestamp.ToNanoseconds; - newEvent.RelativeEndTimestamp = newEvent.EndTimestamp - context.FirstEventTimestamp.ToNanoseconds; + newEvent.RelativeStartTimestamp = newEvent.StartTimestamp - context.FirstEventTimestamp.ToNanoseconds; + newEvent.RelativeEndTimestamp = newEvent.EndTimestamp.HasValue ? + newEvent.EndTimestamp - context.FirstEventTimestamp.ToNanoseconds : + (context.LastEventTimestamp - context.FirstEventTimestamp).ToNanoseconds; + this.ThreadEvents.AddEvent(newEvent); return DataProcessingResult.Processed; diff --git a/PerfettoCds/Pipeline/Tables/PerfettoGenericEventTable.cs b/PerfettoCds/Pipeline/Tables/PerfettoGenericEventTable.cs index bcebfef..f33a30a 100644 --- a/PerfettoCds/Pipeline/Tables/PerfettoGenericEventTable.cs +++ b/PerfettoCds/Pipeline/Tables/PerfettoGenericEventTable.cs @@ -32,6 +32,10 @@ namespace PerfettoCds.Pipeline.Tables new ColumnMetadata(new Guid("{b690f27e-7938-4e86-94ef-d048cbc476cc}"), "Process", "Name of the process"), new UIHints { Width = 210 }); + private static readonly ColumnConfiguration ProcessLabelColConfig = new ColumnConfiguration( + new ColumnMetadata(new Guid("{043E8352-0543-4593-9F7A-04DBD0103A80}"), "ProcessLabel", "Label of the process"), + new UIHints { Width = 210, IsVisible = false }); + private static readonly ColumnConfiguration ThreadNameColConfig = new ColumnConfiguration( new ColumnMetadata(new Guid("{dd1cf3f6-1cab-4012-bbdf-e99e920c4112}"), "Thread", "Name of the thread"), new UIHints { Width = 210 }); @@ -156,6 +160,15 @@ namespace PerfettoCds.Pipeline.Tables genericEventProjection.Compose((genericEvent) => genericEvent.Process)); tableGenerator.AddColumn(processNameColumn); + var processLabelColumn = new DataColumn( + ProcessLabelColConfig, + genericEventProjection.Compose((genericEvent) => genericEvent.ProcessLabel)); + tableGenerator.AddColumn(processLabelColumn); + if (events.Any(f => !String.IsNullOrWhiteSpace(f.ProcessLabel))) + { + ProcessLabelColConfig.DisplayHints.IsVisible = true; + } + var threadNameColumn = new DataColumn( ThreadNameColConfig, genericEventProjection.Compose((genericEvent) => genericEvent.Thread)); @@ -254,6 +267,7 @@ namespace PerfettoCds.Pipeline.Tables { ProviderColConfig, ProcessNameColConfig, + ProcessLabelColConfig, ThreadNameColConfig, TableConfiguration.PivotColumn, // Columns before this get pivotted on EventNameColConfig, @@ -287,14 +301,14 @@ namespace PerfettoCds.Pipeline.Tables // Process-Thread Activity config var processThreadActivityColumns = new List(defaultColumns); processThreadActivityColumns.Remove(StartTimestampColConfig); - processThreadActivityColumns.Insert(8, StartTimestampColConfig); + processThreadActivityColumns.Insert(9, StartTimestampColConfig); processThreadActivityColumns.Remove(EndTimestampColConfig); - processThreadActivityColumns.Insert(9, EndTimestampColConfig); + processThreadActivityColumns.Insert(10, EndTimestampColConfig); processThreadActivityColumns.Add(CountSortedColConfig); // Different sorting than default processThreadActivityColumns.Remove(DurationColConfig); - processThreadActivityColumns.Insert(7, DurationNotSortedColConfig); + processThreadActivityColumns.Insert(8, DurationNotSortedColConfig); DurationNotSortedColConfig.DisplayHints.SortPriority = 1; DurationNotSortedColConfig.DisplayHints.SortOrder = SortOrder.Descending; @@ -306,10 +320,10 @@ namespace PerfettoCds.Pipeline.Tables // Process-Thread-Name config var processThreadNameColumns = new List(defaultColumns); - processThreadNameColumns.Insert(3, ParentDepthLevelColConfig); + processThreadNameColumns.Insert(4, ParentDepthLevelColConfig); processThreadNameColumns.Remove(EventNameColConfig); - processThreadNameColumns.Insert(4, EventNameColConfig); - processThreadNameColumns.Insert(9, ParentEventNameTreeBranchColConfig); + processThreadNameColumns.Insert(5, EventNameColConfig); + processThreadNameColumns.Insert(10, ParentEventNameTreeBranchColConfig); var processThreadNameConfig = new TableConfiguration("Process-Thread-Name") { Columns = processThreadNameColumns, @@ -318,10 +332,10 @@ namespace PerfettoCds.Pipeline.Tables // Process-Thread-Name by StartTime config var processThreadNameByStartTimeColumns = new List(defaultColumns); - processThreadNameByStartTimeColumns.Insert(3, ParentDepthLevelColConfig); + processThreadNameByStartTimeColumns.Insert(4, ParentDepthLevelColConfig); processThreadNameByStartTimeColumns.Remove(EventNameColConfig); - processThreadNameByStartTimeColumns.Insert(4, EventNameColConfig); - processThreadNameByStartTimeColumns.Insert(9, ParentEventNameTreeBranchColConfig); + processThreadNameByStartTimeColumns.Insert(5, EventNameColConfig); + processThreadNameByStartTimeColumns.Insert(10, ParentEventNameTreeBranchColConfig); processThreadNameByStartTimeColumns.Remove(EndTimestampColConfig); processThreadNameByStartTimeColumns.Insert(processThreadNameByStartTimeColumns.Count - 2, EndTimestampColConfig); @@ -333,9 +347,9 @@ namespace PerfettoCds.Pipeline.Tables // Process-Thread-ParentNameTree config var processThreadNameTreeColumns = new List(defaultColumns); - processThreadNameTreeColumns.Insert(3, ParentEventNameTreeBranchColConfig); - processThreadNameTreeColumns.Insert(9, ParentDepthLevelColConfig); - processThreadNameTreeColumns.Insert(10, ParentEventNameTreeBranchColConfig); + processThreadNameTreeColumns.Insert(4, ParentEventNameTreeBranchColConfig); + processThreadNameTreeColumns.Insert(10, ParentDepthLevelColConfig); + processThreadNameTreeColumns.Insert(11, ParentEventNameTreeBranchColConfig); var processThreadParentNameTreeConfig = new TableConfiguration("Process-Thread-ParentNameTree") { Columns = processThreadNameTreeColumns, diff --git a/PerfettoCds/Pipeline/Tables/PerfettoPackageTable.cs b/PerfettoCds/Pipeline/Tables/PerfettoPackageTable.cs new file mode 100644 index 0000000..5903f58 --- /dev/null +++ b/PerfettoCds/Pipeline/Tables/PerfettoPackageTable.cs @@ -0,0 +1,85 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +using System; +using System.Collections.Generic; +using System.Linq; +using Microsoft.Performance.SDK.Extensibility; +using Microsoft.Performance.SDK.Processing; +using PerfettoCds.Pipeline.SourceDataCookers; +using PerfettoProcessor; + +namespace PerfettoCds.Pipeline.Tables +{ + [Table] + public class PerfettoPackageTable + { + public static TableDescriptor TableDescriptor => new TableDescriptor( + Guid.Parse("{82B1AF19-B811-4B51-904E-937F3AEBE9EB}"), + "Packages", + "Metadata about packages installed on the system", + "Perfetto - Android", + defaultLayout: TableLayoutStyle.Table, + requiredDataCookers: new List { PerfettoPluginConstants.PackageListCookerPath } + ); + + private static readonly ColumnConfiguration PackageNameColumn = new ColumnConfiguration( + new ColumnMetadata(new Guid("{53AA9D46-D7F1-4053-A16B-E2CC960FF48B}"), "PackageName", "name of the package, e.g. com.google.android.gm"), + new UIHints { Width = 210 }); + private static readonly ColumnConfiguration UidColumn = new ColumnConfiguration( + new ColumnMetadata(new Guid("{3C32D709-9DE8-407E-A4D2-F0BECA33C0A9}"), "Uid", "UID processes of this package run as"), + new UIHints { Width = 210, IsVisible = false }); + private static readonly ColumnConfiguration DebuggableColumn = new ColumnConfiguration( + new ColumnMetadata(new Guid("{28B2A721-603B-46A7-A7FF-019C46CD7718}"), "Debuggable", "bool whether this app is debuggable"), + new UIHints { Width = 120 }); + private static readonly ColumnConfiguration ProfileableFromShellColumn = new ColumnConfiguration( + new ColumnMetadata(new Guid("{8A072ECE-C5C2-417B-8257-1891F32C4DEC}"), "ProfileableFromShell", "bool whether this app is profileable"), + new UIHints { Width = 210 }); + private static readonly ColumnConfiguration VersionCodeColumn = new ColumnConfiguration( + new ColumnMetadata(new Guid("{1AC6CF0F-46F9-47B6-B91E-BDF3BC558C6B}"), "VersionCode", "versionCode from the APK"), + new UIHints { Width = 210 }); + + + + public static bool IsDataAvailable(IDataExtensionRetrieval tableData) + { + return tableData.QueryOutput>( + new DataOutputPath(PerfettoPluginConstants.PackageListCookerPath, nameof(PerfettoPackageListCooker.PackageListEvents))).Any(); + } + + public static void BuildTable(ITableBuilder tableBuilder, IDataExtensionRetrieval tableData) + { + // Get data from the cooker + var events = tableData.QueryOutput>( + new DataOutputPath(PerfettoPluginConstants.PackageListCookerPath, nameof(PerfettoPackageListCooker.PackageListEvents))); + + var tableGenerator = tableBuilder.SetRowCount((int)events.Count); + var baseProjection = Projection.Index(events); + + tableGenerator.AddColumn(PackageNameColumn, baseProjection.Compose(x => x.PackageName)); + tableGenerator.AddColumn(UidColumn, baseProjection.Compose(x => x.Uid)); + tableGenerator.AddColumn(DebuggableColumn, baseProjection.Compose(x => x.Debuggable)); + tableGenerator.AddColumn(ProfileableFromShellColumn, baseProjection.Compose(x => x.ProfileableFromShell)); + tableGenerator.AddColumn(VersionCodeColumn, baseProjection.Compose(x => x.VersionCode)); + + // Default + List defaultColumns = new List() + { + PackageNameColumn, + TableConfiguration.PivotColumn, // Columns before this get pivotted on + UidColumn, + VersionCodeColumn, + ProfileableFromShellColumn, + TableConfiguration.GraphColumn, + }; + + var packageListDefaultConfig = new TableConfiguration("Default") + { + Columns = defaultColumns, + ChartType = ChartType.Line + }; + + tableBuilder.AddTableConfiguration(packageListDefaultConfig) + .SetDefaultTableConfiguration(packageListDefaultConfig); + } + } +} diff --git a/PerfettoCds/Pipeline/Tables/PerfettoProcessTable.cs b/PerfettoCds/Pipeline/Tables/PerfettoProcessTable.cs new file mode 100644 index 0000000..6c325ce --- /dev/null +++ b/PerfettoCds/Pipeline/Tables/PerfettoProcessTable.cs @@ -0,0 +1,162 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +using System; +using System.Collections.Generic; +using System.Linq; +using Microsoft.Performance.SDK.Extensibility; +using Microsoft.Performance.SDK.Processing; +using PerfettoCds.Pipeline.CompositeDataCookers; +using PerfettoCds.Pipeline.DataOutput; +using Utilities; + +namespace PerfettoCds.Pipeline.Tables +{ + [Table] + public class PerfettoProcessTable + { + public static TableDescriptor TableDescriptor => new TableDescriptor( + Guid.Parse("{B1CB0340-91E6-4BCF-B42D-DD303446CDC8}"), + "Process", + "Contains information of processes seen during the trace", + "Perfetto - System", + defaultLayout: TableLayoutStyle.GraphAndTable, + requiredDataCookers: new List { PerfettoPluginConstants.ProcessEventCookerPath } + ); + + // Set some sort of max to prevent ridiculous field counts + public const int AbsoluteMaxFields = 20; + + private static readonly ColumnConfiguration ProcessNameColumn = new ColumnConfiguration( + new ColumnMetadata(new Guid("{96C00E4C-9544-442D-BA36-8BBE980BF1D6}"), "ProcessName", "The name of the process. Can be populated from many sources (e.g. ftrace, /proc scraping, track event etc)"), + new UIHints { Width = 210 }); + private static readonly ColumnConfiguration ProcessLabelColumn = new ColumnConfiguration( + new ColumnMetadata(new Guid("{57CE4F9E-A687-45C8-9A7B-CA7824773AD0}"), "ProcessLabel", "The process label"), + new UIHints { Width = 210, IsVisible = false }); + private static readonly ColumnConfiguration UpidColumn = new ColumnConfiguration( + new ColumnMetadata(new Guid("{F23C0CBC-5823-4889-9582-31C8C2B724CA}"), "Upid", "Unique process id. This is != the OS pid.This is a monotonic number associated to each process. The OS process id(pid) cannot be used as primary key because tids and pids are recycled by most kernels."), + new UIHints { Width = 210 }); + private static readonly ColumnConfiguration PidColumn = new ColumnConfiguration( + new ColumnMetadata(new Guid("{D359564F-587F-4B5E-8213-1DD96A64772D}"), "Pid", "The OS id for this process. Note: this is not unique over the lifetime of the trace so cannot be used as a primary key."), + new UIHints { Width = 210 }); + private static readonly ColumnConfiguration StartTimestampColumn = new ColumnConfiguration( + new ColumnMetadata(new Guid("{FC213F4F-BCF1-40A2-97D5-983576672EF9}"), "StartTimestamp", "The start timestamp of this process (if known). Isnull in most cases unless a process creation event is enabled (e.g. task_newtask ftrace event on Linux/Android)."), + new UIHints { Width = 120 }); + private static readonly ColumnConfiguration EndTimestampColumn = new ColumnConfiguration( + new ColumnMetadata(new Guid("{FA7FB0B1-8DAF-4155-848E-AEE56F13AF60}"), "EndTimestamp", "The end timestamp of this process (if known). Isnull in most cases unless a process destruction event is enabled (e.g. sched_process_free ftrace event on Linux/Android)."), + new UIHints { Width = 120 }); + + private static readonly ColumnConfiguration ParentUpidColumn = new ColumnConfiguration( + new ColumnMetadata(new Guid("{F0BFB2C2-7A25-464E-BF6C-BEA7B65D3817}"), "ParentUpid", "The upid of the process which caused this process to be spawned"), + new UIHints { Width = 120 }); + private static readonly ColumnConfiguration ParentProcessNameColumn = new ColumnConfiguration( + new ColumnMetadata(new Guid("{8278B256-8A15-4BD2-99EB-3CACBEB7CA75}"), "ParentProcessName", "The name of the process which caused this process to be spawned"), + new UIHints { Width = 120 }); + private static readonly ColumnConfiguration UidColumn = new ColumnConfiguration( + new ColumnMetadata(new Guid("{873E82C4-4B79-480F-A5EF-A9364DBB8E59}"), "Uid", "The Unix user id of the process"), + new UIHints { Width = 120 }); + private static readonly ColumnConfiguration AndroidAppIdColumn = new ColumnConfiguration( + new ColumnMetadata(new Guid("{81963502-B1AA-4F98-9B75-784C57ADE40A}"), "AndroidAppId", "Android appid of this process."), + new UIHints { Width = 120 }); + private static readonly ColumnConfiguration CmdLineColumn = new ColumnConfiguration( + new ColumnMetadata(new Guid("{06771957-9545-4989-BB9C-7EE8A00D9078}"), "CmdLine", "/proc/cmdline for this process."), + new UIHints { Width = 120 }); + + + public static bool IsDataAvailable(IDataExtensionRetrieval tableData) + { + return tableData.QueryOutput>( + new DataOutputPath(PerfettoPluginConstants.ProcessEventCookerPath, nameof(PerfettoProcessEventCooker.ProcessEvents))).Any(); + } + + public static void BuildTable(ITableBuilder tableBuilder, IDataExtensionRetrieval tableData) + { + // Get data from the cooker + var events = tableData.QueryOutput>( + new DataOutputPath(PerfettoPluginConstants.ProcessEventCookerPath, nameof(PerfettoProcessEventCooker.ProcessEvents))); + + int maxArgsFieldCount = Math.Min(AbsoluteMaxFields, tableData.QueryOutput( + new DataOutputPath(PerfettoPluginConstants.ProcessEventCookerPath, nameof(PerfettoProcessEventCooker.MaximumArgsEventFieldCount)))); + + var tableGenerator = tableBuilder.SetRowCount((int)events.Count); + var baseProjection = Projection.Index(events); + + tableGenerator.AddColumn(ProcessNameColumn, baseProjection.Compose(x => x.Name)); + tableGenerator.AddColumn(ProcessLabelColumn, baseProjection.Compose(x => x.Label)); + tableGenerator.AddColumn(StartTimestampColumn, baseProjection.Compose(x => x.StartTimestamp)); + tableGenerator.AddColumn(EndTimestampColumn, baseProjection.Compose(x => x.EndTimestamp)); + tableGenerator.AddColumn(UpidColumn, baseProjection.Compose(x => x.Upid)); + tableGenerator.AddColumn(PidColumn, baseProjection.Compose(x => x.Pid)); + tableGenerator.AddColumn(ParentUpidColumn, baseProjection.Compose(x => x.ParentUpid)); + tableGenerator.AddColumn(ParentProcessNameColumn, baseProjection.Compose(x => x.ParentProcess != null ? x.ParentProcess.Name : String.Empty)); + + tableGenerator.AddColumn(UidColumn, baseProjection.Compose(x => x.Uid)); + tableGenerator.AddColumn(AndroidAppIdColumn, baseProjection.Compose(x => x.AndroidAppId)); + tableGenerator.AddColumn(CmdLineColumn, baseProjection.Compose(x => x.CmdLine)); + + if (events.Any(f => !String.IsNullOrWhiteSpace(f.Label))) + { + ProcessLabelColumn.DisplayHints.IsVisible = true; + } + + List extraProcessArgColumns = new List(); + // Add the field columns, with column names depending on the given event + for (int index = 0; index < maxArgsFieldCount; index++) + { + var colIndex = index; // This seems unncessary but causes odd runtime behavior if not done this way. Compiler is confused perhaps because w/o this func will index=genericEvent.FieldNames.Count every time. index is passed as ref but colIndex as value into func + string fieldName = "Field " + (colIndex + 1); + + var processArgKeysFieldNameProjection = baseProjection.Compose((pe) => colIndex < pe.ArgKeys.Length ? pe.ArgKeys[colIndex] : string.Empty); + + // generate a column configuration + var fieldColumnConfiguration = new ColumnConfiguration( + new ColumnMetadata(Common.GenerateGuidFromName(fieldName), fieldName, processArgKeysFieldNameProjection, fieldName) + { + IsDynamic = true + }, + new UIHints + { + IsVisible = true, + Width = 150, + TextAlignment = TextAlignment.Left, + }); + + // Add this column to the column order + extraProcessArgColumns.Add(fieldColumnConfiguration); + + var argsAsStringProjection = baseProjection.Compose((pe) => colIndex < pe.Values.Length ? pe.Values[colIndex].ToString() : string.Empty); + + tableGenerator.AddColumn(fieldColumnConfiguration, argsAsStringProjection); + } + + // Default + List defaultColumns = new List() + { + ProcessNameColumn, + ProcessLabelColumn, + TableConfiguration.PivotColumn, // Columns before this get pivotted on + CmdLineColumn, + PidColumn, + UpidColumn, + ParentUpidColumn, + ParentProcessNameColumn, + UidColumn, + AndroidAppIdColumn, + }; + defaultColumns.AddRange(extraProcessArgColumns); + defaultColumns.Add(TableConfiguration.GraphColumn); // Columns after this get graphed + defaultColumns.Add(StartTimestampColumn); + defaultColumns.Add(EndTimestampColumn); + + var processDefaultConfig = new TableConfiguration("Default") + { + Columns = defaultColumns, + ChartType = ChartType.Line + }; + processDefaultConfig.AddColumnRole(ColumnRole.StartTime, StartTimestampColumn); + processDefaultConfig.AddColumnRole(ColumnRole.EndTime, EndTimestampColumn);; + + tableBuilder.AddTableConfiguration(processDefaultConfig) + .SetDefaultTableConfiguration(processDefaultConfig); + } + } +} diff --git a/PerfettoCds/ReadMe.md b/PerfettoCds/ReadMe.md index d169549..2494405 100644 --- a/PerfettoCds/ReadMe.md +++ b/PerfettoCds/ReadMe.md @@ -12,13 +12,13 @@ See Perfetto documentation for more information on how to create [config files]( See the ["Record new trace"](https://ui.perfetto.dev/#!/record) menu for a helpful guide on creating custom config files. ### Some config options for some event types we support -* Generic Events +* [Generic Events](https://perfetto.dev/docs/reference/trace-config-proto#DataSourceConfig) * `data_sources: { config { name: "My.Trace.Event" } }` -* CPU Counters (coarse) +* [CPU Counters (coarse)](https://perfetto.dev/docs/reference/trace-config-proto#SysStatsConfig) * `data_sources: { config { name: "linux.sys_stats" @@ -29,7 +29,7 @@ See the ["Record new trace"](https://ui.perfetto.dev/#!/record) menu for a helpf } } }` -* CPU Frequency Scaling +* [CPU Frequency Scaling](https://perfetto.dev/docs/reference/trace-config-proto#FtraceConfig) * `data_sources: { config { name: "linux.ftrace" @@ -40,7 +40,7 @@ See the ["Record new trace"](https://ui.perfetto.dev/#!/record) menu for a helpf } } }` -* CPU Scheduler +* [CPU Scheduler 1](https://perfetto.dev/docs/reference/trace-config-proto#ProcessStatsConfig) [2](https://perfetto.dev/docs/reference/trace-config-proto#FtraceConfig) * `data_sources: { config { name: "linux.process_stats" @@ -66,7 +66,7 @@ data_sources: { } } }` -* Perfetto Process Memory +* [Perfetto Process Memory](https://perfetto.dev/docs/reference/trace-config-proto#SysStatsConfig) * `data_sources: { config { name: "linux.process_stats" @@ -76,7 +76,7 @@ data_sources: { } } }` -* Perfetto System Memory +* [Perfetto System Memory](https://perfetto.dev/docs/reference/trace-config-proto#SysStatsConfig) * `data_sources: { config { name: "linux.sys_stats" @@ -88,12 +88,22 @@ data_sources: { } } }` -* Perfetto Jank Detection +* [Perfetto Jank Detection]() * `data_sources: { config { name: "android.surfaceflinger.frametimeline" } }` +* [Perfetto Processes]() + * No specific config required to capture process info either for Android or Chromium +* [Perfetto Android Packages](https://perfetto.dev/docs/reference/trace-config-proto#PackagesListConfig) + * `data_sources: { + config { + name: "android.packages_list" + packages_list_config { + } + } +}` ## Additional Features diff --git a/PerfettoProcessor/Events/PerfettoAndroidLogEvent.cs b/PerfettoProcessor/Events/PerfettoAndroidLogEvent.cs index b759680..61317c2 100644 --- a/PerfettoProcessor/Events/PerfettoAndroidLogEvent.cs +++ b/PerfettoProcessor/Events/PerfettoAndroidLogEvent.cs @@ -17,7 +17,7 @@ namespace PerfettoProcessor public string PriorityString { get; set; } public string Tag { get; set; } public string Message { get; set; } - public int Utid { get; set; } + public uint Utid { get; set; } public override string GetSqlQuery() { @@ -49,7 +49,7 @@ namespace PerfettoProcessor switch (col) { case "utid": - Utid = (int)longVal; + Utid = (uint)longVal; break; case "ts": Timestamp = longVal; diff --git a/PerfettoProcessor/Events/PerfettoPackageListEvent.cs b/PerfettoProcessor/Events/PerfettoPackageListEvent.cs new file mode 100644 index 0000000..250ece3 --- /dev/null +++ b/PerfettoProcessor/Events/PerfettoPackageListEvent.cs @@ -0,0 +1,110 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +using System; +using Perfetto.Protos; +using Utilities; + +namespace PerfettoProcessor +{ + /// + /// https://perfetto.dev/docs/analysis/sql-tables#package_list + /// + public class PerfettoPackageListEvent : PerfettoSqlEvent + { + public const string Key = "PerfettoPackageListEvent"; + + public const string SqlQuery = "select id, type, package_name, uid, debuggable, profileable_from_shell, version_code from package_list order by id"; + public int Id { get; set; } + public string Type { get; set; } + + /// + /// name of the package, e.g. com.google.android.gm. + /// + public string PackageName { get; set; } + + /// + /// UID processes of this package run as. + /// + public uint Uid { get; set; } // Probably doc'ed incorrectly as a long since every other uid in Perfetto is uint + + /// + /// bool whether this app is debuggable. + /// + public bool Debuggable { get; set; } + + /// + /// bool whether this app is profileable. + /// + public bool ProfileableFromShell { get; set; } + + /// + /// versionCode from the APK. + /// + public long VersionCode { get; set; } + + public override string GetSqlQuery() + { + return SqlQuery; + } + + public override string GetEventKey() + { + return Key; + } + + public override void ProcessCell(string colName, + QueryResult.Types.CellsBatch.Types.CellType cellType, + QueryResult.Types.CellsBatch batch, + string[] stringCells, + CellCounters counters) + { + var col = colName.ToLower(); + switch (cellType) + { + case Perfetto.Protos.QueryResult.Types.CellsBatch.Types.CellType.CellInvalid: + case Perfetto.Protos.QueryResult.Types.CellsBatch.Types.CellType.CellNull: + break; + case Perfetto.Protos.QueryResult.Types.CellsBatch.Types.CellType.CellVarint: + var longVal = batch.VarintCells[counters.IntCounter++]; + switch (col) + { + case "id": + Id = (int)longVal; + break; + case "uid": + Uid = (uint) longVal; + break; + case "debuggable": + Debuggable = Convert.ToBoolean(longVal); + break; + case "profileable_from_shell": + ProfileableFromShell = Convert.ToBoolean(longVal); + break; + case "version_code": + VersionCode = longVal; + break; + } + + break; + case Perfetto.Protos.QueryResult.Types.CellsBatch.Types.CellType.CellFloat64: + break; + case Perfetto.Protos.QueryResult.Types.CellsBatch.Types.CellType.CellString: + var strVal = Common.StringIntern(stringCells[counters.StringCounter++]); + switch (col) + { + case "type": + Type = strVal; + break; + case "package_name": + PackageName = strVal; + break; + } + break; + case Perfetto.Protos.QueryResult.Types.CellsBatch.Types.CellType.CellBlob: + break; + default: + throw new Exception("Unexpected CellType"); + } + } + } +} diff --git a/PerfettoProcessor/Events/PerfettoProcessEvent.cs b/PerfettoProcessor/Events/PerfettoProcessRawEvent.cs similarity index 86% rename from PerfettoProcessor/Events/PerfettoProcessEvent.cs rename to PerfettoProcessor/Events/PerfettoProcessRawEvent.cs index 9446099..e63fdd8 100644 --- a/PerfettoProcessor/Events/PerfettoProcessEvent.cs +++ b/PerfettoProcessor/Events/PerfettoProcessRawEvent.cs @@ -8,23 +8,23 @@ namespace PerfettoProcessor /// /// https://perfetto.dev/docs/analysis/sql-tables#process /// - public class PerfettoProcessEvent : PerfettoSqlEvent + public class PerfettoProcessRawEvent : PerfettoSqlEvent { - public const string Key = "PerfettoProcessEvent"; + public const string Key = "PerfettoProcessRawEvent"; public const string SqlQuery = "select upid, id, type, pid, name, start_ts, end_ts, parent_upid, uid, android_appid, cmdline, arg_set_id from process"; - public long Upid { get; set; } public long Id { get; set; } - public string Type { get; set; } - public long Pid { get; set; } + public string Type { get; set; } + public uint Upid { get; set; } + public uint Pid { get; set; } public string Name { get; set; } public long? StartTimestamp { get; set; } public long? RelativeStartTimestamp { get; set; } public long? EndTimestamp{ get; set; } public long? RelativeEndTimestamp { get; set; } - public long? ParentUpid { get; set; } - public long? Uid { get; set; } - public long? AndroidAppId { get; set; } + public uint? ParentUpid { get; set; } + public uint? Uid { get; set; } + public uint? AndroidAppId { get; set; } public string CmdLine { get; set; } public uint ArgSetId { get; set; } @@ -55,22 +55,22 @@ namespace PerfettoProcessor switch (col) { case "upid": - Upid = longVal; + Upid = (uint) longVal; break; case "id": Id = longVal; break; case "pid": - Pid = longVal; + Pid = (uint) longVal; break; case "uid": - Uid = longVal; + Uid = (uint) longVal; break; case "parent_upid": - ParentUpid = longVal; + ParentUpid = (uint) longVal; break; case "android_appid": - AndroidAppId = longVal; + AndroidAppId = (uint) longVal; break; case "arg_set_id": ArgSetId = (uint)longVal; diff --git a/PerfettoProcessor/Events/PerfettoSchedSliceEvent.cs b/PerfettoProcessor/Events/PerfettoSchedSliceEvent.cs index 9a8e5f4..0b90660 100644 --- a/PerfettoProcessor/Events/PerfettoSchedSliceEvent.cs +++ b/PerfettoProcessor/Events/PerfettoSchedSliceEvent.cs @@ -11,7 +11,7 @@ namespace PerfettoProcessor public const string SqlQuery = "select utid, ts, dur, cpu, end_state, priority from sched_slice"; - public int Utid { get; set; } + public uint Utid { get; set; } public long Timestamp { get; set; } public long RelativeTimestamp { get; set; } public long Duration { get; set; } @@ -111,7 +111,7 @@ namespace PerfettoProcessor switch (col) { case "utid": - Utid = (int)longVal; + Utid = (uint)longVal; break; case "ts": Timestamp = longVal; diff --git a/PerfettoProcessor/Events/PerfettoThreadEvent.cs b/PerfettoProcessor/Events/PerfettoThreadEvent.cs index b01c06c..948ba3e 100644 --- a/PerfettoProcessor/Events/PerfettoThreadEvent.cs +++ b/PerfettoProcessor/Events/PerfettoThreadEvent.cs @@ -10,18 +10,18 @@ namespace PerfettoProcessor public const string Key = "PerfettoThreadEvent"; public const string SqlQuery = "select utid, id, type, tid, name, start_ts, end_ts, upid, is_main_thread from thread"; - public long Utid { get; set; } + public uint Utid { get; set; } public long Id { get; set; } public string Type { get; set; } - public long Tid { get; set; } + public uint Tid { get; set; } public string Name{ get; set; } public long? StartTimestamp { get; set; } public long? RelativeStartTimestamp { get; set; } public long? EndTimestamp { get; set; } public long? RelativeEndTimestamp { get; set; } - public long? Upid { get; set; } - public long? IsMainThread{ get; set; } + public uint? Upid { get; set; } + public uint? IsMainThread{ get; set; } public override string GetSqlQuery() { @@ -50,19 +50,19 @@ namespace PerfettoProcessor switch (col) { case "utid": - Utid = longVal; + Utid = (uint) longVal; break; case "id": Id = longVal; break; case "upid": - Upid = longVal; + Upid = (uint) longVal; break; case "tid": - Tid = longVal; + Tid = (uint) longVal; break; case "is_main_thread": - IsMainThread = longVal; + IsMainThread = (uint) longVal; break; case "start_ts": StartTimestamp = longVal; diff --git a/PerfettoProcessor/PerfettoProcessor.csproj b/PerfettoProcessor/PerfettoProcessor.csproj index 3e5a4b8..73670e5 100644 --- a/PerfettoProcessor/PerfettoProcessor.csproj +++ b/PerfettoProcessor/PerfettoProcessor.csproj @@ -2,7 +2,7 @@ netstandard2.1 - 1.4.0 + 1.5.0 true Microsoft Microsoft Corp. diff --git a/PerfettoProcessor/PerfettoTraceProcessor.cs b/PerfettoProcessor/PerfettoTraceProcessor.cs index 2991688..be3edf5 100644 --- a/PerfettoProcessor/PerfettoTraceProcessor.cs +++ b/PerfettoProcessor/PerfettoTraceProcessor.cs @@ -238,8 +238,8 @@ namespace PerfettoProcessor var rpcResult = SendRpcRequest(rpc); return rpcResult.Msg; - } - + } + /// /// Perform a SQL query against trace_processor_shell to gather Perfetto trace data. Processes the QueryResult /// and returns PerfettoSqlObjects through a callback @@ -247,13 +247,15 @@ namespace PerfettoProcessor /// The query to perform against the loaded trace in trace_processor_shell /// The event key that corresponds to the type of PerfettoSqlEvent to process for this query /// Completed PerfettoSqlEvents will be sent here - public void QueryTraceForEvents(string sqlQuery, string eventKey, Action eventCallback) + /// Count of rows from query result + public long QueryTraceForEvents(string sqlQuery, string eventKey, Action eventCallback) { + long eventCount = 0; var rpcs = QueryTrace(sqlQuery); if (rpcs.Count == 0) { - return; + return eventCount; } // Column information is only available in first result @@ -283,7 +285,7 @@ namespace PerfettoProcessor PerfettoArgEvent.Key => new PerfettoArgEvent(), PerfettoThreadTrackEvent.Key => new PerfettoThreadTrackEvent(), PerfettoThreadEvent.Key => new PerfettoThreadEvent(), - PerfettoProcessEvent.Key => new PerfettoProcessEvent(), + PerfettoProcessRawEvent.Key => new PerfettoProcessRawEvent(), PerfettoSchedSliceEvent.Key => new PerfettoSchedSliceEvent(), PerfettoAndroidLogEvent.Key => new PerfettoAndroidLogEvent(), PerfettoRawEvent.Key => new PerfettoRawEvent(), @@ -303,6 +305,7 @@ namespace PerfettoProcessor PerfettoStackProfileSymbolEvent.Key => new PerfettoStackProfileSymbolEvent(), PerfettoActualFrameEvent.Key => new PerfettoActualFrameEvent(), PerfettoExpectedFrameEvent.Key => new PerfettoExpectedFrameEvent(), + PerfettoPackageListEvent.Key => new PerfettoPackageListEvent(), _ => throw new Exception("Invalid event type"), }; } @@ -320,10 +323,12 @@ namespace PerfettoProcessor eventCallback(ev); ev = null; + eventCount++; } } } } + return eventCount; } public void CloseTraceConnection() diff --git a/PerfettoUnitTest/PerfettoUnitTest.cs b/PerfettoUnitTest/PerfettoUnitTest.cs index 841b15e..1b68ac3 100644 --- a/PerfettoUnitTest/PerfettoUnitTest.cs +++ b/PerfettoUnitTest/PerfettoUnitTest.cs @@ -1,4 +1,6 @@ +using System.Collections.Generic; using System.IO; +using Microsoft.Performance.SDK; using Microsoft.Performance.SDK.Extensibility; using Microsoft.Performance.SDK.Processing; using Microsoft.Performance.Toolkit.Engine; @@ -7,6 +9,7 @@ using PerfettoCds; using PerfettoCds.Pipeline.CompositeDataCookers; using PerfettoCds.Pipeline.DataOutput; using PerfettoCds.Pipeline.SourceDataCookers; +using PerfettoCds.Pipeline.Tables; using PerfettoProcessor; namespace PerfettoUnitTest @@ -44,7 +47,7 @@ namespace PerfettoUnitTest runtime.EnableCooker(PerfettoPluginConstants.ArgCookerPath); runtime.EnableCooker(PerfettoPluginConstants.ThreadTrackCookerPath); runtime.EnableCooker(PerfettoPluginConstants.ThreadCookerPath); - runtime.EnableCooker(PerfettoPluginConstants.ProcessCookerPath); + runtime.EnableCooker(PerfettoPluginConstants.ProcessRawCookerPath); runtime.EnableCooker(PerfettoPluginConstants.SchedSliceCookerPath); runtime.EnableCooker(PerfettoPluginConstants.RawCookerPath); runtime.EnableCooker(PerfettoPluginConstants.CounterCookerPath); @@ -59,9 +62,11 @@ namespace PerfettoUnitTest runtime.EnableCooker(PerfettoPluginConstants.StackProfileSymbolCookerPath); runtime.EnableCooker(PerfettoPluginConstants.ExpectedFrameCookerPath); runtime.EnableCooker(PerfettoPluginConstants.ActualFrameCookerPath); + runtime.EnableCooker(PerfettoPluginConstants.PackageListCookerPath); // Enable the composite data cookers runtime.EnableCooker(PerfettoPluginConstants.GenericEventCookerPath); + runtime.EnableCooker(PerfettoPluginConstants.ProcessEventCookerPath); runtime.EnableCooker(PerfettoPluginConstants.GpuCountersEventCookerPath); runtime.EnableCooker(PerfettoPluginConstants.CpuSchedEventCookerPath); runtime.EnableCooker(PerfettoPluginConstants.LogcatEventCookerPath); @@ -70,6 +75,9 @@ namespace PerfettoUnitTest runtime.EnableCooker(PerfettoPluginConstants.CpuSamplingEventCookerPath); runtime.EnableCooker(PerfettoPluginConstants.FrameEventCookerPath); + // Enable tables + runtime.EnableTable(PerfettoProcessTable.TableDescriptor); + runtime.EnableTable(PerfettoPackageTable.TableDescriptor); // Process our data. RuntimeExecutionResults = runtime.Process(); } @@ -163,6 +171,37 @@ namespace PerfettoUnitTest Assert.IsTrue(cpuSamplingData[0].CallStack.Length == 33); Assert.IsTrue(cpuSamplingData[0].CallStack[0] == "/apex/com.android.runtime/lib64/bionic/libc.so!__libc_init"); Assert.IsTrue(cpuSamplingData[0].CallStack[32] == "/kernel!smp_call_function_many_cond"); + + var processEventData = RuntimeExecutionResults.QueryOutput>( + new DataOutputPath( + PerfettoPluginConstants.ProcessEventCookerPath, + nameof(PerfettoProcessEventCooker.ProcessEvents))); + + Assert.IsTrue(processEventData.Count == 121); + Assert.IsTrue(processEventData[1].AndroidAppId == 10135); + Assert.IsTrue(processEventData[1].Uid == 10135); + Assert.IsTrue(processEventData[1].CmdLine == "com.android.systemui"); + Assert.IsTrue(processEventData[1].ParentUpid == 25); + Assert.IsTrue(processEventData[1].ParentProcess != null && processEventData[1].ParentProcess.Name == "zygote64"); + Assert.IsTrue(processEventData[1].Pid == 980); + Assert.IsTrue(processEventData[1].Upid == 1); + Assert.IsTrue(processEventData[1].StartTimestamp == Timestamp.Zero); // NULL should be at trace start + Assert.IsTrue(processEventData[1].EndTimestamp == new Timestamp(39446647558)); // NULL should be at trace stop + + Assert.IsTrue(processEventData[119].StartTimestamp == new Timestamp(33970357558)); + Assert.IsTrue(processEventData[119].EndTimestamp == new Timestamp(34203203358)); + Assert.IsTrue(processEventData[119].ParentProcess != null && processEventData[119].ParentProcess.Name == "/apex/com.android.adbd/bin/adbd"); + + var processTable = RuntimeExecutionResults.BuildTable(PerfettoProcessTable.TableDescriptor); + Assert.IsTrue(processTable.RowCount == 121); + + var packagesList = RuntimeExecutionResults.QueryOutput>( + new DataOutputPath( + PerfettoPluginConstants.PackageListCookerPath, + nameof(PerfettoPackageListCooker.PackageListEvents))); + Assert.IsTrue(packagesList.Count == 0); + var packageTable = RuntimeExecutionResults.BuildTable(PerfettoPackageTable.TableDescriptor); + Assert.IsTrue(packageTable.RowCount == 0); } [TestMethod] @@ -231,6 +270,26 @@ namespace PerfettoUnitTest Assert.IsTrue(logcatEventData.Count == 43); Assert.IsTrue(logcatEventData[0].Message == "type: 97 score: 0.8\n"); Assert.IsTrue(logcatEventData[1].ProcessName == "Browser"); + + // Processes + var processEventData = RuntimeExecutionResults.QueryOutput>( + new DataOutputPath( + PerfettoPluginConstants.ProcessEventCookerPath, + nameof(PerfettoProcessEventCooker.ProcessEvents))); + + Assert.IsTrue(processEventData.Count == 15); + Assert.IsNull(processEventData[14].AndroidAppId); + Assert.IsNull(processEventData[14].Uid); + Assert.IsNull(processEventData[14].CmdLine); + Assert.IsNull(processEventData[14].ParentUpid); + Assert.IsNull(processEventData[14].ParentProcess); + Assert.IsTrue(processEventData[14].Name == "Renderer"); + Assert.IsTrue(processEventData[14].Pid == 17456); + Assert.IsTrue(processEventData[14].Upid == 14); + Assert.IsTrue(processEventData[14].StartTimestamp == Timestamp.Zero); // NULL should be at trace start + Assert.IsTrue(processEventData[14].EndTimestamp == new Timestamp(40409516000)); // NULL should be at trace stop + var processTable = RuntimeExecutionResults.BuildTable(PerfettoProcessTable.TableDescriptor); + Assert.IsTrue(processTable.RowCount == 15); } [TestMethod] diff --git a/README.md b/README.md index 897dcc1..2ee856c 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ > Tracing supported: - [LTTng](https://lttng.org) (Kernel CPU scheduling, Processes, Threads, Block IO/Disk, Syscalls, File events, etc) - [perf](https://perf.wiki.kernel.org/) CPU Sampling(cpu-clock) -- [Perfetto](https://perfetto.dev/) Android & Chromium (CPU Scheduling, CPU Sampling, CPU Frequency, FTrace, Android Logs, Generic Events / Default Tracks, GPU Counters, Jank Detection) +- [Perfetto](https://perfetto.dev/) Android & Chromium (CPU Scheduling, CPU Sampling, CPU Frequency, FTrace, Android Logs, Generic Events / Default Tracks, GPU Counters, Jank Detection, Processes, Android Packages) > Logs supported: - [Dmesg](https://en.wikipedia.org/wiki/Dmesg)