Supports collecting GCSettingsEvent for gc-collect profile (#4615)

This change supports collecting GCSettingsEvent for gc-collect profile.
The change will work for both dotnet-trace and dotnet-monitor.

Co-authored-by: Noah Falk <noahfalk@users.noreply.github.com>

documentation/design-docs/ipc-protocol.md

@@ -349,6 +349,8 @@ enum class EventPipeCommandId : uint8_t
     StopTracing     = 0x01, // stop a given session
     CollectTracing  = 0x02, // create/start a given session
     CollectTracing2 = 0x03, // create/start a given session with/without rundown
+    CollectTracing3 = 0x04, // create/start a given session with/without collecting stacks
+    CollectTracing4 = 0x05, // create/start a given session with specific rundown keyword
 }
 ```
 See: [EventPipe Commands](#EventPipe-Commands)
@@ -402,6 +404,8 @@ enum class EventPipeCommandId : uint8_t
     StopTracing     = 0x01, // stop a given session
     CollectTracing  = 0x02, // create/start a given session
     CollectTracing2 = 0x03, // create/start a given session with/without rundown
+    CollectTracing3 = 0x04, // create/start a given session with/without collecting stacks
+    CollectTracing4 = 0x05, // create/start a given session with specific rundown keyword
 }
 ```
 EventPipe Payloads are encoded with the following rules:
@@ -566,7 +570,7 @@ A `provider_config` is composed of the following data:
 
 Header: `{ Magic; 28; 0xFF00; 0x0000; }`
 
-`CollectTracing2` returns:
+`CollectTracing3` returns:
 * `ulong sessionId`: the ID for the stream session starting on the current connection
 
 ##### Details:
@@ -600,6 +604,67 @@ Payload
 ```
 Followed by an Optional Continuation of a `nettrace` format stream of events.
 
+### `CollectTracing4`
+
+Command Code: `0x0205`
+
+The `CollectTracing4` command is an extension of the `CollectTracing3` command - its behavior is the same as `CollectTracing3` command, except the requestRundown field is replaced by the rundownKeyword field to allow customizing the set of rundown events to be fired.
+
+> Note available for .NET 9.0 and later.
+
+#### Inputs:
+
+Header: `{ Magic; Size; 0x0205; 0x0000 }`
+
+* `uint circularBufferMB`: The size of the circular buffer used for buffering event data while streaming
+* `uint format`: 0 for the legacy NetPerf format and 1 for the NetTrace format
+* `ulong rundownKeyword`: Indicates the keyword for the rundown provider
+* `array<provider_config> providers`: The providers to turn on for the streaming session
+
+A `provider_config` is composed of the following data:
+* `ulong keywords`: The keywords to turn on with this providers
+* `uint logLevel`: The level of information to turn on
+* `string provider_name`: The name of the provider
+* `string filter_data` (optional): Filter information
+
+> see ETW documentation for a more detailed explanation of Keywords, Filters, and Log Level.
+>
+#### Returns (as an IPC Message Payload):
+
+Header: `{ Magic; 28; 0xFF00; 0x0000; }`
+
+`CollectTracing4` returns:
+* `ulong sessionId`: the ID for the stream session starting on the current connection
+
+##### Details:
+
+Input:
+```
+Payload
+{
+    uint circularBufferMB,
+    uint format,
+    ulong rundownKeyword
+    array<provider_config> providers
+}
+
+provider_config
+{
+    ulong keywords,
+    uint logLevel,
+    string provider_name,
+    string filter_data (optional)
+}
+```
+
+Returns:
+```c
+Payload
+{
+    ulong sessionId
+}
+```
+Followed by an Optional Continuation of a `nettrace` format stream of events.
 
 ### `StopTracing`
 

src/Microsoft.Diagnostics.Monitoring.EventPipe/Configuration/AggregateSourceConfiguration.cs

@@ -23,9 +23,40 @@ namespace Microsoft.Diagnostics.Monitoring.EventPipe
             return _configurations.SelectMany(c => c.GetProviders()).ToList();
         }
 
-        public override bool RequestRundown
+        public override long RundownKeyword
         {
-            get => _configurations.Any(c => c.RequestRundown);
+            get => _configurations.Select(c => c.RundownKeyword).Aggregate((x, y) => x | y);
+            set => throw new NotSupportedException();
+        }
+
+        public override RetryStrategy RetryStrategy
+        {
+            get
+            {
+                RetryStrategy result = RetryStrategy.NothingToRetry;
+                foreach (MonitoringSourceConfiguration configuration in _configurations)
+                {
+                    if (configuration.RetryStrategy == RetryStrategy.ForbiddenToRetry)
+                    {
+                        // Nothing overrides ForbiddenToRetry
+                        return RetryStrategy.ForbiddenToRetry;
+                    }
+                    else if (result == RetryStrategy.NothingToRetry)
+                    {
+                        // Anything override NothingToRetry
+                        result  = configuration.RetryStrategy;
+                    }
+                    else if (result == RetryStrategy.DropKeywordDropRundown)
+                    {
+                        if (configuration.RetryStrategy == RetryStrategy.DropKeywordKeepRundown)
+                        {
+                            // DropKeywordKeepRundown overrides DropKeywordDropRundown
+                            result = RetryStrategy.DropKeywordKeepRundown;
+                        }
+                    }
+                }
+                return result;
+            }
             set => throw new NotSupportedException();
         }
     }

src/Microsoft.Diagnostics.Monitoring.EventPipe/Configuration/AspNetTriggerSourceConfiguration.cs

@@ -17,7 +17,7 @@ namespace Microsoft.Diagnostics.Monitoring.EventPipe
 
         public AspNetTriggerSourceConfiguration(float? heartbeatIntervalSeconds = null)
         {
-            RequestRundown = false;
+            RundownKeyword = 0;
             _heartbeatIntervalSeconds = heartbeatIntervalSeconds;
         }
 

src/Microsoft.Diagnostics.Monitoring.EventPipe/Configuration/EventPipeProviderSourceConfiguration.cs

@@ -12,10 +12,10 @@ namespace Microsoft.Diagnostics.Monitoring.EventPipe
         private readonly IEnumerable<EventPipeProvider> _providers;
         private readonly int _bufferSizeInMB;
 
-        public EventPipeProviderSourceConfiguration(bool requestRundown = true, int bufferSizeInMB = 256, params EventPipeProvider[] providers)
+        public EventPipeProviderSourceConfiguration(long rundownKeyword = EventPipeSession.DefaultRundownKeyword, int bufferSizeInMB = 256, params EventPipeProvider[] providers)
         {
             _providers = providers;
-            RequestRundown = requestRundown;
+            RundownKeyword = rundownKeyword;
             _bufferSizeInMB = bufferSizeInMB;
         }
 

src/Microsoft.Diagnostics.Monitoring.EventPipe/Configuration/GCDumpSourceConfiguration.cs

@@ -12,7 +12,7 @@ namespace Microsoft.Diagnostics.Monitoring.EventPipe
     {
         public GCDumpSourceConfiguration()
         {
-            RequestRundown = false;
+            RundownKeyword = 0;
         }
 
         public override IList<EventPipeProvider> GetProviders()

src/Microsoft.Diagnostics.Monitoring.EventPipe/Configuration/GcCollectConfiguration.cs

@@ -10,7 +10,8 @@ namespace Microsoft.Diagnostics.Monitoring.EventPipe
     {
         public GcCollectConfiguration()
         {
-            RequestRundown = false;
+            RundownKeyword = (long)Tracing.Parsers.ClrTraceEventParser.Keywords.GC;
+            RetryStrategy = RetryStrategy.DropKeywordDropRundown;
         }
 
         public override IList<EventPipeProvider> GetProviders() =>

src/Microsoft.Diagnostics.Monitoring.EventPipe/Configuration/HttpRequestSourceConfiguration.cs

@@ -12,7 +12,7 @@ namespace Microsoft.Diagnostics.Monitoring.EventPipe
         public HttpRequestSourceConfiguration()
         {
             //CONSIDER removing rundown for this scenario.
-            RequestRundown = true;
+            RundownKeyword = EventPipeSession.DefaultRundownKeyword;
         }
 
         // This string is shared between HttpRequestSourceConfiguration and AspNetTriggerSourceConfiguration

src/Microsoft.Diagnostics.Monitoring.EventPipe/Configuration/LoggingSourceConfiguration.cs

@@ -24,7 +24,7 @@ namespace Microsoft.Diagnostics.Monitoring.EventPipe
         public LoggingSourceConfiguration(LogLevel level, LogMessageType messageType, IDictionary<string, LogLevel?> filterSpecs, bool useAppFilters,
             bool collectScopes)
         {
-            RequestRundown = false;
+            RundownKeyword = 0;
             _filterSpecs = ToFilterSpecsString(filterSpecs, useAppFilters);
             _keywords = (long)ToKeywords(messageType);
             _level = ToEventLevel(level);

src/Microsoft.Diagnostics.Monitoring.EventPipe/Configuration/MetricSourceConfiguration.cs

@@ -48,7 +48,7 @@ namespace Microsoft.Diagnostics.Monitoring.EventPipe
                 throw new ArgumentNullException(nameof(providers));
             }
 
-            RequestRundown = false;
+            RundownKeyword = 0;
 
             _eventPipeProviders = providers.Where(provider => provider.Type.HasFlag(MetricType.EventCounter))
                 .Select((MetricEventPipeProvider provider) => new EventPipeProvider(provider.Provider,

src/Microsoft.Diagnostics.Monitoring.EventPipe/Configuration/MonitoringSourceConfiguration.cs

@@ -32,8 +32,10 @@ namespace Microsoft.Diagnostics.Monitoring.EventPipe
 
         public abstract IList<EventPipeProvider> GetProviders();
 
-        public virtual bool RequestRundown { get; set; } = true;
+        public virtual long RundownKeyword { get; set; } = EventPipeSession.DefaultRundownKeyword;
 
         public virtual int BufferSizeInMB => 256;
+
+        public virtual RetryStrategy RetryStrategy { get; set; } = RetryStrategy.NothingToRetry;
     }
 }

src/Microsoft.Diagnostics.Monitoring.EventPipe/Configuration/RetryStrategies.cs

@@ -0,0 +1,25 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+//
+// This class describes the various strategies for retrying a command.
+// The rough idea is that these numbers form a state machine.
+// Any time a command execution fails, a retry will be attempted by matching the
+// condition of the config as well as this strategy number to generate a
+// modified config as well as a modified strategy.
+//
+// This is designed with forward compatibility in mind. We might have newer
+// capabilities that only exists in newer runtimes, but we will never know exactly
+// how we should retry. So this give us a way to encode the retry strategy in the
+// profiles without having to introducing new concepts.
+//
+namespace Microsoft.Diagnostics.Monitoring.EventPipe
+{
+    public enum RetryStrategy
+    {
+        NothingToRetry = 0,
+        DropKeywordKeepRundown = 1,
+        DropKeywordDropRundown = 2,
+        ForbiddenToRetry = 3
+    }
+}

src/Microsoft.Diagnostics.Monitoring.EventPipe/Configuration/SampleProfilerConfiguration.cs

@@ -10,6 +10,11 @@ namespace Microsoft.Diagnostics.Monitoring.EventPipe
 {
     public sealed class SampleProfilerConfiguration : MonitoringSourceConfiguration
     {
+        public SampleProfilerConfiguration()
+        {
+            RundownKeyword = 0;
+        }
+
         public override IList<EventPipeProvider> GetProviders() =>
             new EventPipeProvider[]
             {
@@ -17,11 +22,5 @@ namespace Microsoft.Diagnostics.Monitoring.EventPipe
             };
 
         public override int BufferSizeInMB => 1;
-
-        public override bool RequestRundown
-        {
-            get => false;
-            set => throw new NotSupportedException();
-        }
     }
 }

src/Microsoft.Diagnostics.Monitoring.EventPipe/EventPipeStreamProvider.cs

@@ -2,6 +2,8 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 
 using System;
+using System.Collections.Generic;
+using System.Diagnostics;
 using System.IO;
 using System.Threading;
 using System.Threading.Tasks;
@@ -28,7 +30,45 @@ namespace Microsoft.Diagnostics.Monitoring.EventPipe
             EventPipeSession session = null;
             try
             {
-                session = await client.StartEventPipeSessionAsync(_sourceConfig.GetProviders(), _sourceConfig.RequestRundown, _sourceConfig.BufferSizeInMB, cancellationToken).ConfigureAwait(false);
+                IEnumerable<EventPipeProvider> providers = _sourceConfig.GetProviders();
+                int bufferSizeInMB = _sourceConfig.BufferSizeInMB;
+                long rundownKeyword = _sourceConfig.RundownKeyword;
+                RetryStrategy retryStrategy = _sourceConfig.RetryStrategy;
+                try
+                {
+                    EventPipeSessionConfiguration config = new(providers, bufferSizeInMB, rundownKeyword, true);
+                    session = await client.StartEventPipeSessionAsync(config, cancellationToken).ConfigureAwait(false);
+                }
+                catch (UnsupportedCommandException) when (retryStrategy == RetryStrategy.DropKeywordKeepRundown)
+                {
+                    //
+                    // If you are building new profiles or options, you can test with these asserts to make sure you are writing
+                    // the retry strategies correctly.
+                    //
+                    // If these assert ever fires, it means something is wrong with the option generation logic leading to unnecessary retries.
+                    // unnecessary retries is not fatal.
+                    //
+                    // Debug.Assert(rundownKeyword != 0);
+                    // Debug.Assert(rundownKeyword != EventPipeSession.DefaultRundownKeyword);
+                    //
+                    EventPipeSessionConfiguration config = new(providers, bufferSizeInMB, EventPipeSession.DefaultRundownKeyword, true);
+                    session = await client.StartEventPipeSessionAsync(config, cancellationToken).ConfigureAwait(false);
+                }
+                catch (UnsupportedCommandException) when (retryStrategy == RetryStrategy.DropKeywordDropRundown)
+                {
+                    //
+                    // If you are building new profiles or options, you can test with these asserts to make sure you are writing
+                    // the retry strategies correctly.
+                    //
+                    // If these assert ever fires, it means something is wrong with the option generation logic leading to unnecessary retries.
+                    // unnecessary retries is not fatal.
+                    //
+                    // Debug.Assert(rundownKeyword != 0);
+                    // Debug.Assert(rundownKeyword != EventPipeSession.DefaultRundownKeyword);
+                    //
+                    EventPipeSessionConfiguration config = new(providers, bufferSizeInMB, 0, true);
+                    session = await client.StartEventPipeSessionAsync(config, cancellationToken).ConfigureAwait(false);
+                }
                 if (resumeRuntime)
                 {
                     try

src/Microsoft.Diagnostics.NETCore.Client/DiagnosticsClient/DiagnosticsClient.cs

@@ -89,6 +89,18 @@ namespace Microsoft.Diagnostics.NETCore.Client
             return EventPipeSession.Start(_endpoint, config);
         }
 
+        /// <summary>
+        /// Start tracing the application and return an EventPipeSession object
+        /// </summary>
+        /// <param name="config">The configuration for start tracing.</param>
+        /// <returns>
+        /// An EventPipeSession object representing the EventPipe session that just started.
+        /// </returns>
+        public EventPipeSession StartEventPipeSession(EventPipeSessionConfiguration config)
+        {
+            return EventPipeSession.Start(_endpoint, config);
+        }
+
         /// <summary>
         /// Start tracing the application and return an EventPipeSession object
         /// </summary>

src/Microsoft.Diagnostics.NETCore.Client/DiagnosticsClient/EventPipeSession.cs

@@ -13,6 +13,17 @@ namespace Microsoft.Diagnostics.NETCore.Client
 {
     public class EventPipeSession : IDisposable
     {
+        //! This is CoreCLR specific keywords for native ETW events (ending up in event pipe).
+        //! The keywords below seems to correspond to:
+        //!  GCKeyword                          (0x00000001)
+        //!  LoaderKeyword                      (0x00000008)
+        //!  JitKeyword                         (0x00000010)
+        //!  NgenKeyword                        (0x00000020)
+        //!  unused_keyword                     (0x00000100)
+        //!  JittedMethodILToNativeMapKeyword   (0x00020000)
+        //!  ThreadTransferKeyword              (0x80000000)
+        internal const long DefaultRundownKeyword = 0x80020139;
+
         private ulong _sessionId;
         private IpcEndpoint _endpoint;
         private bool _disposedValue; // To detect redundant calls
@@ -84,10 +95,26 @@ namespace Microsoft.Diagnostics.NETCore.Client
         private static IpcMessage CreateStartMessage(EventPipeSessionConfiguration config)
         {
             // To keep backward compatibility with older runtimes we only use newer serialization format when needed
-            // V3 has added support to disable the stacktraces
-            bool shouldUseV3 = !config.RequestStackwalk;
-            EventPipeCommandId command = shouldUseV3 ? EventPipeCommandId.CollectTracing3 : EventPipeCommandId.CollectTracing2;
-            byte[] payload = shouldUseV3 ? config.SerializeV3() : config.SerializeV2();
+            EventPipeCommandId command;
+            byte[] payload;
+            if (config.RundownKeyword != DefaultRundownKeyword && config.RundownKeyword != 0)
+            {
+                // V4 has added support to specify rundown keyword
+                command = EventPipeCommandId.CollectTracing4;
+                payload = config.SerializeV4();
+            }
+            else if (!config.RequestStackwalk)
+            {
+                // V3 has added support to disable the stacktraces
+                command = EventPipeCommandId.CollectTracing3;
+                payload = config.SerializeV3();
+            }
+            else
+            {
+                command = EventPipeCommandId.CollectTracing2;
+                payload = config.SerializeV2();
+            }
+
             return new IpcMessage(DiagnosticsServerCommandSet.EventPipe, (byte)command, payload);
         }
 

src/Microsoft.Diagnostics.NETCore.Client/DiagnosticsClient/EventPipeSessionConfiguration.cs

@@ -28,15 +28,30 @@ namespace Microsoft.Diagnostics.NETCore.Client
             IEnumerable<EventPipeProvider> providers,
             int circularBufferSizeMB = 256,
             bool requestRundown = true,
-            bool requestStackwalk = true) : this(circularBufferSizeMB, EventPipeSerializationFormat.NetTrace, providers, requestRundown, requestStackwalk)
+            bool requestStackwalk = true) : this(circularBufferSizeMB, EventPipeSerializationFormat.NetTrace, providers, requestStackwalk, (requestRundown ? EventPipeSession.DefaultRundownKeyword : 0))
+        {}
+
+        /// <summary>
+        /// Creates a new configuration object for the EventPipeSession.
+        /// For details, see the documentation of each property of this object.
+        /// </summary>
+        /// <param name="providers">An IEnumerable containing the list of Providers to turn on.</param>
+        /// <param name="circularBufferSizeMB">The size of the runtime's buffer for collecting events in MB</param>
+        /// <param name="rundownKeyword">The keyword for rundown events.</param>
+        /// <param name="requestStackwalk">If true, record a stacktrace for every emitted event.</param>
+        public EventPipeSessionConfiguration(
+            IEnumerable<EventPipeProvider> providers,
+            int circularBufferSizeMB,
+            long rundownKeyword,
+            bool requestStackwalk = true) : this(circularBufferSizeMB, EventPipeSerializationFormat.NetTrace, providers, requestStackwalk, rundownKeyword)
         {}
 
         private EventPipeSessionConfiguration(
             int circularBufferSizeMB,
             EventPipeSerializationFormat format,
             IEnumerable<EventPipeProvider> providers,
-            bool requestRundown,
-            bool requestStackwalk)
+            bool requestStackwalk,
+            long rundownKeyword)
         {
             if (circularBufferSizeMB == 0)
             {
@@ -61,8 +76,8 @@ namespace Microsoft.Diagnostics.NETCore.Client
 
             CircularBufferSizeInMB = circularBufferSizeMB;
             Format = format;
-            RequestRundown = requestRundown;
             RequestStackwalk = requestStackwalk;
+            RundownKeyword = rundownKeyword;
         }
 
         /// <summary>
@@ -73,7 +88,7 @@ namespace Microsoft.Diagnostics.NETCore.Client
         /// <item>Consider to set this parameter to false if you don't need stacktrace information or if you're analyzing events on the fly.</item>
         /// </list>
         /// </summary>
-        public bool RequestRundown { get; }
+        public bool RequestRundown => this.RundownKeyword != 0;
 
         /// <summary>
         /// The size of the runtime's buffer for collecting events in MB.
@@ -92,6 +107,11 @@ namespace Microsoft.Diagnostics.NETCore.Client
         /// </summary>
         public bool RequestStackwalk { get; }
 
+        /// <summary>
+        /// The keywords enabled for the rundown provider.
+        /// </summary>
+        public long RundownKeyword { get; internal set; }
+
         /// <summary>
         /// Providers to enable for this session.
         /// </summary>
@@ -143,6 +163,26 @@ namespace Microsoft.Diagnostics.NETCore.Client
             return serializedData;
         }
 
+        public static byte[] SerializeV4(this EventPipeSessionConfiguration config)
+        {
+            byte[] serializedData = null;
+            using (MemoryStream stream = new())
+            using (BinaryWriter writer = new(stream))
+            {
+                writer.Write(config.CircularBufferSizeInMB);
+                writer.Write((uint)config.Format);
+                writer.Write(config.RundownKeyword);
+                writer.Write(config.RequestStackwalk);
+
+                SerializeProviders(config, writer);
+
+                writer.Flush();
+                serializedData = stream.ToArray();
+            }
+
+            return serializedData;
+        }
+
         private static void SerializeProviders(EventPipeSessionConfiguration config, BinaryWriter writer)
         {
             writer.Write(config.Providers.Count);

src/Microsoft.Diagnostics.NETCore.Client/DiagnosticsIpc/IpcCommands.cs

@@ -26,6 +26,7 @@ namespace Microsoft.Diagnostics.NETCore.Client
         CollectTracing = 0x02,
         CollectTracing2 = 0x03,
         CollectTracing3 = 0x04,
+        CollectTracing4 = 0x05,
     }
 
     internal enum DumpCommandId : byte

src/Tools/dotnet-trace/CommandLine/Commands/CollectCommand.cs

@@ -125,7 +125,8 @@ namespace Microsoft.Diagnostics.Tools.Trace
                     enabledBy[providerCollectionProvider.Name] = "--providers ";
                 }
 
-                bool collectRundownEvents = true;
+                long rundownKeyword = EventPipeSession.DefaultRundownKeyword;
+                RetryStrategy retryStrategy = RetryStrategy.NothingToRetry;
 
                 if (profile.Length != 0)
                 {
@@ -137,14 +138,24 @@ namespace Microsoft.Diagnostics.Tools.Trace
                         return (int)ReturnCode.ArgumentError;
                     }
 
-                    collectRundownEvents = selectedProfile.Rundown;
+                    rundownKeyword = selectedProfile.RundownKeyword;
+                    retryStrategy = selectedProfile.RetryStrategy;
 
                     Profile.MergeProfileAndProviders(selectedProfile, providerCollection, enabledBy);
                 }
 
                 if (rundown.HasValue)
                 {
-                    collectRundownEvents = rundown.Value;
+                    if (rundown.Value)
+                    {
+                        rundownKeyword |= EventPipeSession.DefaultRundownKeyword;
+                        retryStrategy = (rundownKeyword == EventPipeSession.DefaultRundownKeyword) ? RetryStrategy.NothingToRetry : RetryStrategy.DropKeywordKeepRundown;
+                    }
+                    else
+                    {
+                        rundownKeyword = 0;
+                        retryStrategy = RetryStrategy.NothingToRetry;
+                    }
                 }
 
                 // Parse --clrevents parameter
@@ -271,30 +282,65 @@ namespace Microsoft.Diagnostics.Tools.Trace
                         EventPipeSession session = null;
                         try
                         {
-                            session = diagnosticsClient.StartEventPipeSession(providerCollection, collectRundownEvents, (int)buffersize);
-                            if (resumeRuntime)
-                            {
-                                try
-                                {
-                                    diagnosticsClient.ResumeRuntime();
-                                }
-                                catch (UnsupportedCommandException)
-                                {
-                                    // Noop if command is unsupported, since the target is most likely a 3.1 app.
-                                }
-                            }
+                            EventPipeSessionConfiguration config = new(providerCollection, (int)buffersize, rundownKeyword: rundownKeyword, requestStackwalk: true);
+                            session = diagnosticsClient.StartEventPipeSession(config);
                         }
-                        catch (DiagnosticsClientException e)
+                        catch (UnsupportedCommandException e)
                         {
-                            Console.Error.WriteLine($"Unable to start a tracing session: {e}");
-                            return (int)ReturnCode.SessionCreationError;
+                            if (retryStrategy == RetryStrategy.DropKeywordKeepRundown)
+                            {
+                                Console.Error.WriteLine("The runtime version being traced doesn't support the custom rundown feature used by this tracing configuration, retrying with the standard rundown keyword");
+                                //
+                                // If you are building new profiles or options, you can test with these asserts to make sure you are writing
+                                // the retry strategies correctly.
+                                //
+                                // If these assert ever fires, it means something is wrong with the option generation logic leading to unnecessary retries.
+                                // unnecessary retries is not fatal.
+                                //
+                                // Debug.Assert(rundownKeyword != 0);
+                                // Debug.Assert(rundownKeyword != EventPipeSession.DefaultRundownKeyword);
+                                //
+                                EventPipeSessionConfiguration config = new(providerCollection, (int)buffersize, rundownKeyword: EventPipeSession.DefaultRundownKeyword, requestStackwalk: true);
+                                session = diagnosticsClient.StartEventPipeSession(config);
+                            }
+                            else if (retryStrategy == RetryStrategy.DropKeywordDropRundown)
+                            {
+                                Console.Error.WriteLine("The runtime version being traced doesn't support the custom rundown feature used by this tracing configuration, retrying with the rundown omitted");
+                                //
+                                // If you are building new profiles or options, you can test with these asserts to make sure you are writing
+                                // the retry strategies correctly.
+                                //
+                                // If these assert ever fires, it means something is wrong with the option generation logic leading to unnecessary retries.
+                                // unnecessary retries is not fatal.
+                                //
+                                // Debug.Assert(rundownKeyword != 0);
+                                // Debug.Assert(rundownKeyword != EventPipeSession.DefaultRundownKeyword);
+                                //
+                                EventPipeSessionConfiguration config = new(providerCollection, (int)buffersize, rundownKeyword: 0, requestStackwalk: true);
+                                session = diagnosticsClient.StartEventPipeSession(config);
+                            }
+                            else
+                            {
+                                Console.Error.WriteLine($"Unable to start a tracing session: {e}");
+                                return (int)ReturnCode.SessionCreationError;
+                            }
                         }
                         catch (UnauthorizedAccessException e)
                         {
                             Console.Error.WriteLine($"dotnet-trace does not have permission to access the specified app: {e.GetType()}");
                             return (int)ReturnCode.SessionCreationError;
                         }
-
+                        if (resumeRuntime)
+                        {
+                            try
+                            {
+                                diagnosticsClient.ResumeRuntime();
+                            }
+                            catch (UnsupportedCommandException)
+                            {
+                                // Noop if command is unsupported, since the target is most likely a 3.1 app.
+                            }
+                        }
                         if (session == null)
                         {
                             Console.Error.WriteLine("Unable to create session.");

src/Tools/dotnet-trace/CommandLine/Commands/ListProfilesCommandHandler.cs

@@ -58,7 +58,7 @@ namespace Microsoft.Diagnostics.Tools.Trace
                         keywords: (long)ClrTraceEventParser.Keywords.GC |
                                   (long)ClrTraceEventParser.Keywords.GCHandle |
                                   (long)ClrTraceEventParser.Keywords.Exception
-                    ),
+                    )
                 },
                 "Tracks GC collections and samples object allocations."),
             new Profile(
@@ -75,7 +75,7 @@ namespace Microsoft.Diagnostics.Tools.Trace
                         keywords: (long)ClrTraceEventParser.Keywords.GC
                     )
                 },
-                "Tracks GC collections only at very low overhead.") { Rundown = false },
+                "Tracks GC collections only at very low overhead.") { RundownKeyword = (long)ClrTraceEventParser.Keywords.GC, RetryStrategy = RetryStrategy.DropKeywordDropRundown },
             new Profile(
                 "database",
                 new EventPipeProvider[] {

src/Tools/dotnet-trace/Profile.cs

@@ -22,7 +22,9 @@ namespace Microsoft.Diagnostics.Tools.Trace
 
         public string Description { get; }
 
-        public bool Rundown { get; set; } = true;
+        public long RundownKeyword { get; set; }
+
+        public RetryStrategy RetryStrategy { get; set; } = RetryStrategy.NothingToRetry;
 
         public static void MergeProfileAndProviders(Profile selectedProfile, List<EventPipeProvider> providerCollection, Dictionary<string, string> enabledBy)
         {

src/Tools/dotnet-trace/RetryStrategies.cs

@@ -0,0 +1,25 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+//
+// This class describes the various strategies for retrying a command.
+// The rough idea is that these numbers form a state machine.
+// Any time a command execution fails, a retry will be attempted by matching the
+// condition of the config as well as this strategy number to generate a
+// modified config as well as a modified strategy.
+//
+// This is designed with forward compatibility in mind. We might have newer
+// capabilities that only exists in newer runtimes, but we will never know exactly
+// how we should retry. So this give us a way to encode the retry strategy in the
+// profiles without having to introducing new concepts.
+//
+namespace Microsoft.Diagnostics.Tools.Trace
+{
+    internal enum RetryStrategy
+    {
+        NothingToRetry = 0,
+        DropKeywordKeepRundown = 1,
+        DropKeywordDropRundown = 2,
+        ForbiddenToRetry = 3
+    }
+}