Fix duplicated request telemetry when multiple hosts run in the same process (#787)

* Fix duplicated request telemetry when multiple hosts run in the same
process #621

* nits

CHANGELOG.md

@@ -1,6 +1,7 @@
 # Changelog
 
 ## Version 2.6.0-beta3
+- [Fix: Do not track requests by each host in the process](https://github.com/Microsoft/ApplicationInsights-aspnetcore/issues/621)
 - [Fix: Correlation doesn't work for localhost](https://github.com/Microsoft/ApplicationInsights-dotnet-server/issues/1120)
 
 ## Version 2.6.0-beta2

NuGet.config

@@ -5,5 +5,4 @@
     <add key="NuGet" value="https://api.nuget.org/v3/index.json" />
     <add key="applicationinsights" value="https://www.myget.org/F/applicationinsights/api/v3/index.json" />
   </packageSources>
-  <disabledPackageSources />
 </configuration>

src/Microsoft.ApplicationInsights.AspNetCore/DiagnosticListeners/Implementation/HostingDiagnosticListener.cs

@@ -38,7 +38,7 @@
         private readonly bool injectResponseHeaders;
         private readonly bool trackExceptions;
         private readonly bool enableW3CHeaders;
-
+        private static readonly ActiveSubsciptionManager SubscriptionManager = new ActiveSubsciptionManager();
         private const string ActivityCreatedByHostingDiagnosticListener = "ActivityCreatedByHostingDiagnosticListener";
 
         /// <summary>
@@ -49,7 +49,12 @@
         /// <param name="injectResponseHeaders">Flag that indicates that response headers should be injected.</param>
         /// <param name="trackExceptions">Flag that indicates that exceptions should be tracked.</param>
         /// <param name="enableW3CHeaders">Flag that indicates that W3C header parsing should be enabled.</param>
-        public HostingDiagnosticListener(TelemetryClient client, IApplicationIdProvider applicationIdProvider, bool injectResponseHeaders, bool trackExceptions, bool enableW3CHeaders)
+        public HostingDiagnosticListener(
+            TelemetryClient client, 
+            IApplicationIdProvider applicationIdProvider, 
+            bool injectResponseHeaders, 
+            bool trackExceptions, 
+            bool enableW3CHeaders)
         {
             this.client = client ?? throw new ArgumentNullException(nameof(client));
             this.applicationIdProvider = applicationIdProvider;
@@ -58,6 +63,12 @@
             this.enableW3CHeaders = enableW3CHeaders;
         }
 
+        /// <inheritdoc />
+        public void OnSubscribe()
+        {
+            SubscriptionManager.Attach(this);
+        }
+
         /// <inheritdoc/>
         public string ListenerName { get; } = "Microsoft.AspNetCore";
 
@@ -78,6 +89,15 @@
         {
             if (this.client.IsEnabled())
             {
+                // It's possible to host multiple apps (ASP.NET Core or generic hosts) in the same process
+                // Each of this apps has it's own HostingDiagnosticListener and corresponding Http listener.
+                // We should ignore events for all of them except one
+                if (!SubscriptionManager.IsActive(this))
+                {
+                    AspNetCoreEventSource.Instance.NotActiveListenerNoTracking("Microsoft.AspNetCore.Hosting.HttpRequestIn.Start", Activity.Current?.Id);
+                    return;
+                }
+
                 if (Activity.Current == null)
                 {
                     AspNetCoreEventSource.Instance.LogHostingDiagnosticListenerOnHttpRequestInStartActivityNull();
@@ -170,6 +190,16 @@
         {
             if (this.client.IsEnabled() && !IsAspNetCore20)
             {
+                // It's possible to host multiple apps (ASP.NET Core or generic hosts) in the same process
+                // Each of this apps has it's own HostingDiagnosticListener and corresponding Http listener.
+                // We should ignore events for all of them except one
+                if (!SubscriptionManager.IsActive(this))
+                {
+                    AspNetCoreEventSource.Instance.NotActiveListenerNoTracking(
+                        "Microsoft.AspNetCore.Hosting.BeginRequest", Activity.Current?.Id);
+                    return;
+                }
+
                 var activity = new Activity(ActivityCreatedByHostingDiagnosticListener);
                 var isActivityCreatedFromRequestIdHeader = false;
 
@@ -374,6 +404,16 @@
         {
             if (this.client.IsEnabled())
             {
+                // It's possible to host multiple apps (ASP.NET Core or generic hosts) in the same process
+                // Each of this apps has it's own HostingDiagnosticListener and corresponding Http listener.
+                // We should ignore events for all of them except one
+                if (!SubscriptionManager.IsActive(this))
+                {
+                    AspNetCoreEventSource.Instance.NotActiveListenerNoTracking(
+                        "EndRequest", Activity.Current?.Id);
+                    return;
+                }
+
                 var telemetry = httpContext?.Features.Get<RequestTelemetry>();
 
                 if (telemetry == null)
@@ -415,6 +455,16 @@
         {
             if (this.trackExceptions && this.client.IsEnabled())
             {
+                // It's possible to host multiple apps (ASP.NET Core or generic hosts) in the same process
+                // Each of this apps has it's own HostingDiagnosticListener and corresponding Http listener.
+                // We should ignore events for all of them except one
+                if (!SubscriptionManager.IsActive(this))
+                {
+                    AspNetCoreEventSource.Instance.NotActiveListenerNoTracking(
+                        "Exception", Activity.Current?.Id);
+                    return;
+                }
+
                 var telemetry = httpContext?.Features.Get<RequestTelemetry>();
                 if (telemetry != null)
                 {
@@ -504,6 +554,11 @@
             appId = appIds[0];
             return true;
         }
+
+        public void Dispose()
+        {
+            SubscriptionManager.Detach(this);
+        }
     }
 #pragma warning restore 612, 618
 }

src/Microsoft.ApplicationInsights.AspNetCore/DiagnosticListeners/Implementation/IApplicationInsightDiagnosticListener.cs

@@ -1,13 +1,20 @@
 namespace Microsoft.ApplicationInsights.AspNetCore.DiagnosticListeners
 {
+    using System;
+
     /// <summary>
     /// Base diagnostic listener type for Application Insight
     /// </summary>
-    internal interface IApplicationInsightDiagnosticListener
+    internal interface IApplicationInsightDiagnosticListener : IDisposable
     {
         /// <summary>
         /// Gets a value indicating which listener this instance should be subscribed to
         /// </summary>
         string ListenerName { get; }
+
+        /// <summary>
+        /// Notifies listener that it is subscribed to DiagnosticSource
+        /// </summary>
+        void OnSubscribe();
     }
 }

src/Microsoft.ApplicationInsights.AspNetCore/DiagnosticListeners/Implementation/MvcDiagnosticsListener.cs

@@ -35,6 +35,11 @@ namespace Microsoft.ApplicationInsights.AspNetCore.DiagnosticListeners
             }
         }
 
+        /// <inheritdoc />
+        public void OnSubscribe()
+        {
+        }
+
         private string GetNameFromRouteContext(IRouteData routeData)
         {
             string name = null;
@@ -93,6 +98,10 @@ namespace Microsoft.ApplicationInsights.AspNetCore.DiagnosticListeners
             return name;
         }
 
+        public void Dispose()
+        {
+        }
+
         /// <summary>
         /// Proxy interface for <c>RouteData</c> class from Microsoft.AspNetCore.Routing.Abstractions
         /// </summary>

src/Microsoft.ApplicationInsights.AspNetCore/Extensibility/Implementation/Tracing/AspNetEventSource.cs

@@ -146,6 +146,16 @@ namespace Microsoft.ApplicationInsights.AspNetCore.Extensibility.Implementation.
             this.WriteEvent(12, this.ApplicationName);
         }
 
+        [Event(
+            13,
+            Keywords = Keywords.Diagnostics,
+            Message = "Not tracking operation for event = '{0}', id = '{1}', lisener is not active.",
+            Level = EventLevel.Verbose)]
+        public void NotActiveListenerNoTracking(string evntName, string activityId, string appDomainName = "Incorrect")
+        {
+            this.WriteEvent(13, evntName, activityId, this.ApplicationName);
+        }
+
         /// <summary>
         /// Keywords for the AspNetEventSource.
         /// </summary>

src/Microsoft.ApplicationInsights.AspNetCore/RequestTrackingTelemetryModule.cs

@@ -59,8 +59,7 @@ namespace Microsoft.ApplicationInsights.AspNetCore
                     {
                         this.telemetryClient = new TelemetryClient(configuration);
 
-                        this.diagnosticListeners.Add
-                            (new HostingDiagnosticListener(
+                        this.diagnosticListeners.Add(new HostingDiagnosticListener(
                             this.telemetryClient,
                             this.applicationIdProvider,
                             this.CollectionOptions.InjectResponseHeaders,
@@ -92,6 +91,7 @@ namespace Microsoft.ApplicationInsights.AspNetCore
                 if (applicationInsightDiagnosticListener.ListenerName == value.Name)
                 {
                     subs.Add(value.SubscribeWithAdapter(applicationInsightDiagnosticListener));
+                    applicationInsightDiagnosticListener.OnSubscribe();
                 }
             }
         }
@@ -129,6 +129,11 @@ namespace Microsoft.ApplicationInsights.AspNetCore
             {
                 subscription.Dispose();
             }
+
+            foreach (var listener in this.diagnosticListeners)
+            {
+                listener.Dispose();
+            }
         }
     }
 }

test/FunctionalTestUtils20/TelemetryTestsBase.cs

@@ -8,9 +8,6 @@
     using System.Reflection;
     using System.Runtime.CompilerServices;
     using System.Text;
-    using System.Text.RegularExpressions;
-    using System.Threading;
-    using System.Threading.Tasks;
     using AI;
     using Microsoft.ApplicationInsights.DataContracts;
     using Microsoft.Extensions.DependencyInjection;
@@ -138,7 +135,7 @@
 
             using (HttpClient httpClient = new HttpClient(httpClientHandler, true))
             {
-                this.output.WriteLine(string.Format("{0}: Executing request: {1}", DateTime.Now.ToString("MM/dd/yyyy hh:mm:ss.fff tt"), requestPath));
+                this.output.WriteLine($"{DateTime.Now:MM/dd/yyyy hh:mm:ss.fff tt}: Executing request: {requestPath}");
                 HttpRequestMessage request = new HttpRequestMessage(HttpMethod.Get, requestPath);
                 if (headers != null)
                 {
@@ -150,7 +147,7 @@
 
                 var task = httpClient.SendAsync(request);
                 task.Wait(TestListenerTimeoutInMs);
-                this.output.WriteLine(string.Format("{0:MM/dd/yyyy hh:mm:ss.fff tt}: Ended request: {1}", DateTime.Now, requestPath));
+                this.output.WriteLine($"{DateTime.Now:MM/dd/yyyy hh:mm:ss.fff tt}: Ended request: {requestPath}");
 
                 return task.Result;
             }

test/Microsoft.ApplicationInsights.AspNetCore.Tests/ExceptionTrackingMiddlewareTest.cs

@@ -1,5 +1,4 @@
 using Microsoft.ApplicationInsights.AspNetCore.DiagnosticListeners;
-using Microsoft.ApplicationInsights.AspNetCore.Extensions;
 using Microsoft.ApplicationInsights.AspNetCore.Tests.Helpers;
 using Microsoft.ApplicationInsights.Channel;
 using Microsoft.ApplicationInsights.DataContracts;
@@ -16,34 +15,39 @@ namespace Microsoft.ApplicationInsights.AspNetCore.Tests
         [Fact]
         public void InvokeTracksExceptionThrownByNextMiddlewareAsHandledByPlatform()
         {
-            var middleware = new HostingDiagnosticListener(
+            using (var middleware = new HostingDiagnosticListener(
                 CommonMocks.MockTelemetryClient(telemetry => this.sentTelemetry = telemetry),
                 CommonMocks.GetMockApplicationIdProvider(),
                 injectResponseHeaders: true,
                 trackExceptions: true,
-                enableW3CHeaders:false);
+                enableW3CHeaders: false))
+            {
+                middleware.OnSubscribe();
+                middleware.OnHostingException(null, null);
 
-            middleware.OnHostingException(null, null);
-
-            Assert.NotNull(sentTelemetry);
-            Assert.IsType<ExceptionTelemetry>(sentTelemetry);
-            Assert.Equal(ExceptionHandledAt.Platform, ((ExceptionTelemetry)sentTelemetry).HandledAt);
+                Assert.NotNull(sentTelemetry);
+                Assert.IsType<ExceptionTelemetry>(sentTelemetry);
+                Assert.Equal(ExceptionHandledAt.Platform, ((ExceptionTelemetry) sentTelemetry).HandledAt);
+            }
         }
 
         [Fact]
         public void SdkVersionIsPopulatedByMiddleware()
         {
-            var middleware = new HostingDiagnosticListener(
-                CommonMocks.MockTelemetryClient(telemetry => this.sentTelemetry = telemetry), 
+            using (var middleware = new HostingDiagnosticListener(
+                CommonMocks.MockTelemetryClient(telemetry => this.sentTelemetry = telemetry),
                 CommonMocks.GetMockApplicationIdProvider(),
                 injectResponseHeaders: true,
                 trackExceptions: true,
-                enableW3CHeaders: false);
+                enableW3CHeaders: false))
+            {
+                middleware.OnSubscribe();
+                middleware.OnHostingException(null, null);
 
-            middleware.OnHostingException(null, null);
-
-            Assert.NotEmpty(sentTelemetry.Context.GetInternalContext().SdkVersion);
-            Assert.Contains(SdkVersionTestUtils.VersionPrefix, sentTelemetry.Context.GetInternalContext().SdkVersion);
+                Assert.NotEmpty(sentTelemetry.Context.GetInternalContext().SdkVersion);
+                Assert.Contains(SdkVersionTestUtils.VersionPrefix,
+                    sentTelemetry.Context.GetInternalContext().SdkVersion);
+            }
         }
     }
 }

test/Microsoft.ApplicationInsights.AspNetCore.Tests/RequestTrackingMiddlewareTest.cs

@@ -10,6 +10,7 @@
     using Microsoft.ApplicationInsights.AspNetCore.DiagnosticListeners;
     using Microsoft.ApplicationInsights.AspNetCore.Tests.Helpers;
     using Microsoft.ApplicationInsights.Channel;
+    using Microsoft.ApplicationInsights.Common;
     using Microsoft.ApplicationInsights.DataContracts;
     using Microsoft.ApplicationInsights.Extensibility;
     using Microsoft.ApplicationInsights.Extensibility.Implementation;
@@ -69,6 +70,7 @@
         private ConcurrentQueue<ITelemetry> sentTelemetry = new ConcurrentQueue<ITelemetry>();
 
         private HostingDiagnosticListener middleware;
+        private ActiveSubsciptionManager subscriptionManager; 
 
         public RequestTrackingMiddlewareTest()
         {
@@ -78,6 +80,7 @@
                 injectResponseHeaders: true,
                 trackExceptions: true,
                 enableW3CHeaders: false);
+            this.middleware.OnSubscribe();
         }
 
         [Fact]
@@ -536,39 +539,46 @@
         [Fact]
         public void ResponseHeadersAreNotInjectedWhenDisabled()
         {
+            this.middleware.Dispose();
             HttpContext context = CreateContext(HttpRequestScheme, HttpRequestHost);
 
-            var noHeadersMiddleware = new HostingDiagnosticListener(
+            using (var noHeadersMiddleware = new HostingDiagnosticListener(
                 CommonMocks.MockTelemetryClient(telemetry => this.sentTelemetry.Enqueue(telemetry)),
                 CommonMocks.GetMockApplicationIdProvider(),
                 injectResponseHeaders: false,
                 trackExceptions: true,
-                enableW3CHeaders: false);
+                enableW3CHeaders: false))
+            {
+                noHeadersMiddleware.OnSubscribe();
+                noHeadersMiddleware.OnBeginRequest(context, 0);
+                Assert.False(context.Response.Headers.ContainsKey(RequestResponseHeaders.RequestContextHeader));
+                noHeadersMiddleware.OnEndRequest(context, 0);
+                Assert.False(context.Response.Headers.ContainsKey(RequestResponseHeaders.RequestContextHeader));
 
-            noHeadersMiddleware.OnBeginRequest(context, 0);
-            Assert.False(context.Response.Headers.ContainsKey(RequestResponseHeaders.RequestContextHeader));
-            noHeadersMiddleware.OnEndRequest(context, 0);
-            Assert.False(context.Response.Headers.ContainsKey(RequestResponseHeaders.RequestContextHeader));
-
-            Assert.Single(sentTelemetry);
-            Assert.IsType<RequestTelemetry>(this.sentTelemetry.First());
+                Assert.Single(sentTelemetry);
+                Assert.IsType<RequestTelemetry>(this.sentTelemetry.First());
+            }
         }
 
         [Fact]
         public void ExceptionsAreNotTrackedInjectedWhenDisabled()
         {
+            this.middleware.Dispose();
             HttpContext context = CreateContext(HttpRequestScheme, HttpRequestHost);
 
-            var noExceptionsMiddleware = new HostingDiagnosticListener(
+            using (var noExceptionsMiddleware = new HostingDiagnosticListener(
                 CommonMocks.MockTelemetryClient(telemetry => this.sentTelemetry.Enqueue(telemetry)),
                 CommonMocks.GetMockApplicationIdProvider(),
                 injectResponseHeaders: true,
                 trackExceptions: false,
-                enableW3CHeaders: false);
-
-            noExceptionsMiddleware.OnHostingException(context, new Exception("HostingException"));
-            noExceptionsMiddleware.OnDiagnosticsHandledException(context, new Exception("DiagnosticsHandledException"));
-            noExceptionsMiddleware.OnDiagnosticsUnhandledException(context, new Exception("UnhandledException"));
+                enableW3CHeaders: false))
+            {
+                noExceptionsMiddleware.OnSubscribe();
+                noExceptionsMiddleware.OnHostingException(context, new Exception("HostingException"));
+                noExceptionsMiddleware.OnDiagnosticsHandledException(context,
+                    new Exception("DiagnosticsHandledException"));
+                noExceptionsMiddleware.OnDiagnosticsUnhandledException(context, new Exception("UnhandledException"));
+            }
 
             Assert.Empty(sentTelemetry);
         }
@@ -592,230 +602,266 @@
         [Fact]
         public void OnBeginRequestWithW3CHeadersIsTrackedCorrectly()
         {
+            this.middleware.Dispose();
+
             var configuration = TelemetryConfiguration.CreateDefault();
             configuration.TelemetryInitializers.Add(new W3COperationCorrelationTelemetryInitializer());
-            this.middleware = new HostingDiagnosticListener(
+            using (var w3cMiddleware = new HostingDiagnosticListener(
                 CommonMocks.MockTelemetryClient(telemetry => this.sentTelemetry.Enqueue(telemetry), configuration),
                 CommonMocks.GetMockApplicationIdProvider(),
                 injectResponseHeaders: true,
                 trackExceptions: true,
-                enableW3CHeaders: true);
-
-            var context = CreateContext(HttpRequestScheme, HttpRequestHost, "/Test", method: "POST");
-
-            context.Request.Headers[W3CConstants.TraceParentHeader] = "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01";
-            context.Request.Headers[W3CConstants.TraceStateHeader] = "state=some";
-            context.Request.Headers[RequestResponseHeaders.CorrelationContextHeader] = "k=v";
-
-            if (HostingDiagnosticListener.IsAspNetCore20)
+                enableW3CHeaders: true))
             {
-                var activity = new Activity("operation");
-                activity.Start();
+                w3cMiddleware.OnSubscribe();
+                var context = CreateContext(HttpRequestScheme, HttpRequestHost, "/Test", method: "POST");
 
-                middleware.OnHttpRequestInStart(context);
+                context.Request.Headers[W3CConstants.TraceParentHeader] =
+                    "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01";
+                context.Request.Headers[W3CConstants.TraceStateHeader] = "state=some";
+                context.Request.Headers[RequestResponseHeaders.CorrelationContextHeader] = "k=v";
 
-                Assert.NotEqual(Activity.Current, activity);
+                if (HostingDiagnosticListener.IsAspNetCore20)
+                {
+                    var activity = new Activity("operation");
+                    activity.Start();
+
+                    w3cMiddleware.OnHttpRequestInStart(context);
+
+                    Assert.NotEqual(Activity.Current, activity);
+                }
+                else
+                {
+                    w3cMiddleware.OnBeginRequest(context, Stopwatch.GetTimestamp());
+                }
+
+                var activityInitializedByW3CHeader = Activity.Current;
+                Assert.Equal("4bf92f3577b34da6a3ce929d0e0e4736", activityInitializedByW3CHeader.GetTraceId());
+                Assert.Equal("00f067aa0ba902b7", activityInitializedByW3CHeader.GetParentSpanId());
+                Assert.Equal(16, activityInitializedByW3CHeader.GetSpanId().Length);
+                Assert.Equal("state=some", activityInitializedByW3CHeader.GetTracestate());
+                Assert.Equal("v", activityInitializedByW3CHeader.Baggage.Single(t => t.Key == "k").Value);
+
+                if (HostingDiagnosticListener.IsAspNetCore20)
+                {
+                    w3cMiddleware.OnHttpRequestInStop(context);
+                }
+                else
+                {
+                    w3cMiddleware.OnEndRequest(context, Stopwatch.GetTimestamp());
+                }
+
+                Assert.Single(sentTelemetry);
+                var requestTelemetry = (RequestTelemetry) this.sentTelemetry.Single();
+
+                Assert.Equal($"|4bf92f3577b34da6a3ce929d0e0e4736.{activityInitializedByW3CHeader.GetSpanId()}.",
+                    requestTelemetry.Id);
+                Assert.Equal("4bf92f3577b34da6a3ce929d0e0e4736", requestTelemetry.Context.Operation.Id);
+                Assert.Equal("|4bf92f3577b34da6a3ce929d0e0e4736.00f067aa0ba902b7.",
+                    requestTelemetry.Context.Operation.ParentId);
+
+                Assert.True(context.Response.Headers.TryGetValue(RequestResponseHeaders.RequestContextHeader,
+                    out var appId));
+                Assert.Equal($"appId={CommonMocks.TestApplicationId}", appId);
             }
-            else
-            {
-                middleware.OnBeginRequest(context, Stopwatch.GetTimestamp());
-            }
-
-            var activityInitializedByW3CHeader = Activity.Current;
-            Assert.Equal("4bf92f3577b34da6a3ce929d0e0e4736", activityInitializedByW3CHeader.GetTraceId());
-            Assert.Equal("00f067aa0ba902b7", activityInitializedByW3CHeader.GetParentSpanId());
-            Assert.Equal(16, activityInitializedByW3CHeader.GetSpanId().Length);
-            Assert.Equal("state=some", activityInitializedByW3CHeader.GetTracestate());
-            Assert.Equal("v", activityInitializedByW3CHeader.Baggage.Single(t => t.Key == "k").Value);
-
-            if (HostingDiagnosticListener.IsAspNetCore20)
-            {
-                middleware.OnHttpRequestInStop(context);
-            }
-            else
-            {
-                middleware.OnEndRequest(context, Stopwatch.GetTimestamp());
-            }
-
-            Assert.Single(sentTelemetry);
-            var requestTelemetry = (RequestTelemetry)this.sentTelemetry.Single();
-
-            Assert.Equal($"|4bf92f3577b34da6a3ce929d0e0e4736.{activityInitializedByW3CHeader.GetSpanId()}.", requestTelemetry.Id);
-            Assert.Equal("4bf92f3577b34da6a3ce929d0e0e4736", requestTelemetry.Context.Operation.Id);
-            Assert.Equal("|4bf92f3577b34da6a3ce929d0e0e4736.00f067aa0ba902b7.", requestTelemetry.Context.Operation.ParentId);
-
-            Assert.True(context.Response.Headers.TryGetValue(RequestResponseHeaders.RequestContextHeader, out var appId));
-            Assert.Equal($"appId={CommonMocks.TestApplicationId}", appId);
         }
 
         [Fact]
         public void OnBeginRequestWithW3CHeadersAndRequestIdIsTrackedCorrectly()
         {
+            this.middleware.Dispose();
+
             var configuration = TelemetryConfiguration.CreateDefault();
             configuration.TelemetryInitializers.Add(new W3COperationCorrelationTelemetryInitializer());
-            this.middleware = new HostingDiagnosticListener(
+            using (var w3cMiddleware = new HostingDiagnosticListener(
                 CommonMocks.MockTelemetryClient(telemetry => this.sentTelemetry.Enqueue(telemetry), configuration),
                 CommonMocks.GetMockApplicationIdProvider(),
                 injectResponseHeaders: true,
                 trackExceptions: true,
-                enableW3CHeaders: true);
+                enableW3CHeaders: true))
+            {
+                w3cMiddleware.OnSubscribe();
+                var context = CreateContext(HttpRequestScheme, HttpRequestHost, "/Test", method: "POST");
 
-            var context = CreateContext(HttpRequestScheme, HttpRequestHost, "/Test", method: "POST");
+                context.Request.Headers[RequestResponseHeaders.RequestIdHeader] = "|abc.1.2.3.";
+                context.Request.Headers[W3CConstants.TraceParentHeader] =
+                    "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01";
+                context.Request.Headers[W3CConstants.TraceStateHeader] = "state=some";
+                context.Request.Headers[RequestResponseHeaders.CorrelationContextHeader] = "k=v";
 
-            context.Request.Headers[RequestResponseHeaders.RequestIdHeader] = "|abc.1.2.3.";
-            context.Request.Headers[W3CConstants.TraceParentHeader] = "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01";
-            context.Request.Headers[W3CConstants.TraceStateHeader] = "state=some";
-            context.Request.Headers[RequestResponseHeaders.CorrelationContextHeader] = "k=v";
+                w3cMiddleware.OnBeginRequest(context, Stopwatch.GetTimestamp());
+                var activityInitializedByW3CHeader = Activity.Current;
 
-            middleware.OnBeginRequest(context, Stopwatch.GetTimestamp());
-            var activityInitializedByW3CHeader = Activity.Current;
+                Assert.Equal("|abc.1.2.3.", activityInitializedByW3CHeader.ParentId);
+                Assert.Equal("4bf92f3577b34da6a3ce929d0e0e4736", activityInitializedByW3CHeader.GetTraceId());
+                Assert.Equal("00f067aa0ba902b7", activityInitializedByW3CHeader.GetParentSpanId());
+                Assert.Equal(16, activityInitializedByW3CHeader.GetSpanId().Length);
+                Assert.Equal("state=some", activityInitializedByW3CHeader.GetTracestate());
+                Assert.Equal("v", activityInitializedByW3CHeader.Baggage.Single(t => t.Key == "k").Value);
 
-            Assert.Equal("|abc.1.2.3.", activityInitializedByW3CHeader.ParentId);
-            Assert.Equal("4bf92f3577b34da6a3ce929d0e0e4736", activityInitializedByW3CHeader.GetTraceId());
-            Assert.Equal("00f067aa0ba902b7", activityInitializedByW3CHeader.GetParentSpanId());
-            Assert.Equal(16, activityInitializedByW3CHeader.GetSpanId().Length);
-            Assert.Equal("state=some", activityInitializedByW3CHeader.GetTracestate());
-            Assert.Equal("v", activityInitializedByW3CHeader.Baggage.Single(t => t.Key == "k").Value);
+                w3cMiddleware.OnEndRequest(context, Stopwatch.GetTimestamp());
 
-            middleware.OnEndRequest(context, Stopwatch.GetTimestamp());
+                Assert.Single(sentTelemetry);
+                var requestTelemetry = (RequestTelemetry) this.sentTelemetry.Single();
 
-            Assert.Single(sentTelemetry);
-            var requestTelemetry = (RequestTelemetry)this.sentTelemetry.Single();
+                Assert.Equal($"|4bf92f3577b34da6a3ce929d0e0e4736.{activityInitializedByW3CHeader.GetSpanId()}.",
+                    requestTelemetry.Id);
+                Assert.Equal("4bf92f3577b34da6a3ce929d0e0e4736", requestTelemetry.Context.Operation.Id);
+                Assert.Equal("|4bf92f3577b34da6a3ce929d0e0e4736.00f067aa0ba902b7.",
+                    requestTelemetry.Context.Operation.ParentId);
 
-            Assert.Equal($"|4bf92f3577b34da6a3ce929d0e0e4736.{activityInitializedByW3CHeader.GetSpanId()}.", requestTelemetry.Id);
-            Assert.Equal("4bf92f3577b34da6a3ce929d0e0e4736", requestTelemetry.Context.Operation.Id);
-            Assert.Equal("|4bf92f3577b34da6a3ce929d0e0e4736.00f067aa0ba902b7.", requestTelemetry.Context.Operation.ParentId);
+                Assert.True(context.Response.Headers.TryGetValue(RequestResponseHeaders.RequestContextHeader,
+                    out var appId));
+                Assert.Equal($"appId={CommonMocks.TestApplicationId}", appId);
 
-            Assert.True(context.Response.Headers.TryGetValue(RequestResponseHeaders.RequestContextHeader, out var appId));
-            Assert.Equal($"appId={CommonMocks.TestApplicationId}", appId);
-
-            Assert.Equal("abc", requestTelemetry.Properties["ai_legacyRootId"]);
-            Assert.StartsWith("|abc.1.2.3.", requestTelemetry.Properties["ai_legacyRequestId"]);
+                Assert.Equal("abc", requestTelemetry.Properties["ai_legacyRootId"]);
+                Assert.StartsWith("|abc.1.2.3.", requestTelemetry.Properties["ai_legacyRequestId"]);
+            }
         }
 
         [Fact]
         public void OnBeginRequestWithNoW3CHeadersAndRequestIdIsTrackedCorrectly()
         {
+            this.middleware.Dispose();
             var configuration = TelemetryConfiguration.CreateDefault();
             configuration.TelemetryInitializers.Add(new W3COperationCorrelationTelemetryInitializer());
-            this.middleware = new HostingDiagnosticListener(
+            using (var w3cMiddleware = new HostingDiagnosticListener(
                 CommonMocks.MockTelemetryClient(telemetry => this.sentTelemetry.Enqueue(telemetry), configuration),
                 CommonMocks.GetMockApplicationIdProvider(),
                 injectResponseHeaders: true,
                 trackExceptions: true,
-                enableW3CHeaders: true);
+                enableW3CHeaders: true))
+            {
+                w3cMiddleware.OnSubscribe();
+                var context = CreateContext(HttpRequestScheme, HttpRequestHost, "/Test", method: "POST");
 
-            var context = CreateContext(HttpRequestScheme, HttpRequestHost, "/Test", method: "POST");
+                context.Request.Headers[RequestResponseHeaders.RequestIdHeader] = "|abc.1.2.3.";
+                context.Request.Headers[RequestResponseHeaders.CorrelationContextHeader] = "k=v";
 
-            context.Request.Headers[RequestResponseHeaders.RequestIdHeader] = "|abc.1.2.3.";
-            context.Request.Headers[RequestResponseHeaders.CorrelationContextHeader] = "k=v";
+                w3cMiddleware.OnBeginRequest(context, Stopwatch.GetTimestamp());
+                var activityInitializedByW3CHeader = Activity.Current;
 
-            middleware.OnBeginRequest(context, Stopwatch.GetTimestamp());
-            var activityInitializedByW3CHeader = Activity.Current;
+                Assert.Equal("|abc.1.2.3.", activityInitializedByW3CHeader.ParentId);
+                w3cMiddleware.OnEndRequest(context, Stopwatch.GetTimestamp());
 
-            Assert.Equal("|abc.1.2.3.", activityInitializedByW3CHeader.ParentId);
-            middleware.OnEndRequest(context, Stopwatch.GetTimestamp());
+                Assert.Single(sentTelemetry);
+                var requestTelemetry = (RequestTelemetry) this.sentTelemetry.Single();
 
-            Assert.Single(sentTelemetry);
-            var requestTelemetry = (RequestTelemetry)this.sentTelemetry.Single();
+                Assert.Equal(
+                    $"|{activityInitializedByW3CHeader.GetTraceId()}.{activityInitializedByW3CHeader.GetSpanId()}.",
+                    requestTelemetry.Id);
+                Assert.Equal(activityInitializedByW3CHeader.GetTraceId(), requestTelemetry.Context.Operation.Id);
+                Assert.Equal("|abc.1.2.3.", requestTelemetry.Context.Operation.ParentId);
 
-            Assert.Equal($"|{activityInitializedByW3CHeader.GetTraceId()}.{activityInitializedByW3CHeader.GetSpanId()}.", requestTelemetry.Id);
-            Assert.Equal(activityInitializedByW3CHeader.GetTraceId(), requestTelemetry.Context.Operation.Id);
-            Assert.Equal("|abc.1.2.3.", requestTelemetry.Context.Operation.ParentId);
-
-            Assert.Equal("abc", requestTelemetry.Properties["ai_legacyRootId"]);
-            Assert.StartsWith("|abc.1.2.3.", requestTelemetry.Properties["ai_legacyRequestId"]);
+                Assert.Equal("abc", requestTelemetry.Properties["ai_legacyRootId"]);
+                Assert.StartsWith("|abc.1.2.3.", requestTelemetry.Properties["ai_legacyRequestId"]);
+            }
         }
 
         [Fact]
         public void OnBeginRequestWithW3CSupportAndNoHeadersIsTrackedCorrectly()
         {
+            this.middleware.Dispose();
             var configuration = TelemetryConfiguration.CreateDefault();
             configuration.TelemetryInitializers.Add(new W3COperationCorrelationTelemetryInitializer());
-            this.middleware = new HostingDiagnosticListener(
+            using (var w3cMiddleware = new HostingDiagnosticListener(
                 CommonMocks.MockTelemetryClient(telemetry => this.sentTelemetry.Enqueue(telemetry), configuration),
                 CommonMocks.GetMockApplicationIdProvider(),
                 injectResponseHeaders: true,
                 trackExceptions: true,
-                enableW3CHeaders: true);
+                enableW3CHeaders: true))
+            {
+                w3cMiddleware.OnSubscribe();
 
-            var context = CreateContext(HttpRequestScheme, HttpRequestHost, "/Test", method: "POST");
+                var context = CreateContext(HttpRequestScheme, HttpRequestHost, "/Test", method: "POST");
 
-            middleware.OnBeginRequest(context, Stopwatch.GetTimestamp());
+                w3cMiddleware.OnBeginRequest(context, Stopwatch.GetTimestamp());
 
-            var activityInitializedByW3CHeader = Activity.Current;
-            Assert.Null(activityInitializedByW3CHeader.ParentId);
-            Assert.NotNull(activityInitializedByW3CHeader.GetTraceId());
-            Assert.Equal(32, activityInitializedByW3CHeader.GetTraceId().Length);
-            Assert.Equal(16, activityInitializedByW3CHeader.GetSpanId().Length);
-            Assert.Equal($"00-{activityInitializedByW3CHeader.GetTraceId()}-{activityInitializedByW3CHeader.GetSpanId()}-02",
-                activityInitializedByW3CHeader.GetTraceparent());
-            Assert.Null(activityInitializedByW3CHeader.GetTracestate());
-            Assert.Empty(activityInitializedByW3CHeader.Baggage);
+                var activityInitializedByW3CHeader = Activity.Current;
+                Assert.Null(activityInitializedByW3CHeader.ParentId);
+                Assert.NotNull(activityInitializedByW3CHeader.GetTraceId());
+                Assert.Equal(32, activityInitializedByW3CHeader.GetTraceId().Length);
+                Assert.Equal(16, activityInitializedByW3CHeader.GetSpanId().Length);
+                Assert.Equal(
+                    $"00-{activityInitializedByW3CHeader.GetTraceId()}-{activityInitializedByW3CHeader.GetSpanId()}-02",
+                    activityInitializedByW3CHeader.GetTraceparent());
+                Assert.Null(activityInitializedByW3CHeader.GetTracestate());
+                Assert.Empty(activityInitializedByW3CHeader.Baggage);
 
-            middleware.OnEndRequest(context, Stopwatch.GetTimestamp());
+                w3cMiddleware.OnEndRequest(context, Stopwatch.GetTimestamp());
 
-            Assert.Single(sentTelemetry);
-            var requestTelemetry = (RequestTelemetry)this.sentTelemetry.Single();
+                Assert.Single(sentTelemetry);
+                var requestTelemetry = (RequestTelemetry) this.sentTelemetry.Single();
 
-            Assert.Equal($"|{activityInitializedByW3CHeader.GetTraceId()}.{activityInitializedByW3CHeader.GetSpanId()}.", requestTelemetry.Id);
-            Assert.Equal(activityInitializedByW3CHeader.GetTraceId(), requestTelemetry.Context.Operation.Id);
-            Assert.Null(requestTelemetry.Context.Operation.ParentId);
+                Assert.Equal(
+                    $"|{activityInitializedByW3CHeader.GetTraceId()}.{activityInitializedByW3CHeader.GetSpanId()}.",
+                    requestTelemetry.Id);
+                Assert.Equal(activityInitializedByW3CHeader.GetTraceId(), requestTelemetry.Context.Operation.Id);
+                Assert.Null(requestTelemetry.Context.Operation.ParentId);
 
-            Assert.True(context.Response.Headers.TryGetValue(RequestResponseHeaders.RequestContextHeader, out var appId));
-            Assert.Equal($"appId={CommonMocks.TestApplicationId}", appId);
+                Assert.True(context.Response.Headers.TryGetValue(RequestResponseHeaders.RequestContextHeader,
+                    out var appId));
+                Assert.Equal($"appId={CommonMocks.TestApplicationId}", appId);
+            }
         }
 
         [Fact]
         public void OnBeginRequestWithW3CHeadersAndAppIdInState()
         {
+            this.middleware.Dispose();
             var configuration = TelemetryConfiguration.CreateDefault();
             configuration.TelemetryInitializers.Add(new W3COperationCorrelationTelemetryInitializer());
-            this.middleware = new HostingDiagnosticListener(
+            using (var w3cMiddleware = new HostingDiagnosticListener(
                 CommonMocks.MockTelemetryClient(telemetry => this.sentTelemetry.Enqueue(telemetry), configuration),
                 CommonMocks.GetMockApplicationIdProvider(),
                 injectResponseHeaders: true,
                 trackExceptions: true,
-                enableW3CHeaders: true);
-
-            var context = CreateContext(HttpRequestScheme, HttpRequestHost, "/Test", method: "POST");
-
-            context.Request.Headers[W3CConstants.TraceParentHeader] = "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-00";
-            context.Request.Headers[W3CConstants.TraceStateHeader] = $"state=some,{W3CConstants.AzureTracestateNamespace}={ExpectedAppId}";
-
-            if (HostingDiagnosticListener.IsAspNetCore20)
+                enableW3CHeaders: true))
             {
-                var activity = new Activity("operation");
-                activity.Start();
+                w3cMiddleware.OnSubscribe();
 
-                middleware.OnHttpRequestInStart(context);
-                Assert.NotEqual(Activity.Current, activity);
+                var context = CreateContext(HttpRequestScheme, HttpRequestHost, "/Test", method: "POST");
+
+                context.Request.Headers[W3CConstants.TraceParentHeader] =
+                    "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-00";
+                context.Request.Headers[W3CConstants.TraceStateHeader] =
+                    $"state=some,{W3CConstants.AzureTracestateNamespace}={ExpectedAppId}";
+
+                if (HostingDiagnosticListener.IsAspNetCore20)
+                {
+                    var activity = new Activity("operation");
+                    activity.Start();
+
+                    w3cMiddleware.OnHttpRequestInStart(context);
+                    Assert.NotEqual(Activity.Current, activity);
+                }
+                else
+                {
+                    w3cMiddleware.OnBeginRequest(context, Stopwatch.GetTimestamp());
+                }
+
+                var activityInitializedByW3CHeader = Activity.Current;
+
+                Assert.Equal("state=some", activityInitializedByW3CHeader.GetTracestate());
+
+                if (HostingDiagnosticListener.IsAspNetCore20)
+                {
+                    w3cMiddleware.OnHttpRequestInStop(context);
+                }
+                else
+                {
+                    w3cMiddleware.OnEndRequest(context, Stopwatch.GetTimestamp());
+                }
+
+                Assert.Single(sentTelemetry);
+                var requestTelemetry = (RequestTelemetry) this.sentTelemetry.Single();
+
+                Assert.Equal(ExpectedAppId, requestTelemetry.Source);
+
+                Assert.True(context.Response.Headers.TryGetValue(RequestResponseHeaders.RequestContextHeader,
+                    out var appId));
+                Assert.Equal($"appId={CommonMocks.TestApplicationId}", appId);
             }
-            else
-            {
-                middleware.OnBeginRequest(context, Stopwatch.GetTimestamp());
-            }
-
-            var activityInitializedByW3CHeader = Activity.Current;
-
-            Assert.Equal("state=some", activityInitializedByW3CHeader.GetTracestate());
-
-            if (HostingDiagnosticListener.IsAspNetCore20)
-            {
-                middleware.OnHttpRequestInStop(context);
-            }
-            else
-            {
-                middleware.OnEndRequest(context, Stopwatch.GetTimestamp());
-            }
-
-            Assert.Single(sentTelemetry);
-            var requestTelemetry = (RequestTelemetry)this.sentTelemetry.Single();
-
-            Assert.Equal(ExpectedAppId, requestTelemetry.Source);
-
-            Assert.True(context.Response.Headers.TryGetValue(RequestResponseHeaders.RequestContextHeader, out var appId));
-            Assert.Equal($"appId={CommonMocks.TestApplicationId}", appId);
         }
 #pragma warning restore 612, 618
 
@@ -854,6 +900,8 @@
             {
                 Activity.Current.Stop();
             }
+
+            this.middleware?.Dispose();
         }
     }
 }

test/WebApi20.FunctionalTests/FunctionalTest/MultipleWebHostsTests.cs

@@ -1,4 +1,5 @@
 using System.Linq;
+using System.Reflection;
 using System.Runtime.InteropServices;
 using AI;
 using FunctionalTestUtils;
@@ -50,36 +51,34 @@ namespace WebApi20.FunctionalTests20.FunctionalTest
             }
         }
 
-        [Fact(Skip = "We track each request and depednency by each WebHost, issue #621")]
+        [Fact]
         public void TwoWebHostsCreatedInParallel()
         {
             using (var server1 = new InProcessServer(assemblyName, this.output))
             using (var server2 = new InProcessServer(assemblyName, this.output))
             {
                 this.ExecuteRequest(server1.BaseHost + requestPath);
-                var telemetry1 = server1.Listener.ReceiveItems(TestListenerTimeoutInMs);
-                this.DebugTelemetryItems(telemetry1);
-
                 this.ExecuteRequest(server2.BaseHost + requestPath);
+
+                var telemetry1 = server1.Listener.ReceiveItems(TestListenerTimeoutInMs);
                 var telemetry2 = server2.Listener.ReceiveItems(TestListenerTimeoutInMs);
 
+                this.output.WriteLine("~~telemetry1~~");
+                this.DebugTelemetryItems(telemetry1);
+                this.output.WriteLine("~~telemetry2~~");
                 this.DebugTelemetryItems(telemetry2);
-                Assert.Single(telemetry1.Where(t => t is TelemetryItem<RequestData>));
-                Assert.Single(telemetry1.Where(IsServiceDependencyCall));
+
+                Assert.Equal(2, telemetry1.Count(t => t is TelemetryItem<RequestData>));
+                Assert.Equal(2, telemetry1.Count(IsServiceDependencyCall));
                 Assert.DoesNotContain(telemetry1, t => t is TelemetryItem<ExceptionData>);
 
-                var request1 = telemetry1.Single(t => t is TelemetryItem<RequestData>);
+                var request1 = telemetry1.First(t => t is TelemetryItem<RequestData>);
+                var request2 = telemetry1.Last(t => t is TelemetryItem<RequestData>);
                 Assert.Equal("200", ((TelemetryItem<RequestData>)request1).data.baseData.responseCode);
-
-                // Fails here, we track everything twice
-                // it did not happen with the first host because second one has not been really started yet
-                Assert.Single(telemetry2.Where(t => t is TelemetryItem<RequestData>));
-                Assert.Single(telemetry2.Where(IsServiceDependencyCall));
-                Assert.DoesNotContain(telemetry2, t => t is TelemetryItem<ExceptionData>);
-
-                var request2 = telemetry2.Single(t => t is TelemetryItem<RequestData>);
                 Assert.Equal("200", ((TelemetryItem<RequestData>)request2).data.baseData.responseCode);
-            }
+
+                Assert.False(telemetry2.Any());
+           }
         }
 
         [Fact]
@@ -127,7 +126,16 @@ namespace WebApi20.FunctionalTests20.FunctionalTest
                 return;
             }
 
+            // Active config could be used multiple times in the same process before this test
+            // let's reassign it
+
+            TelemetryConfiguration.Active.Dispose();
+            MethodInfo setActive =
+                typeof(TelemetryConfiguration).GetMethod("set_Active", BindingFlags.Static | BindingFlags.NonPublic);
+            setActive.Invoke(null, new object[] { TelemetryConfiguration.CreateDefault() });
+
             var activeConfig = TelemetryConfiguration.Active;
+
             using (var server = new InProcessServer(assemblyName, this.output))
             {
                 this.ExecuteRequest(server.BaseHost + requestPath);
@@ -149,6 +157,7 @@ namespace WebApi20.FunctionalTests20.FunctionalTest
                 Assert.Single(message);
 
                 this.output.WriteLine(((TelemetryItem<MessageData>)message.Single()).data.baseData.message);
+
                 Assert.Equal("some message after web host is disposed", ((TelemetryItem<MessageData>)message.Single()).data.baseData.message);
             }
         }