[Metrics] FHIR metric emitter (#3753)

* Adding default metric emitter

* Fix using statements

* Improve metric emission

* Refactoring metric emitters

* Using statements updated

* New tests to test if methods have the metric emitter filters and remove reference to log in FhirController

* Removed not used code

* Renamining classes with better names

* New metric handler implementation.

* Refactoring interfaces and adding more tests.

* Following coding suggestions.

src/Microsoft.Health.Fhir.Core/Logging/Metrics/BaseMeterMetricHandler.cs

@@ -0,0 +1,24 @@
+// -------------------------------------------------------------------------------------------------
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information.
+// -------------------------------------------------------------------------------------------------
+
+using System.Diagnostics.Metrics;
+using EnsureThat;
+
+namespace Microsoft.Health.Fhir.Core.Logging.Metrics
+{
+    public abstract class BaseMeterMetricHandler
+    {
+        public const string MeterName = "FhirServer";
+
+        protected BaseMeterMetricHandler(IMeterFactory meterFactory)
+        {
+            EnsureArg.IsNotNull(meterFactory, nameof(meterFactory));
+
+            MetricMeter = meterFactory.Create(MeterName);
+        }
+
+        protected Meter MetricMeter { get; }
+    }
+}

src/Microsoft.Health.Fhir.Core/Logging/Metrics/BundleMetricNotification.cs

@@ -0,0 +1,12 @@
+// -------------------------------------------------------------------------------------------------
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information.
+// -------------------------------------------------------------------------------------------------
+
+namespace Microsoft.Health.Fhir.Core.Logging.Metrics
+{
+    public sealed class BundleMetricNotification : ILatencyMetricNotification
+    {
+        public long ElapsedMilliseconds { get; set; }
+    }
+}

src/Microsoft.Health.Fhir.Core/Logging/Metrics/CrudMetricNotification.cs

@@ -0,0 +1,12 @@
+// -------------------------------------------------------------------------------------------------
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information.
+// -------------------------------------------------------------------------------------------------
+
+namespace Microsoft.Health.Fhir.Core.Logging.Metrics
+{
+    public sealed class CrudMetricNotification : ILatencyMetricNotification
+    {
+        public long ElapsedMilliseconds { get; set; }
+    }
+}

src/Microsoft.Health.Fhir.Core/Logging/Metrics/DefaultBundleMetricHandler.cs

@@ -0,0 +1,28 @@
+// -------------------------------------------------------------------------------------------------
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information.
+// -------------------------------------------------------------------------------------------------
+
+using System.Diagnostics.Metrics;
+using EnsureThat;
+
+namespace Microsoft.Health.Fhir.Core.Logging.Metrics
+{
+    public sealed class DefaultBundleMetricHandler : BaseMeterMetricHandler, IBundleMetricHandler
+    {
+        private readonly Counter<long> _bundleLatencyCounter;
+
+        public DefaultBundleMetricHandler(IMeterFactory meterFactory)
+            : base(meterFactory)
+        {
+            _bundleLatencyCounter = MetricMeter.CreateCounter<long>("Bundle.Latency");
+        }
+
+        public void EmitLatency(BundleMetricNotification notification)
+        {
+            EnsureArg.IsNotNull(notification, nameof(notification));
+
+            _bundleLatencyCounter.Add(notification.ElapsedMilliseconds);
+        }
+    }
+}

src/Microsoft.Health.Fhir.Core/Logging/Metrics/DefaultCrudMetricHandler.cs

@@ -0,0 +1,28 @@
+// -------------------------------------------------------------------------------------------------
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information.
+// -------------------------------------------------------------------------------------------------
+
+using System.Diagnostics.Metrics;
+using EnsureThat;
+
+namespace Microsoft.Health.Fhir.Core.Logging.Metrics
+{
+    public sealed class DefaultCrudMetricHandler : BaseMeterMetricHandler, ICrudMetricHandler
+    {
+        private readonly Counter<long> _crudLatencyCounter;
+
+        public DefaultCrudMetricHandler(IMeterFactory meterFactory)
+            : base(meterFactory)
+        {
+            _crudLatencyCounter = MetricMeter.CreateCounter<long>("Crud.Latency");
+        }
+
+        public void EmitLatency(CrudMetricNotification notification)
+        {
+            EnsureArg.IsNotNull(notification, nameof(notification));
+
+            _crudLatencyCounter.Add(notification.ElapsedMilliseconds);
+        }
+    }
+}

src/Microsoft.Health.Fhir.Core/Logging/Metrics/DefaultSearchMetricHandler.cs

@@ -0,0 +1,28 @@
+// -------------------------------------------------------------------------------------------------
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information.
+// -------------------------------------------------------------------------------------------------
+
+using System.Diagnostics.Metrics;
+using EnsureThat;
+
+namespace Microsoft.Health.Fhir.Core.Logging.Metrics
+{
+    public sealed class DefaultSearchMetricHandler : BaseMeterMetricHandler, ISearchMetricHandler
+    {
+        private readonly Counter<long> _searchLatencyCounter;
+
+        public DefaultSearchMetricHandler(IMeterFactory meterFactory)
+            : base(meterFactory)
+        {
+            _searchLatencyCounter = MetricMeter.CreateCounter<long>("Search.Latency");
+        }
+
+        public void EmitLatency(SearchMetricNotification notification)
+        {
+            EnsureArg.IsNotNull(notification, nameof(notification));
+
+            _searchLatencyCounter.Add(notification.ElapsedMilliseconds);
+        }
+    }
+}

src/Microsoft.Health.Fhir.Core/Logging/Metrics/IBundleMetricHandler.cs

@@ -0,0 +1,11 @@
+// -------------------------------------------------------------------------------------------------
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information.
+// -------------------------------------------------------------------------------------------------
+
+namespace Microsoft.Health.Fhir.Core.Logging.Metrics
+{
+    public interface IBundleMetricHandler : ILatencyMetricHandler<BundleMetricNotification>
+    {
+    }
+}

src/Microsoft.Health.Fhir.Core/Logging/Metrics/ICrudMetricHandler.cs

@@ -0,0 +1,11 @@
+// -------------------------------------------------------------------------------------------------
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information.
+// -------------------------------------------------------------------------------------------------
+
+namespace Microsoft.Health.Fhir.Core.Logging.Metrics
+{
+    public interface ICrudMetricHandler : ILatencyMetricHandler<CrudMetricNotification>
+    {
+    }
+}

src/Microsoft.Health.Fhir.Core/Logging/Metrics/ILatencyMetricHandler.cs

@@ -0,0 +1,15 @@
+// -------------------------------------------------------------------------------------------------
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information.
+// -------------------------------------------------------------------------------------------------
+
+using Antlr4.Runtime.Tree.Xpath;
+
+namespace Microsoft.Health.Fhir.Core.Logging.Metrics
+{
+    public interface ILatencyMetricHandler<T>
+         where T : ILatencyMetricNotification
+    {
+        void EmitLatency(T notification);
+    }
+}

src/Microsoft.Health.Fhir.Core/Logging/Metrics/ILatencyMetricNotification.cs

@@ -0,0 +1,12 @@
+// -------------------------------------------------------------------------------------------------
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information.
+// -------------------------------------------------------------------------------------------------
+
+namespace Microsoft.Health.Fhir.Core.Logging.Metrics
+{
+    public interface ILatencyMetricNotification
+    {
+        long ElapsedMilliseconds { get; set; }
+    }
+}

src/Microsoft.Health.Fhir.Core/Logging/Metrics/ISearchMetricHandler.cs

@@ -0,0 +1,11 @@
+// -------------------------------------------------------------------------------------------------
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information.
+// -------------------------------------------------------------------------------------------------
+
+namespace Microsoft.Health.Fhir.Core.Logging.Metrics
+{
+    public interface ISearchMetricHandler : ILatencyMetricHandler<SearchMetricNotification>
+    {
+    }
+}

src/Microsoft.Health.Fhir.Core/Logging/Metrics/SearchMetricNotification.cs

@@ -0,0 +1,12 @@
+// -------------------------------------------------------------------------------------------------
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information.
+// -------------------------------------------------------------------------------------------------
+
+namespace Microsoft.Health.Fhir.Core.Logging.Metrics
+{
+    public sealed class SearchMetricNotification : ILatencyMetricNotification
+    {
+        public long ElapsedMilliseconds { get; set; }
+    }
+}

src/Microsoft.Health.Fhir.Shared.Api.UnitTests/Controllers/FhirControllerTests.cs

@@ -5,10 +5,12 @@
 
 using System;
 using System.Linq;
+using System.Reflection;
 using Microsoft.AspNetCore.Mvc;
 using Microsoft.Health.Api.Features.Audit;
 using Microsoft.Health.Fhir.Api.Controllers;
 using Microsoft.Health.Fhir.Api.Features.Filters;
+using Microsoft.Health.Fhir.Api.Features.Filters.Metrics;
 using Microsoft.Health.Fhir.Tests.Common;
 using Microsoft.Health.Test.Utilities;
 using Xunit;
@@ -19,8 +21,10 @@ namespace Microsoft.Health.Fhir.Api.UnitTests.Controllers
     [Trait(Traits.Category, Categories.Web)]
     public sealed class FhirControllerTests
     {
+        private readonly Type _targetFhirControllerClass = typeof(FhirController);
+
         [Fact]
-        public void WhenProviderAFhirController_CheckIfAllExpectedServiceFilterAttributesArePresent()
+        public void WhenProvidedAFhirController_CheckIfAllExpectedServiceFilterAttributesArePresent()
         {
             Type[] expectedCustomAttributes = new Type[]
             {
@@ -30,7 +34,7 @@ namespace Microsoft.Health.Fhir.Api.UnitTests.Controllers
                 typeof(QueryLatencyOverEfficiencyFilterAttribute),
             };
 
-            ServiceFilterAttribute[] serviceFilterAttributes = Attribute.GetCustomAttributes(typeof(FhirController), typeof(ServiceFilterAttribute))
+            ServiceFilterAttribute[] serviceFilterAttributes = Attribute.GetCustomAttributes(_targetFhirControllerClass, typeof(ServiceFilterAttribute))
                 .Select(a => a as ServiceFilterAttribute)
                 .ToArray();
 
@@ -40,10 +44,56 @@ namespace Microsoft.Health.Fhir.Api.UnitTests.Controllers
 
                 if (!attributeWasFound)
                 {
-                    string errorMessage = $"The custom attribute '{expectedCustomAttribute.ToString()}' is not assigned to '{nameof(FhirController)}'.";
+                    string errorMessage = $"The custom attribute '{expectedCustomAttribute}' is not assigned to '{nameof(FhirController)}'.";
                     Assert.Fail(errorMessage);
                 }
             }
         }
+
+        [Fact]
+        public void WhenProvidedAFhirController_CheckIfTheBundleEndpointHasTheLatencyMetricFilter()
+        {
+            const string methodName = "BatchAndTransactions";
+            Type expectedCustomAttribute = typeof(BundleEndpointMetricEmitterAttribute);
+
+            TestIfTargetMethodContainsCustomAttribute(expectedCustomAttribute, methodName, _targetFhirControllerClass);
+        }
+
+        [Fact]
+        public void WhenProvidedAFhirController_CheckIfTheSearchEndpointsHaveTheLatencyMetricFilter()
+        {
+            Type expectedCustomAttribute = typeof(SearchEndpointMetricEmitterAttribute);
+
+            TestIfTargetMethodContainsCustomAttribute(expectedCustomAttribute, "SearchCompartmentByResourceType", _targetFhirControllerClass);
+            TestIfTargetMethodContainsCustomAttribute(expectedCustomAttribute, "SearchByResourceType", _targetFhirControllerClass);
+            TestIfTargetMethodContainsCustomAttribute(expectedCustomAttribute, "History", _targetFhirControllerClass);
+            TestIfTargetMethodContainsCustomAttribute(expectedCustomAttribute, "TypeHistory", _targetFhirControllerClass);
+            TestIfTargetMethodContainsCustomAttribute(expectedCustomAttribute, "SystemHistory", _targetFhirControllerClass);
+        }
+
+        [Fact]
+        public void WhenProvidedAFhirController_CheckIfTheCrudEndpointsHaveTheLatencyMetricFilter()
+        {
+            Type expectedCustomAttribute = typeof(CrudEndpointMetricEmitterAttribute);
+
+            TestIfTargetMethodContainsCustomAttribute(expectedCustomAttribute, "Create", _targetFhirControllerClass);
+            TestIfTargetMethodContainsCustomAttribute(expectedCustomAttribute, "Update", _targetFhirControllerClass);
+            TestIfTargetMethodContainsCustomAttribute(expectedCustomAttribute, "Read", _targetFhirControllerClass);
+            TestIfTargetMethodContainsCustomAttribute(expectedCustomAttribute, "VRead", _targetFhirControllerClass);
+            TestIfTargetMethodContainsCustomAttribute(expectedCustomAttribute, "Delete", _targetFhirControllerClass);
+        }
+
+        private static void TestIfTargetMethodContainsCustomAttribute(Type expectedCustomAttributeType, string methodName, Type targetClassType)
+        {
+            MethodInfo bundleMethodInfo = targetClassType.GetMethod(methodName);
+            Assert.True(bundleMethodInfo != null, $"The method '{methodName}' was not found in '{targetClassType.Name}'. Was it renamed or removed?");
+
+            TypeFilterAttribute latencyFilter = Attribute.GetCustomAttributes(bundleMethodInfo, typeof(TypeFilterAttribute))
+                .Select(a => a as TypeFilterAttribute)
+                .Where(a => a.ImplementationType == expectedCustomAttributeType)
+                .SingleOrDefault();
+
+            Assert.True(latencyFilter != null, $"The expected filter '{expectedCustomAttributeType.Name}' was not found in the method '{methodName}' from '{targetClassType.Name}'.");
+        }
     }
 }

src/Microsoft.Health.Fhir.Shared.Api.UnitTests/Features/Filters/Metrics/MetricEmitterAttributeTests.cs

@@ -0,0 +1,65 @@
+// -------------------------------------------------------------------------------------------------
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information.
+// -------------------------------------------------------------------------------------------------
+
+using System.Threading.Tasks;
+using Microsoft.Health.Fhir.Api.Features.Filters.Metrics;
+using Microsoft.Health.Fhir.Core.Logging.Metrics;
+using Microsoft.Health.Fhir.Tests.Common;
+using Microsoft.Health.Test.Utilities;
+using NSubstitute;
+using Xunit;
+
+namespace Microsoft.Health.Fhir.Api.UnitTests.Features.Filters.Metrics
+{
+    [Trait(Traits.OwningTeam, OwningTeam.Fhir)]
+    [Trait(Traits.Category, Categories.Operations)]
+    public sealed class MetricEmitterAttributeTests
+    {
+        [Fact]
+        public async void BundleEndpointMetricEmitterAttribute_WhenCalled_EmittsMetrics()
+        {
+            IBundleMetricHandler metricHandler = Substitute.For<IBundleMetricHandler>();
+            metricHandler.EmitLatency(Arg.Any<BundleMetricNotification>());
+
+            var attribute = new BundleEndpointMetricEmitterAttribute(metricHandler);
+
+            attribute.OnActionExecuting(null);
+            await Task.Delay(100);
+            attribute.OnActionExecuted(null);
+
+            metricHandler.Received(1).EmitLatency(Arg.Any<BundleMetricNotification>());
+        }
+
+        [Fact]
+        public async void SearchEndpointMetricEmitterAttribute_WhenCalled_EmittsMetrics()
+        {
+            ISearchMetricHandler metricHandler = Substitute.For<ISearchMetricHandler>();
+            metricHandler.EmitLatency(Arg.Any<SearchMetricNotification>());
+
+            var attribute = new SearchEndpointMetricEmitterAttribute(metricHandler);
+
+            attribute.OnActionExecuting(null);
+            await Task.Delay(100);
+            attribute.OnActionExecuted(null);
+
+            metricHandler.Received(1).EmitLatency(Arg.Any<SearchMetricNotification>());
+        }
+
+        [Fact]
+        public async void CrudEndpointMetricEmitterAttribute_WhenCalled_EmittsMetrics()
+        {
+            ICrudMetricHandler metricHandler = Substitute.For<ICrudMetricHandler>();
+            metricHandler.EmitLatency(Arg.Any<CrudMetricNotification>());
+
+            var attribute = new CrudEndpointMetricEmitterAttribute(metricHandler);
+
+            attribute.OnActionExecuting(null);
+            await Task.Delay(100);
+            attribute.OnActionExecuted(null);
+
+            metricHandler.Received(1).EmitLatency(Arg.Any<CrudMetricNotification>());
+        }
+    }
+}

src/Microsoft.Health.Fhir.Shared.Api.UnitTests/Microsoft.Health.Fhir.Shared.Api.UnitTests.projitems

@@ -35,6 +35,7 @@
     <Compile Include="$(MSBuildThisFileDirectory)Features\Context\FhirRequestContextMiddlewareTests.cs" />
     <Compile Include="$(MSBuildThisFileDirectory)Features\Exceptions\BaseExceptionMiddlewareTests.cs" />
     <Compile Include="$(MSBuildThisFileDirectory)Features\Exceptions\ExceptionNotificationMiddlewareTests.cs" />
+    <Compile Include="$(MSBuildThisFileDirectory)Features\Filters\Metrics\MetricEmitterAttributeTests.cs" />
     <Compile Include="$(MSBuildThisFileDirectory)Features\Filters\QueryLatencyOverEfficiencyFilterAttributeTests.cs" />
     <Compile Include="$(MSBuildThisFileDirectory)Features\Filters\ValidateBulkImportRequestFilterAttributeTests.cs" />
     <Compile Include="$(MSBuildThisFileDirectory)Features\Filters\FhirRequestContextRouteDataPopulatingFilterAttributeTests.cs" />

src/Microsoft.Health.Fhir.Shared.Api/Controllers/FhirController.cs

@@ -5,6 +5,7 @@
 
 using System;
 using System.Collections.Generic;
+using System.Diagnostics;
 using System.Globalization;
 using System.Linq;
 using System.Net;
@@ -30,6 +31,7 @@ using Microsoft.Health.Fhir.Api.Features.ActionConstraints;
 using Microsoft.Health.Fhir.Api.Features.ActionResults;
 using Microsoft.Health.Fhir.Api.Features.AnonymousOperations;
 using Microsoft.Health.Fhir.Api.Features.Filters;
+using Microsoft.Health.Fhir.Api.Features.Filters.Metrics;
 using Microsoft.Health.Fhir.Api.Features.Headers;
 using Microsoft.Health.Fhir.Api.Features.Resources;
 using Microsoft.Health.Fhir.Api.Features.Routing;
@@ -43,6 +45,7 @@ using Microsoft.Health.Fhir.Core.Features.Persistence;
 using Microsoft.Health.Fhir.Core.Features.Persistence.Orchestration;
 using Microsoft.Health.Fhir.Core.Features.Resources.Patch;
 using Microsoft.Health.Fhir.Core.Features.Routing;
+using Microsoft.Health.Fhir.Core.Logging.Metrics;
 using Microsoft.Health.Fhir.Core.Messages.Create;
 using Microsoft.Health.Fhir.Core.Messages.Delete;
 using Microsoft.Health.Fhir.Core.Messages.Get;
@@ -65,7 +68,6 @@ namespace Microsoft.Health.Fhir.Api.Controllers
     public class FhirController : Controller
     {
         private readonly IMediator _mediator;
-        private readonly ILogger<FhirController> _logger;
         private readonly RequestContextAccessor<IFhirRequestContext> _fhirRequestContextAccessor;
         private readonly IUrlResolver _urlResolver;
 
@@ -73,21 +75,18 @@ namespace Microsoft.Health.Fhir.Api.Controllers
         /// Initializes a new instance of the <see cref="FhirController" /> class.
         /// </summary>
         /// <param name="mediator">The mediator.</param>
-        /// <param name="logger">The logger.</param>
         /// <param name="fhirRequestContextAccessor">The FHIR request context accessor.</param>
         /// <param name="urlResolver">The urlResolver.</param>
         /// <param name="uiConfiguration">The UI configuration.</param>
         /// <param name="authorizationService">The authorization service.</param>
         public FhirController(
             IMediator mediator,
-            ILogger<FhirController> logger,
             RequestContextAccessor<IFhirRequestContext> fhirRequestContextAccessor,
             IUrlResolver urlResolver,
             IOptions<FeatureConfiguration> uiConfiguration,
             IAuthorizationService authorizationService)
         {
             EnsureArg.IsNotNull(mediator, nameof(mediator));
-            EnsureArg.IsNotNull(logger, nameof(logger));
             EnsureArg.IsNotNull(fhirRequestContextAccessor, nameof(fhirRequestContextAccessor));
             EnsureArg.IsNotNull(urlResolver, nameof(urlResolver));
             EnsureArg.IsNotNull(uiConfiguration, nameof(uiConfiguration));
@@ -95,7 +94,6 @@ namespace Microsoft.Health.Fhir.Api.Controllers
             EnsureArg.IsNotNull(authorizationService, nameof(authorizationService));
 
             _mediator = mediator;
-            _logger = logger;
             _fhirRequestContextAccessor = fhirRequestContextAccessor;
             _urlResolver = urlResolver;
         }
@@ -163,6 +161,7 @@ namespace Microsoft.Health.Fhir.Api.Controllers
         [Route(KnownRoutes.ResourceType)]
         [AuditEventType(AuditEventSubType.Create)]
         [ServiceFilter(typeof(SearchParameterFilterAttribute))]
+        [TypeFilter(typeof(CrudEndpointMetricEmitterAttribute))]
         public async Task<IActionResult> Create([FromBody] Resource resource)
         {
             RawResourceElement response = await _mediator.CreateResourceAsync(
@@ -218,6 +217,7 @@ namespace Microsoft.Health.Fhir.Api.Controllers
         [ValidateResourceIdFilter]
         [Route(KnownRoutes.ResourceTypeById)]
         [AuditEventType(AuditEventSubType.Update)]
+        [TypeFilter(typeof(CrudEndpointMetricEmitterAttribute))]
         public async Task<IActionResult> Update([FromBody] Resource resource, [ModelBinder(typeof(WeakETagBinder))] WeakETag ifMatchHeader)
         {
             SaveOutcome response = await _mediator.UpsertResourceAsync(
@@ -276,6 +276,7 @@ namespace Microsoft.Health.Fhir.Api.Controllers
         [ValidateIdSegmentAttribute]
         [Route(KnownRoutes.ResourceTypeById, Name = RouteNames.ReadResource)]
         [AuditEventType(AuditEventSubType.Read)]
+        [TypeFilter(typeof(CrudEndpointMetricEmitterAttribute))]
         public async Task<IActionResult> Read(string typeParameter, string idParameter)
         {
             RawResourceElement response = await _mediator.GetResourceAsync(
@@ -294,6 +295,7 @@ namespace Microsoft.Health.Fhir.Api.Controllers
         [HttpGet]
         [Route(KnownRoutes.History, Name = RouteNames.History)]
         [AuditEventType(AuditEventSubType.HistorySystem)]
+        [TypeFilter(typeof(SearchEndpointMetricEmitterAttribute))]
         public async Task<IActionResult> SystemHistory(HistoryModel historyModel)
         {
             ResourceElement response = await _mediator.SearchResourceHistoryAsync(
@@ -317,6 +319,7 @@ namespace Microsoft.Health.Fhir.Api.Controllers
         [HttpGet]
         [Route(KnownRoutes.ResourceTypeHistory, Name = RouteNames.HistoryType)]
         [AuditEventType(AuditEventSubType.HistoryType)]
+        [TypeFilter(typeof(SearchEndpointMetricEmitterAttribute))]
         public async Task<IActionResult> TypeHistory(
             string typeParameter,
             HistoryModel historyModel)
@@ -344,6 +347,7 @@ namespace Microsoft.Health.Fhir.Api.Controllers
         [HttpGet]
         [Route(KnownRoutes.ResourceTypeByIdHistory, Name = RouteNames.HistoryTypeId)]
         [AuditEventType(AuditEventSubType.HistoryInstance)]
+        [TypeFilter(typeof(SearchEndpointMetricEmitterAttribute))]
         public async Task<IActionResult> History(
             string typeParameter,
             string idParameter,
@@ -374,6 +378,7 @@ namespace Microsoft.Health.Fhir.Api.Controllers
         [ValidateIdSegmentAttribute]
         [Route(KnownRoutes.ResourceTypeByIdAndVid, Name = RouteNames.ReadResourceWithVersionRoute)]
         [AuditEventType(AuditEventSubType.VRead)]
+        [TypeFilter(typeof(CrudEndpointMetricEmitterAttribute))]
         public async Task<IActionResult> VRead(string typeParameter, string idParameter, string vidParameter)
         {
             RawResourceElement response = await _mediator.GetResourceAsync(
@@ -396,6 +401,7 @@ namespace Microsoft.Health.Fhir.Api.Controllers
         [ValidateIdSegmentAttribute]
         [Route(KnownRoutes.ResourceTypeById)]
         [AuditEventType(AuditEventSubType.Delete)]
+        [TypeFilter(typeof(CrudEndpointMetricEmitterAttribute))]
         public async Task<IActionResult> Delete(string typeParameter, string idParameter, [FromQuery] bool hardDelete, [FromQuery] bool allowPartialSuccess)
         {
             DeleteResourceResponse response = await _mediator.DeleteResourceAsync(
@@ -577,6 +583,7 @@ namespace Microsoft.Health.Fhir.Api.Controllers
         [HttpGet]
         [Route(KnownRoutes.ResourceType, Name = RouteNames.SearchResources)]
         [AuditEventType(AuditEventSubType.SearchType)]
+        [TypeFilter(typeof(SearchEndpointMetricEmitterAttribute))]
         public async Task<IActionResult> SearchByResourceType(string typeParameter)
         {
             return await PerformSearch(typeParameter, GetQueriesForSearch());
@@ -596,6 +603,7 @@ namespace Microsoft.Health.Fhir.Api.Controllers
         [HttpGet]
         [Route(KnownRoutes.CompartmentTypeByResourceType, Name = RouteNames.SearchCompartmentByResourceType)]
         [AuditEventType(AuditEventSubType.Search)]
+        [TypeFilter(typeof(SearchEndpointMetricEmitterAttribute))]
         public async Task<IActionResult> SearchCompartmentByResourceType(string compartmentTypeParameter, string idParameter, string typeParameter)
         {
             IReadOnlyList<Tuple<string, string>> queries = GetQueriesForSearch();
@@ -663,6 +671,7 @@ namespace Microsoft.Health.Fhir.Api.Controllers
         [HttpPost]
         [Route("", Name = RouteNames.PostBundle)]
         [AuditEventType(AuditEventSubType.BundlePost)]
+        [TypeFilter(typeof(BundleEndpointMetricEmitterAttribute))]
         public async Task<IActionResult> BatchAndTransactions([FromBody] Resource bundle)
         {
             ResourceElement bundleResponse = await _mediator.PostBundle(bundle.ToResourceElement(), HttpContext.RequestAborted);

src/Microsoft.Health.Fhir.Shared.Api/Features/Filters/Metrics/ActionExecutedStatistics.cs

@@ -0,0 +1,12 @@
+// -------------------------------------------------------------------------------------------------
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information.
+// -------------------------------------------------------------------------------------------------
+
+namespace Microsoft.Health.Fhir.Api.Features.Filters.Metrics
+{
+    public sealed class ActionExecutedStatistics
+    {
+        public long ElapsedMilliseconds { get; set; }
+    }
+}

src/Microsoft.Health.Fhir.Shared.Api/Features/Filters/Metrics/BaseEndpointMetricEmitterAttribute.cs

@@ -0,0 +1,31 @@
+// -------------------------------------------------------------------------------------------------
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information.
+// -------------------------------------------------------------------------------------------------
+
+using System.Diagnostics;
+using Microsoft.AspNetCore.Mvc.Filters;
+
+namespace Microsoft.Health.Fhir.Api.Features.Filters.Metrics
+{
+    public abstract class BaseEndpointMetricEmitterAttribute : ActionFilterAttribute
+    {
+        private Stopwatch _stopwatch;
+
+        public override void OnActionExecuting(ActionExecutingContext context)
+        {
+            _stopwatch = Stopwatch.StartNew();
+
+            base.OnActionExecuting(context);
+        }
+
+        public override void OnActionExecuted(ActionExecutedContext context)
+        {
+            base.OnActionExecuted(context);
+
+            EmitMetricOnActionExecuted(context, new ActionExecutedStatistics() { ElapsedMilliseconds = _stopwatch.ElapsedMilliseconds });
+        }
+
+        public abstract void EmitMetricOnActionExecuted(ActionExecutedContext context, ActionExecutedStatistics statistics);
+    }
+}

src/Microsoft.Health.Fhir.Shared.Api/Features/Filters/Metrics/BundleEndpointMetricEmitterAttribute.cs

@@ -0,0 +1,30 @@
+// -------------------------------------------------------------------------------------------------
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information.
+// -------------------------------------------------------------------------------------------------
+
+using System;
+using EnsureThat;
+using Microsoft.AspNetCore.Mvc.Filters;
+using Microsoft.Health.Fhir.Core.Logging.Metrics;
+
+namespace Microsoft.Health.Fhir.Api.Features.Filters.Metrics
+{
+    [AttributeUsage(AttributeTargets.Method)]
+    public sealed class BundleEndpointMetricEmitterAttribute : BaseEndpointMetricEmitterAttribute
+    {
+        private readonly IBundleMetricHandler _metricHandler;
+
+        public BundleEndpointMetricEmitterAttribute(IBundleMetricHandler metricHandler)
+        {
+            EnsureArg.IsNotNull(metricHandler, nameof(metricHandler));
+
+            _metricHandler = metricHandler;
+        }
+
+        public override void EmitMetricOnActionExecuted(ActionExecutedContext context, ActionExecutedStatistics statistics)
+        {
+            _metricHandler.EmitLatency(new BundleMetricNotification { ElapsedMilliseconds = statistics.ElapsedMilliseconds });
+        }
+    }
+}

src/Microsoft.Health.Fhir.Shared.Api/Features/Filters/Metrics/CrudEndpointMetricEmitterAttribute.cs

@@ -0,0 +1,31 @@
+// -------------------------------------------------------------------------------------------------
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information.
+// -------------------------------------------------------------------------------------------------
+
+using System;
+using EnsureThat;
+using Microsoft.AspNetCore.Mvc.Filters;
+using Microsoft.Health.Fhir.Core.Logging.Metrics;
+
+namespace Microsoft.Health.Fhir.Api.Features.Filters.Metrics
+{
+    [AttributeUsage(AttributeTargets.Method)]
+    public sealed class CrudEndpointMetricEmitterAttribute : BaseEndpointMetricEmitterAttribute
+    {
+        private readonly ICrudMetricHandler _metricHandler;
+
+        public CrudEndpointMetricEmitterAttribute(ICrudMetricHandler metricHandler)
+        {
+            EnsureArg.IsNotNull(metricHandler, nameof(metricHandler));
+
+            _metricHandler = metricHandler;
+        }
+
+        // Under the scope of bundle requests, this methid is not called and no nested metrics are emitted.
+        public override void EmitMetricOnActionExecuted(ActionExecutedContext context, ActionExecutedStatistics statistics)
+        {
+            _metricHandler.EmitLatency(new CrudMetricNotification { ElapsedMilliseconds = statistics.ElapsedMilliseconds });
+        }
+    }
+}

src/Microsoft.Health.Fhir.Shared.Api/Features/Filters/Metrics/SearchEndpointMetricEmitterAttribute.cs

@@ -0,0 +1,31 @@
+// -------------------------------------------------------------------------------------------------
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information.
+// -------------------------------------------------------------------------------------------------
+
+using System;
+using EnsureThat;
+using Microsoft.AspNetCore.Mvc.Filters;
+using Microsoft.Health.Fhir.Core.Logging.Metrics;
+
+namespace Microsoft.Health.Fhir.Api.Features.Filters.Metrics
+{
+    [AttributeUsage(AttributeTargets.Method)]
+    public sealed class SearchEndpointMetricEmitterAttribute : BaseEndpointMetricEmitterAttribute
+    {
+        private readonly ISearchMetricHandler _metricHandler;
+
+        public SearchEndpointMetricEmitterAttribute(ISearchMetricHandler metricHandler)
+        {
+            EnsureArg.IsNotNull(metricHandler, nameof(metricHandler));
+
+            _metricHandler = metricHandler;
+        }
+
+        // Under the scope of bundle requests, this methid is not called and no nested metrics are emitted.
+        public override void EmitMetricOnActionExecuted(ActionExecutedContext context, ActionExecutedStatistics statistics)
+        {
+            _metricHandler.EmitLatency(new SearchMetricNotification { ElapsedMilliseconds = statistics.ElapsedMilliseconds });
+        }
+    }
+}

src/Microsoft.Health.Fhir.Shared.Api/Microsoft.Health.Fhir.Shared.Api.projitems

@@ -26,6 +26,11 @@
     <Compile Include="$(MSBuildThisFileDirectory)Features\ActionResults\OperationOutcomeResult.cs" />
     <Compile Include="$(MSBuildThisFileDirectory)Features\ActionResults\OperationSmartConfigurationResult.cs" />
     <Compile Include="$(MSBuildThisFileDirectory)Features\ActionResults\OperationVersionsResult.cs" />
+    <Compile Include="$(MSBuildThisFileDirectory)Features\Filters\Metrics\ActionExecutedStatistics.cs" />
+    <Compile Include="$(MSBuildThisFileDirectory)Features\Filters\Metrics\BaseEndpointMetricEmitterAttribute.cs" />
+    <Compile Include="$(MSBuildThisFileDirectory)Features\Filters\Metrics\BundleEndpointMetricEmitterAttribute.cs" />
+    <Compile Include="$(MSBuildThisFileDirectory)Features\Filters\Metrics\CrudEndpointMetricEmitterAttribute.cs" />
+    <Compile Include="$(MSBuildThisFileDirectory)Features\Filters\Metrics\SearchEndpointMetricEmitterAttribute.cs" />
     <Compile Include="$(MSBuildThisFileDirectory)Features\Filters\QueryLatencyOverEfficiencyFilterAttribute.cs" />
     <Compile Include="$(MSBuildThisFileDirectory)Features\Formatters\FormatParametersValidator.cs" />
     <Compile Include="$(MSBuildThisFileDirectory)Features\Filters\ParameterCompatibleFilter.cs" />

src/Microsoft.Health.Fhir.Shared.Api/Registration/FhirServerServiceCollectionExtensions.cs

@@ -9,12 +9,10 @@ using System.Linq;
 using System.Net;
 using System.Reflection;
 using EnsureThat;
-using Microsoft.AspNetCore.Authorization;
 using Microsoft.AspNetCore.Builder;
 using Microsoft.AspNetCore.Hosting;
-using Microsoft.AspNetCore.Mvc.Authorization;
-using Microsoft.AspNetCore.Routing;
 using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.DependencyInjection.Extensions;
 using Microsoft.Extensions.Hosting;
 using Microsoft.Health.Api.Features.Audit;
 using Microsoft.Health.Api.Features.Headers;
@@ -27,13 +25,12 @@ using Microsoft.Health.Fhir.Api.Features.Exceptions;
 using Microsoft.Health.Fhir.Api.Features.Operations.Export;
 using Microsoft.Health.Fhir.Api.Features.Operations.Import;
 using Microsoft.Health.Fhir.Api.Features.Operations.Reindex;
-using Microsoft.Health.Fhir.Api.Features.Resources.Bundle;
 using Microsoft.Health.Fhir.Api.Features.Routing;
 using Microsoft.Health.Fhir.Api.Features.Throttling;
 using Microsoft.Health.Fhir.Core.Features.Cors;
 using Microsoft.Health.Fhir.Core.Features.Persistence.Orchestration;
+using Microsoft.Health.Fhir.Core.Logging.Metrics;
 using Microsoft.Health.Fhir.Core.Registration;
-using Newtonsoft.Json.Serialization;
 using Polly;
 
 namespace Microsoft.Extensions.DependencyInjection
@@ -120,6 +117,8 @@ namespace Microsoft.Extensions.DependencyInjection
                 }
             }
 
+            AddMetricEmitter(services);
+
             return new FhirServerBuilder(services);
         }
 
@@ -158,6 +157,17 @@ namespace Microsoft.Extensions.DependencyInjection
             return fhirServerBuilder;
         }
 
+        /// <summary>
+        /// Registers the default metric emitter.
+        /// </summary>
+        private static void AddMetricEmitter(IServiceCollection services)
+        {
+            // Register the metric handlers used by the service.
+            services.TryAddSingleton<IBundleMetricHandler, DefaultBundleMetricHandler>();
+            services.TryAddSingleton<ICrudMetricHandler, DefaultCrudMetricHandler>();
+            services.TryAddSingleton<ISearchMetricHandler, DefaultSearchMetricHandler>();
+        }
+
         private class FhirServerBuilder : IFhirServerBuilder
         {
             public FhirServerBuilder(IServiceCollection services)

test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/BundleBatchTests.cs

@@ -5,8 +5,9 @@
 
 using System;
 using System.Collections.Generic;
-using System.Linq;
+using System.IO;
 using System.Net;
+using System.Net.Sockets;
 using System.Threading;
 using Hl7.Fhir.Model;
 using Microsoft.Health.Fhir.Client;
@@ -52,50 +53,76 @@ namespace Microsoft.Health.Fhir.Tests.E2E.Rest
         {
             Skip.If(ModelInfoProvider.Version == FhirSpecification.Stu3, "Patch isn't supported in Bundles by STU3");
 
-            CancellationTokenSource source = new CancellationTokenSource(TimeSpan.FromSeconds(30));
-
-            var requestBundle = Samples.GetBatchWithDuplicatedItems().ToPoco<Bundle>();
-
-            await _client.UpdateAsync(requestBundle.Entry[1].Resource as Patient, cancellationToken: source.Token);
-
-            using FhirResponse<Bundle> fhirResponse = await _client.PostBundleAsync(requestBundle, new FhirBundleOptions() { BundleProcessingLogic = processingLogic }, source.Token);
-            Assert.NotNull(fhirResponse);
-            Assert.Equal(HttpStatusCode.OK, fhirResponse.StatusCode);
-
-            Bundle resource = fhirResponse.Resource;
-
-            Assert.Equal("201", resource.Entry[0].Response.Status);
-
-            // Resources 1, 2 and 3 have the same resource Id.
-            Assert.Equal("200", resource.Entry[1].Response.Status); // PUT
-
-            if (processingLogic == FhirBundleProcessingLogic.Parallel)
+            int attempt = 0;
+            do
             {
-                // Duplicated records. Only one should succeed. As the requests are processed in parallel,
-                // it's not possible to pick the one that will be processed.
-                if (resource.Entry[2].Response.Status == "200")
+                try
                 {
-                    Assert.Equal("200", resource.Entry[2].Response.Status); // PATCH
-                    Assert.Equal("400", resource.Entry[3].Response.Status); // PATCH (Duplicate)
+                    using CancellationTokenSource source = new CancellationTokenSource(TimeSpan.FromSeconds(30));
+
+                    var requestBundle = Samples.GetBatchWithDuplicatedItems().ToPoco<Bundle>();
+
+                    await _client.UpdateAsync(requestBundle.Entry[1].Resource as Patient, cancellationToken: source.Token);
+
+                    using FhirResponse<Bundle> fhirResponse = await _client.PostBundleAsync(requestBundle, new FhirBundleOptions() { BundleProcessingLogic = processingLogic }, source.Token);
+                    Assert.NotNull(fhirResponse);
+                    Assert.Equal(HttpStatusCode.OK, fhirResponse.StatusCode);
+
+                    Bundle resource = fhirResponse.Resource;
+
+                    Assert.Equal("201", resource.Entry[0].Response.Status);
+
+                    // Resources 1, 2 and 3 have the same resource Id.
+                    Assert.Equal("200", resource.Entry[1].Response.Status); // PUT
+
+                    if (processingLogic == FhirBundleProcessingLogic.Parallel)
+                    {
+                        // Duplicated records. Only one should succeed. As the requests are processed in parallel,
+                        // it's not possible to pick the one that will be processed.
+                        if (resource.Entry[2].Response.Status == "200")
+                        {
+                            Assert.Equal("200", resource.Entry[2].Response.Status); // PATCH
+                            Assert.Equal("400", resource.Entry[3].Response.Status); // PATCH (Duplicate)
+                        }
+                        else
+                        {
+                            Assert.Equal("400", resource.Entry[2].Response.Status); // PATCH (Duplicate)
+                            Assert.Equal("200", resource.Entry[3].Response.Status); // PATCH
+                        }
+                    }
+                    else if (processingLogic == FhirBundleProcessingLogic.Sequential)
+                    {
+                        Assert.Equal("200", resource.Entry[2].Response.Status); // PATCH
+                        Assert.Equal("200", resource.Entry[3].Response.Status); // PATCH
+                    }
+
+                    Assert.Equal("204", resource.Entry[4].Response.Status);
+                    Assert.Equal("204", resource.Entry[5].Response.Status);
+
+                    BundleTestsUtil.ValidateOperationOutcome(resource.Entry[6].Response.Status, resource.Entry[6].Response.Outcome as OperationOutcome, _statusCodeMap[HttpStatusCode.NotFound], "The route for \"/ValueSet/$lookup\" was not found.", IssueType.NotFound);
+                    Assert.Equal("200", resource.Entry[7].Response.Status);
+                    BundleTestsUtil.ValidateOperationOutcome(resource.Entry[8].Response.Status, resource.Entry[8].Response.Outcome as OperationOutcome, _statusCodeMap[HttpStatusCode.NotFound], "Resource type 'Patient' with id '12334' couldn't be found.", IssueType.NotFound);
+
+                    break;
                 }
-                else
+                catch (OperationCanceledException oce) when (oce.InnerException is OperationCanceledException)
                 {
-                    Assert.Equal("400", resource.Entry[2].Response.Status); // PATCH (Duplicate)
-                    Assert.Equal("200", resource.Entry[3].Response.Status); // PATCH
+                    if (attempt >= 3)
+                    {
+                        throw;
+                    }
+
+                    if (oce.InnerException.InnerException is IOException ioe && ioe.InnerException is SocketException)
+                    {
+                        // Transient network errors.
+                        attempt++;
+                        continue;
+                    }
+
+                    throw;
                 }
             }
-            else if (processingLogic == FhirBundleProcessingLogic.Sequential)
-            {
-                Assert.Equal("200", resource.Entry[2].Response.Status); // PATCH
-                Assert.Equal("200", resource.Entry[3].Response.Status); // PATCH
-            }
-
-            Assert.Equal("204", resource.Entry[4].Response.Status);
-            Assert.Equal("204", resource.Entry[5].Response.Status);
-
-            BundleTestsUtil.ValidateOperationOutcome(resource.Entry[6].Response.Status, resource.Entry[6].Response.Outcome as OperationOutcome, _statusCodeMap[HttpStatusCode.NotFound], "The route for \"/ValueSet/$lookup\" was not found.", IssueType.NotFound);
-            Assert.Equal("200", resource.Entry[7].Response.Status);
-            BundleTestsUtil.ValidateOperationOutcome(resource.Entry[8].Response.Status, resource.Entry[8].Response.Outcome as OperationOutcome, _statusCodeMap[HttpStatusCode.NotFound], "Resource type 'Patient' with id '12334' couldn't be found.", IssueType.NotFound);
+            while (true);
         }
 
         [Fact]