Enable DimensionCapping API internally (#1259)

* Enable DimensionCapping API and expose it internally only for use in standard metric preaggregation. * Fix test mistake * style cops * changelog
2019-10-28 16:03:01 -07:00 · 2019-10-28 16:03:01 -07:00 · 70e4388489
--- a/BASE/CHANGELOG.md
+++ b/BASE/CHANGELOG.md
@ -2,6 +2,9 @@

 This changelog will be used to generate documentation on [release notes page](http://azure.microsoft.com/documentation/articles/app-insights-release-notes-dotnet/).

+## Version 2.12.0-beta2
+- [Enable Metric DimensionCapping API for Internal use with standard metric aggregation.]https://github.com/microsoft/ApplicationInsights-dotnet/issues/1244)
+
 ## Version 2.12.0-beta1
 - [New: TelemetryConfiguration now supports Connection Strings]https://github.com/microsoft/ApplicationInsights-dotnet/issues/1221)

--- a/BASE/Test/Microsoft.ApplicationInsights.Test/Shared/Metrics/Implementation/ConcurrentDatastructures/MultidimensionalCube2Tests.cs
+++ b/BASE/Test/Microsoft.ApplicationInsights.Test/Shared/Metrics/Implementation/ConcurrentDatastructures/MultidimensionalCube2Tests.cs
@ -377,6 +377,210 @@ namespace Microsoft.ApplicationInsights.Metrics.ConcurrentDatastructures
            Assert.AreEqual(3, cube.TotalPointsCount);
        }

+        /// <summary />
+        [TestMethod]
+        public void TryGetOrCreatePoint_GetsAndCreatesWithDimCap()
+        {
+            string dimCapValue = "999";
+
+            var cube = new MultidimensionalCube2<int>(
+                                        8,
+                                        (vector) => Int32.Parse(vector[0]), true,
+                                        dimCapValue,
+                                        2, 2, 2);
+
+            MultidimensionalPointResult<int> result;
+
+            Assert.AreEqual(0, cube.TotalPointsCount);
+
+            result = cube.TryGetPoint("1", "1", "1");
+
+            Assert.IsFalse(result.IsSuccess);
+            Assert.IsFalse(result.IsPointCreatedNew);
+            Assert.AreEqual(-1, result.FailureCoordinateIndex);
+            Assert.AreEqual(MultidimensionalPointResultCodes.Failure_PointDoesNotExistCreationNotRequested, result.ResultCode);
+            Assert.AreEqual(0, result.Point);
+            Assert.AreEqual(0, cube.TotalPointsCount);
+
+            result = cube.TryGetOrCreatePoint("1", "1", "1");
+
+            Assert.IsTrue(result.IsSuccess);
+            Assert.IsTrue(result.IsPointCreatedNew);
+            Assert.AreEqual(-1, result.FailureCoordinateIndex);
+            Assert.AreEqual(MultidimensionalPointResultCodes.Success_NewPointCreated, result.ResultCode);
+            Assert.AreEqual(1, result.Point);
+            Assert.AreEqual(1, cube.TotalPointsCount);
+
+            result = cube.TryGetOrCreatePoint("1", "1", "1");
+
+            Assert.IsTrue(result.IsSuccess);
+            Assert.IsFalse(result.IsPointCreatedNew);
+            Assert.AreEqual(-1, result.FailureCoordinateIndex);
+            Assert.AreEqual(MultidimensionalPointResultCodes.Success_ExistingPointRetrieved, result.ResultCode);
+            Assert.AreEqual(1, result.Point);
+            Assert.AreEqual(1, cube.TotalPointsCount);
+
+            result = cube.TryGetPoint("1", "1", "1");
+
+            Assert.IsTrue(result.IsSuccess);
+            Assert.IsFalse(result.IsPointCreatedNew);
+            Assert.AreEqual(-1, result.FailureCoordinateIndex);
+            Assert.AreEqual(MultidimensionalPointResultCodes.Success_ExistingPointRetrieved, result.ResultCode);
+            Assert.AreEqual(1, result.Point);
+            Assert.AreEqual(1, cube.TotalPointsCount);
+
+
+            result = cube.TryGetPoint("1", "1", "2");
+
+            Assert.IsFalse(result.IsSuccess);
+            Assert.IsFalse(result.IsPointCreatedNew);
+            Assert.AreEqual(-1, result.FailureCoordinateIndex);
+            Assert.AreEqual(MultidimensionalPointResultCodes.Failure_PointDoesNotExistCreationNotRequested, result.ResultCode);
+            Assert.AreEqual(0, result.Point);
+            Assert.AreEqual(1, cube.TotalPointsCount);
+
+            result = cube.TryGetOrCreatePoint("1", "1", "2");
+
+            Assert.IsTrue(result.IsSuccess);
+            Assert.IsTrue(result.IsPointCreatedNew);
+            Assert.AreEqual(-1, result.FailureCoordinateIndex);
+            Assert.AreEqual(MultidimensionalPointResultCodes.Success_NewPointCreated, result.ResultCode);
+            Assert.AreEqual(1, result.Point);
+            Assert.AreEqual(2, cube.TotalPointsCount);
+
+            result = cube.TryGetOrCreatePoint("1", "1", "2");
+
+            Assert.IsTrue(result.IsSuccess);
+            Assert.IsFalse(result.IsPointCreatedNew);
+            Assert.AreEqual(-1, result.FailureCoordinateIndex);
+            Assert.AreEqual(MultidimensionalPointResultCodes.Success_ExistingPointRetrieved, result.ResultCode);
+            Assert.AreEqual(1, result.Point);
+            Assert.AreEqual(2, cube.TotalPointsCount);
+
+            result = cube.TryGetPoint("1", "1", "3");
+
+            Assert.IsFalse(result.IsSuccess);
+            Assert.IsFalse(result.IsPointCreatedNew);
+            Assert.AreEqual(-1, result.FailureCoordinateIndex);
+            Assert.AreEqual(MultidimensionalPointResultCodes.Failure_PointDoesNotExistCreationNotRequested, result.ResultCode);
+            Assert.AreEqual(0, result.Point);
+            Assert.AreEqual(2, cube.TotalPointsCount);
+
+            // Triggers creation of fallback dimension value.
+            // i.e 3 will be replaced with "dimCapValue"
+            result = cube.TryGetOrCreatePoint("1", "1", "3");
+
+            Assert.IsTrue(result.IsSuccess);
+            Assert.IsTrue(result.IsPointCreatedNew);
+            Assert.AreEqual(-1, result.FailureCoordinateIndex);
+            Assert.AreEqual(MultidimensionalPointResultCodes.Success_NewPointCreatedAboveDimCapLimit, result.ResultCode);
+            Assert.AreEqual(1, result.Point);
+            Assert.AreEqual(3, cube.TotalPointsCount);
+
+            result = cube.TryGetPoint("1", "1", "3");
+
+            Assert.IsFalse(result.IsSuccess);
+            Assert.IsFalse(result.IsPointCreatedNew);
+            Assert.AreEqual(-1, result.FailureCoordinateIndex);
+            Assert.AreEqual(MultidimensionalPointResultCodes.Failure_PointDoesNotExistCreationNotRequested, result.ResultCode);
+            Assert.AreEqual(0, result.Point);
+            Assert.AreEqual(3, cube.TotalPointsCount);
+
+            // Triggers re-use of previously created fallback dimension value.            
+            result = cube.TryGetOrCreatePoint("1", "1", "3");
+
+            Assert.IsTrue(result.IsSuccess);
+            Assert.IsFalse(result.IsPointCreatedNew);
+            Assert.AreEqual(-1, result.FailureCoordinateIndex);
+            Assert.AreEqual(MultidimensionalPointResultCodes.Success_ExistingPointRetrieved, result.ResultCode);
+            Assert.AreEqual(1, result.Point);
+            Assert.AreEqual(3, cube.TotalPointsCount);
+
+            // Force 2nd dimension to reach cap. (preparing for the next case below)
+            result = cube.TryGetOrCreatePoint("1", "2", "1");
+
+            Assert.IsTrue(result.IsSuccess);
+            Assert.IsTrue(result.IsPointCreatedNew);
+            Assert.AreEqual(-1, result.FailureCoordinateIndex);
+            Assert.AreEqual(MultidimensionalPointResultCodes.Success_NewPointCreated, result.ResultCode);
+            Assert.AreEqual(1, result.Point);
+            Assert.AreEqual(4, cube.TotalPointsCount);
+
+
+            // Triggers creation of fallback dimension value.
+            // i.e both 3 will be replaced with "dimCapValue"
+            result = cube.TryGetOrCreatePoint("1", "3", "3");
+
+            Assert.IsTrue(result.IsSuccess);
+            Assert.IsTrue(result.IsPointCreatedNew);
+            Assert.AreEqual(-1, result.FailureCoordinateIndex);
+            Assert.AreEqual(MultidimensionalPointResultCodes.Success_NewPointCreatedAboveDimCapLimit, result.ResultCode);
+            Assert.AreEqual(1, result.Point);
+            Assert.AreEqual(5, cube.TotalPointsCount);
+
+            // Force 1st dimension to reach cap. (preparing for the next case below)
+            result = cube.TryGetOrCreatePoint("2", "2", "1");
+
+            Assert.IsTrue(result.IsSuccess);
+            Assert.IsTrue(result.IsPointCreatedNew);
+            Assert.AreEqual(-1, result.FailureCoordinateIndex);
+            Assert.AreEqual(MultidimensionalPointResultCodes.Success_NewPointCreated, result.ResultCode);
+            Assert.AreEqual(2, result.Point);
+            Assert.AreEqual(6, cube.TotalPointsCount);
+
+
+            // Triggers creation of fallback dimension value.
+            // i.e all 3 will be replaced with "dimCapValue"
+            result = cube.TryGetOrCreatePoint("3", "3", "3");
+
+            Assert.IsTrue(result.IsSuccess);
+            Assert.IsTrue(result.IsPointCreatedNew);
+            Assert.AreEqual(-1, result.FailureCoordinateIndex);
+            Assert.AreEqual(MultidimensionalPointResultCodes.Success_NewPointCreatedAboveDimCapLimit, result.ResultCode);
+            Assert.AreEqual(999, result.Point);
+            Assert.AreEqual(7, cube.TotalPointsCount);
+        }
+
+        /// <summary />
+        [TestMethod]
+        public void TryGetOrCreatePoint_GetsAndCreatesWithDimCapReachingOverallCap()
+        {
+            string dimCapValue = "999";
+
+            var cube = new MultidimensionalCube2<int>(
+                                        4,
+                                        (vector) => Int32.Parse(vector[0]), true,
+                                        dimCapValue,
+                                        2, 2, 2);
+
+            MultidimensionalPointResult<int> result;
+
+            Assert.AreEqual(0, cube.TotalPointsCount);
+
+            result = cube.TryGetOrCreatePoint("1", "1", "1");
+            Assert.IsTrue(result.IsSuccess);
+            Assert.AreEqual(1, cube.TotalPointsCount);
+
+            result = cube.TryGetOrCreatePoint("1", "2", "1");
+            Assert.IsTrue(result.IsSuccess);
+            Assert.AreEqual(2, cube.TotalPointsCount);
+
+            result = cube.TryGetOrCreatePoint("1", "1", "2");
+            Assert.IsTrue(result.IsSuccess);
+            Assert.AreEqual(3, cube.TotalPointsCount);
+
+            result = cube.TryGetOrCreatePoint("2", "1", "1");
+            Assert.IsTrue(result.IsSuccess);
+            Assert.AreEqual(4, cube.TotalPointsCount);
+
+            // At this stage 4 series have been created, and it means overall metric cap is hit.
+            // The following will respect that limit, and will not attempt to do dim capping for the 3rd dimension.
+            result = cube.TryGetOrCreatePoint("1", "1", "3");
+            Assert.IsFalse(result.IsSuccess);
+            Assert.AreEqual(4, cube.TotalPointsCount);
+            Assert.AreEqual(MultidimensionalPointResultCodes.Failure_TotalPointsCountLimitReached, result.ResultCode);
+        }
+
        /// <summary />
        [TestMethod]
        public void GetAllPoints()
--- a/BASE/Test/Microsoft.ApplicationInsights.Test/Shared/Metrics/MetricConfigurationTests.cs
+++ b/BASE/Test/Microsoft.ApplicationInsights.Test/Shared/Metrics/MetricConfigurationTests.cs
@ -16,6 +16,14 @@ namespace Microsoft.ApplicationInsights.Metrics
        [TestMethod]
        public void Ctor()
        {
+            {
+                var config = new MetricConfiguration(
+                                    seriesCountLimit: 1000,
+                                    valuesPerDimensionLimit: 100,
+                                    seriesConfig: new MetricSeriesConfigurationForMeasurement(restrictToUInt32Values: false));
+                Assert.IsFalse(config.ApplyDimensionCapping);
+            }
+
            { 
                var config = new MetricConfiguration(
                                    seriesCountLimit:           1000,
--- a/BASE/src/Microsoft.ApplicationInsights/Metric.cs
+++ b/BASE/src/Microsoft.ApplicationInsights/Metric.cs
@ -61,6 +61,8 @@
                this.metricSeries = new MultidimensionalCube2<MetricSeries>(
                            totalPointsCountLimit: configuration.SeriesCountLimit - 1,
                            pointsFactory: this.CreateNewMetricSeries,
+                            applyDimensionCapping: configuration.ApplyDimensionCapping,
+                            dimensionCapValue: configuration.DimensionCappedString,
                            dimensionValuesCountLimits: dimensionValuesCountLimits);

                this.zeroDimSeriesList = null;
--- a/BASE/src/Microsoft.ApplicationInsights/Metrics/Implementation/AutocollectedMetricsExtraction/MetricTerms.cs
+++ b/BASE/src/Microsoft.ApplicationInsights/Metrics/Implementation/AutocollectedMetricsExtraction/MetricTerms.cs
@ -4,7 +4,8 @@

    internal static class MetricTerms
    {
-        private const string MetricPropertiesNamePrefix = "_MS";
+        internal const string DimensionCapFallbackValue = "DIMENSION_CAPPED";
+        private const string MetricPropertiesNamePrefix = "_MS";        

        public static class Aggregation
        {
--- a/BASE/src/Microsoft.ApplicationInsights/Metrics/Implementation/ConcurrentDatastructures/MultidimensionalCube2.cs
+++ b/BASE/src/Microsoft.ApplicationInsights/Metrics/Implementation/ConcurrentDatastructures/MultidimensionalCube2.cs
@ -33,6 +33,8 @@
        private readonly Func<string[], TPoint> pointsFactory;

        private int totalPointsCount;
+        private bool applyDimensionCapping;
+        private string dimensionCapValue;

        public MultidimensionalCube2(Func<string[], TPoint> pointsFactory, params int[] dimensionValuesCountLimits)
            : this(Int32.MaxValue, pointsFactory, dimensionValuesCountLimits)
@ -83,6 +85,13 @@
            }
        }

+        public MultidimensionalCube2(int totalPointsCountLimit, Func<string[], TPoint> pointsFactory, bool applyDimensionCapping, string dimensionCapValue, params int[] dimensionValuesCountLimits)
+            : this(totalPointsCountLimit, pointsFactory, dimensionValuesCountLimits)
+        {
+            this.dimensionCapValue = dimensionCapValue;
+            this.applyDimensionCapping = applyDimensionCapping;
+        }
+
        public int DimensionsCount
        {
            get { return this.dimensionValuesCountLimits.Length; }
@ -287,6 +296,7 @@

            int reachedValsLimitDim = -1;
            BitArray valueAddedToDims = new BitArray(length: coordinates.Length, defaultValue: false);
+            bool dimensionCappingApplied = false;

            for (int i = 0; i < coordinates.Length; i++)
            {
@ -295,8 +305,17 @@

                if ((dimVals.Count >= this.dimensionValuesCountLimits[i]) && (false == dimVals.Contains(coordinateVal)))
                {
-                    reachedValsLimitDim = i;
-                    break;
+                    if (this.applyDimensionCapping)
+                    {
+                        // throwing away the actual dimension value here as we are asked to apply dimensioncapping.
+                        coordinates[i] = this.dimensionCapValue;
+                        dimensionCappingApplied = true;
+                    }
+                    else
+                    {
+                        reachedValsLimitDim = i;
+                        break;
+                    }
                }

                bool added = dimVals.Add(coordinates[i]);
@ -320,8 +339,23 @@
                return result;
            }

-            // Create new point:
+            if (dimensionCappingApplied)
+            {
+                // We did dimension capping - i.e one or more dimension values were replaced, requiring us to reconstruct the moniker with new values.
+                pointMoniker = BuildPointMoniker(coordinates);

+                // Check again (with new moniker) if the point already exists.
+                hasPoint = this.points.TryGetValue(pointMoniker, out point);
+                if (hasPoint)
+                {
+                    var result = new MultidimensionalPointResult<TPoint>(MultidimensionalPointResultCodes.Success_ExistingPointRetrieved, point);
+                    return result;
+                }
+
+                // Continue with point creation using new moniker.
+            }
+
+            // Create new point:
            try
            {
                point = this.pointsFactory(coordinates);
@ -355,7 +389,9 @@
            this.totalPointsCount++;

            {
-                var result = new MultidimensionalPointResult<TPoint>(MultidimensionalPointResultCodes.Success_NewPointCreated, point);
+                var result = new MultidimensionalPointResult<TPoint>(
+                    dimensionCappingApplied ? MultidimensionalPointResultCodes.Success_NewPointCreatedAboveDimCapLimit : MultidimensionalPointResultCodes.Success_NewPointCreated,
+                    point);
                return result;
            }
 #pragma warning restore SA1509 // Opening braces must not be preceded by blank line
--- a/BASE/src/Microsoft.ApplicationInsights/Metrics/Implementation/ConcurrentDatastructures/MultidimensionalPointResult.cs
+++ b/BASE/src/Microsoft.ApplicationInsights/Metrics/Implementation/ConcurrentDatastructures/MultidimensionalPointResult.cs
@ -39,7 +39,11 @@

        public bool IsPointCreatedNew
        {
-            get { return (this.ResultCode & MultidimensionalPointResultCodes.Success_NewPointCreated) != 0; }
+            get
+            {
+                return ((this.ResultCode & MultidimensionalPointResultCodes.Success_NewPointCreated) != 0)
+                          || ((this.ResultCode & MultidimensionalPointResultCodes.Success_NewPointCreatedAboveDimCapLimit) != 0);
+            }
        }

        public bool IsSuccess
@ -47,7 +51,8 @@
            get
            {
                return ((this.ResultCode & MultidimensionalPointResultCodes.Success_NewPointCreated) != 0)
-                          || ((this.ResultCode & MultidimensionalPointResultCodes.Success_ExistingPointRetrieved) != 0);
+                          || ((this.ResultCode & MultidimensionalPointResultCodes.Success_ExistingPointRetrieved) != 0)
+                          || ((this.ResultCode & MultidimensionalPointResultCodes.Success_NewPointCreatedAboveDimCapLimit) != 0);
            }
        }

--- a/BASE/src/Microsoft.ApplicationInsights/Metrics/Implementation/ConcurrentDatastructures/MultidimensionalPointResultCodes.cs
+++ b/BASE/src/Microsoft.ApplicationInsights/Metrics/Implementation/ConcurrentDatastructures/MultidimensionalPointResultCodes.cs
@ -19,6 +19,13 @@
        /// </summary>
        Success_ExistingPointRetrieved = 2,

+        /// <summary>
+        /// A new point was created and returned in this result.
+        /// The newly created point exceeded the specified dimension values count limit for one or more dimensions,
+        /// but it was capped with a fallback value.
+        /// </summary>
+        Success_NewPointCreatedAboveDimCapLimit = 4,
+
        /// <summary>
        /// A point could not be created becasue the sub-dimsnions count was reached.
        /// </summary>
--- a/BASE/src/Microsoft.ApplicationInsights/Metrics/MetricConfiguration.cs
+++ b/BASE/src/Microsoft.ApplicationInsights/Metrics/MetricConfiguration.cs
@ -118,6 +118,23 @@
        /// <summary>Gets the configuration for how each series of this metric should be aggregated.</summary>
        public IMetricSeriesConfiguration SeriesConfig { get; }

+        /// <summary>Gets or sets a value indicating whether dimension capping should be applied, when any indiviual dimension
+        /// exceeds its limit. If this flag is set, calls to <c>TrackValue(..)</c>, <c>TryGetDataSeries(..)</c> and similar
+        /// that would normally return false when cap is hit will return true, and the actual value of dimension will be replaced
+        /// by <c>DimensionCappedString</c>.
+        /// The metric will continue to track the value, however, users should be beware that any metric filtering or
+        /// splitting involving a dimension which has the value <c>DimensionCappedString</c>, should be ignored.
+        /// Overall metric value, and metric value for dimensions which do not have <c>DimensionCappedString</c> will remain
+        /// accurate.
+        /// </summary>
+        /// <remarks>
+        /// If overall series cap (SeriesCountLimit) is hit, then this dimension capping will not be applied.        
+        /// </remarks>
+        internal bool ApplyDimensionCapping { get; set; }
+
+        /// <summary>Gets or sets value which will be used to represent all dimension values encountered after dimension hits cap.</summary>
+        internal string DimensionCappedString { get; set; }
+
        /// <summary>
        /// Gets the maximum number of distinct values for a dimension identified by the specified 1-based dimension index.
        /// </summary>