Add Network Test Report (#2330)

* Add network controller report and generator; and enabled it in connectivity test

e2e_deployment_files/connectivity_deployment.template.json

@@ -665,6 +665,10 @@
               "ExpectedSource": "directMethodSender2.send",
               "ActualSource": "directMethodReceiver2.receive",
               "TolerancePeriod": "00:00:00.005"
+          },
+          "reportMetadata12": {
+              "TestReportType": "NetworkControllerReport",
+              "Source": "networkController"
           }
         }
       }

test/modules/Modules.Test/TestResultCoordinator/TestReportUtilTest.cs

@@ -118,7 +118,7 @@ namespace Modules.Test.TestResultCoordinator
         }
 
         [Fact]
-        public void ParseReportMetadataList_ParseCountingReport()
+        public void ParseReportMetadataList_ParseCountingReportMetadata()
         {
             const string testDataJson =
                 @"{
@@ -142,7 +142,7 @@ namespace Modules.Test.TestResultCoordinator
         }
 
         [Fact]
-        public void ParseReportMetadataList_ParseTwinCountingReport()
+        public void ParseReportMetadataList_ParseTwinCountingReportMetadata()
         {
             const string testDataJson =
                 @"{
@@ -167,7 +167,7 @@ namespace Modules.Test.TestResultCoordinator
         }
 
         [Fact]
-        public void ParseReportMetadataList_ParseDeploymentTestReport()
+        public void ParseReportMetadataList_ParseDeploymentTestReportMetadata()
         {
             const string testDataJson =
                 @"{
@@ -190,7 +190,7 @@ namespace Modules.Test.TestResultCoordinator
         }
 
         [Fact]
-        public void ParseReportMetadataList_ParseDirectMethodTestReport()
+        public void ParseReportMetadataList_ParseDirectMethodTestReportMetadata()
         {
             const string testDataJson =
                 @"{
@@ -214,6 +214,27 @@ namespace Modules.Test.TestResultCoordinator
             Assert.Equal(new TimeSpan(0, 0, 0, 0, 5), reportMetadata.TolerancePeriod);
         }
 
+        [Fact]
+        public void ParseReportMetadataList_ParseNetworkControllerReportMetadata()
+        {
+            const string testDataJson =
+                @"{
+                    ""reportMetadata"": {
+                        ""TestReportType"": ""NetworkControllerReport"",
+                        ""Source"": ""networkController""
+                    }
+                }";
+
+            List<ITestReportMetadata> results = TestReportUtil.ParseReportMetadataJson(testDataJson, new Mock<ILogger>().Object);
+
+            Assert.Single(results);
+            var reportMetadata = results[0] as NetworkControllerReportMetadata;
+            Assert.NotNull(reportMetadata);
+            Assert.Equal(TestOperationResultType.Network, reportMetadata.TestOperationResultType);
+            Assert.Equal(TestReportType.NetworkControllerReport, reportMetadata.TestReportType);
+            Assert.Equal("networkController", reportMetadata.Source);
+        }
+
         [Fact]
         public void ParseReportMetadataList_ThrowExceptionWhenParseInvalidData()
         {

test/modules/TestResultCoordinator/Reports/CountingReport.cs

@@ -43,6 +43,6 @@ namespace TestResultCoordinator.Reports
 
         public override bool IsPassed => this.TotalExpectCount == this.TotalMatchCount;
 
-        public override string Title => $"Counting Report ({this.ResultType}) between [{this.ExpectedSource}] and [{this.ActualSource}]";
+        public override string Title => $"Counting Report between [{this.ExpectedSource}] and [{this.ActualSource}] ({this.ResultType})";
     }
 }

test/modules/TestResultCoordinator/Reports/CountingReportGenerator.cs

@@ -70,8 +70,8 @@ namespace TestResultCoordinator.Reports
 
             while (hasExpectedResult && hasActualResult)
             {
-                this.ValidateDataSource(this.ExpectedTestResults.Current, this.ExpectedSource);
-                this.ValidateDataSource(this.ActualTestResults.Current, this.ActualSource);
+                this.ValidateResult(this.ExpectedTestResults.Current, this.ExpectedSource);
+                this.ValidateResult(this.ActualTestResults.Current, this.ActualSource);
 
                 // Skip any duplicate actual value
                 while (hasActualResult && this.TestResultComparer.Matches(lastLoadedResult, this.ActualTestResults.Current))
@@ -133,12 +133,17 @@ namespace TestResultCoordinator.Reports
                 unmatchedResults.AsReadOnly());
         }
 
-        void ValidateDataSource(TestOperationResult current, string expectedSource)
+        void ValidateResult(TestOperationResult current, string expectedSource)
         {
             if (!current.Source.Equals(expectedSource, StringComparison.OrdinalIgnoreCase))
             {
                 throw new InvalidDataException($"Result source is '{current.Source}' but expected should be '{expectedSource}'.");
             }
+
+            if (!current.Type.Equals(this.ResultType, StringComparison.Ordinal))
+            {
+                throw new InvalidDataException($"Result type is '{current.Type}' but expected should be '{this.ResultType}'.");
+            }
         }
     }
 }

test/modules/TestResultCoordinator/Reports/CountingReportMetadata.cs

@@ -2,13 +2,14 @@
 namespace TestResultCoordinator.Reports
 {
     using Microsoft.Azure.Devices.Edge.ModuleUtil;
+    using Microsoft.Azure.Devices.Edge.Util;
 
     class CountingReportMetadata : ITestReportMetadata
     {
         public CountingReportMetadata(string expectedSource, string actualSource, TestOperationResultType testOperationResultType, TestReportType testReportType)
         {
-            this.ExpectedSource = expectedSource;
-            this.ActualSource = actualSource;
+            this.ExpectedSource = Preconditions.CheckNonWhiteSpace(expectedSource, nameof(expectedSource));
+            this.ActualSource = Preconditions.CheckNonWhiteSpace(actualSource, nameof(actualSource));
             this.TestOperationResultType = testOperationResultType;
             this.TestReportType = testReportType;
         }

test/modules/TestResultCoordinator/Reports/DeploymentTestReport.cs

@@ -47,6 +47,6 @@ namespace TestResultCoordinator.Reports
 
         public override bool IsPassed => this.TotalExpectedDeployments == this.TotalMatchedDeployments;
 
-        public override string Title => $"Deployment Test Report ({this.ResultType}) between [{this.ExpectedSource}] and [{this.ActualSource}]";
+        public override string Title => $"Deployment Test Report between [{this.ExpectedSource}] and [{this.ActualSource}] ({this.ResultType})";
     }
 }

test/modules/TestResultCoordinator/Reports/DeploymentTestReportGenerator.cs

@@ -76,8 +76,8 @@ namespace TestResultCoordinator.Reports
 
             while (hasExpectedResult && hasActualResult)
             {
-                this.ValidateDataSource(this.ExpectedTestResults.Current, this.ExpectedSource);
-                this.ValidateDataSource(this.ActualTestResults.Current, this.ActualSource);
+                this.ValidateResult(this.ExpectedTestResults.Current, this.ExpectedSource);
+                this.ValidateResult(this.ActualTestResults.Current, this.ActualSource);
 
                 if (this.TestResultComparer.Matches(this.ExpectedTestResults.Current, this.ActualTestResults.Current))
                 {
@@ -145,12 +145,18 @@ namespace TestResultCoordinator.Reports
                 unmatchedResults.AsReadOnly());
         }
 
-        void ValidateDataSource(TestOperationResult current, string expectedSource)
+
+        void ValidateResult(TestOperationResult current, string expectedSource)
         {
             if (!current.Source.Equals(expectedSource, StringComparison.OrdinalIgnoreCase))
             {
                 throw new InvalidDataException($"Result source is '{current.Source}' but expected should be '{expectedSource}'.");
             }
+
+            if (!current.Type.Equals(this.ResultType, StringComparison.Ordinal))
+            {
+                throw new InvalidDataException($"Result type is '{current.Type}' but expected should be '{this.ResultType}'.");
+            }
         }
     }
 }

test/modules/TestResultCoordinator/Reports/DeploymentTestReportMetadata.cs

@@ -2,13 +2,14 @@
 namespace TestResultCoordinator.Reports
 {
     using Microsoft.Azure.Devices.Edge.ModuleUtil;
+    using Microsoft.Azure.Devices.Edge.Util;
 
     class DeploymentTestReportMetadata : ITestReportMetadata
     {
         public DeploymentTestReportMetadata(string expectedSource, string actualSource)
         {
-            this.ExpectedSource = expectedSource;
-            this.ActualSource = actualSource;
+            this.ExpectedSource = Preconditions.CheckNonWhiteSpace(expectedSource, nameof(expectedSource));
+            this.ActualSource = Preconditions.CheckNonWhiteSpace(actualSource, nameof(actualSource));
         }
 
         public string ExpectedSource { get; }

test/modules/TestResultCoordinator/Reports/DirectMethod/DirectMethodReport.cs

@@ -64,7 +64,7 @@ namespace TestResultCoordinator.Reports.DirectMethod
         // MismatchFailure is when the there is a result in ActualStore but no result in ExpectedStore. This should never happen.
         public ulong MismatchFailure { get; }
 
-        public override string Title => $"DirectMethod Report ({this.ResultType}) for [{this.ExpectedSource}] and [{this.ActualSource}]";
+        public override string Title => $"DirectMethod Report for [{this.ExpectedSource}] and [{this.ActualSource}] ({this.ResultType})";
 
         public override bool IsPassed =>
             this.MismatchFailure == 0 && this.NetworkOffFailure == 0 && this.NetworkOnFailure == 0;

test/modules/TestResultCoordinator/Reports/DirectMethod/DirectMethodReportMetadata.cs

@@ -3,14 +3,15 @@ namespace TestResultCoordinator.Reports.DirectMethod
 {
     using System;
     using Microsoft.Azure.Devices.Edge.ModuleUtil;
+    using Microsoft.Azure.Devices.Edge.Util;
     using TestResultCoordinator.Reports;
 
     class DirectMethodReportMetadata : ITestReportMetadata
     {
         public DirectMethodReportMetadata(string expectedSource, string actualSource, TestReportType testReportType, TimeSpan tolerancePeriod)
         {
-            this.ExpectedSource = expectedSource;
-            this.ActualSource = actualSource;
+            this.ExpectedSource = Preconditions.CheckNonWhiteSpace(expectedSource, nameof(expectedSource));
+            this.ActualSource = Preconditions.CheckNonWhiteSpace(actualSource, nameof(actualSource));
             this.TestReportType = testReportType;
             this.TolerancePeriod = tolerancePeriod;
         }

test/modules/TestResultCoordinator/Reports/NetworkControllerReport.cs

@@ -0,0 +1,32 @@
+// Copyright (c) Microsoft. All rights reserved.
+namespace TestResultCoordinator.Reports
+{
+    using System.Collections.Generic;
+    using Microsoft.Azure.Devices.Edge.ModuleUtil;
+    using Microsoft.Azure.Devices.Edge.Util;
+
+    /// <summary>
+    /// This is a network controller report to list out all the network controller status change during test.
+    /// </summary>
+    class NetworkControllerReport : TestResultReportBase
+    {
+        public NetworkControllerReport(
+            string trackingId,
+            string source,
+            string resultType,
+            IReadOnlyList<TestOperationResult> networkControllerResults)
+            : base(trackingId, resultType)
+        {
+            this.Source = Preconditions.CheckNonWhiteSpace(source, nameof(source));
+            this.Results = Preconditions.CheckNotNull(networkControllerResults, nameof(networkControllerResults));
+        }
+
+        public string Source { get; }
+
+        public IReadOnlyList<TestOperationResult> Results { get; }
+
+        public override string Title => $"Network Controller Report for [{this.Source}] ({this.ResultType})";
+
+        public override bool IsPassed => true;
+    }
+}

test/modules/TestResultCoordinator/Reports/NetworkControllerReportGenerator.cs

@@ -0,0 +1,68 @@
+// Copyright (c) Microsoft. All rights reserved.
+namespace TestResultCoordinator.Reports
+{
+    using System;
+    using System.Collections.Generic;
+    using System.IO;
+    using System.Threading.Tasks;
+    using Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Infrastructure;
+    using Microsoft.Azure.Devices.Edge.ModuleUtil;
+    using Microsoft.Azure.Devices.Edge.Util;
+    using Microsoft.Extensions.Logging;
+
+    sealed class NetworkControllerReportGenerator : ITestResultReportGenerator
+    {
+        static readonly ILogger Logger = ModuleUtil.CreateLogger(nameof(DeploymentTestReportGenerator));
+
+        readonly string trackingId;
+
+        internal NetworkControllerReportGenerator(
+            string trackingId,
+            string source,
+            ITestResultCollection<TestOperationResult> testResults)
+        {
+            this.trackingId = Preconditions.CheckNonWhiteSpace(trackingId, nameof(trackingId));
+            this.Source = Preconditions.CheckNonWhiteSpace(source, nameof(source));
+            this.TestResults = Preconditions.CheckNotNull(testResults, nameof(testResults));
+            this.ResultType = TestOperationResultType.Network.ToString();
+        }
+
+        internal string Source { get; }
+
+        internal ITestResultCollection<TestOperationResult> TestResults { get; }
+
+        internal string ResultType { get; }
+
+        public async Task<ITestResultReport> CreateReportAsync()
+        {
+            Logger.LogInformation($"Start to generate report by {nameof(NetworkControllerReportGenerator)} for Source [{this.Source}].");
+
+            var results = new List<TestOperationResult>();
+
+            while (await this.TestResults.MoveNextAsync())
+            {
+                this.ValidateResult(this.TestResults.Current);
+                results.Add(this.TestResults.Current);
+            }
+
+            return new NetworkControllerReport(
+                this.trackingId,
+                this.Source,
+                this.ResultType,
+                results.AsReadOnly());
+        }
+
+        void ValidateResult(TestOperationResult current)
+        {
+            if (!current.Source.Equals(this.Source, StringComparison.OrdinalIgnoreCase))
+            {
+                throw new InvalidDataException($"Result source is '{current.Source}' but expected should be '{this.Source}'.");
+            }
+
+            if (!current.Type.Equals(this.ResultType, StringComparison.Ordinal))
+            {
+                throw new InvalidDataException($"Result type is '{current.Type}' but expected should be '{this.ResultType}'.");
+            }
+        }
+    }
+}

test/modules/TestResultCoordinator/Reports/NetworkControllerReportMetadata.cs

@@ -0,0 +1,27 @@
+// Copyright (c) Microsoft. All rights reserved.
+namespace TestResultCoordinator.Reports
+{
+    using Microsoft.Azure.Devices.Edge.ModuleUtil;
+    using Microsoft.Azure.Devices.Edge.Util;
+
+    class NetworkControllerReportMetadata : ITestReportMetadata
+    {
+        public NetworkControllerReportMetadata(string source)
+        {
+            this.Source = Preconditions.CheckNonWhiteSpace(source, nameof(source));
+        }
+
+        public string Source { get; }
+
+        public TestReportType TestReportType => TestReportType.NetworkControllerReport;
+
+        public TestOperationResultType TestOperationResultType => TestOperationResultType.Network;
+
+        public string[] ResultSources => new string[] { this.Source };
+
+        public override string ToString()
+        {
+            return $"Source: {this.Source}, TestOperationResultType: {this.TestOperationResultType.ToString()}, ReportType: {this.TestReportType.ToString()}";
+        }
+    }
+}

test/modules/TestResultCoordinator/Reports/TestReportGeneratorFactory.cs

@@ -28,39 +28,39 @@ namespace TestResultCoordinator.Reports
             switch (testReportMetadata.TestReportType)
             {
                 case TestReportType.CountingReport:
-                {
-                    var metadata = (CountingReportMetadata)testReportMetadata;
-                    var expectedTestResults = this.GetResults(metadata.ExpectedSource);
-                    var actualTestResults = this.GetResults(metadata.ActualSource);
+                    {
+                        var metadata = (CountingReportMetadata)testReportMetadata;
+                        var expectedTestResults = this.GetResults(metadata.ExpectedSource);
+                        var actualTestResults = this.GetResults(metadata.ActualSource);
 
-                    return new CountingReportGenerator(
-                        trackingId,
-                        metadata.ExpectedSource,
-                        expectedTestResults,
-                        metadata.ActualSource,
-                        actualTestResults,
-                        testReportMetadata.TestOperationResultType.ToString(),
-                        new SimpleTestOperationResultComparer());
-                }
+                        return new CountingReportGenerator(
+                            trackingId,
+                            metadata.ExpectedSource,
+                            expectedTestResults,
+                            metadata.ActualSource,
+                            actualTestResults,
+                            testReportMetadata.TestOperationResultType.ToString(),
+                            new SimpleTestOperationResultComparer());
+                    }
 
                 case TestReportType.TwinCountingReport:
-                {
-                    var metadata = (TwinCountingReportMetadata)testReportMetadata;
-                    var expectedTestResults = this.GetTwinExpectedResults(metadata);
-                    var actualTestResults = this.GetResults(metadata.ActualSource);
+                    {
+                        var metadata = (TwinCountingReportMetadata)testReportMetadata;
+                        var expectedTestResults = this.GetTwinExpectedResults(metadata);
+                        var actualTestResults = this.GetResults(metadata.ActualSource);
 
-                    return new TwinCountingReportGenerator(
-                        trackingId,
-                        metadata.ExpectedSource,
-                        expectedTestResults,
-                        metadata.ActualSource,
-                        actualTestResults,
-                        testReportMetadata.TestOperationResultType.ToString(),
-                        new SimpleTestOperationResultComparer());
-                }
+                        return new TwinCountingReportGenerator(
+                            trackingId,
+                            metadata.ExpectedSource,
+                            expectedTestResults,
+                            metadata.ActualSource,
+                            actualTestResults,
+                            testReportMetadata.TestOperationResultType.ToString(),
+                            new SimpleTestOperationResultComparer());
+                    }
 
                 case TestReportType.DeploymentTestReport:
-                {
+                    {
                         var metadata = (DeploymentTestReportMetadata)testReportMetadata;
                         var expectedTestResults = this.GetResults(metadata.ExpectedSource);
                         var actualTestResults = this.GetResults(metadata.ActualSource);
@@ -71,31 +71,42 @@ namespace TestResultCoordinator.Reports
                         expectedTestResults,
                         metadata.ActualSource,
                         actualTestResults);
-                }
+                    }
 
                 case TestReportType.DirectMethodReport:
-                {
-                    var metadata = (DirectMethodReportMetadata)testReportMetadata;
-                    var expectedTestResults = this.GetResults(metadata.ExpectedSource);
-                    var actualTestResults = this.GetResults(metadata.ActualSource);
-                    var tolerancePeriod = metadata.TolerancePeriod;
-                    var networkStatusTimeline = await this.GetNetworkStatusTimelineAsync(tolerancePeriod);
+                    {
+                        var metadata = (DirectMethodReportMetadata)testReportMetadata;
+                        var expectedTestResults = this.GetResults(metadata.ExpectedSource);
+                        var actualTestResults = this.GetResults(metadata.ActualSource);
+                        var tolerancePeriod = metadata.TolerancePeriod;
+                        var networkStatusTimeline = await this.GetNetworkStatusTimelineAsync(tolerancePeriod);
 
-                    return new DirectMethodReportGenerator(
-                        trackingId,
-                        metadata.ExpectedSource,
-                        expectedTestResults,
-                        metadata.ActualSource,
-                        actualTestResults,
-                        metadata.TestOperationResultType.ToString(),
-                        new DirectMethodTestOperationResultComparer(),
-                        networkStatusTimeline);
-                }
+                        return new DirectMethodReportGenerator(
+                            trackingId,
+                            metadata.ExpectedSource,
+                            expectedTestResults,
+                            metadata.ActualSource,
+                            actualTestResults,
+                            metadata.TestOperationResultType.ToString(),
+                            new DirectMethodTestOperationResultComparer(),
+                            networkStatusTimeline);
+                    }
+
+                case TestReportType.NetworkControllerReport:
+                    {
+                        var metadata = (NetworkControllerReportMetadata)testReportMetadata;
+                        var testResults = this.GetResults(metadata.Source);
+
+                        return new NetworkControllerReportGenerator(
+                            trackingId,
+                            metadata.Source,
+                            testResults);
+                    }
 
                 default:
-                {
-                    throw new NotSupportedException($"Report type {testReportMetadata.TestReportType} is not supported.");
-                }
+                    {
+                        throw new NotSupportedException($"Report type {testReportMetadata.TestReportType} is not supported.");
+                    }
             }
         }
 

test/modules/TestResultCoordinator/Reports/TestReportType.cs

@@ -6,6 +6,7 @@ namespace TestResultCoordinator.Reports
         CountingReport,
         TwinCountingReport,
         DeploymentTestReport,
-        DirectMethodReport
+        DirectMethodReport,
+        NetworkControllerReport
     }
 }

test/modules/TestResultCoordinator/Reports/TwinCountingReport.cs

@@ -34,6 +34,6 @@ namespace TestResultCoordinator.Reports
 
         public override bool IsPassed => this.TotalExpectCount == this.TotalMatchCount;
 
-        public override string Title => $"Twin Counting Report ({this.ResultType}) between [{this.ExpectedSource}] and [{this.ActualSource}]";
+        public override string Title => $"Twin Counting Report between [{this.ExpectedSource}] and [{this.ActualSource}] ({this.ResultType})";
     }
 }

test/modules/TestResultCoordinator/Settings.cs

@@ -149,7 +149,6 @@ namespace TestResultCoordinator
             HashSet<string> sources = (await this.GetReportMetadataListAsync(logger)).SelectMany(r => r.ResultSources).ToHashSet();
             string[] additionalResultSources = new string[]
             {
-                "networkController",
                 "directMethodSender3.send",
                 "directMethodSender3.send"
             };

test/modules/TestResultCoordinator/TestReportUtil.cs

@@ -86,6 +86,9 @@ namespace TestResultCoordinator
                         case TestReportType.DirectMethodReport:
                             reportMetadataList.Add(JsonConvert.DeserializeObject<DirectMethodReportMetadata>(((JProperty)metadata).Value.ToString()));
                             break;
+                        case TestReportType.NetworkControllerReport:
+                            reportMetadataList.Add(JsonConvert.DeserializeObject<NetworkControllerReportMetadata>(((JProperty)metadata).Value.ToString()));
+                            break;
                         default:
                             throw new NotImplementedException("{testReportType} doesn't implement to construct report metadata from Twin.");
                     }