simplifying result reporting and other minor changes

csharp/Samples/Microsoft.Spark.CSharp/CommandlineArgumentProcessor.cs

@@ -2,14 +2,13 @@
 // Licensed under the MIT license. See LICENSE file in the project root for full license information.
 
 using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
 using Microsoft.Spark.CSharp.Services;
 
 namespace Microsoft.Spark.CSharp.Samples
 {
+    /// <summary>
+    /// Simple commandline argument parser
+    /// </summary>
     internal class CommandlineArgumentProcessor
     {
         private static readonly ILoggerService logger = LoggerServiceFactory.GetLogger(typeof(CommandlineArgumentProcessor));
@@ -22,7 +21,7 @@ namespace Microsoft.Spark.CSharp.Samples
             }
 
             var configuration = new Configuration();
-            logger.LogInfo(string.Format("Arguments to SparkCLRSamples are {0}", string.Join(",", args)));
+            logger.LogInfo(string.Format("Arguments to SparkCLRSamples.exe are {0}", string.Join(",", args)));
             for (int i = 0; i < args.Length; i++)
             {
                 if (args[i].Equals("--help", StringComparison.InvariantCultureIgnoreCase)
@@ -69,24 +68,24 @@ namespace Microsoft.Spark.CSharp.Samples
 
         private static void PrintUsage()
         {
-            var p = AppDomain.CurrentDomain.FriendlyName;
+            const string exeName = "SparkCLRSamples.exe";
             Console.WriteLine("   ");
-            Console.WriteLine(" {0} supports following options:", p);
+            Console.WriteLine(" {0} supports following options:", exeName);
             Console.WriteLine("   ");
             Console.WriteLine("   [--temp | spark.local.dir] <TEMP_DIR>                 TEMP_DIR is the directory used as \"scratch\" space in Spark, including map output files and RDDs that get stored on disk. ");
             Console.WriteLine("                                                         See http://spark.apache.org/docs/latest/configuration.html for details.");
             Console.WriteLine("   ");
-            Console.WriteLine("   [--data | sparkclr.sampledata.loc] <SAMPLE_DATA_DIR>  SAMPLE_DATA_DIR is the directory where Sample data resides. ");
+            Console.WriteLine("   [--data | sparkclr.sampledata.loc] <SAMPLE_DATA_DIR>  SAMPLE_DATA_DIR is the directory where data files used by samples reside. ");
             Console.WriteLine("   ");
             Console.WriteLine("   [--torun | sparkclr.samples.torun] <SAMPLE_LIST>      SAMPLE_LIST specifies a list of samples to run. ");
             Console.WriteLine("                                                         Case-insensitive command line wild card matching by default. Or, use \"/\" (forward slash) to enclose regular expression. ");
-            Console.WriteLine("   ");
+            Console.WriteLine("   "); 
             Console.WriteLine("   [--cat | sparkclr.samples.category] <SAMPLE_CATEGORY> SAMPLE_CATEGORY can be \"all\", \"default\", \"experimental\" or any new categories. ");
             Console.WriteLine("                                                         Case-insensitive command line wild card matching by default. Or, use \"/\" (forward slash) to enclose regular expression. ");
             Console.WriteLine("   ");
-            Console.WriteLine("   [--validate | sparkclr.enablevalidation]              Enable validation. ");
+            Console.WriteLine("   [--validate | sparkclr.enablevalidation]              Enables validation of results produced in each sample. ");
             Console.WriteLine("   ");
-            Console.WriteLine("   [--dryrun | sparkclr.dryrun]                          Dry-run mode. ");
+            Console.WriteLine("   [--dryrun | sparkclr.dryrun]                          Dry-run mode. Just lists the samples that will be executed with given parameters without running them");
             Console.WriteLine("   ");
             Console.WriteLine("   [--help | -h | -?]                                    Display usage. ");
             Console.WriteLine("   ");
@@ -95,31 +94,31 @@ namespace Microsoft.Spark.CSharp.Samples
             Console.WriteLine("   ");
             Console.WriteLine("   Example 1 - run default samples:");
             Console.WriteLine("   ");
-            Console.WriteLine(@"     {0} --temp C:\gitsrc\SparkCLR\run\Temp --data C:\gitsrc\SparkCLR\run\data ", p);
+            Console.WriteLine(@"     {0} --temp C:\gitsrc\SparkCLR\run\Temp --data C:\gitsrc\SparkCLR\run\data ", exeName);
             Console.WriteLine("   ");
             Console.WriteLine("   Example 2 - dryrun default samples:");
             Console.WriteLine("   ");
-            Console.WriteLine(@"     {0} --dryrun ", p);
+            Console.WriteLine(@"     {0} --dryrun ", exeName);
             Console.WriteLine("   ");
             Console.WriteLine("   Example 3 - dryrun all samples:");
             Console.WriteLine("   ");
-            Console.WriteLine(@"     {0} --dryrun --cat all ", p);
+            Console.WriteLine(@"     {0} --dryrun --cat all ", exeName);
             Console.WriteLine("   ");
             Console.WriteLine("   Example 4 - dryrun PiSample (commandline wildcard matching, case-insensitive):");
             Console.WriteLine("   ");
-            Console.WriteLine(@"     {0} --dryrun --torun pi*", p);
+            Console.WriteLine(@"     {0} --dryrun --torun pi*", exeName);
             Console.WriteLine("   ");
             Console.WriteLine("   Example 5 - dryrun all DF* samples (commandline wildcard matching, case-insensitive):");
             Console.WriteLine("   ");
-            Console.WriteLine(@"     {0} --dryrun --cat a* --torun DF*", p);
+            Console.WriteLine(@"     {0} --dryrun --cat a* --torun DF*", exeName);
             Console.WriteLine("   ");
             Console.WriteLine("   Example 6 - dryrun all RD* samples (regular expression):");
             Console.WriteLine("   ");
-            Console.WriteLine(@"     {0} --dryrun --cat a* --torun /\bRD.*Sample.*\b/", p);
+            Console.WriteLine(@"     {0} --dryrun --cat a* --torun /\bRD.*Sample.*\b/", exeName);
             Console.WriteLine("   ");
             Console.WriteLine("   Example 7 - dryrun specific samples (case insensitive): ");
             Console.WriteLine("   ");
-            Console.WriteLine("     {0} --dryrun --torun \"DFShowSchemaSample,DFHeadSample\"", p);
+            Console.WriteLine("     {0} --dryrun --torun \"DFShowSchemaSample,DFHeadSample\"", exeName);
             Console.WriteLine("   ");
         }
     }

csharp/Samples/Microsoft.Spark.CSharp/MiscSamples.cs

@@ -51,7 +51,7 @@ namespace Microsoft.Spark.CSharp.Samples
 
             if (SparkCLRSamples.Configuration.IsValidationEnabled)
             {
-                Assert.IsTrue(Math.Abs(approximatePiValue - 3.14) <= 0.01);
+                Assert.IsTrue(Math.Abs(approximatePiValue - 3.14) <= 0.019);
             }
         }
 

csharp/Samples/Microsoft.Spark.CSharp/Program.cs

@@ -2,13 +2,9 @@
 // Licensed under the MIT license. See LICENSE file in the project root for full license information.
 
 using System;
-using System.Collections.Generic;
 using System.Diagnostics;
 using System.IO;
-using System.Linq;
 using System.Reflection;
-using System.Text;
-using System.Text.RegularExpressions;
 using Microsoft.Spark.CSharp.Core;
 using Microsoft.Spark.CSharp.Services;
 
@@ -38,11 +34,14 @@ namespace Microsoft.Spark.CSharp.Samples
             {
                 SparkContext = CreateSparkContext();
                 SparkContext.SetCheckpointDir(Path.GetTempPath()); 
+
                 SamplesRunner.RunSamples();
 
                 PrintLogLocation();
-                ConsoleWriteLine("Main", "Completed RunSamples. Calling SparkContext.Stop() to tear down ...");
+                ConsoleWriteLine("Completed running samples. Calling SparkContext.Stop() to tear down ...");
-                ConsoleWriteLine("Main", "If the program does not terminate in 10 seconds, please manually terminate java process !!!");
+                //following comment is necessary due to known issue in Spark. See https://issues.apache.org/jira/browse/SPARK-8333
+                ConsoleWriteLine("If this program (SparkCLRSamples.exe) does not terminate in 10 seconds, please manually terminate java process launched by this program!!!");
+                //TODO - add instructions to terminate java process
                 SparkContext.Stop();
             }
         }
@@ -60,15 +59,17 @@ namespace Microsoft.Spark.CSharp.Samples
 
         private static void PrintLogLocation()
         {
-            ConsoleWriteLine("Main",
+            ConsoleWriteLine(string.Format(@"Logs by SparkCLR and Apache Spark are available at {0}\SparkCLRLogs",
-                             string.Format(@"Logs by SparkCLR and Apache Spark are available at {0}\SparkCLRLogs",
                                            Environment.GetEnvironmentVariable("TEMP")));
         }
 
-        private static void ConsoleWriteLine(string functionName, string message)
+        private static void ConsoleWriteLine(string message)
         {
-            var p = AppDomain.CurrentDomain.FriendlyName;
+            const string exeName = "SparkCLRSamples.exe";
-            Console.WriteLine("[{0}.{1}] {2}", p, functionName, message);
+            var callingMethod = new StackTrace().GetFrames()[1].GetMethod();
+            var callingMethodName = callingMethod.Name;
+            var typeName = callingMethod.ReflectedType;
+            Console.WriteLine("[{0}.{1}.{2}] {3}", exeName, typeName, callingMethodName, message);
         }
     }
 }

csharp/Samples/Microsoft.Spark.CSharp/SampleAttribute.cs

@@ -1,9 +1,8 @@
-using System;
+// Copyright (c) Microsoft. All rights reserved.
-using System.Collections.Generic;
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
-using System.Linq;
+
-using System.Text;
+using System;
 using System.Text.RegularExpressions;
-using System.Threading.Tasks;
 
 namespace Microsoft.Spark.CSharp.Samples
 {

csharp/Samples/Microsoft.Spark.CSharp/SamplesRunner.cs

@@ -1,24 +1,26 @@
-using System;
+// Copyright (c) Microsoft. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+using System;
 using System.Collections.Generic;
 using System.Diagnostics;
 using System.Linq;
 using System.Reflection;
 using System.Text;
 using System.Text.RegularExpressions;
-using System.Threading.Tasks;
 
 namespace Microsoft.Spark.CSharp.Samples
 {
-    //finds all methods that are marked with [Sample] attribute and 
+    /// <summary>
-    //runs all of them if sparkclr.samples.torun commandline arg is not used
+    /// Runs samples
-    //or just runs the ones that are provided as comma separated list
+    /// </summary>
     internal class SamplesRunner
     {
         private static Regex samplesToRunRegex;
         private static Regex samplesCategoryRegex;
         private static Stopwatch stopWatch;
         // track <SampleName, Category, SampleSucceeded, Duration> for reporting
-        private static readonly List<Tuple<string, string, bool, TimeSpan>> samplesRunResultList = new List<Tuple<string, string, bool, TimeSpan>>();
+        private static readonly List<Tuple<string, string, bool, TimeSpan>> samplesRunInfoList = new List<Tuple<string, string, bool, TimeSpan>>();
 
         internal static void RunSamples()
         {
@@ -33,10 +35,10 @@ namespace Microsoft.Spark.CSharp.Samples
 
             foreach (var sample in samples)
             {
-                var sampleRunResult = RunSample(sample);
+                var sampleRunInfo = RunSample(sample);
-                if (sampleRunResult != null)
+                if (sampleRunInfo != null)
                 {
-                    samplesRunResultList.Add(sampleRunResult);
+                    samplesRunInfoList.Add(sampleRunInfo);
                 }
             }
             stopWatch.Stop();
@@ -135,43 +137,38 @@ namespace Microsoft.Spark.CSharp.Samples
 
         private static void ReportOutcome()
         {
+            var succeededSamples = samplesRunInfoList.Where(x => x.Item3).ToList();
+            var failedSamples = samplesRunInfoList.Where(x => !x.Item3).ToList();
+
             var msg = new StringBuilder();
 
             msg.Append("----- ")
                 .Append("Finished running ")
-                .Append(string.Format("{0} samples(s)", samplesRunResultList.Count))
+                .Append(string.Format("{0} samples(s) [succeeded={1}, failed={2}]", samplesRunInfoList.Count, succeededSamples.Count, failedSamples.Count))
                 .Append(" in ").Append(stopWatch.Elapsed)
                 .AppendLine(" -----");
 
-            var completed = samplesRunResultList.Where(x => x.Item3).ToList();
+            if (succeededSamples.Count > 0)
-            var errors = samplesRunResultList.Where(x => !x.Item3).ToList();
-
-            msg.Append("----- ")
-                .Append(" Completion counts:")
-                .Append(" Success=").Append(completed.Count)
-                .Append(" Failed=").Append(errors.Count)
-                .AppendLine(" -----");
-
-            msg.AppendLine("Successful samples:");
-            foreach (var s in completed)
             {
-                msg.Append("    ").AppendLine(string.Format("{0} (category: {1}), duration={2}", s.Item1, s.Item2, s.Item3));
+                msg.AppendLine("Successfully completed samples:");
+                foreach (var s in succeededSamples)
+                {
+                    msg.Append("    ")
+                        .AppendLine(string.Format("{0} (category: {1}), duration={2}", s.Item1, s.Item2, s.Item4));
+                }
             }
 
-            msg.AppendLine("Failed samples:");
+            if (failedSamples.Count > 0)
-            foreach (var s in errors)
             {
-                msg.Append("    ").AppendLine(string.Format("{0} (category: {1}), duration={2}", s.Item1, s.Item2, s.Item3));
+                msg.AppendLine("Failed samples:");
+                foreach (var s in failedSamples)
+                {
+                    msg.Append("    ")
+                        .AppendLine(string.Format("{0} (category: {1}), duration={2}", s.Item1, s.Item2, s.Item4));
+                }
             }
 
-            if (errors.Count == 0)
+            Console.WriteLine(msg.ToString());
-            {
-                Console.WriteLine(msg.ToString());
-            }
-            else
-            {
-                Console.WriteLine("[Warning]{0}", msg);
-            }
         }
     }
 }