microsoft
/
Mobius
зеркало из
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

simplifying result reporting and other minor changes

This commit is contained in:
skaarthik 2015-12-03 09:32:44 -08:00
Родитель 9056eea715
Коммит 421821b491
5 изменённых файлов: 64 добавлений и 68 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

35
csharp/Samples/Microsoft.Spark.CSharp/CommandlineArgumentProcessor.cs
Просмотреть файл

@ -2,14 +2,13 @@
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Microsoft.Spark.CSharp.Services;
namespace Microsoft.Spark.CSharp.Samples
{
/// <summary>
/// Simple commandline argument parser
/// </summary>
internal class CommandlineArgumentProcessor
{
private static readonly ILoggerService logger = LoggerServiceFactory.GetLogger(typeof(CommandlineArgumentProcessor));
@ -22,7 +21,7 @@ namespace Microsoft.Spark.CSharp.Samples
}
var configuration = new Configuration();
logger.LogInfo(string.Format("Arguments to SparkCLRSamples are {0}", string.Join(",", args)));
logger.LogInfo(string.Format("Arguments to SparkCLRSamples.exe are {0}", string.Join(",", args)));
for (int i = 0; i < args.Length; i++)
{
if (args[i].Equals("--help", StringComparison.InvariantCultureIgnoreCase)
@ -69,24 +68,24 @@ namespace Microsoft.Spark.CSharp.Samples
private static void PrintUsage()
{
var p = AppDomain.CurrentDomain.FriendlyName;
const string exeName = "SparkCLRSamples.exe";
Console.WriteLine(" ");
Console.WriteLine(" {0} supports following options:", p);
Console.WriteLine(" {0} supports following options:", exeName);
Console.WriteLine(" ");
Console.WriteLine(" [--temp | spark.local.dir] <TEMP_DIR> TEMP_DIR is the directory used as \"scratch\" space in Spark, including map output files and RDDs that get stored on disk. ");
Console.WriteLine(" See http://spark.apache.org/docs/latest/configuration.html for details.");
Console.WriteLine(" ");
Console.WriteLine(" [--data | sparkclr.sampledata.loc] <SAMPLE_DATA_DIR> SAMPLE_DATA_DIR is the directory where Sample data resides. ");
Console.WriteLine(" [--data | sparkclr.sampledata.loc] <SAMPLE_DATA_DIR> SAMPLE_DATA_DIR is the directory where data files used by samples reside. ");
Console.WriteLine(" ");
Console.WriteLine(" [--torun | sparkclr.samples.torun] <SAMPLE_LIST> SAMPLE_LIST specifies a list of samples to run. ");
Console.WriteLine(" Case-insensitive command line wild card matching by default. Or, use \"/\" (forward slash) to enclose regular expression. ");
Console.WriteLine(" ");
Console.WriteLine(" ");
Console.WriteLine(" [--cat | sparkclr.samples.category] <SAMPLE_CATEGORY> SAMPLE_CATEGORY can be \"all\", \"default\", \"experimental\" or any new categories. ");
Console.WriteLine(" Case-insensitive command line wild card matching by default. Or, use \"/\" (forward slash) to enclose regular expression. ");
Console.WriteLine(" ");
Console.WriteLine(" [--validate | sparkclr.enablevalidation] Enable validation. ");
Console.WriteLine(" [--validate | sparkclr.enablevalidation] Enables validation of results produced in each sample. ");
Console.WriteLine(" ");
Console.WriteLine(" [--dryrun | sparkclr.dryrun] Dry-run mode. ");
Console.WriteLine(" [--dryrun | sparkclr.dryrun] Dry-run mode. Just lists the samples that will be executed with given parameters without running them");
Console.WriteLine(" ");
Console.WriteLine(" [--help | -h | -?] Display usage. ");
Console.WriteLine(" ");
@ -95,31 +94,31 @@ namespace Microsoft.Spark.CSharp.Samples
Console.WriteLine(" ");
Console.WriteLine(" Example 1 - run default samples:");
Console.WriteLine(" ");
Console.WriteLine(@" {0} --temp C:\gitsrc\SparkCLR\run\Temp --data C:\gitsrc\SparkCLR\run\data ", p);
Console.WriteLine(@" {0} --temp C:\gitsrc\SparkCLR\run\Temp --data C:\gitsrc\SparkCLR\run\data ", exeName);
Console.WriteLine(" ");
Console.WriteLine(" Example 2 - dryrun default samples:");
Console.WriteLine(" ");
Console.WriteLine(@" {0} --dryrun ", p);
Console.WriteLine(@" {0} --dryrun ", exeName);
Console.WriteLine(" ");
Console.WriteLine(" Example 3 - dryrun all samples:");
Console.WriteLine(" ");
Console.WriteLine(@" {0} --dryrun --cat all ", p);
Console.WriteLine(@" {0} --dryrun --cat all ", exeName);
Console.WriteLine(" ");
Console.WriteLine(" Example 4 - dryrun PiSample (commandline wildcard matching, case-insensitive):");
Console.WriteLine(" ");
Console.WriteLine(@" {0} --dryrun --torun pi*", p);
Console.WriteLine(@" {0} --dryrun --torun pi*", exeName);
Console.WriteLine(" ");
Console.WriteLine(" Example 5 - dryrun all DF* samples (commandline wildcard matching, case-insensitive):");
Console.WriteLine(" ");
Console.WriteLine(@" {0} --dryrun --cat a* --torun DF*", p);
Console.WriteLine(@" {0} --dryrun --cat a* --torun DF*", exeName);
Console.WriteLine(" ");
Console.WriteLine(" Example 6 - dryrun all RD* samples (regular expression):");
Console.WriteLine(" ");
Console.WriteLine(@" {0} --dryrun --cat a* --torun /\bRD.*Sample.*\b/", p);
Console.WriteLine(@" {0} --dryrun --cat a* --torun /\bRD.*Sample.*\b/", exeName);
Console.WriteLine(" ");
Console.WriteLine(" Example 7 - dryrun specific samples (case insensitive): ");
Console.WriteLine(" ");
Console.WriteLine(" {0} --dryrun --torun \"DFShowSchemaSample,DFHeadSample\"", p);
Console.WriteLine(" {0} --dryrun --torun \"DFShowSchemaSample,DFHeadSample\"", exeName);
Console.WriteLine(" ");
}
}

2
csharp/Samples/Microsoft.Spark.CSharp/MiscSamples.cs
Просмотреть файл

@ -51,7 +51,7 @@ namespace Microsoft.Spark.CSharp.Samples
if (SparkCLRSamples.Configuration.IsValidationEnabled)
{
Assert.IsTrue(Math.Abs(approximatePiValue - 3.14) <= 0.01);
Assert.IsTrue(Math.Abs(approximatePiValue - 3.14) <= 0.019);
}
}

23
csharp/Samples/Microsoft.Spark.CSharp/Program.cs
Просмотреть файл

@ -2,13 +2,9 @@
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Reflection;
using System.Text;
using System.Text.RegularExpressions;
using Microsoft.Spark.CSharp.Core;
using Microsoft.Spark.CSharp.Services;
@ -38,11 +34,14 @@ namespace Microsoft.Spark.CSharp.Samples
{
SparkContext = CreateSparkContext();
SparkContext.SetCheckpointDir(Path.GetTempPath());
SamplesRunner.RunSamples();
PrintLogLocation();
ConsoleWriteLine("Main", "Completed RunSamples. Calling SparkContext.Stop() to tear down ...");
ConsoleWriteLine("Main", "If the program does not terminate in 10 seconds, please manually terminate java process !!!");
ConsoleWriteLine("Completed running samples. Calling SparkContext.Stop() to tear down ...");
//following comment is necessary due to known issue in Spark. See https://issues.apache.org/jira/browse/SPARK-8333
ConsoleWriteLine("If this program (SparkCLRSamples.exe) does not terminate in 10 seconds, please manually terminate java process launched by this program!!!");
//TODO - add instructions to terminate java process
SparkContext.Stop();
}
}
@ -60,15 +59,17 @@ namespace Microsoft.Spark.CSharp.Samples
private static void PrintLogLocation()
{
ConsoleWriteLine("Main",
string.Format(@"Logs by SparkCLR and Apache Spark are available at {0}\SparkCLRLogs",
ConsoleWriteLine(string.Format(@"Logs by SparkCLR and Apache Spark are available at {0}\SparkCLRLogs",
Environment.GetEnvironmentVariable("TEMP")));
}
private static void ConsoleWriteLine(string functionName, string message)
private static void ConsoleWriteLine(string message)
{
var p = AppDomain.CurrentDomain.FriendlyName;
Console.WriteLine("[{0}.{1}] {2}", p, functionName, message);
const string exeName = "SparkCLRSamples.exe";
var callingMethod = new StackTrace().GetFrames()[1].GetMethod();
var callingMethodName = callingMethod.Name;
var typeName = callingMethod.ReflectedType;
Console.WriteLine("[{0}.{1}.{2}] {3}", exeName, typeName, callingMethodName, message);
}
}
}

9
csharp/Samples/Microsoft.Spark.CSharp/SampleAttribute.cs
Просмотреть файл

@ -1,9 +1,8 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
using System;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
namespace Microsoft.Spark.CSharp.Samples
{

63
csharp/Samples/Microsoft.Spark.CSharp/SamplesRunner.cs
Просмотреть файл

@ -1,24 +1,26 @@
using System;
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Reflection;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
namespace Microsoft.Spark.CSharp.Samples
{
//finds all methods that are marked with [Sample] attribute and
//runs all of them if sparkclr.samples.torun commandline arg is not used
//or just runs the ones that are provided as comma separated list
/// <summary>
/// Runs samples
/// </summary>
internal class SamplesRunner
{
private static Regex samplesToRunRegex;
private static Regex samplesCategoryRegex;
private static Stopwatch stopWatch;
// track <SampleName, Category, SampleSucceeded, Duration> for reporting
private static readonly List<Tuple<string, string, bool, TimeSpan>> samplesRunResultList = new List<Tuple<string, string, bool, TimeSpan>>();
private static readonly List<Tuple<string, string, bool, TimeSpan>> samplesRunInfoList = new List<Tuple<string, string, bool, TimeSpan>>();
internal static void RunSamples()
{
@ -33,10 +35,10 @@ namespace Microsoft.Spark.CSharp.Samples
foreach (var sample in samples)
{
var sampleRunResult = RunSample(sample);
if (sampleRunResult != null)
var sampleRunInfo = RunSample(sample);
if (sampleRunInfo != null)
{
samplesRunResultList.Add(sampleRunResult);
samplesRunInfoList.Add(sampleRunInfo);
}
}
stopWatch.Stop();
@ -135,43 +137,38 @@ namespace Microsoft.Spark.CSharp.Samples
private static void ReportOutcome()
{
var succeededSamples = samplesRunInfoList.Where(x => x.Item3).ToList();
var failedSamples = samplesRunInfoList.Where(x => !x.Item3).ToList();
var msg = new StringBuilder();
msg.Append("----- ")
.Append("Finished running ")
.Append(string.Format("{0} samples(s)", samplesRunResultList.Count))
.Append(string.Format("{0} samples(s) [succeeded={1}, failed={2}]", samplesRunInfoList.Count, succeededSamples.Count, failedSamples.Count))
.Append(" in ").Append(stopWatch.Elapsed)
.AppendLine(" -----");
var completed = samplesRunResultList.Where(x => x.Item3).ToList();
var errors = samplesRunResultList.Where(x => !x.Item3).ToList();
msg.Append("----- ")
.Append(" Completion counts:")
.Append(" Success=").Append(completed.Count)
.Append(" Failed=").Append(errors.Count)
.AppendLine(" -----");
msg.AppendLine("Successful samples:");
foreach (var s in completed)
if (succeededSamples.Count > 0)
{
msg.Append(" ").AppendLine(string.Format("{0} (category: {1}), duration={2}", s.Item1, s.Item2, s.Item3));
msg.AppendLine("Successfully completed samples:");
foreach (var s in succeededSamples)
{
msg.Append(" ")
.AppendLine(string.Format("{0} (category: {1}), duration={2}", s.Item1, s.Item2, s.Item4));
}
}
msg.AppendLine("Failed samples:");
foreach (var s in errors)
if (failedSamples.Count > 0)
{
msg.Append(" ").AppendLine(string.Format("{0} (category: {1}), duration={2}", s.Item1, s.Item2, s.Item3));
msg.AppendLine("Failed samples:");
foreach (var s in failedSamples)
{
msg.Append(" ")
.AppendLine(string.Format("{0} (category: {1}), duration={2}", s.Item1, s.Item2, s.Item4));
}
}
if (errors.Count == 0)
{
Console.WriteLine(msg.ToString());
}
else
{
Console.WriteLine("[Warning]{0}", msg);
}
Console.WriteLine(msg.ToString());
}
}
}