# This is a combination of 3 commits.
# The first commit's message is: convert PiSample to PiExample consuming SparkCLR nuget package # The 2nd commit message will be skipped: # incorproate feedback # The 3rd commit message will be skipped: # replace sample with example
This commit is contained in:
danielli90
Родитель
4a0ff8bf37
Коммит
1d48924f59
|
|
@ -0,0 +1,20 @@
|
|||
|
||||
Microsoft Visual Studio Solution File, Format Version 12.00
|
||||
# Visual Studio 2013
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Pi", "Pi\Pi.csproj", "{913E6A56-9839-4379-8B3C-855BA9341663}"
|
||||
EndProject
|
||||
Global
|
||||
GlobalSection(SolutionConfigurationPlatforms) = preSolution
|
||||
Debug|Any CPU = Debug|Any CPU
|
||||
Release|Any CPU = Release|Any CPU
|
||||
EndGlobalSection
|
||||
GlobalSection(ProjectConfigurationPlatforms) = postSolution
|
||||
{913E6A56-9839-4379-8B3C-855BA9341663}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{913E6A56-9839-4379-8B3C-855BA9341663}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{913E6A56-9839-4379-8B3C-855BA9341663}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{913E6A56-9839-4379-8B3C-855BA9341663}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
EndGlobalSection
|
||||
GlobalSection(SolutionProperties) = preSolution
|
||||
HideSolutionNode = FALSE
|
||||
EndGlobalSection
|
||||
EndGlobal
|
||||
|
|
@ -0,0 +1,39 @@
|
|||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<configuration>
|
||||
<!--*************************************************************************************-->
|
||||
<!-- Log4Net is used by the example -->
|
||||
<!--**************************************************************************************-->
|
||||
<configSections>
|
||||
<section name="log4net" type="log4net.Config.Log4NetConfigurationSectionHandler, log4net" />
|
||||
</configSections>
|
||||
|
||||
<log4net>
|
||||
<root>
|
||||
<level value="DEBUG" />
|
||||
<appender-ref ref="ConsoleAppender" />
|
||||
<!--<appender-ref ref="LogFileAppender" />-->
|
||||
</root>
|
||||
|
||||
<appender name="ConsoleAppender" type="log4net.Appender.ConsoleAppender">
|
||||
<layout type="log4net.Layout.PatternLayout">
|
||||
<conversionPattern value="[%date] [%thread] [%-5level] [%logger] - %message%newline" />
|
||||
</layout>
|
||||
</appender>
|
||||
|
||||
<appender name="LogFileAppender" type="log4net.Appender.RollingFileAppender">
|
||||
<file type="log4net.Util.PatternString">
|
||||
<conversionPattern value="%env{TEMP}\\SparkCLRLogs\\SparkCLR-Pi_%env{COMPUTERNAME}[%processid].log" />
|
||||
</file>
|
||||
<param name="AppendToFile" value="true" />
|
||||
<param name="MaxSizeRollBackups" value="2000" />
|
||||
<param name="MaxFileSize" value="51200000" />
|
||||
<param name="StaticLogFileName" value="false" />
|
||||
<param name="DatePattern" value=".yyyy_MM_dd_hh" />
|
||||
<param name="RollingStyle" value="Composite" />
|
||||
<layout type="log4net.Layout.PatternLayout">
|
||||
<conversionPattern value="[%date] [%thread] [%-5level] [%logger] - %message%newline" />
|
||||
</layout>
|
||||
</appender>
|
||||
|
||||
</log4net>
|
||||
</configuration>
|
||||
|
|
@ -0,0 +1,93 @@
|
|||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<Project ToolsVersion="12.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
|
||||
<Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
|
||||
<PropertyGroup>
|
||||
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
|
||||
<Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
|
||||
<ProjectGuid>{913E6A56-9839-4379-8B3C-855BA9341663}</ProjectGuid>
|
||||
<OutputType>Exe</OutputType>
|
||||
<AppDesignerFolder>Properties</AppDesignerFolder>
|
||||
<RootNamespace>Microsoft.Spark.CSharp.Examples</RootNamespace>
|
||||
<AssemblyName>Pi</AssemblyName>
|
||||
<TargetFrameworkVersion>v4.5</TargetFrameworkVersion>
|
||||
<FileAlignment>512</FileAlignment>
|
||||
<SolutionDir Condition="$(SolutionDir) == '' Or $(SolutionDir) == '*Undefined*'">.\</SolutionDir>
|
||||
<RestorePackages>true</RestorePackages>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
|
||||
<PlatformTarget>AnyCPU</PlatformTarget>
|
||||
<DebugSymbols>true</DebugSymbols>
|
||||
<DebugType>full</DebugType>
|
||||
<Optimize>false</Optimize>
|
||||
<OutputPath>bin\Debug\</OutputPath>
|
||||
<DefineConstants>DEBUG;TRACE</DefineConstants>
|
||||
<ErrorReport>prompt</ErrorReport>
|
||||
<WarningLevel>4</WarningLevel>
|
||||
<Prefer32Bit>false</Prefer32Bit>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
|
||||
<PlatformTarget>AnyCPU</PlatformTarget>
|
||||
<DebugType>pdbonly</DebugType>
|
||||
<Optimize>true</Optimize>
|
||||
<OutputPath>bin\Release\</OutputPath>
|
||||
<DefineConstants>TRACE</DefineConstants>
|
||||
<ErrorReport>prompt</ErrorReport>
|
||||
<WarningLevel>4</WarningLevel>
|
||||
</PropertyGroup>
|
||||
<ItemGroup>
|
||||
<Reference Include="Newtonsoft.Json">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\packages\Newtonsoft.Json.7.0.1\lib\net45\Newtonsoft.Json.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="CSharpWorker">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\packages\Microsoft.SparkCLR.1.5.2-SNAPSHOT\lib\net45\CSharpWorker.exe</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="log4net">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\packages\log4net.2.0.5\lib\net45-full\log4net.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.Spark.CSharp.Adapter">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\packages\Microsoft.SparkCLR.1.5.2-SNAPSHOT\lib\net45\Microsoft.Spark.CSharp.Adapter.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Razorvine.Pyrolite">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\packages\Razorvine.Pyrolite.4.10.0.0\lib\net40\Razorvine.Pyrolite.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Razorvine.Serpent">
|
||||
<HintPath>..\packages\Razorvine.Serpent.1.12.0.0\lib\net40\Razorvine.Serpent.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="System" />
|
||||
<Reference Include="System.Configuration" />
|
||||
<Reference Include="System.Core" />
|
||||
<Reference Include="System.Xml.Linq" />
|
||||
<Reference Include="System.Data.DataSetExtensions" />
|
||||
<Reference Include="Microsoft.CSharp" />
|
||||
<Reference Include="System.Data" />
|
||||
<Reference Include="System.Xml" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<Compile Include="Program.cs" />
|
||||
<Compile Include="Properties\AssemblyInfo.cs" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<None Include="App.config">
|
||||
<SubType>Designer</SubType>
|
||||
</None>
|
||||
<None Include="packages.config" />
|
||||
<None Include="..\packages\Microsoft.SparkCLR.1.5.2-SNAPSHOT\lib\net45\CSharpWorker.exe.config">
|
||||
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
|
||||
</None>
|
||||
</ItemGroup>
|
||||
<ItemGroup />
|
||||
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
|
||||
<Import Project="$(SolutionDir)\.nuget\NuGet.targets" Condition="Exists('$(SolutionDir)\.nuget\NuGet.targets')" />
|
||||
<!-- To modify your build process, add your task inside one of the targets below and uncomment it.
|
||||
Other similar extension points exist, see Microsoft.Common.targets.
|
||||
<Target Name="BeforeBuild">
|
||||
</Target>
|
||||
<Target Name="AfterBuild">
|
||||
</Target>
|
||||
-->
|
||||
</Project>
|
||||
|
|
@ -0,0 +1,134 @@
|
|||
// Copyright (c) Microsoft. All rights reserved.
|
||||
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics;
|
||||
using System.IO;
|
||||
using System.Reflection;
|
||||
using Microsoft.Spark.CSharp.Core;
|
||||
|
||||
[assembly: log4net.Config.XmlConfigurator(Watch = true)]
|
||||
|
||||
namespace Microsoft.Spark.CSharp.Examples
|
||||
{
|
||||
/// <summary>
|
||||
/// SparkCLR Pi example
|
||||
/// </summary>
|
||||
public static class PiExample
|
||||
{
|
||||
internal static log4net.ILog Logger { get { return log4net.LogManager.GetLogger(typeof(PiExample)); } }
|
||||
internal static SparkContext SparkContext;
|
||||
|
||||
private static void Main(string[] args)
|
||||
{
|
||||
var success = true;
|
||||
|
||||
SparkContext = CreateSparkContext();
|
||||
SparkContext.SetCheckpointDir(Path.GetTempPath());
|
||||
|
||||
var stopWatch = Stopwatch.StartNew();
|
||||
var clockStart = stopWatch.Elapsed;
|
||||
try
|
||||
{
|
||||
Logger.Info("----- Running Pi example -----");
|
||||
|
||||
Pi();
|
||||
|
||||
var duration = stopWatch.Elapsed - clockStart;
|
||||
Logger.InfoFormat("----- Successfully finished running Pi example (duration={0}) -----", duration);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
success = false;
|
||||
var duration = stopWatch.Elapsed - clockStart;
|
||||
Logger.InfoFormat("----- Error running Pi example (duration={0}) -----{1}{2}", duration, Environment.NewLine, ex);
|
||||
}
|
||||
|
||||
Logger.Info("Completed running examples. Calling SparkContext.Stop() to tear down ...");
|
||||
// following comment is necessary due to known issue in Spark. See https://issues.apache.org/jira/browse/SPARK-8333
|
||||
Logger.Info("If this program (SparkCLRExamples.exe) does not terminate in 10 seconds, please manually terminate java process launched by this program!!!");
|
||||
|
||||
SparkContext.Stop();
|
||||
|
||||
if (!success)
|
||||
{
|
||||
Environment.Exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Calculate Pi
|
||||
/// Reference: https://github.com/apache/spark/blob/branch-1.5/examples/src/main/scala/org/apache/spark/examples/SparkPi.scala
|
||||
/// </summary>
|
||||
private static void Pi()
|
||||
{
|
||||
const int slices = 3;
|
||||
var n = (int)Math.Min(100000L * slices, int.MaxValue);
|
||||
var values = new List<int>(n);
|
||||
for (var i = 0; i <= n; i++)
|
||||
{
|
||||
values.Add(i);
|
||||
}
|
||||
|
||||
//
|
||||
// Anonymous method approach
|
||||
//
|
||||
var count = SparkContext.Parallelize(values, slices)
|
||||
.Map(i =>
|
||||
{
|
||||
var random = new Random();
|
||||
var x = random.NextDouble() * 2 - 1;
|
||||
var y = random.NextDouble() * 2 - 1;
|
||||
|
||||
return (x * x + y * y) < 1 ? 1 : 0;
|
||||
}
|
||||
).Reduce((x, y) => x + y);
|
||||
Logger.InfoFormat("(anonymous method approach) Pi is roughly {0}.", 4.0 * (int)count / n);
|
||||
|
||||
//
|
||||
// Serialized class approach, an alternative to the anonymous method approach above
|
||||
//
|
||||
var countComputedUsingAnotherApproach = SparkContext.Parallelize(values, slices).Map(new PiHelper().Execute).Reduce((x, y) => x + y);
|
||||
var approximatePiValue = 4.0 * countComputedUsingAnotherApproach / n;
|
||||
Logger.InfoFormat("(serialized class approach) Pi is roughly {0}.", approximatePiValue);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Serialized class used in RDD Map Transformation
|
||||
/// </summary>
|
||||
[Serializable]
|
||||
private class PiHelper
|
||||
{
|
||||
private readonly Random random = new Random();
|
||||
public int Execute(int input)
|
||||
{
|
||||
var x = random.NextDouble() * 2 - 1;
|
||||
var y = random.NextDouble() * 2 - 1;
|
||||
|
||||
return (x * x + y * y) < 1 ? 1 : 0;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates and returns a context
|
||||
/// </summary>
|
||||
/// <returns>SparkContext</returns>
|
||||
private static SparkContext CreateSparkContext()
|
||||
{
|
||||
var conf = new SparkConf();
|
||||
|
||||
// set up local directory
|
||||
var tempDir = Environment.GetEnvironmentVariable("spark.local.dir");
|
||||
if (string.IsNullOrEmpty(tempDir))
|
||||
{
|
||||
tempDir = Path.GetTempPath();
|
||||
}
|
||||
|
||||
conf.Set("spark.local.dir", tempDir);
|
||||
Logger.DebugFormat("spark.local.dir is set to {0}", tempDir);
|
||||
|
||||
return new SparkContext(conf);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,34 @@
|
|||
// Copyright (c) Microsoft. All rights reserved.
|
||||
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
|
||||
using System;
|
||||
using System.Reflection;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
|
||||
// General Information about an assembly is controlled through the following
|
||||
// set of attributes. Change these attribute values to modify the information
|
||||
// associated with an assembly.
|
||||
[assembly: AssemblyTitle("SparkCLR-Pi")]
|
||||
[assembly: AssemblyDescription("SparkCLR Pi example.")]
|
||||
[assembly: AssemblyConfiguration("")]
|
||||
[assembly: AssemblyCompany("Microsoft Corporation")]
|
||||
[assembly: AssemblyProduct("Microsoft SparkCLR")]
|
||||
[assembly: AssemblyCopyright("Copyright © Microsoft Corporation 2015")]
|
||||
[assembly: AssemblyTrademark("")]
|
||||
[assembly: AssemblyCulture("")]
|
||||
|
||||
// Setting ComVisible to false makes the types in this assembly not visible
|
||||
// to COM components. If you need to access a type in this assembly from
|
||||
// COM, set the ComVisible attribute to true on that type.
|
||||
[assembly: ComVisible(false)]
|
||||
// [assembly: CLSCompliant(true)]
|
||||
|
||||
// Version information for an assembly consists of the following four values:
|
||||
//
|
||||
// Major Version
|
||||
// Minor Version
|
||||
// Build Number
|
||||
// Revision
|
||||
//
|
||||
[assembly: AssemblyVersion("1.5.2.0")]
|
||||
[assembly: AssemblyFileVersion("1.5.2.0")]
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<packages>
|
||||
<package id="log4net" version="2.0.5" targetFramework="net45" />
|
||||
<package id="Microsoft.SparkCLR" version="1.5.2-SNAPSHOT" targetFramework="net45" />
|
||||
<package id="Newtonsoft.Json" version="7.0.1" targetFramework="net45" />
|
||||
<package id="Razorvine.Pyrolite" version="4.10.0.0" targetFramework="net45" />
|
||||
<package id="Razorvine.Serpent" version="1.12.0.0" targetFramework="net45" />
|
||||
</packages>
|
||||
Загрузка…
Ссылка в новой задаче