Sample word count project in F#
This commit is contained in:
Isaac Abraham
Kaarthik Sivashanmugam
Родитель
1208b1d925
Коммит
07b704f8aa
|
|
@ -145,7 +145,7 @@ call Clean.cmd
|
|||
call Build.cmd
|
||||
|
||||
if %ERRORLEVEL% NEQ 0 (
|
||||
@echo Build Mobius C# examples failed, stop building.
|
||||
@echo Build Mobius .NET examples failed, stop building.
|
||||
popd
|
||||
goto :eof
|
||||
)
|
||||
|
|
|
|||
|
|
@ -86,7 +86,7 @@ pushd "$FWDIR/../examples"
|
|||
|
||||
if [ $? -ne 0 ];
|
||||
then
|
||||
echo "Build Mobius C# Examples failed, stop building."
|
||||
echo "Build Mobius .NET Examples failed, stop building."
|
||||
popd
|
||||
exit 1
|
||||
fi
|
||||
|
|
|
|||
|
|
@ -0,0 +1,45 @@
|
|||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<configuration>
|
||||
<!--*************************************************************************************-->
|
||||
<!-- Log4Net is used by the example -->
|
||||
<!--**************************************************************************************-->
|
||||
<configSections>
|
||||
<section name="log4net" type="log4net.Config.Log4NetConfigurationSectionHandler, log4net" />
|
||||
</configSections>
|
||||
<log4net>
|
||||
<root>
|
||||
<level value="DEBUG" />
|
||||
<appender-ref ref="ConsoleAppender" />
|
||||
<!--<appender-ref ref="LogFileAppender" />-->
|
||||
</root>
|
||||
<appender name="ConsoleAppender" type="log4net.Appender.ConsoleAppender">
|
||||
<layout type="log4net.Layout.PatternLayout">
|
||||
<conversionPattern value="[%date] [%thread] [%-5level] [%logger] - %message%newline" />
|
||||
</layout>
|
||||
</appender>
|
||||
<appender name="LogFileAppender" type="log4net.Appender.RollingFileAppender">
|
||||
<file type="log4net.Util.PatternString">
|
||||
<conversionPattern value="%env{TEMP}\\SparkCLRLogs\\SparkCLR-WordCount_%env{COMPUTERNAME}[%processid].log" />
|
||||
</file>
|
||||
<param name="AppendToFile" value="true" />
|
||||
<param name="MaxSizeRollBackups" value="2000" />
|
||||
<param name="MaxFileSize" value="51200000" />
|
||||
<param name="StaticLogFileName" value="false" />
|
||||
<param name="DatePattern" value=".yyyy_MM_dd_hh" />
|
||||
<param name="RollingStyle" value="Composite" />
|
||||
<layout type="log4net.Layout.PatternLayout">
|
||||
<conversionPattern value="[%date] [%thread] [%-5level] [%logger] - %message%newline" />
|
||||
</layout>
|
||||
</appender>
|
||||
</log4net>
|
||||
<appSettings>
|
||||
<!--********************************************************************************************************-->
|
||||
<!--** Uncomment the following settings to run Spark driver executable in **local** or **debug** modes ** -->
|
||||
<!--** In debug mode, the driver is not launched by CSharpRunner but launched from VS or command prompt not configured for SparkCLR ** -->
|
||||
<!--** CSharpBackend should be launched in debug mode as well and the port number from that should be used below ** -->
|
||||
<!--** Command to launch CSharpBackend in debug mode is "sparkclr-submit.cmd debug" ** -->
|
||||
<!--********************************************************************************************************-->
|
||||
<!--<add key="CSharpWorkerPath" value="path.to.CSharpWorker.exe"/>
|
||||
<add key="CSharpBackendPortNumber" value="11097"/>-->
|
||||
</appSettings>
|
||||
</configuration>
|
||||
|
|
@ -0,0 +1,41 @@
|
|||
namespace WordCountFSharp.AssemblyInfo
|
||||
|
||||
open System.Reflection
|
||||
open System.Runtime.CompilerServices
|
||||
open System.Runtime.InteropServices
|
||||
|
||||
// General Information about an assembly is controlled through the following
|
||||
// set of attributes. Change these attribute values to modify the information
|
||||
// associated with an assembly.
|
||||
[<assembly: AssemblyTitle("WordCountFSharp")>]
|
||||
[<assembly: AssemblyDescription("")>]
|
||||
[<assembly: AssemblyConfiguration("")>]
|
||||
[<assembly: AssemblyCompany("")>]
|
||||
[<assembly: AssemblyProduct("WordCountFSharp")>]
|
||||
[<assembly: AssemblyCopyright("Copyright © 2016")>]
|
||||
[<assembly: AssemblyTrademark("")>]
|
||||
[<assembly: AssemblyCulture("")>]
|
||||
|
||||
// Setting ComVisible to false makes the types in this assembly not visible
|
||||
// to COM components. If you need to access a type in this assembly from
|
||||
// COM, set the ComVisible attribute to true on that type.
|
||||
[<assembly: ComVisible(false)>]
|
||||
|
||||
// The following GUID is for the ID of the typelib if this project is exposed to COM
|
||||
[<assembly: Guid("17b63d32-efc8-4ef5-831a-197a4fc29f06")>]
|
||||
|
||||
// Version information for an assembly consists of the following four values:
|
||||
//
|
||||
// Major Version
|
||||
// Minor Version
|
||||
// Build Number
|
||||
// Revision
|
||||
//
|
||||
// You can specify all the values or you can default the Build and Revision Numbers
|
||||
// by using the '*' as shown below:
|
||||
// [<assembly: AssemblyVersion("1.0.*")>]
|
||||
[<assembly: AssemblyVersion("1.0.0.0")>]
|
||||
[<assembly: AssemblyFileVersion("1.0.0.0")>]
|
||||
|
||||
do
|
||||
()
|
||||
|
|
@ -0,0 +1,35 @@
|
|||
open Microsoft.Spark.CSharp.Core
|
||||
open Microsoft.Spark.CSharp.Services
|
||||
open System.Reflection
|
||||
open System.Collections.Generic
|
||||
|
||||
[<EntryPoint>]
|
||||
let main args =
|
||||
match args with
|
||||
| [| filePath |] ->
|
||||
let logger =
|
||||
LoggerServiceFactory.SetLoggerService Log4NetLoggerService.Instance
|
||||
LoggerServiceFactory.GetLogger (MethodInfo.GetCurrentMethod().DeclaringType)
|
||||
|
||||
let sparkContext = SparkContext(SparkConf().SetAppName "MobiusWordCount")
|
||||
logger.LogInfo (sprintf "Reading from file %s" filePath)
|
||||
|
||||
try
|
||||
let lines = sparkContext.TextFile filePath
|
||||
let counts =
|
||||
lines.FlatMap(fun x -> x.Split ' ' :> _)
|
||||
.Map(fun w -> KeyValuePair(w, 1))
|
||||
.ReduceByKey(fun x y -> x + y)
|
||||
.Collect()
|
||||
for count in counts do
|
||||
printfn "%s: %d" count.Key count.Value
|
||||
with
|
||||
| ex ->
|
||||
logger.LogError "Error performing Word Count"
|
||||
logger.LogException ex
|
||||
|
||||
sparkContext.Stop()
|
||||
0
|
||||
| _ ->
|
||||
printfn "Usage: WordCount <file>"
|
||||
1
|
||||
|
|
@ -0,0 +1,111 @@
|
|||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<Project ToolsVersion="12.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
|
||||
<Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
|
||||
<PropertyGroup>
|
||||
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
|
||||
<Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
|
||||
<SchemaVersion>2.0</SchemaVersion>
|
||||
<ProjectGuid>17b63d32-efc8-4ef5-831a-197a4fc29f06</ProjectGuid>
|
||||
<OutputType>Exe</OutputType>
|
||||
<RootNamespace>WordCountFSharp</RootNamespace>
|
||||
<AssemblyName>WordCountFSharp</AssemblyName>
|
||||
<TargetFrameworkVersion>v4.5</TargetFrameworkVersion>
|
||||
<TargetFSharpCoreVersion>4.4.0.0</TargetFSharpCoreVersion>
|
||||
<AutoGenerateBindingRedirects>true</AutoGenerateBindingRedirects>
|
||||
<Name>WordCountFSharp</Name>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
|
||||
<DebugSymbols>true</DebugSymbols>
|
||||
<DebugType>full</DebugType>
|
||||
<Optimize>false</Optimize>
|
||||
<Tailcalls>false</Tailcalls>
|
||||
<OutputPath>bin\Debug\</OutputPath>
|
||||
<DefineConstants>DEBUG;TRACE</DefineConstants>
|
||||
<WarningLevel>3</WarningLevel>
|
||||
<DocumentationFile>bin\Debug\WordCountFSharp.XML</DocumentationFile>
|
||||
<StartArguments>
|
||||
</StartArguments>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
|
||||
<DebugType>pdbonly</DebugType>
|
||||
<Optimize>true</Optimize>
|
||||
<Tailcalls>true</Tailcalls>
|
||||
<OutputPath>bin\Release\</OutputPath>
|
||||
<DefineConstants>TRACE</DefineConstants>
|
||||
<WarningLevel>3</WarningLevel>
|
||||
<DocumentationFile>bin\Release\WordCountFSharp.XML</DocumentationFile>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup>
|
||||
<MinimumVisualStudioVersion Condition="'$(MinimumVisualStudioVersion)' == ''">11</MinimumVisualStudioVersion>
|
||||
</PropertyGroup>
|
||||
<Choose>
|
||||
<When Condition="'$(VisualStudioVersion)' != '11.0' and Exists('$(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v$(VisualStudioVersion)\FSharp\Microsoft.FSharp.Targets')">
|
||||
<PropertyGroup>
|
||||
<FSharpTargetsPath>$(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v$(VisualStudioVersion)\FSharp\Microsoft.FSharp.Targets</FSharpTargetsPath>
|
||||
</PropertyGroup>
|
||||
</When>
|
||||
<When Condition="Exists('$(MSBuildExtensionsPath32)\..\Microsoft SDKs\F#\3.1\Framework\v4.0\Microsoft.FSharp.Targets')">
|
||||
<PropertyGroup>
|
||||
<FSharpTargetsPath>$(MSBuildExtensionsPath32)\..\Microsoft SDKs\F#\3.1\Framework\v4.0\Microsoft.FSharp.Targets</FSharpTargetsPath>
|
||||
</PropertyGroup>
|
||||
</When>
|
||||
<When Condition="Exists('$(MSBuildExtensionsPath32)\..\Microsoft SDKs\F#\3.0\Framework\v4.0\Microsoft.FSharp.Targets')">
|
||||
<PropertyGroup>
|
||||
<FSharpTargetsPath>$(MSBuildExtensionsPath32)\..\Microsoft SDKs\F#\3.0\Framework\v4.0\Microsoft.FSharp.Targets</FSharpTargetsPath>
|
||||
</PropertyGroup>
|
||||
</When>
|
||||
<When Condition="Exists('$(MSBuildExtensionsPath32)\..\Microsoft SDKs\F#\4.0\Framework\v4.0\Microsoft.FSharp.Targets')">
|
||||
<PropertyGroup>
|
||||
<FSharpTargetsPath>$(MSBuildExtensionsPath32)\..\Microsoft SDKs\F#\4.0\Framework\v4.0\Microsoft.FSharp.Targets</FSharpTargetsPath>
|
||||
</PropertyGroup>
|
||||
</When>
|
||||
</Choose>
|
||||
<Import Project="$(FSharpTargetsPath)" />
|
||||
<ItemGroup>
|
||||
<Compile Include="AssemblyInfo.fs" />
|
||||
<Compile Include="WordCountExample.fs" />
|
||||
<Content Include="App.config" />
|
||||
<Content Include="packages.config" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<Reference Include="CSharpWorker">
|
||||
<HintPath>..\..\packages\Microsoft.SparkCLR.1.6.100\lib\net45\CSharpWorker.exe</HintPath>
|
||||
<Private>True</Private>
|
||||
</Reference>
|
||||
<Reference Include="FSharp.Core">
|
||||
<HintPath>..\..\packages\FSharp.Core.4.0.0.1\lib\net40\FSharp.Core.dll</HintPath>
|
||||
<Private>True</Private>
|
||||
</Reference>
|
||||
<Reference Include="log4net">
|
||||
<HintPath>..\..\packages\log4net.2.0.5\lib\net45-full\log4net.dll</HintPath>
|
||||
<Private>True</Private>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.Spark.CSharp.Adapter">
|
||||
<HintPath>..\..\packages\Microsoft.SparkCLR.1.6.100\lib\net45\Microsoft.Spark.CSharp.Adapter.dll</HintPath>
|
||||
<Private>True</Private>
|
||||
</Reference>
|
||||
<Reference Include="mscorlib" />
|
||||
<Reference Include="Newtonsoft.Json">
|
||||
<HintPath>..\..\packages\Newtonsoft.Json.7.0.1\lib\net45\Newtonsoft.Json.dll</HintPath>
|
||||
<Private>True</Private>
|
||||
</Reference>
|
||||
<Reference Include="Razorvine.Pyrolite">
|
||||
<HintPath>..\..\packages\Razorvine.Pyrolite.4.10.0.0\lib\net40\Razorvine.Pyrolite.dll</HintPath>
|
||||
<Private>True</Private>
|
||||
</Reference>
|
||||
<Reference Include="Razorvine.Serpent">
|
||||
<HintPath>..\..\packages\Razorvine.Serpent.1.12.0.0\lib\net40\Razorvine.Serpent.dll</HintPath>
|
||||
<Private>True</Private>
|
||||
</Reference>
|
||||
<Reference Include="System" />
|
||||
<Reference Include="System.Core" />
|
||||
<Reference Include="System.Numerics" />
|
||||
</ItemGroup>
|
||||
<!-- To modify your build process, add your task inside one of the targets below and uncomment it.
|
||||
Other similar extension points exist, see Microsoft.Common.targets.
|
||||
<Target Name="BeforeBuild">
|
||||
</Target>
|
||||
<Target Name="AfterBuild">
|
||||
</Target>
|
||||
-->
|
||||
</Project>
|
||||
|
|
@ -0,0 +1,9 @@
|
|||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<packages>
|
||||
<package id="FSharp.Core" version="4.0.0.1" targetFramework="net45" />
|
||||
<package id="log4net" version="2.0.5" targetFramework="net45" />
|
||||
<package id="Microsoft.SparkCLR" version="1.6.100" targetFramework="net45" />
|
||||
<package id="Newtonsoft.Json" version="7.0.1" targetFramework="net45" />
|
||||
<package id="Razorvine.Pyrolite" version="4.10.0.0" targetFramework="net45" />
|
||||
<package id="Razorvine.Serpent" version="1.12.0.0" targetFramework="net45" />
|
||||
</packages>
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
|
||||
|
||||
Microsoft Visual Studio Solution File, Format Version 12.00
|
||||
# Visual Studio 2013
|
||||
VisualStudioVersion = 12.0.30501.0
|
||||
|
|
@ -21,6 +21,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Pi", "Batch\pi\Pi.csproj",
|
|||
EndProject
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "WordCount", "Batch\WordCount\WordCount.csproj", "{17E4C27F-8441-425A-B82B-23BA5E313CC4}"
|
||||
EndProject
|
||||
Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "WordCountFSharp", "Batch\WordCountFSharp\WordCountFSharp.fsproj", "{17B63D32-EFC8-4EF5-831A-197A4FC29F06}"
|
||||
EndProject
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Kafka", "Streaming\Kafka\Kafka.csproj", "{8764EAAA-9D32-4549-A64F-C7C89B014EA6}"
|
||||
EndProject
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "HiveDataFrame", "Sql\HiveDataFrame\HiveDataFrame.csproj", "{5C97498A-C4DB-43DD-86AD-4E50DEE8D405}"
|
||||
|
|
@ -55,6 +57,10 @@ Global
|
|||
{17E4C27F-8441-425A-B82B-23BA5E313CC4}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{17E4C27F-8441-425A-B82B-23BA5E313CC4}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{17E4C27F-8441-425A-B82B-23BA5E313CC4}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
{17B63D32-EFC8-4EF5-831A-197A4FC29F06}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{17B63D32-EFC8-4EF5-831A-197A4FC29F06}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{17B63D32-EFC8-4EF5-831A-197A4FC29F06}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{17B63D32-EFC8-4EF5-831A-197A4FC29F06}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
{8764EAAA-9D32-4549-A64F-C7C89B014EA6}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{8764EAAA-9D32-4549-A64F-C7C89B014EA6}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{8764EAAA-9D32-4549-A64F-C7C89B014EA6}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
|
|
@ -74,6 +80,7 @@ Global
|
|||
{DF840BFB-B3A3-493D-B667-4CF21ADBFCAD} = {6F90310A-2DA2-4E81-A062-8D8A9F47C25B}
|
||||
{913E6A56-9839-4379-8B3C-855BA9341663} = {AE001E84-471E-4D02-BDDE-40B85915CEAE}
|
||||
{17E4C27F-8441-425A-B82B-23BA5E313CC4} = {AE001E84-471E-4D02-BDDE-40B85915CEAE}
|
||||
{17B63D32-EFC8-4EF5-831A-197A4FC29F06} = {AE001E84-471E-4D02-BDDE-40B85915CEAE}
|
||||
{8764EAAA-9D32-4549-A64F-C7C89B014EA6} = {6F90310A-2DA2-4E81-A062-8D8A9F47C25B}
|
||||
{5C97498A-C4DB-43DD-86AD-4E50DEE8D405} = {28600A86-E011-41C9-AB41-591580EDB9F1}
|
||||
EndGlobalSection
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче