Sample word count project in F#
This commit is contained in:
Родитель
1208b1d925
Коммит
07b704f8aa
|
@ -145,7 +145,7 @@ call Clean.cmd
|
||||||
call Build.cmd
|
call Build.cmd
|
||||||
|
|
||||||
if %ERRORLEVEL% NEQ 0 (
|
if %ERRORLEVEL% NEQ 0 (
|
||||||
@echo Build Mobius C# examples failed, stop building.
|
@echo Build Mobius .NET examples failed, stop building.
|
||||||
popd
|
popd
|
||||||
goto :eof
|
goto :eof
|
||||||
)
|
)
|
||||||
|
|
|
@ -86,7 +86,7 @@ pushd "$FWDIR/../examples"
|
||||||
|
|
||||||
if [ $? -ne 0 ];
|
if [ $? -ne 0 ];
|
||||||
then
|
then
|
||||||
echo "Build Mobius C# Examples failed, stop building."
|
echo "Build Mobius .NET Examples failed, stop building."
|
||||||
popd
|
popd
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
|
|
@ -0,0 +1,45 @@
|
||||||
|
<?xml version="1.0" encoding="utf-8"?>
|
||||||
|
<configuration>
|
||||||
|
<!--*************************************************************************************-->
|
||||||
|
<!-- Log4Net is used by the example -->
|
||||||
|
<!--**************************************************************************************-->
|
||||||
|
<configSections>
|
||||||
|
<section name="log4net" type="log4net.Config.Log4NetConfigurationSectionHandler, log4net" />
|
||||||
|
</configSections>
|
||||||
|
<log4net>
|
||||||
|
<root>
|
||||||
|
<level value="DEBUG" />
|
||||||
|
<appender-ref ref="ConsoleAppender" />
|
||||||
|
<!--<appender-ref ref="LogFileAppender" />-->
|
||||||
|
</root>
|
||||||
|
<appender name="ConsoleAppender" type="log4net.Appender.ConsoleAppender">
|
||||||
|
<layout type="log4net.Layout.PatternLayout">
|
||||||
|
<conversionPattern value="[%date] [%thread] [%-5level] [%logger] - %message%newline" />
|
||||||
|
</layout>
|
||||||
|
</appender>
|
||||||
|
<appender name="LogFileAppender" type="log4net.Appender.RollingFileAppender">
|
||||||
|
<file type="log4net.Util.PatternString">
|
||||||
|
<conversionPattern value="%env{TEMP}\\SparkCLRLogs\\SparkCLR-WordCount_%env{COMPUTERNAME}[%processid].log" />
|
||||||
|
</file>
|
||||||
|
<param name="AppendToFile" value="true" />
|
||||||
|
<param name="MaxSizeRollBackups" value="2000" />
|
||||||
|
<param name="MaxFileSize" value="51200000" />
|
||||||
|
<param name="StaticLogFileName" value="false" />
|
||||||
|
<param name="DatePattern" value=".yyyy_MM_dd_hh" />
|
||||||
|
<param name="RollingStyle" value="Composite" />
|
||||||
|
<layout type="log4net.Layout.PatternLayout">
|
||||||
|
<conversionPattern value="[%date] [%thread] [%-5level] [%logger] - %message%newline" />
|
||||||
|
</layout>
|
||||||
|
</appender>
|
||||||
|
</log4net>
|
||||||
|
<appSettings>
|
||||||
|
<!--********************************************************************************************************-->
|
||||||
|
<!--** Uncomment the following settings to run Spark driver executable in **local** or **debug** modes ** -->
|
||||||
|
<!--** In debug mode, the driver is not launched by CSharpRunner but launched from VS or command prompt not configured for SparkCLR ** -->
|
||||||
|
<!--** CSharpBackend should be launched in debug mode as well and the port number from that should be used below ** -->
|
||||||
|
<!--** Command to launch CSharpBackend in debug mode is "sparkclr-submit.cmd debug" ** -->
|
||||||
|
<!--********************************************************************************************************-->
|
||||||
|
<!--<add key="CSharpWorkerPath" value="path.to.CSharpWorker.exe"/>
|
||||||
|
<add key="CSharpBackendPortNumber" value="11097"/>-->
|
||||||
|
</appSettings>
|
||||||
|
</configuration>
|
|
@ -0,0 +1,41 @@
|
||||||
|
namespace WordCountFSharp.AssemblyInfo
|
||||||
|
|
||||||
|
open System.Reflection
|
||||||
|
open System.Runtime.CompilerServices
|
||||||
|
open System.Runtime.InteropServices
|
||||||
|
|
||||||
|
// General Information about an assembly is controlled through the following
|
||||||
|
// set of attributes. Change these attribute values to modify the information
|
||||||
|
// associated with an assembly.
|
||||||
|
[<assembly: AssemblyTitle("WordCountFSharp")>]
|
||||||
|
[<assembly: AssemblyDescription("")>]
|
||||||
|
[<assembly: AssemblyConfiguration("")>]
|
||||||
|
[<assembly: AssemblyCompany("")>]
|
||||||
|
[<assembly: AssemblyProduct("WordCountFSharp")>]
|
||||||
|
[<assembly: AssemblyCopyright("Copyright © 2016")>]
|
||||||
|
[<assembly: AssemblyTrademark("")>]
|
||||||
|
[<assembly: AssemblyCulture("")>]
|
||||||
|
|
||||||
|
// Setting ComVisible to false makes the types in this assembly not visible
|
||||||
|
// to COM components. If you need to access a type in this assembly from
|
||||||
|
// COM, set the ComVisible attribute to true on that type.
|
||||||
|
[<assembly: ComVisible(false)>]
|
||||||
|
|
||||||
|
// The following GUID is for the ID of the typelib if this project is exposed to COM
|
||||||
|
[<assembly: Guid("17b63d32-efc8-4ef5-831a-197a4fc29f06")>]
|
||||||
|
|
||||||
|
// Version information for an assembly consists of the following four values:
|
||||||
|
//
|
||||||
|
// Major Version
|
||||||
|
// Minor Version
|
||||||
|
// Build Number
|
||||||
|
// Revision
|
||||||
|
//
|
||||||
|
// You can specify all the values or you can default the Build and Revision Numbers
|
||||||
|
// by using the '*' as shown below:
|
||||||
|
// [<assembly: AssemblyVersion("1.0.*")>]
|
||||||
|
[<assembly: AssemblyVersion("1.0.0.0")>]
|
||||||
|
[<assembly: AssemblyFileVersion("1.0.0.0")>]
|
||||||
|
|
||||||
|
do
|
||||||
|
()
|
|
@ -0,0 +1,35 @@
|
||||||
|
open Microsoft.Spark.CSharp.Core
|
||||||
|
open Microsoft.Spark.CSharp.Services
|
||||||
|
open System.Reflection
|
||||||
|
open System.Collections.Generic
|
||||||
|
|
||||||
|
[<EntryPoint>]
|
||||||
|
let main args =
|
||||||
|
match args with
|
||||||
|
| [| filePath |] ->
|
||||||
|
let logger =
|
||||||
|
LoggerServiceFactory.SetLoggerService Log4NetLoggerService.Instance
|
||||||
|
LoggerServiceFactory.GetLogger (MethodInfo.GetCurrentMethod().DeclaringType)
|
||||||
|
|
||||||
|
let sparkContext = SparkContext(SparkConf().SetAppName "MobiusWordCount")
|
||||||
|
logger.LogInfo (sprintf "Reading from file %s" filePath)
|
||||||
|
|
||||||
|
try
|
||||||
|
let lines = sparkContext.TextFile filePath
|
||||||
|
let counts =
|
||||||
|
lines.FlatMap(fun x -> x.Split ' ' :> _)
|
||||||
|
.Map(fun w -> KeyValuePair(w, 1))
|
||||||
|
.ReduceByKey(fun x y -> x + y)
|
||||||
|
.Collect()
|
||||||
|
for count in counts do
|
||||||
|
printfn "%s: %d" count.Key count.Value
|
||||||
|
with
|
||||||
|
| ex ->
|
||||||
|
logger.LogError "Error performing Word Count"
|
||||||
|
logger.LogException ex
|
||||||
|
|
||||||
|
sparkContext.Stop()
|
||||||
|
0
|
||||||
|
| _ ->
|
||||||
|
printfn "Usage: WordCount <file>"
|
||||||
|
1
|
|
@ -0,0 +1,111 @@
|
||||||
|
<?xml version="1.0" encoding="utf-8"?>
|
||||||
|
<Project ToolsVersion="12.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
|
||||||
|
<Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
|
||||||
|
<PropertyGroup>
|
||||||
|
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
|
||||||
|
<Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
|
||||||
|
<SchemaVersion>2.0</SchemaVersion>
|
||||||
|
<ProjectGuid>17b63d32-efc8-4ef5-831a-197a4fc29f06</ProjectGuid>
|
||||||
|
<OutputType>Exe</OutputType>
|
||||||
|
<RootNamespace>WordCountFSharp</RootNamespace>
|
||||||
|
<AssemblyName>WordCountFSharp</AssemblyName>
|
||||||
|
<TargetFrameworkVersion>v4.5</TargetFrameworkVersion>
|
||||||
|
<TargetFSharpCoreVersion>4.4.0.0</TargetFSharpCoreVersion>
|
||||||
|
<AutoGenerateBindingRedirects>true</AutoGenerateBindingRedirects>
|
||||||
|
<Name>WordCountFSharp</Name>
|
||||||
|
</PropertyGroup>
|
||||||
|
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
|
||||||
|
<DebugSymbols>true</DebugSymbols>
|
||||||
|
<DebugType>full</DebugType>
|
||||||
|
<Optimize>false</Optimize>
|
||||||
|
<Tailcalls>false</Tailcalls>
|
||||||
|
<OutputPath>bin\Debug\</OutputPath>
|
||||||
|
<DefineConstants>DEBUG;TRACE</DefineConstants>
|
||||||
|
<WarningLevel>3</WarningLevel>
|
||||||
|
<DocumentationFile>bin\Debug\WordCountFSharp.XML</DocumentationFile>
|
||||||
|
<StartArguments>
|
||||||
|
</StartArguments>
|
||||||
|
</PropertyGroup>
|
||||||
|
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
|
||||||
|
<DebugType>pdbonly</DebugType>
|
||||||
|
<Optimize>true</Optimize>
|
||||||
|
<Tailcalls>true</Tailcalls>
|
||||||
|
<OutputPath>bin\Release\</OutputPath>
|
||||||
|
<DefineConstants>TRACE</DefineConstants>
|
||||||
|
<WarningLevel>3</WarningLevel>
|
||||||
|
<DocumentationFile>bin\Release\WordCountFSharp.XML</DocumentationFile>
|
||||||
|
</PropertyGroup>
|
||||||
|
<PropertyGroup>
|
||||||
|
<MinimumVisualStudioVersion Condition="'$(MinimumVisualStudioVersion)' == ''">11</MinimumVisualStudioVersion>
|
||||||
|
</PropertyGroup>
|
||||||
|
<Choose>
|
||||||
|
<When Condition="'$(VisualStudioVersion)' != '11.0' and Exists('$(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v$(VisualStudioVersion)\FSharp\Microsoft.FSharp.Targets')">
|
||||||
|
<PropertyGroup>
|
||||||
|
<FSharpTargetsPath>$(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v$(VisualStudioVersion)\FSharp\Microsoft.FSharp.Targets</FSharpTargetsPath>
|
||||||
|
</PropertyGroup>
|
||||||
|
</When>
|
||||||
|
<When Condition="Exists('$(MSBuildExtensionsPath32)\..\Microsoft SDKs\F#\3.1\Framework\v4.0\Microsoft.FSharp.Targets')">
|
||||||
|
<PropertyGroup>
|
||||||
|
<FSharpTargetsPath>$(MSBuildExtensionsPath32)\..\Microsoft SDKs\F#\3.1\Framework\v4.0\Microsoft.FSharp.Targets</FSharpTargetsPath>
|
||||||
|
</PropertyGroup>
|
||||||
|
</When>
|
||||||
|
<When Condition="Exists('$(MSBuildExtensionsPath32)\..\Microsoft SDKs\F#\3.0\Framework\v4.0\Microsoft.FSharp.Targets')">
|
||||||
|
<PropertyGroup>
|
||||||
|
<FSharpTargetsPath>$(MSBuildExtensionsPath32)\..\Microsoft SDKs\F#\3.0\Framework\v4.0\Microsoft.FSharp.Targets</FSharpTargetsPath>
|
||||||
|
</PropertyGroup>
|
||||||
|
</When>
|
||||||
|
<When Condition="Exists('$(MSBuildExtensionsPath32)\..\Microsoft SDKs\F#\4.0\Framework\v4.0\Microsoft.FSharp.Targets')">
|
||||||
|
<PropertyGroup>
|
||||||
|
<FSharpTargetsPath>$(MSBuildExtensionsPath32)\..\Microsoft SDKs\F#\4.0\Framework\v4.0\Microsoft.FSharp.Targets</FSharpTargetsPath>
|
||||||
|
</PropertyGroup>
|
||||||
|
</When>
|
||||||
|
</Choose>
|
||||||
|
<Import Project="$(FSharpTargetsPath)" />
|
||||||
|
<ItemGroup>
|
||||||
|
<Compile Include="AssemblyInfo.fs" />
|
||||||
|
<Compile Include="WordCountExample.fs" />
|
||||||
|
<Content Include="App.config" />
|
||||||
|
<Content Include="packages.config" />
|
||||||
|
</ItemGroup>
|
||||||
|
<ItemGroup>
|
||||||
|
<Reference Include="CSharpWorker">
|
||||||
|
<HintPath>..\..\packages\Microsoft.SparkCLR.1.6.100\lib\net45\CSharpWorker.exe</HintPath>
|
||||||
|
<Private>True</Private>
|
||||||
|
</Reference>
|
||||||
|
<Reference Include="FSharp.Core">
|
||||||
|
<HintPath>..\..\packages\FSharp.Core.4.0.0.1\lib\net40\FSharp.Core.dll</HintPath>
|
||||||
|
<Private>True</Private>
|
||||||
|
</Reference>
|
||||||
|
<Reference Include="log4net">
|
||||||
|
<HintPath>..\..\packages\log4net.2.0.5\lib\net45-full\log4net.dll</HintPath>
|
||||||
|
<Private>True</Private>
|
||||||
|
</Reference>
|
||||||
|
<Reference Include="Microsoft.Spark.CSharp.Adapter">
|
||||||
|
<HintPath>..\..\packages\Microsoft.SparkCLR.1.6.100\lib\net45\Microsoft.Spark.CSharp.Adapter.dll</HintPath>
|
||||||
|
<Private>True</Private>
|
||||||
|
</Reference>
|
||||||
|
<Reference Include="mscorlib" />
|
||||||
|
<Reference Include="Newtonsoft.Json">
|
||||||
|
<HintPath>..\..\packages\Newtonsoft.Json.7.0.1\lib\net45\Newtonsoft.Json.dll</HintPath>
|
||||||
|
<Private>True</Private>
|
||||||
|
</Reference>
|
||||||
|
<Reference Include="Razorvine.Pyrolite">
|
||||||
|
<HintPath>..\..\packages\Razorvine.Pyrolite.4.10.0.0\lib\net40\Razorvine.Pyrolite.dll</HintPath>
|
||||||
|
<Private>True</Private>
|
||||||
|
</Reference>
|
||||||
|
<Reference Include="Razorvine.Serpent">
|
||||||
|
<HintPath>..\..\packages\Razorvine.Serpent.1.12.0.0\lib\net40\Razorvine.Serpent.dll</HintPath>
|
||||||
|
<Private>True</Private>
|
||||||
|
</Reference>
|
||||||
|
<Reference Include="System" />
|
||||||
|
<Reference Include="System.Core" />
|
||||||
|
<Reference Include="System.Numerics" />
|
||||||
|
</ItemGroup>
|
||||||
|
<!-- To modify your build process, add your task inside one of the targets below and uncomment it.
|
||||||
|
Other similar extension points exist, see Microsoft.Common.targets.
|
||||||
|
<Target Name="BeforeBuild">
|
||||||
|
</Target>
|
||||||
|
<Target Name="AfterBuild">
|
||||||
|
</Target>
|
||||||
|
-->
|
||||||
|
</Project>
|
|
@ -0,0 +1,9 @@
|
||||||
|
<?xml version="1.0" encoding="utf-8"?>
|
||||||
|
<packages>
|
||||||
|
<package id="FSharp.Core" version="4.0.0.1" targetFramework="net45" />
|
||||||
|
<package id="log4net" version="2.0.5" targetFramework="net45" />
|
||||||
|
<package id="Microsoft.SparkCLR" version="1.6.100" targetFramework="net45" />
|
||||||
|
<package id="Newtonsoft.Json" version="7.0.1" targetFramework="net45" />
|
||||||
|
<package id="Razorvine.Pyrolite" version="4.10.0.0" targetFramework="net45" />
|
||||||
|
<package id="Razorvine.Serpent" version="1.12.0.0" targetFramework="net45" />
|
||||||
|
</packages>
|
|
@ -1,4 +1,4 @@
|
||||||
|
|
||||||
Microsoft Visual Studio Solution File, Format Version 12.00
|
Microsoft Visual Studio Solution File, Format Version 12.00
|
||||||
# Visual Studio 2013
|
# Visual Studio 2013
|
||||||
VisualStudioVersion = 12.0.30501.0
|
VisualStudioVersion = 12.0.30501.0
|
||||||
|
@ -21,6 +21,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Pi", "Batch\pi\Pi.csproj",
|
||||||
EndProject
|
EndProject
|
||||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "WordCount", "Batch\WordCount\WordCount.csproj", "{17E4C27F-8441-425A-B82B-23BA5E313CC4}"
|
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "WordCount", "Batch\WordCount\WordCount.csproj", "{17E4C27F-8441-425A-B82B-23BA5E313CC4}"
|
||||||
EndProject
|
EndProject
|
||||||
|
Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "WordCountFSharp", "Batch\WordCountFSharp\WordCountFSharp.fsproj", "{17B63D32-EFC8-4EF5-831A-197A4FC29F06}"
|
||||||
|
EndProject
|
||||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Kafka", "Streaming\Kafka\Kafka.csproj", "{8764EAAA-9D32-4549-A64F-C7C89B014EA6}"
|
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Kafka", "Streaming\Kafka\Kafka.csproj", "{8764EAAA-9D32-4549-A64F-C7C89B014EA6}"
|
||||||
EndProject
|
EndProject
|
||||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "HiveDataFrame", "Sql\HiveDataFrame\HiveDataFrame.csproj", "{5C97498A-C4DB-43DD-86AD-4E50DEE8D405}"
|
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "HiveDataFrame", "Sql\HiveDataFrame\HiveDataFrame.csproj", "{5C97498A-C4DB-43DD-86AD-4E50DEE8D405}"
|
||||||
|
@ -55,6 +57,10 @@ Global
|
||||||
{17E4C27F-8441-425A-B82B-23BA5E313CC4}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
{17E4C27F-8441-425A-B82B-23BA5E313CC4}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||||
{17E4C27F-8441-425A-B82B-23BA5E313CC4}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
{17E4C27F-8441-425A-B82B-23BA5E313CC4}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||||
{17E4C27F-8441-425A-B82B-23BA5E313CC4}.Release|Any CPU.Build.0 = Release|Any CPU
|
{17E4C27F-8441-425A-B82B-23BA5E313CC4}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||||
|
{17B63D32-EFC8-4EF5-831A-197A4FC29F06}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||||
|
{17B63D32-EFC8-4EF5-831A-197A4FC29F06}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||||
|
{17B63D32-EFC8-4EF5-831A-197A4FC29F06}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||||
|
{17B63D32-EFC8-4EF5-831A-197A4FC29F06}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||||
{8764EAAA-9D32-4549-A64F-C7C89B014EA6}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
{8764EAAA-9D32-4549-A64F-C7C89B014EA6}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||||
{8764EAAA-9D32-4549-A64F-C7C89B014EA6}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
{8764EAAA-9D32-4549-A64F-C7C89B014EA6}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||||
{8764EAAA-9D32-4549-A64F-C7C89B014EA6}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
{8764EAAA-9D32-4549-A64F-C7C89B014EA6}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||||
|
@ -74,6 +80,7 @@ Global
|
||||||
{DF840BFB-B3A3-493D-B667-4CF21ADBFCAD} = {6F90310A-2DA2-4E81-A062-8D8A9F47C25B}
|
{DF840BFB-B3A3-493D-B667-4CF21ADBFCAD} = {6F90310A-2DA2-4E81-A062-8D8A9F47C25B}
|
||||||
{913E6A56-9839-4379-8B3C-855BA9341663} = {AE001E84-471E-4D02-BDDE-40B85915CEAE}
|
{913E6A56-9839-4379-8B3C-855BA9341663} = {AE001E84-471E-4D02-BDDE-40B85915CEAE}
|
||||||
{17E4C27F-8441-425A-B82B-23BA5E313CC4} = {AE001E84-471E-4D02-BDDE-40B85915CEAE}
|
{17E4C27F-8441-425A-B82B-23BA5E313CC4} = {AE001E84-471E-4D02-BDDE-40B85915CEAE}
|
||||||
|
{17B63D32-EFC8-4EF5-831A-197A4FC29F06} = {AE001E84-471E-4D02-BDDE-40B85915CEAE}
|
||||||
{8764EAAA-9D32-4549-A64F-C7C89B014EA6} = {6F90310A-2DA2-4E81-A062-8D8A9F47C25B}
|
{8764EAAA-9D32-4549-A64F-C7C89B014EA6} = {6F90310A-2DA2-4E81-A062-8D8A9F47C25B}
|
||||||
{5C97498A-C4DB-43DD-86AD-4E50DEE8D405} = {28600A86-E011-41C9-AB41-591580EDB9F1}
|
{5C97498A-C4DB-43DD-86AD-4E50DEE8D405} = {28600A86-E011-41C9-AB41-591580EDB9F1}
|
||||||
EndGlobalSection
|
EndGlobalSection
|
||||||
|
|
Загрузка…
Ссылка в новой задаче