updated examples to use latest version of Mobius
This commit is contained in:
Родитель
41e1836ff7
Коммит
e829864490
|
@ -7675,6 +7675,11 @@
|
||||||
<param name="json">The Json object used to construct a StructType</param>
|
<param name="json">The Json object used to construct a StructType</param>
|
||||||
<returns>A new StructType instance</returns>
|
<returns>A new StructType instance</returns>
|
||||||
</member>
|
</member>
|
||||||
|
<member name="T:Microsoft.Spark.CSharp.Sql.UdfRegistration">
|
||||||
|
<summary>
|
||||||
|
Used for registering User Defined Functions. SparkSession.Udf is used to access instance of this type.
|
||||||
|
</summary>
|
||||||
|
</member>
|
||||||
<member name="M:Microsoft.Spark.CSharp.Sql.UdfRegistration.RegisterFunction``1(System.String,System.Func{``0})">
|
<member name="M:Microsoft.Spark.CSharp.Sql.UdfRegistration.RegisterFunction``1(System.String,System.Func{``0})">
|
||||||
<summary>
|
<summary>
|
||||||
Register UDF with no input argument, e.g:
|
Register UDF with no input argument, e.g:
|
||||||
|
|
|
@ -1047,6 +1047,20 @@
|
||||||
---
|
---
|
||||||
|
|
||||||
|
|
||||||
|
###<font color="#68228B">Microsoft.Spark.CSharp.Sql.UdfRegistration</font>
|
||||||
|
####Summary
|
||||||
|
|
||||||
|
|
||||||
|
Used for registering User Defined Functions. SparkSession.Udf is used to access instance of this type.
|
||||||
|
|
||||||
|
|
||||||
|
####Methods
|
||||||
|
|
||||||
|
<table><tr><th>Name</th><th>Description</th></tr><tr><td><font color="blue">RegisterFunction``1</font></td><td>Register UDF with no input argument, e.g: SqlContext.RegisterFunction<bool>("MyFilter", () => true); sqlContext.Sql("SELECT * FROM MyTable where MyFilter()");</td></tr><tr><td><font color="blue">RegisterFunction``2</font></td><td>Register UDF with 1 input argument, e.g: SqlContext.RegisterFunction<bool, string>("MyFilter", (arg1) => arg1 != null); sqlContext.Sql("SELECT * FROM MyTable where MyFilter(columnName1)");</td></tr><tr><td><font color="blue">RegisterFunction``3</font></td><td>Register UDF with 2 input arguments, e.g: SqlContext.RegisterFunction<bool, string, string>("MyFilter", (arg1, arg2) => arg1 != null && arg2 != null); sqlContext.Sql("SELECT * FROM MyTable where MyFilter(columnName1, columnName2)");</td></tr><tr><td><font color="blue">RegisterFunction``4</font></td><td>Register UDF with 3 input arguments, e.g: SqlContext.RegisterFunction<bool, string, string, string>("MyFilter", (arg1, arg2, arg3) => arg1 != null && arg2 != null && arg3 != null); sqlContext.Sql("SELECT * FROM MyTable where MyFilter(columnName1, columnName2, columnName3)");</td></tr><tr><td><font color="blue">RegisterFunction``5</font></td><td>Register UDF with 4 input arguments, e.g: SqlContext.RegisterFunction<bool, string, string, ..., string>("MyFilter", (arg1, arg2, ..., arg4) => arg1 != null && arg2 != null && ... && arg3 != null); sqlContext.Sql("SELECT * FROM MyTable where MyFilter(columnName1, columnName2, ..., columnName4)");</td></tr><tr><td><font color="blue">RegisterFunction``6</font></td><td>Register UDF with 5 input arguments, e.g: SqlContext.RegisterFunction<bool, string, string, ..., string>("MyFilter", (arg1, arg2, ..., arg5) => arg1 != null && arg2 != null && ... && arg5 != null); sqlContext.Sql("SELECT * FROM MyTable where MyFilter(columnName1, columnName2, ..., columnName5)");</td></tr><tr><td><font color="blue">RegisterFunction``7</font></td><td>Register UDF with 6 input arguments, e.g: SqlContext.RegisterFunction<bool, string, string, ..., string>("MyFilter", (arg1, arg2, ..., arg6) => arg1 != null && arg2 != null && ... && arg6 != null); sqlContext.Sql("SELECT * FROM MyTable where MyFilter(columnName1, columnName2, ..., columnName6)");</td></tr><tr><td><font color="blue">RegisterFunction``8</font></td><td>Register UDF with 7 input arguments, e.g: SqlContext.RegisterFunction<bool, string, string, ..., string>("MyFilter", (arg1, arg2, ..., arg7) => arg1 != null && arg2 != null && ... && arg7 != null); sqlContext.Sql("SELECT * FROM MyTable where MyFilter(columnName1, columnName2, ..., columnName7)");</td></tr><tr><td><font color="blue">RegisterFunction``9</font></td><td>Register UDF with 8 input arguments, e.g: SqlContext.RegisterFunction<bool, string, string, ..., string>("MyFilter", (arg1, arg2, ..., arg8) => arg1 != null && arg2 != null && ... && arg8 != null); sqlContext.Sql("SELECT * FROM MyTable where MyFilter(columnName1, columnName2, ..., columnName8)");</td></tr><tr><td><font color="blue">RegisterFunction``10</font></td><td>Register UDF with 9 input arguments, e.g: SqlContext.RegisterFunction<bool, string, string, ..., string>("MyFilter", (arg1, arg2, ..., arg9) => arg1 != null && arg2 != null && ... && arg9 != null); sqlContext.Sql("SELECT * FROM MyTable where MyFilter(columnName1, columnName2, ..., columnName9)");</td></tr><tr><td><font color="blue">RegisterFunction``11</font></td><td>Register UDF with 10 input arguments, e.g: SqlContext.RegisterFunction<bool, string, string, ..., string>("MyFilter", (arg1, arg2, ..., arg10) => arg1 != null && arg2 != null && ... && arg10 != null); sqlContext.Sql("SELECT * FROM MyTable where MyFilter(columnName1, columnName2, ..., columnName10)");</td></tr></table>
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
|
||||||
###<font color="#68228B">Microsoft.Spark.CSharp.Streaming.ConstantInputDStream`1</font>
|
###<font color="#68228B">Microsoft.Spark.CSharp.Streaming.ConstantInputDStream`1</font>
|
||||||
####Summary
|
####Summary
|
||||||
|
|
||||||
|
|
|
@ -34,12 +34,12 @@ namespace Microsoft.Spark.CSharp.Examples
|
||||||
var lines = sparkContext.TextFile(args[0]);
|
var lines = sparkContext.TextFile(args[0]);
|
||||||
var counts = lines
|
var counts = lines
|
||||||
.FlatMap(x => x.Split(' '))
|
.FlatMap(x => x.Split(' '))
|
||||||
.Map(w => new KeyValuePair<string, int>(w, 1))
|
.Map(w => new Tuple<string, int>(w, 1))
|
||||||
.ReduceByKey((x, y) => x + y);
|
.ReduceByKey((x, y) => x + y);
|
||||||
|
|
||||||
foreach (var wordcount in counts.Collect())
|
foreach (var wordcount in counts.Collect())
|
||||||
{
|
{
|
||||||
Console.WriteLine("{0}: {1}", wordcount.Key, wordcount.Value);
|
Console.WriteLine("{0}: {1}", wordcount.Item1, wordcount.Item2);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
catch (Exception ex)
|
catch (Exception ex)
|
||||||
|
|
|
@ -34,7 +34,7 @@
|
||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
<Reference Include="CSharpWorker">
|
<Reference Include="CSharpWorker">
|
||||||
<SpecificVersion>False</SpecificVersion>
|
<SpecificVersion>False</SpecificVersion>
|
||||||
<HintPath>..\..\packages\Microsoft.SparkCLR.2.0.0-PREVIEW-2\lib\net45\CSharpWorker.exe</HintPath>
|
<HintPath>..\..\packages\Microsoft.SparkCLR.2.0.200\lib\net45\CSharpWorker.exe</HintPath>
|
||||||
</Reference>
|
</Reference>
|
||||||
<Reference Include="log4net, Version=1.2.15.0, Culture=neutral, PublicKeyToken=669e0ddf0bb1aa2a, processorArchitecture=MSIL">
|
<Reference Include="log4net, Version=1.2.15.0, Culture=neutral, PublicKeyToken=669e0ddf0bb1aa2a, processorArchitecture=MSIL">
|
||||||
<SpecificVersion>False</SpecificVersion>
|
<SpecificVersion>False</SpecificVersion>
|
||||||
|
@ -42,7 +42,7 @@
|
||||||
</Reference>
|
</Reference>
|
||||||
<Reference Include="Microsoft.Spark.CSharp.Adapter">
|
<Reference Include="Microsoft.Spark.CSharp.Adapter">
|
||||||
<SpecificVersion>False</SpecificVersion>
|
<SpecificVersion>False</SpecificVersion>
|
||||||
<HintPath>..\..\packages\Microsoft.SparkCLR.2.0.0-PREVIEW-2\lib\net45\Microsoft.Spark.CSharp.Adapter.dll</HintPath>
|
<HintPath>..\..\packages\Microsoft.SparkCLR.2.0.200\lib\net45\Microsoft.Spark.CSharp.Adapter.dll</HintPath>
|
||||||
</Reference>
|
</Reference>
|
||||||
<Reference Include="Newtonsoft.Json, Version=4.5.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL">
|
<Reference Include="Newtonsoft.Json, Version=4.5.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL">
|
||||||
<SpecificVersion>False</SpecificVersion>
|
<SpecificVersion>False</SpecificVersion>
|
||||||
|
@ -67,7 +67,7 @@
|
||||||
<Compile Include="Properties\AssemblyInfo.cs" />
|
<Compile Include="Properties\AssemblyInfo.cs" />
|
||||||
</ItemGroup>
|
</ItemGroup>
|
||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
<None Include="..\..\packages\Microsoft.SparkCLR.2.0.0-PREVIEW-2\lib\net45\CSharpWorker.exe.config">
|
<None Include="..\..\packages\Microsoft.SparkCLR.2.0.200\lib\net45\CSharpWorker.exe.config">
|
||||||
<Link>CSharpWorker.exe.config</Link>
|
<Link>CSharpWorker.exe.config</Link>
|
||||||
</None>
|
</None>
|
||||||
<None Include="..\..\App.config">
|
<None Include="..\..\App.config">
|
||||||
|
|
|
@ -4,5 +4,5 @@
|
||||||
<package id="Newtonsoft.Json" version="7.0.1" targetFramework="net45" />
|
<package id="Newtonsoft.Json" version="7.0.1" targetFramework="net45" />
|
||||||
<package id="Razorvine.Pyrolite" version="4.10.0.0" targetFramework="net45" />
|
<package id="Razorvine.Pyrolite" version="4.10.0.0" targetFramework="net45" />
|
||||||
<package id="Razorvine.Serpent" version="1.12.0.0" targetFramework="net45" />
|
<package id="Razorvine.Serpent" version="1.12.0.0" targetFramework="net45" />
|
||||||
<package id="Microsoft.SparkCLR" version="2.0.0-PREVIEW-2" targetFramework="net45" />
|
<package id="Microsoft.SparkCLR" version="2.0.200" targetFramework="net45" />
|
||||||
</packages>
|
</packages>
|
||||||
|
|
|
@ -37,7 +37,7 @@
|
||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
<Reference Include="CSharpWorker">
|
<Reference Include="CSharpWorker">
|
||||||
<SpecificVersion>False</SpecificVersion>
|
<SpecificVersion>False</SpecificVersion>
|
||||||
<HintPath>..\..\packages\Microsoft.SparkCLR.2.0.0-PREVIEW-2\lib\net45\CSharpWorker.exe</HintPath>
|
<HintPath>..\..\packages\Microsoft.SparkCLR.2.0.200\lib\net45\CSharpWorker.exe</HintPath>
|
||||||
</Reference>
|
</Reference>
|
||||||
<Reference Include="log4net, Version=1.2.15.0, Culture=neutral, PublicKeyToken=669e0ddf0bb1aa2a, processorArchitecture=MSIL">
|
<Reference Include="log4net, Version=1.2.15.0, Culture=neutral, PublicKeyToken=669e0ddf0bb1aa2a, processorArchitecture=MSIL">
|
||||||
<SpecificVersion>False</SpecificVersion>
|
<SpecificVersion>False</SpecificVersion>
|
||||||
|
@ -45,7 +45,7 @@
|
||||||
</Reference>
|
</Reference>
|
||||||
<Reference Include="Microsoft.Spark.CSharp.Adapter">
|
<Reference Include="Microsoft.Spark.CSharp.Adapter">
|
||||||
<SpecificVersion>False</SpecificVersion>
|
<SpecificVersion>False</SpecificVersion>
|
||||||
<HintPath>..\..\packages\Microsoft.SparkCLR.2.0.0-PREVIEW-2\lib\net45\Microsoft.Spark.CSharp.Adapter.dll</HintPath>
|
<HintPath>..\..\packages\Microsoft.SparkCLR.2.0.200\lib\net45\Microsoft.Spark.CSharp.Adapter.dll</HintPath>
|
||||||
</Reference>
|
</Reference>
|
||||||
<Reference Include="Newtonsoft.Json, Version=4.5.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL">
|
<Reference Include="Newtonsoft.Json, Version=4.5.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL">
|
||||||
<SpecificVersion>False</SpecificVersion>
|
<SpecificVersion>False</SpecificVersion>
|
||||||
|
@ -66,7 +66,7 @@
|
||||||
<Compile Include="Properties\AssemblyInfo.cs" />
|
<Compile Include="Properties\AssemblyInfo.cs" />
|
||||||
</ItemGroup>
|
</ItemGroup>
|
||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
<None Include="..\..\packages\Microsoft.SparkCLR.2.0.0-PREVIEW-2\lib\net45\CSharpWorker.exe.config">
|
<None Include="..\..\packages\Microsoft.SparkCLR.2.0.200\lib\net45\CSharpWorker.exe.config">
|
||||||
<Link>CSharpWorker.exe.config</Link>
|
<Link>CSharpWorker.exe.config</Link>
|
||||||
</None>
|
</None>
|
||||||
<None Include="..\..\App.config">
|
<None Include="..\..\App.config">
|
||||||
|
@ -84,4 +84,4 @@
|
||||||
<Target Name="AfterBuild">
|
<Target Name="AfterBuild">
|
||||||
</Target>
|
</Target>
|
||||||
-->
|
-->
|
||||||
</Project>
|
</Project>
|
||||||
|
|
|
@ -1,7 +1,7 @@
|
||||||
<?xml version="1.0" encoding="utf-8"?>
|
<?xml version="1.0" encoding="utf-8"?>
|
||||||
<packages>
|
<packages>
|
||||||
<package id="log4net" version="2.0.5" targetFramework="net45" />
|
<package id="log4net" version="2.0.5" targetFramework="net45" />
|
||||||
<package id="Microsoft.SparkCLR" version="2.0.0-PREVIEW-2" targetFramework="net45" />
|
<package id="Microsoft.SparkCLR" version="2.0.200" targetFramework="net45" />
|
||||||
<package id="Newtonsoft.Json" version="7.0.1" targetFramework="net45" />
|
<package id="Newtonsoft.Json" version="7.0.1" targetFramework="net45" />
|
||||||
<package id="Razorvine.Pyrolite" version="4.10.0.0" targetFramework="net45" />
|
<package id="Razorvine.Pyrolite" version="4.10.0.0" targetFramework="net45" />
|
||||||
<package id="Razorvine.Serpent" version="1.12.0.0" targetFramework="net45" />
|
<package id="Razorvine.Serpent" version="1.12.0.0" targetFramework="net45" />
|
||||||
|
|
|
@ -35,13 +35,13 @@
|
||||||
</PropertyGroup>
|
</PropertyGroup>
|
||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
<Reference Include="CSharpWorker">
|
<Reference Include="CSharpWorker">
|
||||||
<HintPath>..\..\packages\Microsoft.SparkCLR.2.0.0-PREVIEW-2\lib\net45\CSharpWorker.exe</HintPath>
|
<HintPath>..\..\packages\Microsoft.SparkCLR.2.0.200\lib\net45\CSharpWorker.exe</HintPath>
|
||||||
</Reference>
|
</Reference>
|
||||||
<Reference Include="log4net">
|
<Reference Include="log4net">
|
||||||
<HintPath>..\..\packages\log4net.2.0.5\lib\net45-full\log4net.dll</HintPath>
|
<HintPath>..\..\packages\log4net.2.0.5\lib\net45-full\log4net.dll</HintPath>
|
||||||
</Reference>
|
</Reference>
|
||||||
<Reference Include="Microsoft.Spark.CSharp.Adapter">
|
<Reference Include="Microsoft.Spark.CSharp.Adapter">
|
||||||
<HintPath>..\..\packages\Microsoft.SparkCLR.2.0.0-PREVIEW-2\lib\net45\Microsoft.Spark.CSharp.Adapter.dll</HintPath>
|
<HintPath>..\..\packages\Microsoft.SparkCLR.2.0.200\lib\net45\Microsoft.Spark.CSharp.Adapter.dll</HintPath>
|
||||||
</Reference>
|
</Reference>
|
||||||
<Reference Include="Newtonsoft.Json, Version=4.5.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL">
|
<Reference Include="Newtonsoft.Json, Version=4.5.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL">
|
||||||
<SpecificVersion>False</SpecificVersion>
|
<SpecificVersion>False</SpecificVersion>
|
||||||
|
|
|
@ -4,5 +4,5 @@
|
||||||
<package id="Newtonsoft.Json" version="7.0.1" targetFramework="net45" />
|
<package id="Newtonsoft.Json" version="7.0.1" targetFramework="net45" />
|
||||||
<package id="Razorvine.Pyrolite" version="4.10.0.0" targetFramework="net45" />
|
<package id="Razorvine.Pyrolite" version="4.10.0.0" targetFramework="net45" />
|
||||||
<package id="Razorvine.Serpent" version="1.12.0.0" targetFramework="net45" />
|
<package id="Razorvine.Serpent" version="1.12.0.0" targetFramework="net45" />
|
||||||
<package id="Microsoft.SparkCLR" version="2.0.0-PREVIEW-2" targetFramework="net45" />
|
<package id="Microsoft.SparkCLR" version="2.0.200" targetFramework="net45" />
|
||||||
</packages>
|
</packages>
|
||||||
|
|
|
@ -38,11 +38,11 @@
|
||||||
<HintPath>..\..\packages\log4net.2.0.5\lib\net45-full\log4net.dll</HintPath>
|
<HintPath>..\..\packages\log4net.2.0.5\lib\net45-full\log4net.dll</HintPath>
|
||||||
</Reference>
|
</Reference>
|
||||||
<Reference Include="CSharpWorker">
|
<Reference Include="CSharpWorker">
|
||||||
<HintPath>..\..\packages\Microsoft.SparkCLR.2.0.0-PREVIEW-2\lib\net45\CSharpWorker.exe</HintPath>
|
<HintPath>..\..\packages\Microsoft.SparkCLR.2.0.200\lib\net45\CSharpWorker.exe</HintPath>
|
||||||
<Private>True</Private>
|
<Private>True</Private>
|
||||||
</Reference>
|
</Reference>
|
||||||
<Reference Include="Microsoft.Spark.CSharp.Adapter">
|
<Reference Include="Microsoft.Spark.CSharp.Adapter">
|
||||||
<HintPath>..\..\packages\Microsoft.SparkCLR.2.0.0-PREVIEW-2\lib\net45\Microsoft.Spark.CSharp.Adapter.dll</HintPath>
|
<HintPath>..\..\packages\Microsoft.SparkCLR.2.0.200\lib\net45\Microsoft.Spark.CSharp.Adapter.dll</HintPath>
|
||||||
<Private>True</Private>
|
<Private>True</Private>
|
||||||
</Reference>
|
</Reference>
|
||||||
<Reference Include="System" />
|
<Reference Include="System" />
|
||||||
|
|
|
@ -1,7 +1,7 @@
|
||||||
<?xml version="1.0" encoding="utf-8"?>
|
<?xml version="1.0" encoding="utf-8"?>
|
||||||
<packages>
|
<packages>
|
||||||
<package id="log4net" version="2.0.5" targetFramework="net45" />
|
<package id="log4net" version="2.0.5" targetFramework="net45" />
|
||||||
<package id="Microsoft.SparkCLR" version="2.0.0-PREVIEW-2" targetFramework="net452" />
|
<package id="Microsoft.SparkCLR" version="2.0.200" targetFramework="net452" />
|
||||||
<package id="Newtonsoft.Json" version="7.0.1" targetFramework="net45" />
|
<package id="Newtonsoft.Json" version="7.0.1" targetFramework="net45" />
|
||||||
<package id="Razorvine.Pyrolite" version="4.10.0.0" targetFramework="net45" />
|
<package id="Razorvine.Pyrolite" version="4.10.0.0" targetFramework="net45" />
|
||||||
<package id="Razorvine.Serpent" version="1.12.0.0" targetFramework="net45" />
|
<package id="Razorvine.Serpent" version="1.12.0.0" targetFramework="net45" />
|
||||||
|
|
|
@ -36,7 +36,7 @@
|
||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
<Reference Include="CSharpWorker">
|
<Reference Include="CSharpWorker">
|
||||||
<SpecificVersion>False</SpecificVersion>
|
<SpecificVersion>False</SpecificVersion>
|
||||||
<HintPath>..\..\packages\Microsoft.SparkCLR.2.0.0-PREVIEW-2\lib\net45\CSharpWorker.exe</HintPath>
|
<HintPath>..\..\packages\Microsoft.SparkCLR.2.0.200\lib\net45\CSharpWorker.exe</HintPath>
|
||||||
</Reference>
|
</Reference>
|
||||||
<Reference Include="log4net, Version=1.2.15.0, Culture=neutral, PublicKeyToken=669e0ddf0bb1aa2a, processorArchitecture=MSIL">
|
<Reference Include="log4net, Version=1.2.15.0, Culture=neutral, PublicKeyToken=669e0ddf0bb1aa2a, processorArchitecture=MSIL">
|
||||||
<SpecificVersion>False</SpecificVersion>
|
<SpecificVersion>False</SpecificVersion>
|
||||||
|
@ -44,7 +44,7 @@
|
||||||
</Reference>
|
</Reference>
|
||||||
<Reference Include="Microsoft.Spark.CSharp.Adapter">
|
<Reference Include="Microsoft.Spark.CSharp.Adapter">
|
||||||
<SpecificVersion>False</SpecificVersion>
|
<SpecificVersion>False</SpecificVersion>
|
||||||
<HintPath>..\..\packages\Microsoft.SparkCLR.2.0.0-PREVIEW-2\lib\net45\Microsoft.Spark.CSharp.Adapter.dll</HintPath>
|
<HintPath>..\..\packages\Microsoft.SparkCLR.2.0.200\lib\net45\Microsoft.Spark.CSharp.Adapter.dll</HintPath>
|
||||||
</Reference>
|
</Reference>
|
||||||
<Reference Include="Newtonsoft.Json, Version=4.5.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL">
|
<Reference Include="Newtonsoft.Json, Version=4.5.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL">
|
||||||
<SpecificVersion>False</SpecificVersion>
|
<SpecificVersion>False</SpecificVersion>
|
||||||
|
@ -65,7 +65,7 @@
|
||||||
<Compile Include="Properties\AssemblyInfo.cs" />
|
<Compile Include="Properties\AssemblyInfo.cs" />
|
||||||
</ItemGroup>
|
</ItemGroup>
|
||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
<None Include="..\..\packages\Microsoft.SparkCLR.2.0.0-PREVIEW-2\lib\net45\CSharpWorker.exe.config">
|
<None Include="..\..\packages\Microsoft.SparkCLR.2.0.200\lib\net45\CSharpWorker.exe.config">
|
||||||
<Link>CSharpWorker.exe.config</Link>
|
<Link>CSharpWorker.exe.config</Link>
|
||||||
</None>
|
</None>
|
||||||
<None Include="..\..\App.config">
|
<None Include="..\..\App.config">
|
||||||
|
|
|
@ -4,5 +4,5 @@
|
||||||
<package id="Newtonsoft.Json" version="7.0.1" targetFramework="net45" />
|
<package id="Newtonsoft.Json" version="7.0.1" targetFramework="net45" />
|
||||||
<package id="Razorvine.Pyrolite" version="4.10.0.0" targetFramework="net45" />
|
<package id="Razorvine.Pyrolite" version="4.10.0.0" targetFramework="net45" />
|
||||||
<package id="Razorvine.Serpent" version="1.12.0.0" targetFramework="net45" />
|
<package id="Razorvine.Serpent" version="1.12.0.0" targetFramework="net45" />
|
||||||
<package id="Microsoft.SparkCLR" version="2.0.0-PREVIEW-2" targetFramework="net45" />
|
<package id="Microsoft.SparkCLR" version="2.0.200" targetFramework="net45" />
|
||||||
</packages>
|
</packages>
|
||||||
|
|
|
@ -36,7 +36,7 @@
|
||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
<Reference Include="CSharpWorker">
|
<Reference Include="CSharpWorker">
|
||||||
<SpecificVersion>False</SpecificVersion>
|
<SpecificVersion>False</SpecificVersion>
|
||||||
<HintPath>..\..\packages\Microsoft.SparkCLR.2.0.0-PREVIEW-2\lib\net45\CSharpWorker.exe</HintPath>
|
<HintPath>..\..\packages\Microsoft.SparkCLR.2.0.200\lib\net45\CSharpWorker.exe</HintPath>
|
||||||
</Reference>
|
</Reference>
|
||||||
<Reference Include="log4net, Version=1.2.15.0, Culture=neutral, PublicKeyToken=669e0ddf0bb1aa2a, processorArchitecture=MSIL">
|
<Reference Include="log4net, Version=1.2.15.0, Culture=neutral, PublicKeyToken=669e0ddf0bb1aa2a, processorArchitecture=MSIL">
|
||||||
<SpecificVersion>False</SpecificVersion>
|
<SpecificVersion>False</SpecificVersion>
|
||||||
|
@ -44,7 +44,7 @@
|
||||||
</Reference>
|
</Reference>
|
||||||
<Reference Include="Microsoft.Spark.CSharp.Adapter">
|
<Reference Include="Microsoft.Spark.CSharp.Adapter">
|
||||||
<SpecificVersion>False</SpecificVersion>
|
<SpecificVersion>False</SpecificVersion>
|
||||||
<HintPath>..\..\packages\Microsoft.SparkCLR.2.0.0-PREVIEW-2\lib\net45\Microsoft.Spark.CSharp.Adapter.dll</HintPath>
|
<HintPath>..\..\packages\Microsoft.SparkCLR.2.0.200\lib\net45\Microsoft.Spark.CSharp.Adapter.dll</HintPath>
|
||||||
</Reference>
|
</Reference>
|
||||||
<Reference Include="Newtonsoft.Json, Version=4.5.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL">
|
<Reference Include="Newtonsoft.Json, Version=4.5.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL">
|
||||||
<SpecificVersion>False</SpecificVersion>
|
<SpecificVersion>False</SpecificVersion>
|
||||||
|
@ -65,7 +65,7 @@
|
||||||
<Compile Include="Properties\AssemblyInfo.cs" />
|
<Compile Include="Properties\AssemblyInfo.cs" />
|
||||||
</ItemGroup>
|
</ItemGroup>
|
||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
<None Include="..\..\packages\Microsoft.SparkCLR.2.0.0-PREVIEW-2\lib\net45\CSharpWorker.exe.config">
|
<None Include="..\..\packages\Microsoft.SparkCLR.2.0.200\lib\net45\CSharpWorker.exe.config">
|
||||||
<Link>CSharpWorker.exe.config</Link>
|
<Link>CSharpWorker.exe.config</Link>
|
||||||
</None>
|
</None>
|
||||||
<None Include="..\..\App.config">
|
<None Include="..\..\App.config">
|
||||||
|
|
|
@ -4,5 +4,5 @@
|
||||||
<package id="Newtonsoft.Json" version="7.0.1" targetFramework="net45" />
|
<package id="Newtonsoft.Json" version="7.0.1" targetFramework="net45" />
|
||||||
<package id="Razorvine.Pyrolite" version="4.10.0.0" targetFramework="net45" />
|
<package id="Razorvine.Pyrolite" version="4.10.0.0" targetFramework="net45" />
|
||||||
<package id="Razorvine.Serpent" version="1.12.0.0" targetFramework="net45" />
|
<package id="Razorvine.Serpent" version="1.12.0.0" targetFramework="net45" />
|
||||||
<package id="Microsoft.SparkCLR" version="2.0.0-PREVIEW-2" targetFramework="net45" />
|
<package id="Microsoft.SparkCLR" version="2.0.200" targetFramework="net45" />
|
||||||
</packages>
|
</packages>
|
||||||
|
|
|
@ -35,7 +35,7 @@
|
||||||
</PropertyGroup>
|
</PropertyGroup>
|
||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
<Reference Include="CSharpWorker">
|
<Reference Include="CSharpWorker">
|
||||||
<HintPath>..\..\packages\Microsoft.SparkCLR.2.0.0-PREVIEW-2\lib\net45\CSharpWorker.exe</HintPath>
|
<HintPath>..\..\packages\Microsoft.SparkCLR.2.0.200\lib\net45\CSharpWorker.exe</HintPath>
|
||||||
</Reference>
|
</Reference>
|
||||||
<Reference Include="log4net, Version=1.2.15.0, Culture=neutral, PublicKeyToken=669e0ddf0bb1aa2a, processorArchitecture=MSIL">
|
<Reference Include="log4net, Version=1.2.15.0, Culture=neutral, PublicKeyToken=669e0ddf0bb1aa2a, processorArchitecture=MSIL">
|
||||||
<SpecificVersion>False</SpecificVersion>
|
<SpecificVersion>False</SpecificVersion>
|
||||||
|
@ -43,7 +43,7 @@
|
||||||
</Reference>
|
</Reference>
|
||||||
<Reference Include="Microsoft.CSharp" />
|
<Reference Include="Microsoft.CSharp" />
|
||||||
<Reference Include="Microsoft.Spark.CSharp.Adapter">
|
<Reference Include="Microsoft.Spark.CSharp.Adapter">
|
||||||
<HintPath>..\..\packages\Microsoft.SparkCLR.2.0.0-PREVIEW-2\lib\net45\Microsoft.Spark.CSharp.Adapter.dll</HintPath>
|
<HintPath>..\..\packages\Microsoft.SparkCLR.2.0.200\lib\net45\Microsoft.Spark.CSharp.Adapter.dll</HintPath>
|
||||||
</Reference>
|
</Reference>
|
||||||
<Reference Include="Newtonsoft.Json, Version=4.5.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL">
|
<Reference Include="Newtonsoft.Json, Version=4.5.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL">
|
||||||
<SpecificVersion>False</SpecificVersion>
|
<SpecificVersion>False</SpecificVersion>
|
||||||
|
@ -68,7 +68,7 @@
|
||||||
<Compile Include="EventPublisher.cs" />
|
<Compile Include="EventPublisher.cs" />
|
||||||
</ItemGroup>
|
</ItemGroup>
|
||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
<None Include="..\..\packages\Microsoft.SparkCLR.2.0.0-PREVIEW-2\lib\net45\CSharpWorker.exe.config">
|
<None Include="..\..\packages\Microsoft.SparkCLR.2.0.200\lib\net45\CSharpWorker.exe.config">
|
||||||
<Link>CSharpWorker.exe.config</Link>
|
<Link>CSharpWorker.exe.config</Link>
|
||||||
</None>
|
</None>
|
||||||
<None Include="..\..\App.config">
|
<None Include="..\..\App.config">
|
||||||
|
|
|
@ -3,6 +3,7 @@
|
||||||
|
|
||||||
using System;
|
using System;
|
||||||
using System.Collections.Generic;
|
using System.Collections.Generic;
|
||||||
|
using System.Linq;
|
||||||
using System.Text;
|
using System.Text;
|
||||||
using Microsoft.Spark.CSharp.Core;
|
using Microsoft.Spark.CSharp.Core;
|
||||||
using Microsoft.Spark.CSharp.Streaming;
|
using Microsoft.Spark.CSharp.Streaming;
|
||||||
|
@ -42,14 +43,14 @@ namespace Microsoft.Spark.CSharp.Examples
|
||||||
var ssc = new StreamingContext(sparkContext, slideDurationInMillis);
|
var ssc = new StreamingContext(sparkContext, slideDurationInMillis);
|
||||||
ssc.Checkpoint(checkpointPath);
|
ssc.Checkpoint(checkpointPath);
|
||||||
|
|
||||||
var stream = EventHubsUtils.CreateUnionStream(ssc, eventhubsParams);
|
var stream = EventHubsUtils.CreateUnionStream(ssc, eventhubsParams.Select(v => new Tuple<string, string>(v.Key, v.Value)));
|
||||||
var countByLogLevelAndTime = stream
|
var countByLogLevelAndTime = stream
|
||||||
.Map(bytes => Encoding.UTF8.GetString(bytes))
|
.Map(bytes => Encoding.UTF8.GetString(bytes))
|
||||||
.Filter(line => line.Contains(","))
|
.Filter(line => line.Contains(","))
|
||||||
.Map(line => line.Split(','))
|
.Map(line => line.Split(','))
|
||||||
.Map(columns => new KeyValuePair<string, int>(string.Format("{0},{1}", columns[0], columns[1]), 1))
|
.Map(columns => new Tuple<string, int>(string.Format("{0},{1}", columns[0], columns[1]), 1))
|
||||||
.ReduceByKeyAndWindow((x, y) => x + y, (x, y) => x - y, windowDurationInSecs, slideDurationInSecs, 3)
|
.ReduceByKeyAndWindow((x, y) => x + y, (x, y) => x - y, windowDurationInSecs, slideDurationInSecs, 3)
|
||||||
.Map(logLevelCountPair => string.Format("{0},{1}", logLevelCountPair.Key, logLevelCountPair.Value));
|
.Map(logLevelCountPair => string.Format("{0},{1}", logLevelCountPair.Item1, logLevelCountPair.Item2));
|
||||||
|
|
||||||
countByLogLevelAndTime.ForeachRDD(countByLogLevel =>
|
countByLogLevelAndTime.ForeachRDD(countByLogLevel =>
|
||||||
{
|
{
|
||||||
|
|
|
@ -4,7 +4,7 @@
|
||||||
<package id="Newtonsoft.Json" version="7.0.1" targetFramework="net45" />
|
<package id="Newtonsoft.Json" version="7.0.1" targetFramework="net45" />
|
||||||
<package id="Razorvine.Pyrolite" version="4.10.0.0" targetFramework="net45" />
|
<package id="Razorvine.Pyrolite" version="4.10.0.0" targetFramework="net45" />
|
||||||
<package id="Razorvine.Serpent" version="1.12.0.0" targetFramework="net45" />
|
<package id="Razorvine.Serpent" version="1.12.0.0" targetFramework="net45" />
|
||||||
<package id="Microsoft.SparkCLR" version="2.0.0-PREVIEW-2" targetFramework="net45" />
|
<package id="Microsoft.SparkCLR" version="2.0.200" targetFramework="net45" />
|
||||||
|
|
||||||
<!-- *** ****************************************************************** *** -->
|
<!-- *** ****************************************************************** *** -->
|
||||||
<!-- *** Following references are needed for publishing events to EventHubs *** -->
|
<!-- *** Following references are needed for publishing events to EventHubs *** -->
|
||||||
|
|
|
@ -38,7 +38,7 @@
|
||||||
</Reference>
|
</Reference>
|
||||||
<Reference Include="CSharpWorker">
|
<Reference Include="CSharpWorker">
|
||||||
<SpecificVersion>False</SpecificVersion>
|
<SpecificVersion>False</SpecificVersion>
|
||||||
<HintPath>..\..\packages\Microsoft.SparkCLR.2.0.0-PREVIEW-2\lib\net45\CSharpWorker.exe</HintPath>
|
<HintPath>..\..\packages\Microsoft.SparkCLR.2.0.200\lib\net45\CSharpWorker.exe</HintPath>
|
||||||
</Reference>
|
</Reference>
|
||||||
<Reference Include="log4net">
|
<Reference Include="log4net">
|
||||||
<SpecificVersion>False</SpecificVersion>
|
<SpecificVersion>False</SpecificVersion>
|
||||||
|
@ -46,7 +46,7 @@
|
||||||
</Reference>
|
</Reference>
|
||||||
<Reference Include="Microsoft.Spark.CSharp.Adapter">
|
<Reference Include="Microsoft.Spark.CSharp.Adapter">
|
||||||
<SpecificVersion>False</SpecificVersion>
|
<SpecificVersion>False</SpecificVersion>
|
||||||
<HintPath>..\..\packages\Microsoft.SparkCLR.2.0.0-PREVIEW-2\lib\net45\Microsoft.Spark.CSharp.Adapter.dll</HintPath>
|
<HintPath>..\..\packages\Microsoft.SparkCLR.2.0.200\lib\net45\Microsoft.Spark.CSharp.Adapter.dll</HintPath>
|
||||||
</Reference>
|
</Reference>
|
||||||
<Reference Include="Razorvine.Pyrolite">
|
<Reference Include="Razorvine.Pyrolite">
|
||||||
<SpecificVersion>False</SpecificVersion>
|
<SpecificVersion>False</SpecificVersion>
|
||||||
|
@ -64,7 +64,7 @@
|
||||||
<Compile Include="Properties\AssemblyInfo.cs" />
|
<Compile Include="Properties\AssemblyInfo.cs" />
|
||||||
</ItemGroup>
|
</ItemGroup>
|
||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
<None Include="..\..\packages\Microsoft.SparkCLR.2.0.0-PREVIEW-2\lib\net45\CSharpWorker.exe.config">
|
<None Include="..\..\packages\Microsoft.SparkCLR.2.0.200\lib\net45\CSharpWorker.exe.config">
|
||||||
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
|
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
|
||||||
</None>
|
</None>
|
||||||
<None Include="..\..\App.config">
|
<None Include="..\..\App.config">
|
||||||
|
|
|
@ -40,7 +40,7 @@ namespace Microsoft.Spark.CSharp.Examples
|
||||||
|
|
||||||
var lines = context.TextFileStream(inputDir);
|
var lines = context.TextFileStream(inputDir);
|
||||||
var words = lines.FlatMap(l => l.Split(' '));
|
var words = lines.FlatMap(l => l.Split(' '));
|
||||||
var pairs = words.Map(w => new KeyValuePair<string, int>(w, 1));
|
var pairs = words.Map(w => new Tuple<string, int>(w, 1));
|
||||||
var wordCounts = pairs.ReduceByKey((x, y) => x + y);
|
var wordCounts = pairs.ReduceByKey((x, y) => x + y);
|
||||||
|
|
||||||
wordCounts.ForeachRDD((time, rdd) =>
|
wordCounts.ForeachRDD((time, rdd) =>
|
||||||
|
|
|
@ -1,7 +1,7 @@
|
||||||
<?xml version="1.0" encoding="utf-8"?>
|
<?xml version="1.0" encoding="utf-8"?>
|
||||||
<packages>
|
<packages>
|
||||||
<package id="log4net" version="2.0.5" targetFramework="net45" />
|
<package id="log4net" version="2.0.5" targetFramework="net45" />
|
||||||
<package id="Microsoft.SparkCLR" version="2.0.0-PREVIEW-2" targetFramework="net45" />
|
<package id="Microsoft.SparkCLR" version="2.0.200" targetFramework="net45" />
|
||||||
<package id="Newtonsoft.Json" version="7.0.1" targetFramework="net45" />
|
<package id="Newtonsoft.Json" version="7.0.1" targetFramework="net45" />
|
||||||
<package id="Razorvine.Pyrolite" version="4.10.0.0" targetFramework="net45" />
|
<package id="Razorvine.Pyrolite" version="4.10.0.0" targetFramework="net45" />
|
||||||
<package id="Razorvine.Serpent" version="1.12.0.0" targetFramework="net45" />
|
<package id="Razorvine.Serpent" version="1.12.0.0" targetFramework="net45" />
|
||||||
|
|
|
@ -33,14 +33,14 @@
|
||||||
</PropertyGroup>
|
</PropertyGroup>
|
||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
<Reference Include="CSharpWorker">
|
<Reference Include="CSharpWorker">
|
||||||
<HintPath>..\..\packages\Microsoft.SparkCLR.2.0.0-PREVIEW-2\lib\net45\CSharpWorker.exe</HintPath>
|
<HintPath>..\..\packages\Microsoft.SparkCLR.2.0.200\lib\net45\CSharpWorker.exe</HintPath>
|
||||||
</Reference>
|
</Reference>
|
||||||
<Reference Include="log4net, Version=1.2.10.0, Culture=neutral, PublicKeyToken=1b44e1d426115821, processorArchitecture=MSIL">
|
<Reference Include="log4net, Version=1.2.10.0, Culture=neutral, PublicKeyToken=1b44e1d426115821, processorArchitecture=MSIL">
|
||||||
<SpecificVersion>False</SpecificVersion>
|
<SpecificVersion>False</SpecificVersion>
|
||||||
<HintPath>..\..\packages\log4net.2.0.5\lib\net45-full\log4net.dll</HintPath>
|
<HintPath>..\..\packages\log4net.2.0.5\lib\net45-full\log4net.dll</HintPath>
|
||||||
</Reference>
|
</Reference>
|
||||||
<Reference Include="Microsoft.Spark.CSharp.Adapter">
|
<Reference Include="Microsoft.Spark.CSharp.Adapter">
|
||||||
<HintPath>..\..\packages\Microsoft.SparkCLR.2.0.0-PREVIEW-2\lib\net45\Microsoft.Spark.CSharp.Adapter.dll</HintPath>
|
<HintPath>..\..\packages\Microsoft.SparkCLR.2.0.200\lib\net45\Microsoft.Spark.CSharp.Adapter.dll</HintPath>
|
||||||
</Reference>
|
</Reference>
|
||||||
<Reference Include="Newtonsoft.Json, Version=4.5.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL">
|
<Reference Include="Newtonsoft.Json, Version=4.5.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL">
|
||||||
<SpecificVersion>False</SpecificVersion>
|
<SpecificVersion>False</SpecificVersion>
|
||||||
|
|
|
@ -42,14 +42,14 @@ namespace Microsoft.Spark.CSharp.Examples
|
||||||
var ssc = new StreamingContext(sparkContext, slideDurationInMillis);
|
var ssc = new StreamingContext(sparkContext, slideDurationInMillis);
|
||||||
ssc.Checkpoint(checkpointPath);
|
ssc.Checkpoint(checkpointPath);
|
||||||
|
|
||||||
var stream = KafkaUtils.CreateDirectStream(ssc, topicList, kafkaParams, perTopicPartitionKafkaOffsets);
|
var stream = KafkaUtils.CreateDirectStream(ssc, topicList, kafkaParams.Select(v => new Tuple<string, string>(v.Key, v.Value)), perTopicPartitionKafkaOffsets.Select(v => new Tuple<string, long>(v.Key, v.Value)));
|
||||||
var countByLogLevelAndTime = stream
|
var countByLogLevelAndTime = stream
|
||||||
.Map(kvp => Encoding.UTF8.GetString(kvp.Value))
|
.Map(tuple => Encoding.UTF8.GetString(tuple.Item2))
|
||||||
.Filter(line => line.Contains(","))
|
.Filter(line => line.Contains(","))
|
||||||
.Map(line => line.Split(','))
|
.Map(line => line.Split(','))
|
||||||
.Map(columns => new KeyValuePair<string, int>(string.Format("{0},{1}", columns[0], columns[1]), 1))
|
.Map(columns => new Tuple<string, int>(string.Format("{0},{1}", columns[0], columns[1]), 1))
|
||||||
.ReduceByKeyAndWindow((x, y) => x + y, (x, y) => x - y, windowDurationInSecs, slideDurationInSecs, 3)
|
.ReduceByKeyAndWindow((x, y) => x + y, (x, y) => x - y, windowDurationInSecs, slideDurationInSecs, 3)
|
||||||
.Map(logLevelCountPair => string.Format("{0},{1}", logLevelCountPair.Key, logLevelCountPair.Value));
|
.Map(logLevelCountPair => string.Format("{0},{1}", logLevelCountPair.Item1, logLevelCountPair.Item2));
|
||||||
|
|
||||||
countByLogLevelAndTime.ForeachRDD(countByLogLevel =>
|
countByLogLevelAndTime.ForeachRDD(countByLogLevel =>
|
||||||
{
|
{
|
||||||
|
|
|
@ -1,7 +1,7 @@
|
||||||
<?xml version="1.0" encoding="utf-8"?>
|
<?xml version="1.0" encoding="utf-8"?>
|
||||||
<packages>
|
<packages>
|
||||||
<package id="log4net" version="2.0.5" targetFramework="net45" />
|
<package id="log4net" version="2.0.5" targetFramework="net45" />
|
||||||
<package id="Microsoft.SparkCLR" version="2.0.0-PREVIEW-2" targetFramework="net45" />
|
<package id="Microsoft.SparkCLR" version="2.0.200" targetFramework="net45" />
|
||||||
<package id="Newtonsoft.Json" version="7.0.1" targetFramework="net45" />
|
<package id="Newtonsoft.Json" version="7.0.1" targetFramework="net45" />
|
||||||
<package id="Razorvine.Pyrolite" version="4.10.0.0" targetFramework="net45" />
|
<package id="Razorvine.Pyrolite" version="4.10.0.0" targetFramework="net45" />
|
||||||
<package id="Razorvine.Serpent" version="1.12.0.0" targetFramework="net45" />
|
<package id="Razorvine.Serpent" version="1.12.0.0" targetFramework="net45" />
|
||||||
|
|
|
@ -66,13 +66,13 @@
|
||||||
</ItemGroup>
|
</ItemGroup>
|
||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
<Reference Include="CSharpWorker">
|
<Reference Include="CSharpWorker">
|
||||||
<HintPath>..\..\packages\Microsoft.SparkCLR.2.0.0-PREVIEW-2\lib\net45\CSharpWorker.exe</HintPath>
|
<HintPath>..\..\packages\Microsoft.SparkCLR.2.0.200\lib\net45\CSharpWorker.exe</HintPath>
|
||||||
</Reference>
|
</Reference>
|
||||||
<Reference Include="log4net">
|
<Reference Include="log4net">
|
||||||
<HintPath>..\..\packages\log4net.2.0.5\lib\net45-full\log4net.dll</HintPath>
|
<HintPath>..\..\packages\log4net.2.0.5\lib\net45-full\log4net.dll</HintPath>
|
||||||
</Reference>
|
</Reference>
|
||||||
<Reference Include="Microsoft.Spark.CSharp.Adapter">
|
<Reference Include="Microsoft.Spark.CSharp.Adapter">
|
||||||
<HintPath>..\..\packages\Microsoft.SparkCLR.2.0.0-PREVIEW-2\lib\net45\Microsoft.Spark.CSharp.Adapter.dll</HintPath>
|
<HintPath>..\..\packages\Microsoft.SparkCLR.2.0.200\lib\net45\Microsoft.Spark.CSharp.Adapter.dll</HintPath>
|
||||||
</Reference>
|
</Reference>
|
||||||
<Reference Include="mscorlib" />
|
<Reference Include="mscorlib" />
|
||||||
<Reference Include="FSharp.Core, Version=$(TargetFSharpCoreVersion), Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a">
|
<Reference Include="FSharp.Core, Version=$(TargetFSharpCoreVersion), Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a">
|
||||||
|
|
|
@ -2,7 +2,7 @@
|
||||||
<packages>
|
<packages>
|
||||||
<package id="FSharp.Core" version="4.0.0.1" targetFramework="net45" />
|
<package id="FSharp.Core" version="4.0.0.1" targetFramework="net45" />
|
||||||
<package id="log4net" version="2.0.5" targetFramework="net45" />
|
<package id="log4net" version="2.0.5" targetFramework="net45" />
|
||||||
<package id="Microsoft.SparkCLR" version="2.0.0-PREVIEW-2" targetFramework="net45" />
|
<package id="Microsoft.SparkCLR" version="2.0.200" targetFramework="net45" />
|
||||||
<package id="Newtonsoft.Json" version="7.0.1" targetFramework="net45" />
|
<package id="Newtonsoft.Json" version="7.0.1" targetFramework="net45" />
|
||||||
<package id="Razorvine.Pyrolite" version="4.10.0.0" targetFramework="net45" />
|
<package id="Razorvine.Pyrolite" version="4.10.0.0" targetFramework="net45" />
|
||||||
<package id="Razorvine.Serpent" version="1.12.0.0" targetFramework="net45" />
|
<package id="Razorvine.Serpent" version="1.12.0.0" targetFramework="net45" />
|
||||||
|
|
|
@ -21,11 +21,11 @@ let main args =
|
||||||
let lines = sparkContext.TextFile filePath
|
let lines = sparkContext.TextFile filePath
|
||||||
let counts =
|
let counts =
|
||||||
lines.FlatMap(fun x -> x.Split ' ' :> _)
|
lines.FlatMap(fun x -> x.Split ' ' :> _)
|
||||||
.Map(fun w -> KeyValuePair(w, 1))
|
.Map(fun w -> (w, 1))
|
||||||
.ReduceByKey(fun x y -> x + y)
|
.ReduceByKey(fun x y -> x + y)
|
||||||
.Collect()
|
.Collect()
|
||||||
for count in counts do
|
for (word,count) in counts do
|
||||||
printfn "%s: %d" count.Key count.Value
|
printfn "%s: %d" word count
|
||||||
with
|
with
|
||||||
| ex ->
|
| ex ->
|
||||||
logger.LogError "Error performing Word Count"
|
logger.LogError "Error performing Word Count"
|
||||||
|
|
|
@ -71,7 +71,7 @@
|
||||||
</ItemGroup>
|
</ItemGroup>
|
||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
<Reference Include="CSharpWorker">
|
<Reference Include="CSharpWorker">
|
||||||
<HintPath>..\..\packages\Microsoft.SparkCLR.2.0.0-PREVIEW-2\lib\net45\CSharpWorker.exe</HintPath>
|
<HintPath>..\..\packages\Microsoft.SparkCLR.2.0.200\lib\net45\CSharpWorker.exe</HintPath>
|
||||||
<Private>True</Private>
|
<Private>True</Private>
|
||||||
</Reference>
|
</Reference>
|
||||||
<Reference Include="FSharp.Core">
|
<Reference Include="FSharp.Core">
|
||||||
|
@ -83,7 +83,7 @@
|
||||||
<Private>True</Private>
|
<Private>True</Private>
|
||||||
</Reference>
|
</Reference>
|
||||||
<Reference Include="Microsoft.Spark.CSharp.Adapter">
|
<Reference Include="Microsoft.Spark.CSharp.Adapter">
|
||||||
<HintPath>..\..\packages\Microsoft.SparkCLR.2.0.0-PREVIEW-2\lib\net45\Microsoft.Spark.CSharp.Adapter.dll</HintPath>
|
<HintPath>..\..\packages\Microsoft.SparkCLR.2.0.200\lib\net45\Microsoft.Spark.CSharp.Adapter.dll</HintPath>
|
||||||
<Private>True</Private>
|
<Private>True</Private>
|
||||||
</Reference>
|
</Reference>
|
||||||
<Reference Include="mscorlib" />
|
<Reference Include="mscorlib" />
|
||||||
|
|
|
@ -2,7 +2,7 @@
|
||||||
<packages>
|
<packages>
|
||||||
<package id="FSharp.Core" version="4.0.0.1" targetFramework="net45" />
|
<package id="FSharp.Core" version="4.0.0.1" targetFramework="net45" />
|
||||||
<package id="log4net" version="2.0.5" targetFramework="net45" />
|
<package id="log4net" version="2.0.5" targetFramework="net45" />
|
||||||
<package id="Microsoft.SparkCLR" version="2.0.0-PREVIEW-2" targetFramework="net45" />
|
<package id="Microsoft.SparkCLR" version="2.0.200" targetFramework="net45" />
|
||||||
<package id="Newtonsoft.Json" version="7.0.1" targetFramework="net45" />
|
<package id="Newtonsoft.Json" version="7.0.1" targetFramework="net45" />
|
||||||
<package id="Razorvine.Pyrolite" version="4.10.0.0" targetFramework="net45" />
|
<package id="Razorvine.Pyrolite" version="4.10.0.0" targetFramework="net45" />
|
||||||
<package id="Razorvine.Serpent" version="1.12.0.0" targetFramework="net45" />
|
<package id="Razorvine.Serpent" version="1.12.0.0" targetFramework="net45" />
|
||||||
|
|
|
@ -42,7 +42,7 @@ if not exist "%SPARK_JARS_DIR%" (
|
||||||
|
|
||||||
set SPARK_JARS_CLASSPATH=%SPARK_JARS_DIR%\*
|
set SPARK_JARS_CLASSPATH=%SPARK_JARS_DIR%\*
|
||||||
|
|
||||||
if not defined SPARKCLR_JAR (set SPARKCLR_JAR=spark-clr_2.11-2.0.200-SNAPSHOT.jar)
|
if not defined SPARKCLR_JAR (set SPARKCLR_JAR=spark-clr_2.11-2.0.200.jar)
|
||||||
echo [sparkclr-submit.cmd] SPARKCLR_JAR=%SPARKCLR_JAR%
|
echo [sparkclr-submit.cmd] SPARKCLR_JAR=%SPARKCLR_JAR%
|
||||||
set SPARKCLR_CLASSPATH=%SPARKCLR_HOME%\lib\%SPARKCLR_JAR%
|
set SPARKCLR_CLASSPATH=%SPARKCLR_HOME%\lib\%SPARKCLR_JAR%
|
||||||
REM SPARKCLR_DEBUGMODE_EXT_JARS environment variable is used to specify external dependencies to use in debug mode
|
REM SPARKCLR_DEBUGMODE_EXT_JARS environment variable is used to specify external dependencies to use in debug mode
|
||||||
|
|
|
@ -57,7 +57,7 @@ fi
|
||||||
|
|
||||||
export SPARK_JARS_CLASSPATH="$SPARK_JARS_DIR/*"
|
export SPARK_JARS_CLASSPATH="$SPARK_JARS_DIR/*"
|
||||||
|
|
||||||
export SPARKCLR_JAR=spark-clr_2.11-2.0.200-SNAPSHOT.jar
|
export SPARKCLR_JAR=spark-clr_2.11-2.0.200.jar
|
||||||
export SPARKCLR_CLASSPATH="$SPARKCLR_HOME/lib/$SPARKCLR_JAR"
|
export SPARKCLR_CLASSPATH="$SPARKCLR_HOME/lib/$SPARKCLR_JAR"
|
||||||
# SPARKCLR_DEBUGMODE_EXT_JARS environment variable is used to specify external dependencies to use in debug mode
|
# SPARKCLR_DEBUGMODE_EXT_JARS environment variable is used to specify external dependencies to use in debug mode
|
||||||
[ ! "$SPARKCLR_DEBUGMODE_EXT_JARS" = "" ] && export SPARKCLR_CLASSPATH="$SPARKCLR_CLASSPATH:$SPARKCLR_DEBUGMODE_EXT_JARS"
|
[ ! "$SPARKCLR_DEBUGMODE_EXT_JARS" = "" ] && export SPARKCLR_CLASSPATH="$SPARKCLR_CLASSPATH:$SPARKCLR_DEBUGMODE_EXT_JARS"
|
||||||
|
|
Загрузка…
Ссылка в новой задаче