зеркало из https://github.com/microsoft/spark.git
Uniform whitespace across scala examples
This commit is contained in:
Родитель
adba773fab
Коммит
f1d8871ca1
|
@ -10,73 +10,73 @@ import scala.collection.mutable.HashSet
|
|||
* K-means clustering.
|
||||
*/
|
||||
object LocalKMeans {
|
||||
val N = 1000
|
||||
val R = 1000 // Scaling factor
|
||||
val D = 10
|
||||
val K = 10
|
||||
val convergeDist = 0.001
|
||||
val rand = new Random(42)
|
||||
|
||||
def generateData = {
|
||||
def generatePoint(i: Int) = {
|
||||
Vector(D, _ => rand.nextDouble * R)
|
||||
}
|
||||
Array.tabulate(N)(generatePoint)
|
||||
}
|
||||
|
||||
def closestPoint(p: Vector, centers: HashMap[Int, Vector]): Int = {
|
||||
var index = 0
|
||||
var bestIndex = 0
|
||||
var closest = Double.PositiveInfinity
|
||||
|
||||
for (i <- 1 to centers.size) {
|
||||
val vCurr = centers.get(i).get
|
||||
val tempDist = p.squaredDist(vCurr)
|
||||
if (tempDist < closest) {
|
||||
closest = tempDist
|
||||
bestIndex = i
|
||||
}
|
||||
}
|
||||
|
||||
return bestIndex
|
||||
}
|
||||
val N = 1000
|
||||
val R = 1000 // Scaling factor
|
||||
val D = 10
|
||||
val K = 10
|
||||
val convergeDist = 0.001
|
||||
val rand = new Random(42)
|
||||
|
||||
def main(args: Array[String]) {
|
||||
val data = generateData
|
||||
var points = new HashSet[Vector]
|
||||
var kPoints = new HashMap[Int, Vector]
|
||||
var tempDist = 1.0
|
||||
|
||||
while (points.size < K) {
|
||||
points.add(data(rand.nextInt(N)))
|
||||
}
|
||||
|
||||
val iter = points.iterator
|
||||
for (i <- 1 to points.size) {
|
||||
kPoints.put(i, iter.next())
|
||||
}
|
||||
def generateData = {
|
||||
def generatePoint(i: Int) = {
|
||||
Vector(D, _ => rand.nextDouble * R)
|
||||
}
|
||||
Array.tabulate(N)(generatePoint)
|
||||
}
|
||||
|
||||
println("Initial centers: " + kPoints)
|
||||
def closestPoint(p: Vector, centers: HashMap[Int, Vector]): Int = {
|
||||
var index = 0
|
||||
var bestIndex = 0
|
||||
var closest = Double.PositiveInfinity
|
||||
|
||||
while(tempDist > convergeDist) {
|
||||
var closest = data.map (p => (closestPoint(p, kPoints), (p, 1)))
|
||||
|
||||
var mappings = closest.groupBy[Int] (x => x._1)
|
||||
|
||||
var pointStats = mappings.map(pair => pair._2.reduceLeft [(Int, (Vector, Int))] {case ((id1, (x1, y1)), (id2, (x2, y2))) => (id1, (x1 + x2, y1+y2))})
|
||||
|
||||
var newPoints = pointStats.map {mapping => (mapping._1, mapping._2._1/mapping._2._2)}
|
||||
|
||||
tempDist = 0.0
|
||||
for (mapping <- newPoints) {
|
||||
tempDist += kPoints.get(mapping._1).get.squaredDist(mapping._2)
|
||||
}
|
||||
|
||||
for (newP <- newPoints) {
|
||||
kPoints.put(newP._1, newP._2)
|
||||
}
|
||||
}
|
||||
for (i <- 1 to centers.size) {
|
||||
val vCurr = centers.get(i).get
|
||||
val tempDist = p.squaredDist(vCurr)
|
||||
if (tempDist < closest) {
|
||||
closest = tempDist
|
||||
bestIndex = i
|
||||
}
|
||||
}
|
||||
|
||||
println("Final centers: " + kPoints)
|
||||
}
|
||||
return bestIndex
|
||||
}
|
||||
|
||||
def main(args: Array[String]) {
|
||||
val data = generateData
|
||||
var points = new HashSet[Vector]
|
||||
var kPoints = new HashMap[Int, Vector]
|
||||
var tempDist = 1.0
|
||||
|
||||
while (points.size < K) {
|
||||
points.add(data(rand.nextInt(N)))
|
||||
}
|
||||
|
||||
val iter = points.iterator
|
||||
for (i <- 1 to points.size) {
|
||||
kPoints.put(i, iter.next())
|
||||
}
|
||||
|
||||
println("Initial centers: " + kPoints)
|
||||
|
||||
while(tempDist > convergeDist) {
|
||||
var closest = data.map (p => (closestPoint(p, kPoints), (p, 1)))
|
||||
|
||||
var mappings = closest.groupBy[Int] (x => x._1)
|
||||
|
||||
var pointStats = mappings.map(pair => pair._2.reduceLeft [(Int, (Vector, Int))] {case ((id1, (x1, y1)), (id2, (x2, y2))) => (id1, (x1 + x2, y1+y2))})
|
||||
|
||||
var newPoints = pointStats.map {mapping => (mapping._1, mapping._2._1/mapping._2._2)}
|
||||
|
||||
tempDist = 0.0
|
||||
for (mapping <- newPoints) {
|
||||
tempDist += kPoints.get(mapping._1).get.squaredDist(mapping._2)
|
||||
}
|
||||
|
||||
for (newP <- newPoints) {
|
||||
kPoints.put(newP._1, newP._2)
|
||||
}
|
||||
}
|
||||
|
||||
println("Final centers: " + kPoints)
|
||||
}
|
||||
}
|
||||
|
|
|
@ -8,7 +8,7 @@ object MultiBroadcastTest {
|
|||
System.err.println("Usage: BroadcastTest <master> [<slices>] [numElem]")
|
||||
System.exit(1)
|
||||
}
|
||||
|
||||
|
||||
val sc = new SparkContext(args(0), "Broadcast Test",
|
||||
System.getenv("SPARK_HOME"), Seq(System.getenv("SPARK_EXAMPLES_JAR")))
|
||||
|
||||
|
@ -19,7 +19,7 @@ object MultiBroadcastTest {
|
|||
for (i <- 0 until arr1.length) {
|
||||
arr1(i) = i
|
||||
}
|
||||
|
||||
|
||||
var arr2 = new Array[Int](num)
|
||||
for (i <- 0 until arr2.length) {
|
||||
arr2(i) = i
|
||||
|
@ -30,7 +30,7 @@ object MultiBroadcastTest {
|
|||
sc.parallelize(1 to 10, slices).foreach {
|
||||
i => println(barr1.value.size + barr2.value.size)
|
||||
}
|
||||
|
||||
|
||||
System.exit(0)
|
||||
}
|
||||
}
|
||||
|
|
|
@ -11,7 +11,7 @@ object SimpleSkewedGroupByTest {
|
|||
"[numMappers] [numKVPairs] [valSize] [numReducers] [ratio]")
|
||||
System.exit(1)
|
||||
}
|
||||
|
||||
|
||||
var numMappers = if (args.length > 1) args(1).toInt else 2
|
||||
var numKVPairs = if (args.length > 2) args(2).toInt else 1000
|
||||
var valSize = if (args.length > 3) args(3).toInt else 1000
|
||||
|
@ -20,7 +20,7 @@ object SimpleSkewedGroupByTest {
|
|||
|
||||
val sc = new SparkContext(args(0), "GroupBy Test",
|
||||
System.getenv("SPARK_HOME"), Seq(System.getenv("SPARK_EXAMPLES_JAR")))
|
||||
|
||||
|
||||
val pairs1 = sc.parallelize(0 until numMappers, numMappers).flatMap { p =>
|
||||
val ranGen = new Random
|
||||
var result = new Array[(Int, Array[Byte])](numKVPairs)
|
||||
|
|
|
@ -10,7 +10,7 @@ object SkewedGroupByTest {
|
|||
System.err.println("Usage: GroupByTest <master> [numMappers] [numKVPairs] [KeySize] [numReducers]")
|
||||
System.exit(1)
|
||||
}
|
||||
|
||||
|
||||
var numMappers = if (args.length > 1) args(1).toInt else 2
|
||||
var numKVPairs = if (args.length > 2) args(2).toInt else 1000
|
||||
var valSize = if (args.length > 3) args(3).toInt else 1000
|
||||
|
@ -18,7 +18,7 @@ object SkewedGroupByTest {
|
|||
|
||||
val sc = new SparkContext(args(0), "GroupBy Test",
|
||||
System.getenv("SPARK_HOME"), Seq(System.getenv("SPARK_EXAMPLES_JAR")))
|
||||
|
||||
|
||||
val pairs1 = sc.parallelize(0 until numMappers, numMappers).flatMap { p =>
|
||||
val ranGen = new Random
|
||||
|
||||
|
|
Загрузка…
Ссылка в новой задаче