зеркало из https://github.com/microsoft/spark.git
Make ShuffledRDD.prev transient
This commit is contained in:
Родитель
ec5e553b41
Коммит
a64c107449
|
@ -29,7 +29,7 @@ private[spark] case class NarrowCoGroupSplitDep(
|
|||
private[spark] case class ShuffleCoGroupSplitDep(shuffleId: Int) extends CoGroupSplitDep
|
||||
|
||||
private[spark]
|
||||
class CoGroupPartition(idx: Int, val deps: Seq[CoGroupSplitDep])
|
||||
class CoGroupPartition(idx: Int, val deps: Array[CoGroupSplitDep])
|
||||
extends Partition with Serializable {
|
||||
override val index: Int = idx
|
||||
override def hashCode(): Int = idx
|
||||
|
@ -88,7 +88,7 @@ class CoGroupedRDD[K](
|
|||
case _ =>
|
||||
new NarrowCoGroupSplitDep(rdd, i, rdd.partitions(i))
|
||||
}
|
||||
}.toList)
|
||||
}.toArray)
|
||||
}
|
||||
array
|
||||
}
|
||||
|
|
|
@ -16,7 +16,7 @@ private[spark] class ShuffledRDDPartition(val idx: Int) extends Partition {
|
|||
* @tparam V the value class.
|
||||
*/
|
||||
class ShuffledRDD[K, V](
|
||||
prev: RDD[(K, V)],
|
||||
@transient prev: RDD[(K, V)],
|
||||
part: Partitioner)
|
||||
extends RDD[(K, V)](prev.context, List(new ShuffleDependency(prev, part))) {
|
||||
|
||||
|
|
|
@ -56,7 +56,7 @@ private[spark] class SubtractedRDD[K: ClassManifest, V: ClassManifest, W: ClassM
|
|||
case _ =>
|
||||
new NarrowCoGroupSplitDep(rdd, i, rdd.partitions(i))
|
||||
}
|
||||
}.toList)
|
||||
}.toArray)
|
||||
}
|
||||
array
|
||||
}
|
||||
|
|
Загрузка…
Ссылка в новой задаче