зеркало из https://github.com/microsoft/spark.git
Merge pull request #620 from jerryshao/master
Fix CheckpointRDD java.io.FileNotFoundException when calling getPreferredLocations
This commit is contained in:
Коммит
3db1e17baa
|
@ -43,7 +43,7 @@ class CheckpointRDD[T: ClassManifest](sc: SparkContext, val checkpointPath: Stri
|
||||||
checkpointData.get.cpFile = Some(checkpointPath)
|
checkpointData.get.cpFile = Some(checkpointPath)
|
||||||
|
|
||||||
override def getPreferredLocations(split: Partition): Seq[String] = {
|
override def getPreferredLocations(split: Partition): Seq[String] = {
|
||||||
val status = fs.getFileStatus(new Path(checkpointPath))
|
val status = fs.getFileStatus(new Path(checkpointPath, CheckpointRDD.splitIdToFile(split.index)))
|
||||||
val locations = fs.getFileBlockLocations(status, 0, status.getLen)
|
val locations = fs.getFileBlockLocations(status, 0, status.getLen)
|
||||||
locations.headOption.toList.flatMap(_.getHosts).filter(_ != "localhost")
|
locations.headOption.toList.flatMap(_.getHosts).filter(_ != "localhost")
|
||||||
}
|
}
|
||||||
|
|
Загрузка…
Ссылка в новой задаче