fix CheckpointRDD getPreferredLocations java.io.FileNotFoundException

This commit is contained in:
jerryshao 2013-05-24 14:26:19 +08:00
Родитель a2b0a7975c
Коммит bd3ea8f2a6
1 изменённых файлов: 1 добавлений и 1 удалений

Просмотреть файл

@ -43,7 +43,7 @@ class CheckpointRDD[T: ClassManifest](sc: SparkContext, val checkpointPath: Stri
checkpointData.get.cpFile = Some(checkpointPath)
override def getPreferredLocations(split: Partition): Seq[String] = {
val status = fs.getFileStatus(new Path(checkpointPath))
val status = fs.getFileStatus(new Path(checkpointPath, CheckpointRDD.splitIdToFile(split.index)))
val locations = fs.getFileBlockLocations(status, 0, status.getLen)
locations.headOption.toList.flatMap(_.getHosts).filter(_ != "localhost")
}