Hendrik/xgboost
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

impose shuffle when creating training RDD (#1531)

Browse Source
This commit is contained in:
Nan Zhu 2016-08-31 07:34:10 -04:00 committed by GitHub
parent 3f198b9fef
commit 7fb3fbf577
1 changed files with 1 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/XGBoost.scala
View File

@ -70,12 +70,9 @@ object XGBoost extends Serializable {
useExternalMemory: Boolean, missing: Float = Float.NaN): RDD[Booster] = { useExternalMemory: Boolean, missing: Float = Float.NaN): RDD[Booster] = {
import DataUtils._ import DataUtils._
val partitionedData = { val partitionedData = {
if (numWorkers > trainingData.partitions.length) { if (numWorkers != trainingData.partitions.length) {
logger.info(s"repartitioning training set to $numWorkers partitions") logger.info(s"repartitioning training set to $numWorkers partitions")
trainingData.repartition(numWorkers) trainingData.repartition(numWorkers)
} else if (numWorkers < trainingData.partitions.length) {
logger.info(s"repartitioning training set to $numWorkers partitions")
trainingData.coalesce(numWorkers)
} else { } else {
trainingData trainingData
} }