From 86ed01c4bbecef66e1bc4d02fb13116bd6130fae Mon Sep 17 00:00:00 2001 From: Chen Qin Date: Fri, 18 Oct 2019 12:07:31 -0700 Subject: [PATCH] [jvm-packages] update rabit, surface new changes to spark, add parity and failure tests (#4876) * Expose sets of rabit configurations to spark layer --- CMakeLists.txt | 40 ++----- jvm-packages/.gitignore | 3 +- jvm-packages/create_jni.py | 3 +- jvm-packages/pom.xml | 42 +++++++ jvm-packages/scalastyle-config.xml | 2 +- .../scala/example/BasicWalkThrough.scala | 2 +- .../scala/example/BoostFromPrediction.scala | 2 +- .../scala/example/CrossValidation.scala | 2 +- .../scala/example/CustomObjective.scala | 2 +- .../scala/example/ExternalMemory.scala | 2 +- .../example/GeneralizedLinearModel.scala | 2 +- .../scala/example/PredictFirstNTree.scala | 2 +- .../scala/example/PredictLeafIndices.scala | 2 +- .../example/flink/DistTrainWithFlink.scala | 2 +- .../example/spark/SparkMLlibPipeline.scala | 2 +- .../scala/example/spark/SparkTraining.scala | 2 +- .../scala/example/util/CustomEval.scala | 2 +- .../dmlc/xgboost4j/scala/flink/XGBoost.scala | 2 +- .../xgboost4j/scala/flink/XGBoostModel.scala | 2 +- .../scala/spark/CheckpointManager.scala | 2 +- .../xgboost4j/scala/spark/DataUtils.scala | 2 +- .../dmlc/xgboost4j/scala/spark/XGBoost.scala | 37 ++++-- .../scala/spark/XGBoostClassifier.scala | 4 +- .../scala/spark/XGBoostEstimatorCommon.scala | 4 +- .../scala/spark/XGBoostRegressor.scala | 4 +- .../scala/spark/XGBoostTrainingSummary.scala | 2 +- .../dmlc/xgboost4j/scala/spark/package.scala | 2 +- .../scala/spark/params/BoosterParams.scala | 2 +- .../scala/spark/params/CustomParams.scala | 2 +- .../params/DefaultXGBoostParamsReader.scala | 2 +- .../params/DefaultXGBoostParamsWriter.scala | 2 +- .../scala/spark/params/GeneralParams.scala | 4 +- .../scala/spark/params/InferenceParams.scala | 2 +- .../spark/params/LearningTaskParams.scala | 2 +- .../spark/params/NonParamVariables.scala | 2 +- .../scala/spark/params/RabitParams.scala | 62 ++++++++++ .../xgboost4j/scala/spark/params/Utils.scala | 2 +- .../spark/SparkParallelismTracker.scala | 2 +- .../scala/spark/CheckpointManagerSuite.scala | 2 +- .../DeterministicPartitioningSuite.scala | 2 +- .../xgboost4j/scala/spark/EvalError.scala | 2 +- .../spark/MissingValueHandlingSuite.scala | 2 +- .../scala/spark/ParameterSuite.scala | 2 +- .../dmlc/xgboost4j/scala/spark/PerTest.scala | 2 +- .../scala/spark/PersistenceSuite.scala | 2 +- .../scala/spark/RabitRobustnessSuite.scala | 2 +- .../scala/spark/TmpFolderPerSuite.scala | 2 +- .../xgboost4j/scala/spark/TrainTestData.scala | 2 +- .../scala/spark/XGBoostClassifierSuite.scala | 2 +- .../scala/spark/XGBoostConfigureSuite.scala | 11 +- .../scala/spark/XGBoostGeneralSuite.scala | 2 +- .../spark/XGBoostRabitRegressionSuite.scala | 110 ++++++++++++++++++ .../scala/spark/XGBoostRegressorSuite.scala | 2 +- .../spark/SparkParallelismTrackerSuite.scala | 2 +- jvm-packages/xgboost4j/pom.xml | 1 + .../java/ml/dmlc/xgboost4j/java/Rabit.java | 13 ++- .../ml/dmlc/xgboost4j/LabeledPoint.scala | 2 +- .../ml/dmlc/xgboost4j/scala/Booster.scala | 2 +- .../ml/dmlc/xgboost4j/scala/DMatrix.scala | 2 +- .../ml/dmlc/xgboost4j/scala/EvalTrait.scala | 2 +- .../dmlc/xgboost4j/scala/ObjectiveTrait.scala | 2 +- .../ml/dmlc/xgboost4j/scala/XGBoost.scala | 2 +- .../xgboost4j/scala/rabit/RabitTracker.scala | 2 +- .../rabit/handler/RabitTrackerHandler.scala | 2 +- .../rabit/handler/RabitWorkerHandler.scala | 2 +- .../xgboost4j/scala/rabit/util/LinkMap.scala | 2 +- .../rabit/util/RabitTrackerHelpers.scala | 2 +- .../dmlc/xgboost4j/scala/DMatrixSuite.scala | 2 +- .../scala/ScalaBoosterImplSuite.scala | 2 +- .../RabitTrackerConnectionHandlerTest.scala | 2 +- rabit | 2 +- tests/ci_build/build_jvm_packages.sh | 2 +- tests/travis/run_test.sh | 4 +- 73 files changed, 343 insertions(+), 115 deletions(-) create mode 100644 jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/params/RabitParams.scala create mode 100644 jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/XGBoostRabitRegressionSuite.scala diff --git a/CMakeLists.txt b/CMakeLists.txt index ceb44a597..5bd6b4f63 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -94,36 +94,16 @@ set_target_properties(dmlc PROPERTIES list(APPEND LINKED_LIBRARIES_PRIVATE dmlc) # rabit -# full rabit doesn't build on windows, so we can't import it as subdirectory -if(MINGW OR R_LIB OR WIN32) - set(RABIT_SOURCES - rabit/src/engine_empty.cc - rabit/src/c_api.cc) -else () - if(RABIT_MOCK) - set(RABIT_SOURCES - rabit/src/allreduce_base.cc - rabit/src/allreduce_robust.cc - rabit/src/engine_mock.cc - rabit/src/c_api.cc) - else() - set(RABIT_SOURCES - rabit/src/allreduce_base.cc - rabit/src/allreduce_robust.cc - rabit/src/engine.cc - rabit/src/c_api.cc) - endif(RABIT_MOCK) -endif (MINGW OR R_LIB OR WIN32) -add_library(rabit STATIC ${RABIT_SOURCES}) -target_include_directories(rabit PRIVATE - $ - $) -set_target_properties(rabit - PROPERTIES - CXX_STANDARD 11 - CXX_STANDARD_REQUIRED ON - POSITION_INDEPENDENT_CODE ON) -list(APPEND LINKED_LIBRARIES_PRIVATE rabit) +set(RABIT_BUILD_DMLC OFF) +set(DMLC_ROOT ${xgboost_SOURCE_DIR}/dmlc-core) +set(RABIT_WITH_R_LIB ${R_LIB}) +add_subdirectory(rabit) + +if (RABIT_MOCK) + list(APPEND LINKED_LIBRARIES_PRIVATE rabit_mock_static) +else() + list(APPEND LINKED_LIBRARIES_PRIVATE rabit) +endif(RABIT_MOCK) # Exports some R specific definitions and objects if (R_LIB) diff --git a/jvm-packages/.gitignore b/jvm-packages/.gitignore index d1d4390d6..becd9e300 100644 --- a/jvm-packages/.gitignore +++ b/jvm-packages/.gitignore @@ -1,2 +1,3 @@ tracker.py -build.sh \ No newline at end of file +build.sh + diff --git a/jvm-packages/create_jni.py b/jvm-packages/create_jni.py index 4d627cb7c..0357b3a43 100755 --- a/jvm-packages/create_jni.py +++ b/jvm-packages/create_jni.py @@ -18,7 +18,7 @@ CONFIG = { "USE_HDFS": "OFF", "USE_AZURE": "OFF", "USE_S3": "OFF", - + "RABIT_MOCK": "OFF", "USE_CUDA": "OFF", "JVM_BINDINGS": "ON" } @@ -68,6 +68,7 @@ def normpath(path): if __name__ == "__main__": + CONFIG["RABIT_MOCK"] = str(sys.argv[1]) if sys.platform == "darwin": # Enable of your compiler supports OpenMP. CONFIG["USE_OPENMP"] = "OFF" diff --git a/jvm-packages/pom.xml b/jvm-packages/pom.xml index 628f7ca7a..b37547e1c 100644 --- a/jvm-packages/pom.xml +++ b/jvm-packages/pom.xml @@ -37,6 +37,7 @@ 2.4.3 2.12.8 2.12 + OFF @@ -52,6 +53,47 @@ xgboost4j-flink + + dev + + ON + + + + + org.apache.maven.plugins + maven-jar-plugin + 3.0.2 + + + empty-javadoc-jar + package + + jar + + + javadoc + ${basedir}/javadoc + + + + + + org.apache.maven.plugins + maven-source-plugin + 2.2.1 + + + attach-sources + + jar-no-fork + + + + + + + release diff --git a/jvm-packages/scalastyle-config.xml b/jvm-packages/scalastyle-config.xml index fd4d22e9a..310471926 100644 --- a/jvm-packages/scalastyle-config.xml +++ b/jvm-packages/scalastyle-config.xml @@ -49,7 +49,7 @@ This file is divided into 3 sections: PyRabitTracker} import ml.dmlc.xgboost4j.scala.rabit.RabitTracker import ml.dmlc.xgboost4j.scala.spark.CheckpointManager.CheckpointParam -import ml.dmlc.xgboost4j.scala.spark.XGBoost.logger import ml.dmlc.xgboost4j.scala.spark.params.LearningTaskParams import ml.dmlc.xgboost4j.scala.{XGBoost => SXGBoost, _} import ml.dmlc.xgboost4j.{LabeledPoint => XGBLabeledPoint} @@ -155,7 +155,7 @@ private[this] class XGBoostExecutionParamsFactory(rawParams: Map[String, Any], s overridedParams } - def buildXGBRuntimeParams: XGBoostExecutionParams = { + private[spark] def buildXGBRuntimeParams: XGBoostExecutionParams = { val nWorkers = overridedParams("num_workers").asInstanceOf[Int] val round = overridedParams("num_round").asInstanceOf[Int] val useExternalMemory = overridedParams("use_external_memory").asInstanceOf[Boolean] @@ -221,6 +221,25 @@ private[this] class XGBoostExecutionParamsFactory(rawParams: Map[String, Any], s xgbExecParam.setRawParamMap(overridedParams) xgbExecParam } + + private[spark] def buildRabitParams: Map[String, String] = Map( + "rabit_reduce_ring_mincount" -> + overridedParams.getOrElse("rabit_reduce_ring_mincount", 32<<10).toString, + "rabit_reduce_buffer" -> + overridedParams.getOrElse("rabit_reduce_buffer", "256MB").toString, + "rabit_bootstrap_cache" -> + overridedParams.getOrElse("rabit_bootstrap_cache", false).toString, + "rabit_debug" -> + overridedParams.getOrElse("rabit_debug", false).toString, + "rabit_timeout" -> + overridedParams.getOrElse("rabit_timeout", false).toString, + "rabit_timeout_sec" -> + overridedParams.getOrElse("rabit_timeout_sec", 1800).toString, + "DMLC_WORKER_CONNECT_RETRY" -> + overridedParams.getOrElse("dmlc_worker_connect_retry", 5).toString, + "DMLC_WORKER_STOP_PROCESS_ON_ERROR" -> + overridedParams.getOrElse("dmlc_worker_stop_process_on_error", false).toString + ) } /** @@ -321,7 +340,6 @@ object XGBoost extends Serializable { } val taskId = TaskContext.getPartitionId().toString rabitEnv.put("DMLC_TASK_ID", taskId) - rabitEnv.put("DMLC_WORKER_STOP_PROCESS_ON_ERROR", "false") try { Rabit.init(rabitEnv) @@ -490,8 +508,9 @@ object XGBoost extends Serializable { evalSetsMap: Map[String, RDD[XGBLabeledPoint]] = Map()): (Booster, Map[String, Array[Float]]) = { logger.info(s"Running XGBoost ${spark.VERSION} with parameters:\n${params.mkString("\n")}") - val xgbExecParams = new XGBoostExecutionParamsFactory(params, trainingData.sparkContext). - buildXGBRuntimeParams + val xgbParamsFactory = new XGBoostExecutionParamsFactory(params, trainingData.sparkContext) + val xgbExecParams = xgbParamsFactory.buildXGBRuntimeParams + val xgbRabitParams = xgbParamsFactory.buildRabitParams val sc = trainingData.sparkContext val checkpointManager = new CheckpointManager(sc, xgbExecParams.checkpointParam. checkpointPath) @@ -510,12 +529,12 @@ object XGBoost extends Serializable { val parallelismTracker = new SparkParallelismTracker(sc, xgbExecParams.timeoutRequestWorkers, xgbExecParams.numWorkers) - val rabitEnv = tracker.getWorkerEnvs + val rabitEnv = tracker.getWorkerEnvs.asScala ++ xgbRabitParams val boostersAndMetrics = if (hasGroup) { - trainForRanking(transformedTrainingData.left.get, xgbExecParams, rabitEnv, + trainForRanking(transformedTrainingData.left.get, xgbExecParams, rabitEnv.asJava, checkpointRound, prevBooster, evalSetsMap) } else { - trainForNonRanking(transformedTrainingData.right.get, xgbExecParams, rabitEnv, + trainForNonRanking(transformedTrainingData.right.get, xgbExecParams, rabitEnv.asJava, checkpointRound, prevBooster, evalSetsMap) } val sparkJobThread = new Thread() { diff --git a/jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/XGBoostClassifier.scala b/jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/XGBoostClassifier.scala index db4936430..5c59b0e40 100644 --- a/jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/XGBoostClassifier.scala +++ b/jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/XGBoostClassifier.scala @@ -1,5 +1,5 @@ /* - Copyright (c) 2014 by Contributors + Copyright (c) 2014 - 2019 by Contributors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -50,7 +50,7 @@ class XGBoostClassifier ( def this(xgboostParams: Map[String, Any]) = this( Identifiable.randomUID("xgbc"), xgboostParams) - XGBoostToMLlibParams(xgboostParams) + XGBoost2MLlibParams(xgboostParams) def setWeightCol(value: String): this.type = set(weightCol, value) diff --git a/jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/XGBoostEstimatorCommon.scala b/jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/XGBoostEstimatorCommon.scala index 1213a8f72..39e4d000e 100644 --- a/jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/XGBoostEstimatorCommon.scala +++ b/jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/XGBoostEstimatorCommon.scala @@ -1,5 +1,5 @@ /* - Copyright (c) 2014 by Contributors + Copyright (c) 2014 - 2019 by Contributors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -21,7 +21,7 @@ import ml.dmlc.xgboost4j.scala.spark.params._ import org.apache.spark.ml.param.shared.HasWeightCol private[spark] sealed trait XGBoostEstimatorCommon extends GeneralParams with LearningTaskParams - with BoosterParams with ParamMapFuncs with NonParamVariables { + with BoosterParams with RabitParams with ParamMapFuncs with NonParamVariables { def needDeterministicRepartitioning: Boolean = { getCheckpointPath.nonEmpty && getCheckpointInterval > 0 diff --git a/jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/XGBoostRegressor.scala b/jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/XGBoostRegressor.scala index e2f22c7af..6006bb4ce 100644 --- a/jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/XGBoostRegressor.scala +++ b/jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/XGBoostRegressor.scala @@ -1,5 +1,5 @@ /* - Copyright (c) 2014 by Contributors + Copyright (c) 2014 - 2019 by Contributors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -54,7 +54,7 @@ class XGBoostRegressor ( def this(xgboostParams: Map[String, Any]) = this( Identifiable.randomUID("xgbr"), xgboostParams) - XGBoostToMLlibParams(xgboostParams) + XGBoost2MLlibParams(xgboostParams) def setWeightCol(value: String): this.type = set(weightCol, value) diff --git a/jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/XGBoostTrainingSummary.scala b/jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/XGBoostTrainingSummary.scala index 9454befc2..6e0d04e90 100644 --- a/jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/XGBoostTrainingSummary.scala +++ b/jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/XGBoostTrainingSummary.scala @@ -1,5 +1,5 @@ /* - Copyright (c) 2014 by Contributors + Copyright (c) 2014 - 2019 by Contributors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/package.scala b/jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/package.scala index df5e3bcc0..410a75856 100644 --- a/jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/package.scala +++ b/jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/package.scala @@ -1,5 +1,5 @@ /* - Copyright (c) 2014 by Contributors + Copyright (c) 2014 - 2019 by Contributors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/params/BoosterParams.scala b/jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/params/BoosterParams.scala index 6ec588d4b..5048ce2b3 100644 --- a/jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/params/BoosterParams.scala +++ b/jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/params/BoosterParams.scala @@ -1,5 +1,5 @@ /* - Copyright (c) 2014 by Contributors + Copyright (c) 2014 - 2019 by Contributors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/params/CustomParams.scala b/jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/params/CustomParams.scala index b7f696a23..47bb89577 100644 --- a/jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/params/CustomParams.scala +++ b/jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/params/CustomParams.scala @@ -1,5 +1,5 @@ /* - Copyright (c) 2014 by Contributors + Copyright (c) 2014 - 2019 by Contributors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/params/DefaultXGBoostParamsReader.scala b/jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/params/DefaultXGBoostParamsReader.scala index bb75bb342..3bd5b4bb7 100644 --- a/jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/params/DefaultXGBoostParamsReader.scala +++ b/jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/params/DefaultXGBoostParamsReader.scala @@ -1,5 +1,5 @@ /* - Copyright (c) 2014 by Contributors + Copyright (c) 2014 - 2019 by Contributors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/params/DefaultXGBoostParamsWriter.scala b/jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/params/DefaultXGBoostParamsWriter.scala index acf6815ec..a38f9400e 100644 --- a/jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/params/DefaultXGBoostParamsWriter.scala +++ b/jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/params/DefaultXGBoostParamsWriter.scala @@ -1,5 +1,5 @@ /* - Copyright (c) 2014 by Contributors + Copyright (c) 2014 - 2019 by Contributors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/params/GeneralParams.scala b/jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/params/GeneralParams.scala index 076dff42e..d90e137c4 100644 --- a/jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/params/GeneralParams.scala +++ b/jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/params/GeneralParams.scala @@ -1,5 +1,5 @@ /* - Copyright (c) 2014 by Contributors + Copyright (c) 2014 - 2019 by Contributors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -241,7 +241,7 @@ trait HasNumClass extends Params { private[spark] trait ParamMapFuncs extends Params { - def XGBoostToMLlibParams(xgboostParams: Map[String, Any]): Unit = { + def XGBoost2MLlibParams(xgboostParams: Map[String, Any]): Unit = { for ((paramName, paramValue) <- xgboostParams) { if ((paramName == "booster" && paramValue != "gbtree") || (paramName == "updater" && paramValue != "grow_histmaker,prune" && diff --git a/jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/params/InferenceParams.scala b/jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/params/InferenceParams.scala index abfe777d3..bf5eb415d 100644 --- a/jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/params/InferenceParams.scala +++ b/jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/params/InferenceParams.scala @@ -1,5 +1,5 @@ /* - Copyright (c) 2014 by Contributors + Copyright (c) 2014 - 2019 by Contributors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/params/LearningTaskParams.scala b/jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/params/LearningTaskParams.scala index 1512c85d0..af2561840 100644 --- a/jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/params/LearningTaskParams.scala +++ b/jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/params/LearningTaskParams.scala @@ -1,5 +1,5 @@ /* - Copyright (c) 2014 by Contributors + Copyright (c) 2014 - 2019 by Contributors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/params/NonParamVariables.scala b/jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/params/NonParamVariables.scala index 276a938e0..76837074e 100644 --- a/jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/params/NonParamVariables.scala +++ b/jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/params/NonParamVariables.scala @@ -1,5 +1,5 @@ /* - Copyright (c) 2014 by Contributors + Copyright (c) 2014 - 2019 by Contributors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/params/RabitParams.scala b/jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/params/RabitParams.scala new file mode 100644 index 000000000..b48afb74e --- /dev/null +++ b/jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/params/RabitParams.scala @@ -0,0 +1,62 @@ +/* + Copyright (c) 2014 - 2019 by Contributors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +package ml.dmlc.xgboost4j.scala.spark.params + +import org.apache.spark.ml.param._ + +private[spark] trait RabitParams extends Params { + /** + * Rabit worker configurations. These parameters were passed to Rabit.Init and decide + * rabit_reduce_ring_mincount - threshold of enable ring based allreduce/broadcast operations. + * rabit_reduce_buffer - buffer size to recv and run reduction + * rabit_bootstrap_cache - enable save allreduce cache before loadcheckpoint + * rabit_debug - enable more verbose rabit logging to stdout + * rabit_timeout - enable sidecar thread after rabit observed failures + * rabit_timeout_sec - wait interval before exit after rabit observed failures + * dmlc_worker_connect_retry - number of retrys to tracker + * dmlc_worker_stop_process_on_error - exit process when rabit see assert/error + */ + final val ringReduceMin = new IntParam(this, "rabitReduceRingMincount", + "minimal counts of enable allreduce/broadcast with ring based topology", + ParamValidators.gtEq(1)) + + final def reduceBuffer: Param[String] = new Param[String](this, "rabitReduceBuffer", + "buffer size (MB/GB) allocated to each xgb trainner recv and run reduction", + (buf: String) => buf.contains("MB") || buf.contains("GB")) + + final def bootstrapCache: BooleanParam = new BooleanParam(this, "rabitBootstrapCache", + "enable save allreduce cache before loadcheckpoint, used to allow failed task retry") + + final def rabitDebug: BooleanParam = new BooleanParam(this, "rabitDebug", + "enable more verbose rabit logging to stdout") + + final def rabitTimeout: BooleanParam = new BooleanParam(this, "rabitTimeout", + "enable failure timeout sidecar threads") + + final def timeoutInterval: IntParam = new IntParam(this, "rabitTimeoutSec", + "timeout threshold after rabit observed failures", (interval: Int) => interval > 0) + + final def connectRetry: IntParam = new IntParam(this, "dmlcWorkerConnectRetry", + "number of retry worker do before fail", ParamValidators.gtEq(1)) + + final def exitOnError: BooleanParam = new BooleanParam(this, "dmlcWorkerStopProcessOnError", + "exit process when rabit see assert error") + + setDefault(ringReduceMin -> (32 << 10), reduceBuffer -> "256MB", bootstrapCache -> false, + rabitDebug -> false, connectRetry -> 5, rabitTimeout -> false, timeoutInterval -> 1800, + exitOnError -> false) +} diff --git a/jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/params/Utils.scala b/jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/params/Utils.scala index 7d6e7b9ed..b1c5e6c5e 100644 --- a/jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/params/Utils.scala +++ b/jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/params/Utils.scala @@ -1,5 +1,5 @@ /* - Copyright (c) 2014 by Contributors + Copyright (c) 2014 - 2019 by Contributors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/jvm-packages/xgboost4j-spark/src/main/scala/org/apache/spark/SparkParallelismTracker.scala b/jvm-packages/xgboost4j-spark/src/main/scala/org/apache/spark/SparkParallelismTracker.scala index bd0ab4d6d..c17a61be9 100644 --- a/jvm-packages/xgboost4j-spark/src/main/scala/org/apache/spark/SparkParallelismTracker.scala +++ b/jvm-packages/xgboost4j-spark/src/main/scala/org/apache/spark/SparkParallelismTracker.scala @@ -1,5 +1,5 @@ /* - Copyright (c) 2014 by Contributors + Copyright (c) 2014 - 2019 by Contributors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/CheckpointManagerSuite.scala b/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/CheckpointManagerSuite.scala index ddeb48241..1b35763c0 100755 --- a/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/CheckpointManagerSuite.scala +++ b/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/CheckpointManagerSuite.scala @@ -1,5 +1,5 @@ /* - Copyright (c) 2014 by Contributors + Copyright (c) 2014 - 2019 by Contributors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/DeterministicPartitioningSuite.scala b/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/DeterministicPartitioningSuite.scala index 986b0843b..62229ee69 100644 --- a/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/DeterministicPartitioningSuite.scala +++ b/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/DeterministicPartitioningSuite.scala @@ -1,5 +1,5 @@ /* - Copyright (c) 2014 by Contributors + Copyright (c) 2014 - 2019 by Contributors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/EvalError.scala b/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/EvalError.scala index 91a840911..7ec0826f4 100644 --- a/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/EvalError.scala +++ b/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/EvalError.scala @@ -1,5 +1,5 @@ /* - Copyright (c) 2014 by Contributors + Copyright (c) 2014 - 2019 by Contributors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/MissingValueHandlingSuite.scala b/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/MissingValueHandlingSuite.scala index e09f10a4c..28c34c32b 100644 --- a/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/MissingValueHandlingSuite.scala +++ b/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/MissingValueHandlingSuite.scala @@ -1,5 +1,5 @@ /* - Copyright (c) 2014 by Contributors + Copyright (c) 2014 - 2019 by Contributors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/ParameterSuite.scala b/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/ParameterSuite.scala index e0450f90d..92f92dda1 100644 --- a/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/ParameterSuite.scala +++ b/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/ParameterSuite.scala @@ -1,5 +1,5 @@ /* - Copyright (c) 2014 by Contributors + Copyright (c) 2014 - 2019 by Contributors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/PerTest.scala b/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/PerTest.scala index 341db97bc..9da916a1f 100644 --- a/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/PerTest.scala +++ b/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/PerTest.scala @@ -1,5 +1,5 @@ /* - Copyright (c) 2014 by Contributors + Copyright (c) 2014 - 2019 by Contributors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/PersistenceSuite.scala b/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/PersistenceSuite.scala index a3841772c..3cf889b5c 100755 --- a/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/PersistenceSuite.scala +++ b/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/PersistenceSuite.scala @@ -1,5 +1,5 @@ /* - Copyright (c) 2014 by Contributors + Copyright (c) 2014 - 2019 by Contributors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/RabitRobustnessSuite.scala b/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/RabitRobustnessSuite.scala index 9655104c3..eca7ffad7 100644 --- a/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/RabitRobustnessSuite.scala +++ b/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/RabitRobustnessSuite.scala @@ -1,5 +1,5 @@ /* - Copyright (c) 2014 by Contributors + Copyright (c) 2014 - 2019 by Contributors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/TmpFolderPerSuite.scala b/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/TmpFolderPerSuite.scala index 96b74d679..2a9f52bd6 100755 --- a/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/TmpFolderPerSuite.scala +++ b/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/TmpFolderPerSuite.scala @@ -1,5 +1,5 @@ /* - Copyright (c) 2014 by Contributors + Copyright (c) 2014 - 2019 by Contributors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/TrainTestData.scala b/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/TrainTestData.scala index 1a64e2d03..2737c341a 100644 --- a/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/TrainTestData.scala +++ b/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/TrainTestData.scala @@ -1,5 +1,5 @@ /* - Copyright (c) 2014 by Contributors + Copyright (c) 2014 - 2019 by Contributors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/XGBoostClassifierSuite.scala b/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/XGBoostClassifierSuite.scala index f2ccdd44e..e930df9e5 100644 --- a/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/XGBoostClassifierSuite.scala +++ b/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/XGBoostClassifierSuite.scala @@ -1,5 +1,5 @@ /* - Copyright (c) 2014 by Contributors + Copyright (c) 2014 - 2019 by Contributors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/XGBoostConfigureSuite.scala b/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/XGBoostConfigureSuite.scala index fe16bcda5..9e777a9d8 100644 --- a/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/XGBoostConfigureSuite.scala +++ b/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/XGBoostConfigureSuite.scala @@ -1,5 +1,5 @@ /* - Copyright (c) 2014 by Contributors + Copyright (c) 2014 - 2019 by Contributors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -16,7 +16,10 @@ package ml.dmlc.xgboost4j.scala.spark +import ml.dmlc.xgboost4j.java.Rabit import ml.dmlc.xgboost4j.scala.{Booster, DMatrix} + +import scala.collection.JavaConverters._ import org.apache.spark.sql._ import org.scalatest.FunSuite @@ -28,7 +31,7 @@ class XGBoostConfigureSuite extends FunSuite with PerTest { test("nthread configuration must be no larger than spark.task.cpus") { val training = buildDataFrame(Classification.train) - val paramMap = Map("eta" -> "1", "max_depth" -> "2", "silent" -> "1", + val paramMap = Map("eta" -> "1", "max_depth" -> "2", "verbosity" -> "1", "objective" -> "binary:logistic", "num_workers" -> numWorkers, "nthread" -> (sc.getConf.getInt("spark.task.cpus", 1) + 1)) intercept[IllegalArgumentException] { @@ -40,7 +43,7 @@ class XGBoostConfigureSuite extends FunSuite with PerTest { // TODO write an isolated test for Booster. val training = buildDataFrame(Classification.train) val testDM = new DMatrix(Classification.test.iterator, null) - val paramMap = Map("eta" -> "1", "max_depth" -> "2", "silent" -> "1", + val paramMap = Map("eta" -> "1", "max_depth" -> "2", "verbosity" -> "1", "objective" -> "binary:logistic", "num_round" -> 5, "num_workers" -> numWorkers) val model = new XGBoostClassifier(paramMap).fit(training) @@ -52,7 +55,7 @@ class XGBoostConfigureSuite extends FunSuite with PerTest { val originalSslConfOpt = ss.conf.getOption("spark.ssl.enabled") ss.conf.set("spark.ssl.enabled", true) - val paramMap = Map("eta" -> "1", "max_depth" -> "2", "silent" -> "1", + val paramMap = Map("eta" -> "1", "max_depth" -> "2", "verbosity" -> "1", "objective" -> "binary:logistic", "num_round" -> 2, "num_workers" -> numWorkers) val training = buildDataFrame(Classification.train) diff --git a/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/XGBoostGeneralSuite.scala b/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/XGBoostGeneralSuite.scala index f3492b2e3..383edb72f 100755 --- a/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/XGBoostGeneralSuite.scala +++ b/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/XGBoostGeneralSuite.scala @@ -1,5 +1,5 @@ /* - Copyright (c) 2014 by Contributors + Copyright (c) 2014 - 2019 by Contributors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/XGBoostRabitRegressionSuite.scala b/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/XGBoostRabitRegressionSuite.scala new file mode 100644 index 000000000..a092afd7f --- /dev/null +++ b/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/XGBoostRabitRegressionSuite.scala @@ -0,0 +1,110 @@ +/* + Copyright (c) 2014 - 2019 by Contributors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +package ml.dmlc.xgboost4j.scala.spark + +import ml.dmlc.xgboost4j.java.{Rabit, XGBoostError} +import ml.dmlc.xgboost4j.scala.{Booster, DMatrix} + +import scala.collection.JavaConverters._ +import org.apache.spark.sql._ +import org.scalatest.FunSuite + +class XGBoostRabitRegressionSuite extends FunSuite with PerTest { + override def sparkSessionBuilder: SparkSession.Builder = super.sparkSessionBuilder + .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer") + .config("spark.kryo.classesToRegister", classOf[Booster].getName) + + test("test parity classification prediction") { + val training = buildDataFrame(Classification.train) + val testDF = buildDataFrame(Classification.test) + + val model1 = new XGBoostClassifier(Map("eta" -> "1", "max_depth" -> "2", "verbosity" -> "1", + "objective" -> "binary:logistic", "num_round" -> 5, "num_workers" -> numWorkers) + ).fit(training) + val prediction1 = model1.transform(testDF).select("prediction").collect() + + val model2 = new XGBoostClassifier(Map("eta" -> "1", "max_depth" -> "2", "verbosity" -> "1", + "objective" -> "binary:logistic", "num_round" -> 5, "num_workers" -> numWorkers, + "rabit_bootstrap_cache" -> true, "rabit_debug" -> true, "rabit_reduce_ring_mincount" -> 100, + "rabit_reduce_buffer" -> "2MB", "DMLC_WORKER_CONNECT_RETRY" -> 1, + "rabit_timeout" -> true, "rabit_timeout_sec" -> 5)).fit(training) + + assert(Rabit.rabitEnvs.asScala.size > 7) + Rabit.rabitEnvs.asScala.foreach( item => { + if (item._1.toString == "rabit_bootstrap_cache") assert(item._2 == "true") + if (item._1.toString == "rabit_debug") assert(item._2 == "true") + if (item._1.toString == "rabit_reduce_ring_mincount") assert(item._2 == "100") + if (item._1.toString == "rabit_reduce_buffer") assert(item._2 == "2MB") + if (item._1.toString == "dmlc_worker_connect_retry") assert(item._2 == "1") + if (item._1.toString == "rabit_timeout") assert(item._2 == "true") + if (item._1.toString == "rabit_timeout_sec") assert(item._2 == "5") + }) + + val prediction2 = model2.transform(testDF).select("prediction").collect() + // check parity w/o rabit cache + prediction1.zip(prediction2).foreach { case (Row(p1: Double), Row(p2: Double)) => + assert(p1 == p2) + } + } + + test("test parity regression prediction") { + val training = buildDataFrame(Regression.train) + val testDM = new DMatrix(Regression.test.iterator, null) + val testDF = buildDataFrame(Classification.test) + + val model1 = new XGBoostRegressor(Map("eta" -> "1", "max_depth" -> "2", "verbosity" -> "1", + "objective" -> "reg:squarederror", "num_round" -> 5, "num_workers" -> numWorkers) + ).fit(training) + val prediction1 = model1.transform(testDF).select("prediction").collect() + + val model2 = new XGBoostRegressor(Map("eta" -> "1", "max_depth" -> "2", "verbosity" -> "1", + "objective" -> "reg:squarederror", "num_round" -> 5, "num_workers" -> numWorkers, + "rabit_bootstrap_cache" -> true, "rabit_debug" -> true, "rabit_reduce_ring_mincount" -> 100, + "rabit_reduce_buffer" -> "2MB", "DMLC_WORKER_CONNECT_RETRY" -> 1, + "rabit_timeout" -> true, "rabit_timeout_sec" -> 5)).fit(training) + assert(Rabit.rabitEnvs.asScala.size > 7) + Rabit.rabitEnvs.asScala.foreach( item => { + if (item._1.toString == "rabit_bootstrap_cache") assert(item._2 == "true") + if (item._1.toString == "rabit_debug") assert(item._2 == "true") + if (item._1.toString == "rabit_reduce_ring_mincount") assert(item._2 == "100") + if (item._1.toString == "rabit_reduce_buffer") assert(item._2 == "2MB") + if (item._1.toString == "dmlc_worker_connect_retry") assert(item._2 == "true") + if (item._1.toString == "rabit_timeout") assert(item._2 == "true") + if (item._1.toString == "rabit_timeout_sec") assert(item._2 == "5") + if (item._1.toString == "DMLC_WORKER_STOP_PROCESS_ON_ERROR") assert(item._2 == "false") + }) + // check the equality of single instance prediction + val prediction2 = model2.transform(testDF).select("prediction").collect() + // check parity w/o rabit cache + prediction1.zip(prediction2).foreach { case (Row(p1: Double), Row(p2: Double)) => + assert(math.abs(p1 - p2) < 0.00001f) + } + } + + test("test graceful failure handle") { + val training = buildDataFrame(Classification.train) + val testDF = buildDataFrame(Classification.test) + // mock rank 0 failure during 4th allreduce synchronization + Rabit.mockList = Array("0,4,0,0").toList.asJava + intercept[XGBoostError] { + new XGBoostClassifier(Map("eta" -> "1", "max_depth" -> "2", "verbosity" -> "1", + "objective" -> "binary:logistic", "num_round" -> 5, "num_workers" -> numWorkers, + "rabit_timeout" -> true, "rabit_timeout_sec" -> 1, + "DMLC_WORKER_STOP_PROCESS_ON_ERROR" -> false)).fit(training) + } + } +} diff --git a/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/XGBoostRegressorSuite.scala b/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/XGBoostRegressorSuite.scala index ff88ff328..5105335be 100644 --- a/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/XGBoostRegressorSuite.scala +++ b/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/XGBoostRegressorSuite.scala @@ -1,5 +1,5 @@ /* - Copyright (c) 2014 by Contributors + Copyright (c) 2014 - 2019 by Contributors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/jvm-packages/xgboost4j-spark/src/test/scala/org/apache/spark/SparkParallelismTrackerSuite.scala b/jvm-packages/xgboost4j-spark/src/test/scala/org/apache/spark/SparkParallelismTrackerSuite.scala index 7f344674f..92cbca23c 100644 --- a/jvm-packages/xgboost4j-spark/src/test/scala/org/apache/spark/SparkParallelismTrackerSuite.scala +++ b/jvm-packages/xgboost4j-spark/src/test/scala/org/apache/spark/SparkParallelismTrackerSuite.scala @@ -1,5 +1,5 @@ /* - Copyright (c) 2014 by Contributors + Copyright (c) 2014 - 2019 by Contributors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/jvm-packages/xgboost4j/pom.xml b/jvm-packages/xgboost4j/pom.xml index 4cc2ab20a..805e275f3 100644 --- a/jvm-packages/xgboost4j/pom.xml +++ b/jvm-packages/xgboost4j/pom.xml @@ -66,6 +66,7 @@ python create_jni.py + ${rabit.mock} ${user.dir} diff --git a/jvm-packages/xgboost4j/src/main/java/ml/dmlc/xgboost4j/java/Rabit.java b/jvm-packages/xgboost4j/src/main/java/ml/dmlc/xgboost4j/java/Rabit.java index 35b500757..7e019dc65 100644 --- a/jvm-packages/xgboost4j/src/main/java/ml/dmlc/xgboost4j/java/Rabit.java +++ b/jvm-packages/xgboost4j/src/main/java/ml/dmlc/xgboost4j/java/Rabit.java @@ -3,6 +3,8 @@ package ml.dmlc.xgboost4j.java; import java.io.Serializable; import java.nio.ByteBuffer; import java.nio.ByteOrder; +import java.util.LinkedList; +import java.util.List; import java.util.Map; /** @@ -51,18 +53,25 @@ public class Rabit { throw new XGBoostError(XGBoostJNI.XGBGetLastError()); } } - + // used as way to test/debug passed rabit init parameters + public static Map rabitEnvs; + public static List mockList = new LinkedList<>(); /** * Initialize the rabit library on current working thread. * @param envs The additional environment variables to pass to rabit. * @throws XGBoostError */ public static void init(Map envs) throws XGBoostError { - String[] args = new String[envs.size()]; + rabitEnvs = envs; + String[] args = new String[envs.size() + mockList.size()]; int idx = 0; for (java.util.Map.Entry e : envs.entrySet()) { args[idx++] = e.getKey() + '=' + e.getValue(); } + // pass list of rabit mock strings eg mock=0,1,0,0 + for(String mock : mockList) { + args[idx++] = "mock=" + mock; + } checkCall(XGBoostJNI.RabitInit(args)); } diff --git a/jvm-packages/xgboost4j/src/main/scala/ml/dmlc/xgboost4j/LabeledPoint.scala b/jvm-packages/xgboost4j/src/main/scala/ml/dmlc/xgboost4j/LabeledPoint.scala index 9a92d1b91..fbeb5e0e1 100644 --- a/jvm-packages/xgboost4j/src/main/scala/ml/dmlc/xgboost4j/LabeledPoint.scala +++ b/jvm-packages/xgboost4j/src/main/scala/ml/dmlc/xgboost4j/LabeledPoint.scala @@ -1,5 +1,5 @@ /* - Copyright (c) 2014 by Contributors + Copyright (c) 2014 - 2019 by Contributors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/jvm-packages/xgboost4j/src/main/scala/ml/dmlc/xgboost4j/scala/Booster.scala b/jvm-packages/xgboost4j/src/main/scala/ml/dmlc/xgboost4j/scala/Booster.scala index bb2d5e9e5..43faf0553 100644 --- a/jvm-packages/xgboost4j/src/main/scala/ml/dmlc/xgboost4j/scala/Booster.scala +++ b/jvm-packages/xgboost4j/src/main/scala/ml/dmlc/xgboost4j/scala/Booster.scala @@ -1,5 +1,5 @@ /* - Copyright (c) 2014 by Contributors + Copyright (c) 2014 - 2019 by Contributors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/jvm-packages/xgboost4j/src/main/scala/ml/dmlc/xgboost4j/scala/DMatrix.scala b/jvm-packages/xgboost4j/src/main/scala/ml/dmlc/xgboost4j/scala/DMatrix.scala index 629a39dbf..b30b4fe03 100644 --- a/jvm-packages/xgboost4j/src/main/scala/ml/dmlc/xgboost4j/scala/DMatrix.scala +++ b/jvm-packages/xgboost4j/src/main/scala/ml/dmlc/xgboost4j/scala/DMatrix.scala @@ -1,5 +1,5 @@ /* - Copyright (c) 2014 by Contributors + Copyright (c) 2014 - 2019 by Contributors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/jvm-packages/xgboost4j/src/main/scala/ml/dmlc/xgboost4j/scala/EvalTrait.scala b/jvm-packages/xgboost4j/src/main/scala/ml/dmlc/xgboost4j/scala/EvalTrait.scala index 587ace352..85747fde4 100644 --- a/jvm-packages/xgboost4j/src/main/scala/ml/dmlc/xgboost4j/scala/EvalTrait.scala +++ b/jvm-packages/xgboost4j/src/main/scala/ml/dmlc/xgboost4j/scala/EvalTrait.scala @@ -1,5 +1,5 @@ /* - Copyright (c) 2014 by Contributors + Copyright (c) 2014 - 2019 by Contributors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/jvm-packages/xgboost4j/src/main/scala/ml/dmlc/xgboost4j/scala/ObjectiveTrait.scala b/jvm-packages/xgboost4j/src/main/scala/ml/dmlc/xgboost4j/scala/ObjectiveTrait.scala index 24e603762..b46f4afc5 100644 --- a/jvm-packages/xgboost4j/src/main/scala/ml/dmlc/xgboost4j/scala/ObjectiveTrait.scala +++ b/jvm-packages/xgboost4j/src/main/scala/ml/dmlc/xgboost4j/scala/ObjectiveTrait.scala @@ -1,5 +1,5 @@ /* - Copyright (c) 2014 by Contributors + Copyright (c) 2014 - 2019 by Contributors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/jvm-packages/xgboost4j/src/main/scala/ml/dmlc/xgboost4j/scala/XGBoost.scala b/jvm-packages/xgboost4j/src/main/scala/ml/dmlc/xgboost4j/scala/XGBoost.scala index 609d7b2cd..2b0c2a095 100644 --- a/jvm-packages/xgboost4j/src/main/scala/ml/dmlc/xgboost4j/scala/XGBoost.scala +++ b/jvm-packages/xgboost4j/src/main/scala/ml/dmlc/xgboost4j/scala/XGBoost.scala @@ -1,5 +1,5 @@ /* - Copyright (c) 2014 by Contributors + Copyright (c) 2014 - 2019 by Contributors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/jvm-packages/xgboost4j/src/main/scala/ml/dmlc/xgboost4j/scala/rabit/RabitTracker.scala b/jvm-packages/xgboost4j/src/main/scala/ml/dmlc/xgboost4j/scala/rabit/RabitTracker.scala index fb388d083..ddab56d5e 100644 --- a/jvm-packages/xgboost4j/src/main/scala/ml/dmlc/xgboost4j/scala/rabit/RabitTracker.scala +++ b/jvm-packages/xgboost4j/src/main/scala/ml/dmlc/xgboost4j/scala/rabit/RabitTracker.scala @@ -1,5 +1,5 @@ /* - Copyright (c) 2014 by Contributors + Copyright (c) 2014 - 2019 by Contributors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/jvm-packages/xgboost4j/src/main/scala/ml/dmlc/xgboost4j/scala/rabit/handler/RabitTrackerHandler.scala b/jvm-packages/xgboost4j/src/main/scala/ml/dmlc/xgboost4j/scala/rabit/handler/RabitTrackerHandler.scala index f9de71746..cf680df68 100644 --- a/jvm-packages/xgboost4j/src/main/scala/ml/dmlc/xgboost4j/scala/rabit/handler/RabitTrackerHandler.scala +++ b/jvm-packages/xgboost4j/src/main/scala/ml/dmlc/xgboost4j/scala/rabit/handler/RabitTrackerHandler.scala @@ -1,5 +1,5 @@ /* - Copyright (c) 2014 by Contributors + Copyright (c) 2014 - 2019 by Contributors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/jvm-packages/xgboost4j/src/main/scala/ml/dmlc/xgboost4j/scala/rabit/handler/RabitWorkerHandler.scala b/jvm-packages/xgboost4j/src/main/scala/ml/dmlc/xgboost4j/scala/rabit/handler/RabitWorkerHandler.scala index 234c4d25a..3621dcc5b 100644 --- a/jvm-packages/xgboost4j/src/main/scala/ml/dmlc/xgboost4j/scala/rabit/handler/RabitWorkerHandler.scala +++ b/jvm-packages/xgboost4j/src/main/scala/ml/dmlc/xgboost4j/scala/rabit/handler/RabitWorkerHandler.scala @@ -1,5 +1,5 @@ /* - Copyright (c) 2014 by Contributors + Copyright (c) 2014 - 2019 by Contributors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/jvm-packages/xgboost4j/src/main/scala/ml/dmlc/xgboost4j/scala/rabit/util/LinkMap.scala b/jvm-packages/xgboost4j/src/main/scala/ml/dmlc/xgboost4j/scala/rabit/util/LinkMap.scala index edec4931b..0901099fa 100644 --- a/jvm-packages/xgboost4j/src/main/scala/ml/dmlc/xgboost4j/scala/rabit/util/LinkMap.scala +++ b/jvm-packages/xgboost4j/src/main/scala/ml/dmlc/xgboost4j/scala/rabit/util/LinkMap.scala @@ -1,5 +1,5 @@ /* - Copyright (c) 2014 by Contributors + Copyright (c) 2014 - 2019 by Contributors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/jvm-packages/xgboost4j/src/main/scala/ml/dmlc/xgboost4j/scala/rabit/util/RabitTrackerHelpers.scala b/jvm-packages/xgboost4j/src/main/scala/ml/dmlc/xgboost4j/scala/rabit/util/RabitTrackerHelpers.scala index 3d7be618d..55459144c 100644 --- a/jvm-packages/xgboost4j/src/main/scala/ml/dmlc/xgboost4j/scala/rabit/util/RabitTrackerHelpers.scala +++ b/jvm-packages/xgboost4j/src/main/scala/ml/dmlc/xgboost4j/scala/rabit/util/RabitTrackerHelpers.scala @@ -1,5 +1,5 @@ /* - Copyright (c) 2014 by Contributors + Copyright (c) 2014 - 2019 by Contributors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/jvm-packages/xgboost4j/src/test/scala/ml/dmlc/xgboost4j/scala/DMatrixSuite.scala b/jvm-packages/xgboost4j/src/test/scala/ml/dmlc/xgboost4j/scala/DMatrixSuite.scala index 87ff8e006..291642d21 100644 --- a/jvm-packages/xgboost4j/src/test/scala/ml/dmlc/xgboost4j/scala/DMatrixSuite.scala +++ b/jvm-packages/xgboost4j/src/test/scala/ml/dmlc/xgboost4j/scala/DMatrixSuite.scala @@ -1,5 +1,5 @@ /* - Copyright (c) 2014 by Contributors + Copyright (c) 2014 - 2019 by Contributors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/jvm-packages/xgboost4j/src/test/scala/ml/dmlc/xgboost4j/scala/ScalaBoosterImplSuite.scala b/jvm-packages/xgboost4j/src/test/scala/ml/dmlc/xgboost4j/scala/ScalaBoosterImplSuite.scala index adea1b1ec..066166d02 100644 --- a/jvm-packages/xgboost4j/src/test/scala/ml/dmlc/xgboost4j/scala/ScalaBoosterImplSuite.scala +++ b/jvm-packages/xgboost4j/src/test/scala/ml/dmlc/xgboost4j/scala/ScalaBoosterImplSuite.scala @@ -1,5 +1,5 @@ /* - Copyright (c) 2014 by Contributors + Copyright (c) 2014 - 2019 by Contributors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/jvm-packages/xgboost4j/src/test/scala/ml/dmlc/xgboost4j/scala/rabit/RabitTrackerConnectionHandlerTest.scala b/jvm-packages/xgboost4j/src/test/scala/ml/dmlc/xgboost4j/scala/rabit/RabitTrackerConnectionHandlerTest.scala index cd9016812..f4979cd7d 100644 --- a/jvm-packages/xgboost4j/src/test/scala/ml/dmlc/xgboost4j/scala/rabit/RabitTrackerConnectionHandlerTest.scala +++ b/jvm-packages/xgboost4j/src/test/scala/ml/dmlc/xgboost4j/scala/rabit/RabitTrackerConnectionHandlerTest.scala @@ -1,5 +1,5 @@ /* - Copyright (c) 2014 by Contributors + Copyright (c) 2014 - 2019 by Contributors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/rabit b/rabit index 9a7ac85d7..d22e0809a 160000 --- a/rabit +++ b/rabit @@ -1 +1 @@ -Subproject commit 9a7ac85d7eb65b1a0b904e1fa8d5a01b910adda4 +Subproject commit d22e0809a890ce0bc7af8d76c3c504b333d62f49 diff --git a/tests/ci_build/build_jvm_packages.sh b/tests/ci_build/build_jvm_packages.sh index e342c8f89..87b1ea742 100755 --- a/tests/ci_build/build_jvm_packages.sh +++ b/tests/ci_build/build_jvm_packages.sh @@ -16,7 +16,7 @@ spark_version=$1 rm -rf build/ cd jvm-packages -mvn --no-transfer-progress package -Dspark.version=${spark_version} +mvn --no-transfer-progress package -Dspark.version=${spark_version} -Pdev set +x set +e diff --git a/tests/travis/run_test.sh b/tests/travis/run_test.sh index 8890c15bd..ae73ece2b 100755 --- a/tests/travis/run_test.sh +++ b/tests/travis/run_test.sh @@ -27,8 +27,8 @@ fi if [ ${TASK} == "java_test" ]; then set -e cd jvm-packages - mvn -q clean install -DskipTests -Dmaven.test.skip - mvn -q test + mvn -q clean install -DskipTests -Dmaven.test.skip -Pdev + mvn -q test -Pdev fi if [ ${TASK} == "cmake_test" ]; then