diff --git a/CMakeLists.txt b/CMakeLists.txt index 5bd6b4f63..ceb44a597 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -94,16 +94,36 @@ set_target_properties(dmlc PROPERTIES list(APPEND LINKED_LIBRARIES_PRIVATE dmlc) # rabit -set(RABIT_BUILD_DMLC OFF) -set(DMLC_ROOT ${xgboost_SOURCE_DIR}/dmlc-core) -set(RABIT_WITH_R_LIB ${R_LIB}) -add_subdirectory(rabit) - -if (RABIT_MOCK) - list(APPEND LINKED_LIBRARIES_PRIVATE rabit_mock_static) -else() - list(APPEND LINKED_LIBRARIES_PRIVATE rabit) -endif(RABIT_MOCK) +# full rabit doesn't build on windows, so we can't import it as subdirectory +if(MINGW OR R_LIB OR WIN32) + set(RABIT_SOURCES + rabit/src/engine_empty.cc + rabit/src/c_api.cc) +else () + if(RABIT_MOCK) + set(RABIT_SOURCES + rabit/src/allreduce_base.cc + rabit/src/allreduce_robust.cc + rabit/src/engine_mock.cc + rabit/src/c_api.cc) + else() + set(RABIT_SOURCES + rabit/src/allreduce_base.cc + rabit/src/allreduce_robust.cc + rabit/src/engine.cc + rabit/src/c_api.cc) + endif(RABIT_MOCK) +endif (MINGW OR R_LIB OR WIN32) +add_library(rabit STATIC ${RABIT_SOURCES}) +target_include_directories(rabit PRIVATE + $ + $) +set_target_properties(rabit + PROPERTIES + CXX_STANDARD 11 + CXX_STANDARD_REQUIRED ON + POSITION_INDEPENDENT_CODE ON) +list(APPEND LINKED_LIBRARIES_PRIVATE rabit) # Exports some R specific definitions and objects if (R_LIB) diff --git a/jvm-packages/.gitignore b/jvm-packages/.gitignore index becd9e300..d1d4390d6 100644 --- a/jvm-packages/.gitignore +++ b/jvm-packages/.gitignore @@ -1,3 +1,2 @@ tracker.py -build.sh - +build.sh \ No newline at end of file diff --git a/jvm-packages/create_jni.py b/jvm-packages/create_jni.py index 0357b3a43..4d627cb7c 100755 --- a/jvm-packages/create_jni.py +++ b/jvm-packages/create_jni.py @@ -18,7 +18,7 @@ CONFIG = { "USE_HDFS": "OFF", "USE_AZURE": "OFF", "USE_S3": "OFF", - "RABIT_MOCK": "OFF", + "USE_CUDA": "OFF", "JVM_BINDINGS": "ON" } @@ -68,7 +68,6 @@ def normpath(path): if __name__ == "__main__": - CONFIG["RABIT_MOCK"] = str(sys.argv[1]) if sys.platform == "darwin": # Enable of your compiler supports OpenMP. CONFIG["USE_OPENMP"] = "OFF" diff --git a/jvm-packages/pom.xml b/jvm-packages/pom.xml index b37547e1c..628f7ca7a 100644 --- a/jvm-packages/pom.xml +++ b/jvm-packages/pom.xml @@ -37,7 +37,6 @@ 2.4.3 2.12.8 2.12 - OFF @@ -53,47 +52,6 @@ xgboost4j-flink - - dev - - ON - - - - - org.apache.maven.plugins - maven-jar-plugin - 3.0.2 - - - empty-javadoc-jar - package - - jar - - - javadoc - ${basedir}/javadoc - - - - - - org.apache.maven.plugins - maven-source-plugin - 2.2.1 - - - attach-sources - - jar-no-fork - - - - - - - release diff --git a/jvm-packages/scalastyle-config.xml b/jvm-packages/scalastyle-config.xml index 310471926..fd4d22e9a 100644 --- a/jvm-packages/scalastyle-config.xml +++ b/jvm-packages/scalastyle-config.xml @@ -49,7 +49,7 @@ This file is divided into 3 sections: PyRabitTracker} import ml.dmlc.xgboost4j.scala.rabit.RabitTracker import ml.dmlc.xgboost4j.scala.spark.CheckpointManager.CheckpointParam +import ml.dmlc.xgboost4j.scala.spark.XGBoost.logger import ml.dmlc.xgboost4j.scala.spark.params.LearningTaskParams import ml.dmlc.xgboost4j.scala.{XGBoost => SXGBoost, _} import ml.dmlc.xgboost4j.{LabeledPoint => XGBLabeledPoint} @@ -155,7 +155,7 @@ private[this] class XGBoostExecutionParamsFactory(rawParams: Map[String, Any], s overridedParams } - private[spark] def buildXGBRuntimeParams: XGBoostExecutionParams = { + def buildXGBRuntimeParams: XGBoostExecutionParams = { val nWorkers = overridedParams("num_workers").asInstanceOf[Int] val round = overridedParams("num_round").asInstanceOf[Int] val useExternalMemory = overridedParams("use_external_memory").asInstanceOf[Boolean] @@ -221,25 +221,6 @@ private[this] class XGBoostExecutionParamsFactory(rawParams: Map[String, Any], s xgbExecParam.setRawParamMap(overridedParams) xgbExecParam } - - private[spark] def buildRabitParams: Map[String, String] = Map( - "rabit_reduce_ring_mincount" -> - overridedParams.getOrElse("rabit_reduce_ring_mincount", 32<<10).toString, - "rabit_reduce_buffer" -> - overridedParams.getOrElse("rabit_reduce_buffer", "256MB").toString, - "rabit_bootstrap_cache" -> - overridedParams.getOrElse("rabit_bootstrap_cache", false).toString, - "rabit_debug" -> - overridedParams.getOrElse("rabit_debug", false).toString, - "rabit_timeout" -> - overridedParams.getOrElse("rabit_timeout", false).toString, - "rabit_timeout_sec" -> - overridedParams.getOrElse("rabit_timeout_sec", 1800).toString, - "DMLC_WORKER_CONNECT_RETRY" -> - overridedParams.getOrElse("dmlc_worker_connect_retry", 5).toString, - "DMLC_WORKER_STOP_PROCESS_ON_ERROR" -> - overridedParams.getOrElse("dmlc_worker_stop_process_on_error", false).toString - ) } /** @@ -340,6 +321,7 @@ object XGBoost extends Serializable { } val taskId = TaskContext.getPartitionId().toString rabitEnv.put("DMLC_TASK_ID", taskId) + rabitEnv.put("DMLC_WORKER_STOP_PROCESS_ON_ERROR", "false") try { Rabit.init(rabitEnv) @@ -508,9 +490,8 @@ object XGBoost extends Serializable { evalSetsMap: Map[String, RDD[XGBLabeledPoint]] = Map()): (Booster, Map[String, Array[Float]]) = { logger.info(s"Running XGBoost ${spark.VERSION} with parameters:\n${params.mkString("\n")}") - val xgbParamsFactory = new XGBoostExecutionParamsFactory(params, trainingData.sparkContext) - val xgbExecParams = xgbParamsFactory.buildXGBRuntimeParams - val xgbRabitParams = xgbParamsFactory.buildRabitParams + val xgbExecParams = new XGBoostExecutionParamsFactory(params, trainingData.sparkContext). + buildXGBRuntimeParams val sc = trainingData.sparkContext val checkpointManager = new CheckpointManager(sc, xgbExecParams.checkpointParam. checkpointPath) @@ -529,12 +510,12 @@ object XGBoost extends Serializable { val parallelismTracker = new SparkParallelismTracker(sc, xgbExecParams.timeoutRequestWorkers, xgbExecParams.numWorkers) - val rabitEnv = tracker.getWorkerEnvs.asScala ++ xgbRabitParams + val rabitEnv = tracker.getWorkerEnvs val boostersAndMetrics = if (hasGroup) { - trainForRanking(transformedTrainingData.left.get, xgbExecParams, rabitEnv.asJava, + trainForRanking(transformedTrainingData.left.get, xgbExecParams, rabitEnv, checkpointRound, prevBooster, evalSetsMap) } else { - trainForNonRanking(transformedTrainingData.right.get, xgbExecParams, rabitEnv.asJava, + trainForNonRanking(transformedTrainingData.right.get, xgbExecParams, rabitEnv, checkpointRound, prevBooster, evalSetsMap) } val sparkJobThread = new Thread() { diff --git a/jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/XGBoostClassifier.scala b/jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/XGBoostClassifier.scala index 5c59b0e40..db4936430 100644 --- a/jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/XGBoostClassifier.scala +++ b/jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/XGBoostClassifier.scala @@ -1,5 +1,5 @@ /* - Copyright (c) 2014 - 2019 by Contributors + Copyright (c) 2014 by Contributors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -50,7 +50,7 @@ class XGBoostClassifier ( def this(xgboostParams: Map[String, Any]) = this( Identifiable.randomUID("xgbc"), xgboostParams) - XGBoost2MLlibParams(xgboostParams) + XGBoostToMLlibParams(xgboostParams) def setWeightCol(value: String): this.type = set(weightCol, value) diff --git a/jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/XGBoostEstimatorCommon.scala b/jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/XGBoostEstimatorCommon.scala index 39e4d000e..1213a8f72 100644 --- a/jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/XGBoostEstimatorCommon.scala +++ b/jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/XGBoostEstimatorCommon.scala @@ -1,5 +1,5 @@ /* - Copyright (c) 2014 - 2019 by Contributors + Copyright (c) 2014 by Contributors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -21,7 +21,7 @@ import ml.dmlc.xgboost4j.scala.spark.params._ import org.apache.spark.ml.param.shared.HasWeightCol private[spark] sealed trait XGBoostEstimatorCommon extends GeneralParams with LearningTaskParams - with BoosterParams with RabitParams with ParamMapFuncs with NonParamVariables { + with BoosterParams with ParamMapFuncs with NonParamVariables { def needDeterministicRepartitioning: Boolean = { getCheckpointPath.nonEmpty && getCheckpointInterval > 0 diff --git a/jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/XGBoostRegressor.scala b/jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/XGBoostRegressor.scala index 6006bb4ce..e2f22c7af 100644 --- a/jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/XGBoostRegressor.scala +++ b/jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/XGBoostRegressor.scala @@ -1,5 +1,5 @@ /* - Copyright (c) 2014 - 2019 by Contributors + Copyright (c) 2014 by Contributors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -54,7 +54,7 @@ class XGBoostRegressor ( def this(xgboostParams: Map[String, Any]) = this( Identifiable.randomUID("xgbr"), xgboostParams) - XGBoost2MLlibParams(xgboostParams) + XGBoostToMLlibParams(xgboostParams) def setWeightCol(value: String): this.type = set(weightCol, value) diff --git a/jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/XGBoostTrainingSummary.scala b/jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/XGBoostTrainingSummary.scala index 6e0d04e90..9454befc2 100644 --- a/jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/XGBoostTrainingSummary.scala +++ b/jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/XGBoostTrainingSummary.scala @@ -1,5 +1,5 @@ /* - Copyright (c) 2014 - 2019 by Contributors + Copyright (c) 2014 by Contributors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/package.scala b/jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/package.scala index 410a75856..df5e3bcc0 100644 --- a/jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/package.scala +++ b/jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/package.scala @@ -1,5 +1,5 @@ /* - Copyright (c) 2014 - 2019 by Contributors + Copyright (c) 2014 by Contributors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/params/BoosterParams.scala b/jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/params/BoosterParams.scala index 5048ce2b3..6ec588d4b 100644 --- a/jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/params/BoosterParams.scala +++ b/jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/params/BoosterParams.scala @@ -1,5 +1,5 @@ /* - Copyright (c) 2014 - 2019 by Contributors + Copyright (c) 2014 by Contributors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/params/CustomParams.scala b/jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/params/CustomParams.scala index 47bb89577..b7f696a23 100644 --- a/jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/params/CustomParams.scala +++ b/jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/params/CustomParams.scala @@ -1,5 +1,5 @@ /* - Copyright (c) 2014 - 2019 by Contributors + Copyright (c) 2014 by Contributors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/params/DefaultXGBoostParamsReader.scala b/jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/params/DefaultXGBoostParamsReader.scala index 3bd5b4bb7..bb75bb342 100644 --- a/jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/params/DefaultXGBoostParamsReader.scala +++ b/jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/params/DefaultXGBoostParamsReader.scala @@ -1,5 +1,5 @@ /* - Copyright (c) 2014 - 2019 by Contributors + Copyright (c) 2014 by Contributors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/params/DefaultXGBoostParamsWriter.scala b/jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/params/DefaultXGBoostParamsWriter.scala index a38f9400e..acf6815ec 100644 --- a/jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/params/DefaultXGBoostParamsWriter.scala +++ b/jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/params/DefaultXGBoostParamsWriter.scala @@ -1,5 +1,5 @@ /* - Copyright (c) 2014 - 2019 by Contributors + Copyright (c) 2014 by Contributors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/params/GeneralParams.scala b/jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/params/GeneralParams.scala index d90e137c4..076dff42e 100644 --- a/jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/params/GeneralParams.scala +++ b/jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/params/GeneralParams.scala @@ -1,5 +1,5 @@ /* - Copyright (c) 2014 - 2019 by Contributors + Copyright (c) 2014 by Contributors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -241,7 +241,7 @@ trait HasNumClass extends Params { private[spark] trait ParamMapFuncs extends Params { - def XGBoost2MLlibParams(xgboostParams: Map[String, Any]): Unit = { + def XGBoostToMLlibParams(xgboostParams: Map[String, Any]): Unit = { for ((paramName, paramValue) <- xgboostParams) { if ((paramName == "booster" && paramValue != "gbtree") || (paramName == "updater" && paramValue != "grow_histmaker,prune" && diff --git a/jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/params/InferenceParams.scala b/jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/params/InferenceParams.scala index bf5eb415d..abfe777d3 100644 --- a/jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/params/InferenceParams.scala +++ b/jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/params/InferenceParams.scala @@ -1,5 +1,5 @@ /* - Copyright (c) 2014 - 2019 by Contributors + Copyright (c) 2014 by Contributors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/params/LearningTaskParams.scala b/jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/params/LearningTaskParams.scala index af2561840..1512c85d0 100644 --- a/jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/params/LearningTaskParams.scala +++ b/jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/params/LearningTaskParams.scala @@ -1,5 +1,5 @@ /* - Copyright (c) 2014 - 2019 by Contributors + Copyright (c) 2014 by Contributors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/params/NonParamVariables.scala b/jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/params/NonParamVariables.scala index 76837074e..276a938e0 100644 --- a/jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/params/NonParamVariables.scala +++ b/jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/params/NonParamVariables.scala @@ -1,5 +1,5 @@ /* - Copyright (c) 2014 - 2019 by Contributors + Copyright (c) 2014 by Contributors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/params/RabitParams.scala b/jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/params/RabitParams.scala deleted file mode 100644 index b48afb74e..000000000 --- a/jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/params/RabitParams.scala +++ /dev/null @@ -1,62 +0,0 @@ -/* - Copyright (c) 2014 - 2019 by Contributors - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - */ - -package ml.dmlc.xgboost4j.scala.spark.params - -import org.apache.spark.ml.param._ - -private[spark] trait RabitParams extends Params { - /** - * Rabit worker configurations. These parameters were passed to Rabit.Init and decide - * rabit_reduce_ring_mincount - threshold of enable ring based allreduce/broadcast operations. - * rabit_reduce_buffer - buffer size to recv and run reduction - * rabit_bootstrap_cache - enable save allreduce cache before loadcheckpoint - * rabit_debug - enable more verbose rabit logging to stdout - * rabit_timeout - enable sidecar thread after rabit observed failures - * rabit_timeout_sec - wait interval before exit after rabit observed failures - * dmlc_worker_connect_retry - number of retrys to tracker - * dmlc_worker_stop_process_on_error - exit process when rabit see assert/error - */ - final val ringReduceMin = new IntParam(this, "rabitReduceRingMincount", - "minimal counts of enable allreduce/broadcast with ring based topology", - ParamValidators.gtEq(1)) - - final def reduceBuffer: Param[String] = new Param[String](this, "rabitReduceBuffer", - "buffer size (MB/GB) allocated to each xgb trainner recv and run reduction", - (buf: String) => buf.contains("MB") || buf.contains("GB")) - - final def bootstrapCache: BooleanParam = new BooleanParam(this, "rabitBootstrapCache", - "enable save allreduce cache before loadcheckpoint, used to allow failed task retry") - - final def rabitDebug: BooleanParam = new BooleanParam(this, "rabitDebug", - "enable more verbose rabit logging to stdout") - - final def rabitTimeout: BooleanParam = new BooleanParam(this, "rabitTimeout", - "enable failure timeout sidecar threads") - - final def timeoutInterval: IntParam = new IntParam(this, "rabitTimeoutSec", - "timeout threshold after rabit observed failures", (interval: Int) => interval > 0) - - final def connectRetry: IntParam = new IntParam(this, "dmlcWorkerConnectRetry", - "number of retry worker do before fail", ParamValidators.gtEq(1)) - - final def exitOnError: BooleanParam = new BooleanParam(this, "dmlcWorkerStopProcessOnError", - "exit process when rabit see assert error") - - setDefault(ringReduceMin -> (32 << 10), reduceBuffer -> "256MB", bootstrapCache -> false, - rabitDebug -> false, connectRetry -> 5, rabitTimeout -> false, timeoutInterval -> 1800, - exitOnError -> false) -} diff --git a/jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/params/Utils.scala b/jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/params/Utils.scala index b1c5e6c5e..7d6e7b9ed 100644 --- a/jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/params/Utils.scala +++ b/jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/params/Utils.scala @@ -1,5 +1,5 @@ /* - Copyright (c) 2014 - 2019 by Contributors + Copyright (c) 2014 by Contributors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/jvm-packages/xgboost4j-spark/src/main/scala/org/apache/spark/SparkParallelismTracker.scala b/jvm-packages/xgboost4j-spark/src/main/scala/org/apache/spark/SparkParallelismTracker.scala index c17a61be9..bd0ab4d6d 100644 --- a/jvm-packages/xgboost4j-spark/src/main/scala/org/apache/spark/SparkParallelismTracker.scala +++ b/jvm-packages/xgboost4j-spark/src/main/scala/org/apache/spark/SparkParallelismTracker.scala @@ -1,5 +1,5 @@ /* - Copyright (c) 2014 - 2019 by Contributors + Copyright (c) 2014 by Contributors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/CheckpointManagerSuite.scala b/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/CheckpointManagerSuite.scala index 1b35763c0..ddeb48241 100755 --- a/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/CheckpointManagerSuite.scala +++ b/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/CheckpointManagerSuite.scala @@ -1,5 +1,5 @@ /* - Copyright (c) 2014 - 2019 by Contributors + Copyright (c) 2014 by Contributors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/DeterministicPartitioningSuite.scala b/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/DeterministicPartitioningSuite.scala index 62229ee69..986b0843b 100644 --- a/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/DeterministicPartitioningSuite.scala +++ b/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/DeterministicPartitioningSuite.scala @@ -1,5 +1,5 @@ /* - Copyright (c) 2014 - 2019 by Contributors + Copyright (c) 2014 by Contributors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/EvalError.scala b/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/EvalError.scala index 7ec0826f4..91a840911 100644 --- a/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/EvalError.scala +++ b/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/EvalError.scala @@ -1,5 +1,5 @@ /* - Copyright (c) 2014 - 2019 by Contributors + Copyright (c) 2014 by Contributors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/MissingValueHandlingSuite.scala b/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/MissingValueHandlingSuite.scala index 28c34c32b..e09f10a4c 100644 --- a/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/MissingValueHandlingSuite.scala +++ b/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/MissingValueHandlingSuite.scala @@ -1,5 +1,5 @@ /* - Copyright (c) 2014 - 2019 by Contributors + Copyright (c) 2014 by Contributors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/ParameterSuite.scala b/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/ParameterSuite.scala index 92f92dda1..e0450f90d 100644 --- a/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/ParameterSuite.scala +++ b/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/ParameterSuite.scala @@ -1,5 +1,5 @@ /* - Copyright (c) 2014 - 2019 by Contributors + Copyright (c) 2014 by Contributors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/PerTest.scala b/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/PerTest.scala index 9da916a1f..341db97bc 100644 --- a/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/PerTest.scala +++ b/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/PerTest.scala @@ -1,5 +1,5 @@ /* - Copyright (c) 2014 - 2019 by Contributors + Copyright (c) 2014 by Contributors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/PersistenceSuite.scala b/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/PersistenceSuite.scala index 3cf889b5c..a3841772c 100755 --- a/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/PersistenceSuite.scala +++ b/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/PersistenceSuite.scala @@ -1,5 +1,5 @@ /* - Copyright (c) 2014 - 2019 by Contributors + Copyright (c) 2014 by Contributors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/RabitRobustnessSuite.scala b/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/RabitRobustnessSuite.scala index eca7ffad7..9655104c3 100644 --- a/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/RabitRobustnessSuite.scala +++ b/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/RabitRobustnessSuite.scala @@ -1,5 +1,5 @@ /* - Copyright (c) 2014 - 2019 by Contributors + Copyright (c) 2014 by Contributors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/TmpFolderPerSuite.scala b/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/TmpFolderPerSuite.scala index 2a9f52bd6..96b74d679 100755 --- a/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/TmpFolderPerSuite.scala +++ b/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/TmpFolderPerSuite.scala @@ -1,5 +1,5 @@ /* - Copyright (c) 2014 - 2019 by Contributors + Copyright (c) 2014 by Contributors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/TrainTestData.scala b/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/TrainTestData.scala index 2737c341a..1a64e2d03 100644 --- a/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/TrainTestData.scala +++ b/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/TrainTestData.scala @@ -1,5 +1,5 @@ /* - Copyright (c) 2014 - 2019 by Contributors + Copyright (c) 2014 by Contributors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/XGBoostClassifierSuite.scala b/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/XGBoostClassifierSuite.scala index e930df9e5..f2ccdd44e 100644 --- a/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/XGBoostClassifierSuite.scala +++ b/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/XGBoostClassifierSuite.scala @@ -1,5 +1,5 @@ /* - Copyright (c) 2014 - 2019 by Contributors + Copyright (c) 2014 by Contributors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/XGBoostConfigureSuite.scala b/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/XGBoostConfigureSuite.scala index 9e777a9d8..fe16bcda5 100644 --- a/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/XGBoostConfigureSuite.scala +++ b/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/XGBoostConfigureSuite.scala @@ -1,5 +1,5 @@ /* - Copyright (c) 2014 - 2019 by Contributors + Copyright (c) 2014 by Contributors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -16,10 +16,7 @@ package ml.dmlc.xgboost4j.scala.spark -import ml.dmlc.xgboost4j.java.Rabit import ml.dmlc.xgboost4j.scala.{Booster, DMatrix} - -import scala.collection.JavaConverters._ import org.apache.spark.sql._ import org.scalatest.FunSuite @@ -31,7 +28,7 @@ class XGBoostConfigureSuite extends FunSuite with PerTest { test("nthread configuration must be no larger than spark.task.cpus") { val training = buildDataFrame(Classification.train) - val paramMap = Map("eta" -> "1", "max_depth" -> "2", "verbosity" -> "1", + val paramMap = Map("eta" -> "1", "max_depth" -> "2", "silent" -> "1", "objective" -> "binary:logistic", "num_workers" -> numWorkers, "nthread" -> (sc.getConf.getInt("spark.task.cpus", 1) + 1)) intercept[IllegalArgumentException] { @@ -43,7 +40,7 @@ class XGBoostConfigureSuite extends FunSuite with PerTest { // TODO write an isolated test for Booster. val training = buildDataFrame(Classification.train) val testDM = new DMatrix(Classification.test.iterator, null) - val paramMap = Map("eta" -> "1", "max_depth" -> "2", "verbosity" -> "1", + val paramMap = Map("eta" -> "1", "max_depth" -> "2", "silent" -> "1", "objective" -> "binary:logistic", "num_round" -> 5, "num_workers" -> numWorkers) val model = new XGBoostClassifier(paramMap).fit(training) @@ -55,7 +52,7 @@ class XGBoostConfigureSuite extends FunSuite with PerTest { val originalSslConfOpt = ss.conf.getOption("spark.ssl.enabled") ss.conf.set("spark.ssl.enabled", true) - val paramMap = Map("eta" -> "1", "max_depth" -> "2", "verbosity" -> "1", + val paramMap = Map("eta" -> "1", "max_depth" -> "2", "silent" -> "1", "objective" -> "binary:logistic", "num_round" -> 2, "num_workers" -> numWorkers) val training = buildDataFrame(Classification.train) diff --git a/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/XGBoostGeneralSuite.scala b/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/XGBoostGeneralSuite.scala index 383edb72f..f3492b2e3 100755 --- a/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/XGBoostGeneralSuite.scala +++ b/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/XGBoostGeneralSuite.scala @@ -1,5 +1,5 @@ /* - Copyright (c) 2014 - 2019 by Contributors + Copyright (c) 2014 by Contributors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/XGBoostRabitRegressionSuite.scala b/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/XGBoostRabitRegressionSuite.scala deleted file mode 100644 index a092afd7f..000000000 --- a/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/XGBoostRabitRegressionSuite.scala +++ /dev/null @@ -1,110 +0,0 @@ -/* - Copyright (c) 2014 - 2019 by Contributors - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - */ - -package ml.dmlc.xgboost4j.scala.spark - -import ml.dmlc.xgboost4j.java.{Rabit, XGBoostError} -import ml.dmlc.xgboost4j.scala.{Booster, DMatrix} - -import scala.collection.JavaConverters._ -import org.apache.spark.sql._ -import org.scalatest.FunSuite - -class XGBoostRabitRegressionSuite extends FunSuite with PerTest { - override def sparkSessionBuilder: SparkSession.Builder = super.sparkSessionBuilder - .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer") - .config("spark.kryo.classesToRegister", classOf[Booster].getName) - - test("test parity classification prediction") { - val training = buildDataFrame(Classification.train) - val testDF = buildDataFrame(Classification.test) - - val model1 = new XGBoostClassifier(Map("eta" -> "1", "max_depth" -> "2", "verbosity" -> "1", - "objective" -> "binary:logistic", "num_round" -> 5, "num_workers" -> numWorkers) - ).fit(training) - val prediction1 = model1.transform(testDF).select("prediction").collect() - - val model2 = new XGBoostClassifier(Map("eta" -> "1", "max_depth" -> "2", "verbosity" -> "1", - "objective" -> "binary:logistic", "num_round" -> 5, "num_workers" -> numWorkers, - "rabit_bootstrap_cache" -> true, "rabit_debug" -> true, "rabit_reduce_ring_mincount" -> 100, - "rabit_reduce_buffer" -> "2MB", "DMLC_WORKER_CONNECT_RETRY" -> 1, - "rabit_timeout" -> true, "rabit_timeout_sec" -> 5)).fit(training) - - assert(Rabit.rabitEnvs.asScala.size > 7) - Rabit.rabitEnvs.asScala.foreach( item => { - if (item._1.toString == "rabit_bootstrap_cache") assert(item._2 == "true") - if (item._1.toString == "rabit_debug") assert(item._2 == "true") - if (item._1.toString == "rabit_reduce_ring_mincount") assert(item._2 == "100") - if (item._1.toString == "rabit_reduce_buffer") assert(item._2 == "2MB") - if (item._1.toString == "dmlc_worker_connect_retry") assert(item._2 == "1") - if (item._1.toString == "rabit_timeout") assert(item._2 == "true") - if (item._1.toString == "rabit_timeout_sec") assert(item._2 == "5") - }) - - val prediction2 = model2.transform(testDF).select("prediction").collect() - // check parity w/o rabit cache - prediction1.zip(prediction2).foreach { case (Row(p1: Double), Row(p2: Double)) => - assert(p1 == p2) - } - } - - test("test parity regression prediction") { - val training = buildDataFrame(Regression.train) - val testDM = new DMatrix(Regression.test.iterator, null) - val testDF = buildDataFrame(Classification.test) - - val model1 = new XGBoostRegressor(Map("eta" -> "1", "max_depth" -> "2", "verbosity" -> "1", - "objective" -> "reg:squarederror", "num_round" -> 5, "num_workers" -> numWorkers) - ).fit(training) - val prediction1 = model1.transform(testDF).select("prediction").collect() - - val model2 = new XGBoostRegressor(Map("eta" -> "1", "max_depth" -> "2", "verbosity" -> "1", - "objective" -> "reg:squarederror", "num_round" -> 5, "num_workers" -> numWorkers, - "rabit_bootstrap_cache" -> true, "rabit_debug" -> true, "rabit_reduce_ring_mincount" -> 100, - "rabit_reduce_buffer" -> "2MB", "DMLC_WORKER_CONNECT_RETRY" -> 1, - "rabit_timeout" -> true, "rabit_timeout_sec" -> 5)).fit(training) - assert(Rabit.rabitEnvs.asScala.size > 7) - Rabit.rabitEnvs.asScala.foreach( item => { - if (item._1.toString == "rabit_bootstrap_cache") assert(item._2 == "true") - if (item._1.toString == "rabit_debug") assert(item._2 == "true") - if (item._1.toString == "rabit_reduce_ring_mincount") assert(item._2 == "100") - if (item._1.toString == "rabit_reduce_buffer") assert(item._2 == "2MB") - if (item._1.toString == "dmlc_worker_connect_retry") assert(item._2 == "true") - if (item._1.toString == "rabit_timeout") assert(item._2 == "true") - if (item._1.toString == "rabit_timeout_sec") assert(item._2 == "5") - if (item._1.toString == "DMLC_WORKER_STOP_PROCESS_ON_ERROR") assert(item._2 == "false") - }) - // check the equality of single instance prediction - val prediction2 = model2.transform(testDF).select("prediction").collect() - // check parity w/o rabit cache - prediction1.zip(prediction2).foreach { case (Row(p1: Double), Row(p2: Double)) => - assert(math.abs(p1 - p2) < 0.00001f) - } - } - - test("test graceful failure handle") { - val training = buildDataFrame(Classification.train) - val testDF = buildDataFrame(Classification.test) - // mock rank 0 failure during 4th allreduce synchronization - Rabit.mockList = Array("0,4,0,0").toList.asJava - intercept[XGBoostError] { - new XGBoostClassifier(Map("eta" -> "1", "max_depth" -> "2", "verbosity" -> "1", - "objective" -> "binary:logistic", "num_round" -> 5, "num_workers" -> numWorkers, - "rabit_timeout" -> true, "rabit_timeout_sec" -> 1, - "DMLC_WORKER_STOP_PROCESS_ON_ERROR" -> false)).fit(training) - } - } -} diff --git a/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/XGBoostRegressorSuite.scala b/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/XGBoostRegressorSuite.scala index 5105335be..ff88ff328 100644 --- a/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/XGBoostRegressorSuite.scala +++ b/jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/XGBoostRegressorSuite.scala @@ -1,5 +1,5 @@ /* - Copyright (c) 2014 - 2019 by Contributors + Copyright (c) 2014 by Contributors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/jvm-packages/xgboost4j-spark/src/test/scala/org/apache/spark/SparkParallelismTrackerSuite.scala b/jvm-packages/xgboost4j-spark/src/test/scala/org/apache/spark/SparkParallelismTrackerSuite.scala index 92cbca23c..7f344674f 100644 --- a/jvm-packages/xgboost4j-spark/src/test/scala/org/apache/spark/SparkParallelismTrackerSuite.scala +++ b/jvm-packages/xgboost4j-spark/src/test/scala/org/apache/spark/SparkParallelismTrackerSuite.scala @@ -1,5 +1,5 @@ /* - Copyright (c) 2014 - 2019 by Contributors + Copyright (c) 2014 by Contributors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/jvm-packages/xgboost4j/pom.xml b/jvm-packages/xgboost4j/pom.xml index 805e275f3..4cc2ab20a 100644 --- a/jvm-packages/xgboost4j/pom.xml +++ b/jvm-packages/xgboost4j/pom.xml @@ -66,7 +66,6 @@ python create_jni.py - ${rabit.mock} ${user.dir} diff --git a/jvm-packages/xgboost4j/src/main/java/ml/dmlc/xgboost4j/java/Rabit.java b/jvm-packages/xgboost4j/src/main/java/ml/dmlc/xgboost4j/java/Rabit.java index 7e019dc65..35b500757 100644 --- a/jvm-packages/xgboost4j/src/main/java/ml/dmlc/xgboost4j/java/Rabit.java +++ b/jvm-packages/xgboost4j/src/main/java/ml/dmlc/xgboost4j/java/Rabit.java @@ -3,8 +3,6 @@ package ml.dmlc.xgboost4j.java; import java.io.Serializable; import java.nio.ByteBuffer; import java.nio.ByteOrder; -import java.util.LinkedList; -import java.util.List; import java.util.Map; /** @@ -53,25 +51,18 @@ public class Rabit { throw new XGBoostError(XGBoostJNI.XGBGetLastError()); } } - // used as way to test/debug passed rabit init parameters - public static Map rabitEnvs; - public static List mockList = new LinkedList<>(); + /** * Initialize the rabit library on current working thread. * @param envs The additional environment variables to pass to rabit. * @throws XGBoostError */ public static void init(Map envs) throws XGBoostError { - rabitEnvs = envs; - String[] args = new String[envs.size() + mockList.size()]; + String[] args = new String[envs.size()]; int idx = 0; for (java.util.Map.Entry e : envs.entrySet()) { args[idx++] = e.getKey() + '=' + e.getValue(); } - // pass list of rabit mock strings eg mock=0,1,0,0 - for(String mock : mockList) { - args[idx++] = "mock=" + mock; - } checkCall(XGBoostJNI.RabitInit(args)); } diff --git a/jvm-packages/xgboost4j/src/main/scala/ml/dmlc/xgboost4j/LabeledPoint.scala b/jvm-packages/xgboost4j/src/main/scala/ml/dmlc/xgboost4j/LabeledPoint.scala index fbeb5e0e1..9a92d1b91 100644 --- a/jvm-packages/xgboost4j/src/main/scala/ml/dmlc/xgboost4j/LabeledPoint.scala +++ b/jvm-packages/xgboost4j/src/main/scala/ml/dmlc/xgboost4j/LabeledPoint.scala @@ -1,5 +1,5 @@ /* - Copyright (c) 2014 - 2019 by Contributors + Copyright (c) 2014 by Contributors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/jvm-packages/xgboost4j/src/main/scala/ml/dmlc/xgboost4j/scala/Booster.scala b/jvm-packages/xgboost4j/src/main/scala/ml/dmlc/xgboost4j/scala/Booster.scala index 43faf0553..bb2d5e9e5 100644 --- a/jvm-packages/xgboost4j/src/main/scala/ml/dmlc/xgboost4j/scala/Booster.scala +++ b/jvm-packages/xgboost4j/src/main/scala/ml/dmlc/xgboost4j/scala/Booster.scala @@ -1,5 +1,5 @@ /* - Copyright (c) 2014 - 2019 by Contributors + Copyright (c) 2014 by Contributors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/jvm-packages/xgboost4j/src/main/scala/ml/dmlc/xgboost4j/scala/DMatrix.scala b/jvm-packages/xgboost4j/src/main/scala/ml/dmlc/xgboost4j/scala/DMatrix.scala index b30b4fe03..629a39dbf 100644 --- a/jvm-packages/xgboost4j/src/main/scala/ml/dmlc/xgboost4j/scala/DMatrix.scala +++ b/jvm-packages/xgboost4j/src/main/scala/ml/dmlc/xgboost4j/scala/DMatrix.scala @@ -1,5 +1,5 @@ /* - Copyright (c) 2014 - 2019 by Contributors + Copyright (c) 2014 by Contributors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/jvm-packages/xgboost4j/src/main/scala/ml/dmlc/xgboost4j/scala/EvalTrait.scala b/jvm-packages/xgboost4j/src/main/scala/ml/dmlc/xgboost4j/scala/EvalTrait.scala index 85747fde4..587ace352 100644 --- a/jvm-packages/xgboost4j/src/main/scala/ml/dmlc/xgboost4j/scala/EvalTrait.scala +++ b/jvm-packages/xgboost4j/src/main/scala/ml/dmlc/xgboost4j/scala/EvalTrait.scala @@ -1,5 +1,5 @@ /* - Copyright (c) 2014 - 2019 by Contributors + Copyright (c) 2014 by Contributors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/jvm-packages/xgboost4j/src/main/scala/ml/dmlc/xgboost4j/scala/ObjectiveTrait.scala b/jvm-packages/xgboost4j/src/main/scala/ml/dmlc/xgboost4j/scala/ObjectiveTrait.scala index b46f4afc5..24e603762 100644 --- a/jvm-packages/xgboost4j/src/main/scala/ml/dmlc/xgboost4j/scala/ObjectiveTrait.scala +++ b/jvm-packages/xgboost4j/src/main/scala/ml/dmlc/xgboost4j/scala/ObjectiveTrait.scala @@ -1,5 +1,5 @@ /* - Copyright (c) 2014 - 2019 by Contributors + Copyright (c) 2014 by Contributors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/jvm-packages/xgboost4j/src/main/scala/ml/dmlc/xgboost4j/scala/XGBoost.scala b/jvm-packages/xgboost4j/src/main/scala/ml/dmlc/xgboost4j/scala/XGBoost.scala index 2b0c2a095..609d7b2cd 100644 --- a/jvm-packages/xgboost4j/src/main/scala/ml/dmlc/xgboost4j/scala/XGBoost.scala +++ b/jvm-packages/xgboost4j/src/main/scala/ml/dmlc/xgboost4j/scala/XGBoost.scala @@ -1,5 +1,5 @@ /* - Copyright (c) 2014 - 2019 by Contributors + Copyright (c) 2014 by Contributors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/jvm-packages/xgboost4j/src/main/scala/ml/dmlc/xgboost4j/scala/rabit/RabitTracker.scala b/jvm-packages/xgboost4j/src/main/scala/ml/dmlc/xgboost4j/scala/rabit/RabitTracker.scala index ddab56d5e..fb388d083 100644 --- a/jvm-packages/xgboost4j/src/main/scala/ml/dmlc/xgboost4j/scala/rabit/RabitTracker.scala +++ b/jvm-packages/xgboost4j/src/main/scala/ml/dmlc/xgboost4j/scala/rabit/RabitTracker.scala @@ -1,5 +1,5 @@ /* - Copyright (c) 2014 - 2019 by Contributors + Copyright (c) 2014 by Contributors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/jvm-packages/xgboost4j/src/main/scala/ml/dmlc/xgboost4j/scala/rabit/handler/RabitTrackerHandler.scala b/jvm-packages/xgboost4j/src/main/scala/ml/dmlc/xgboost4j/scala/rabit/handler/RabitTrackerHandler.scala index cf680df68..f9de71746 100644 --- a/jvm-packages/xgboost4j/src/main/scala/ml/dmlc/xgboost4j/scala/rabit/handler/RabitTrackerHandler.scala +++ b/jvm-packages/xgboost4j/src/main/scala/ml/dmlc/xgboost4j/scala/rabit/handler/RabitTrackerHandler.scala @@ -1,5 +1,5 @@ /* - Copyright (c) 2014 - 2019 by Contributors + Copyright (c) 2014 by Contributors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/jvm-packages/xgboost4j/src/main/scala/ml/dmlc/xgboost4j/scala/rabit/handler/RabitWorkerHandler.scala b/jvm-packages/xgboost4j/src/main/scala/ml/dmlc/xgboost4j/scala/rabit/handler/RabitWorkerHandler.scala index 3621dcc5b..234c4d25a 100644 --- a/jvm-packages/xgboost4j/src/main/scala/ml/dmlc/xgboost4j/scala/rabit/handler/RabitWorkerHandler.scala +++ b/jvm-packages/xgboost4j/src/main/scala/ml/dmlc/xgboost4j/scala/rabit/handler/RabitWorkerHandler.scala @@ -1,5 +1,5 @@ /* - Copyright (c) 2014 - 2019 by Contributors + Copyright (c) 2014 by Contributors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/jvm-packages/xgboost4j/src/main/scala/ml/dmlc/xgboost4j/scala/rabit/util/LinkMap.scala b/jvm-packages/xgboost4j/src/main/scala/ml/dmlc/xgboost4j/scala/rabit/util/LinkMap.scala index 0901099fa..edec4931b 100644 --- a/jvm-packages/xgboost4j/src/main/scala/ml/dmlc/xgboost4j/scala/rabit/util/LinkMap.scala +++ b/jvm-packages/xgboost4j/src/main/scala/ml/dmlc/xgboost4j/scala/rabit/util/LinkMap.scala @@ -1,5 +1,5 @@ /* - Copyright (c) 2014 - 2019 by Contributors + Copyright (c) 2014 by Contributors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/jvm-packages/xgboost4j/src/main/scala/ml/dmlc/xgboost4j/scala/rabit/util/RabitTrackerHelpers.scala b/jvm-packages/xgboost4j/src/main/scala/ml/dmlc/xgboost4j/scala/rabit/util/RabitTrackerHelpers.scala index 55459144c..3d7be618d 100644 --- a/jvm-packages/xgboost4j/src/main/scala/ml/dmlc/xgboost4j/scala/rabit/util/RabitTrackerHelpers.scala +++ b/jvm-packages/xgboost4j/src/main/scala/ml/dmlc/xgboost4j/scala/rabit/util/RabitTrackerHelpers.scala @@ -1,5 +1,5 @@ /* - Copyright (c) 2014 - 2019 by Contributors + Copyright (c) 2014 by Contributors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/jvm-packages/xgboost4j/src/test/scala/ml/dmlc/xgboost4j/scala/DMatrixSuite.scala b/jvm-packages/xgboost4j/src/test/scala/ml/dmlc/xgboost4j/scala/DMatrixSuite.scala index 291642d21..87ff8e006 100644 --- a/jvm-packages/xgboost4j/src/test/scala/ml/dmlc/xgboost4j/scala/DMatrixSuite.scala +++ b/jvm-packages/xgboost4j/src/test/scala/ml/dmlc/xgboost4j/scala/DMatrixSuite.scala @@ -1,5 +1,5 @@ /* - Copyright (c) 2014 - 2019 by Contributors + Copyright (c) 2014 by Contributors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/jvm-packages/xgboost4j/src/test/scala/ml/dmlc/xgboost4j/scala/ScalaBoosterImplSuite.scala b/jvm-packages/xgboost4j/src/test/scala/ml/dmlc/xgboost4j/scala/ScalaBoosterImplSuite.scala index 066166d02..adea1b1ec 100644 --- a/jvm-packages/xgboost4j/src/test/scala/ml/dmlc/xgboost4j/scala/ScalaBoosterImplSuite.scala +++ b/jvm-packages/xgboost4j/src/test/scala/ml/dmlc/xgboost4j/scala/ScalaBoosterImplSuite.scala @@ -1,5 +1,5 @@ /* - Copyright (c) 2014 - 2019 by Contributors + Copyright (c) 2014 by Contributors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/jvm-packages/xgboost4j/src/test/scala/ml/dmlc/xgboost4j/scala/rabit/RabitTrackerConnectionHandlerTest.scala b/jvm-packages/xgboost4j/src/test/scala/ml/dmlc/xgboost4j/scala/rabit/RabitTrackerConnectionHandlerTest.scala index f4979cd7d..cd9016812 100644 --- a/jvm-packages/xgboost4j/src/test/scala/ml/dmlc/xgboost4j/scala/rabit/RabitTrackerConnectionHandlerTest.scala +++ b/jvm-packages/xgboost4j/src/test/scala/ml/dmlc/xgboost4j/scala/rabit/RabitTrackerConnectionHandlerTest.scala @@ -1,5 +1,5 @@ /* - Copyright (c) 2014 - 2019 by Contributors + Copyright (c) 2014 by Contributors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/rabit b/rabit index d22e0809a..9a7ac85d7 160000 --- a/rabit +++ b/rabit @@ -1 +1 @@ -Subproject commit d22e0809a890ce0bc7af8d76c3c504b333d62f49 +Subproject commit 9a7ac85d7eb65b1a0b904e1fa8d5a01b910adda4 diff --git a/tests/ci_build/build_jvm_packages.sh b/tests/ci_build/build_jvm_packages.sh index 87b1ea742..e342c8f89 100755 --- a/tests/ci_build/build_jvm_packages.sh +++ b/tests/ci_build/build_jvm_packages.sh @@ -16,7 +16,7 @@ spark_version=$1 rm -rf build/ cd jvm-packages -mvn --no-transfer-progress package -Dspark.version=${spark_version} -Pdev +mvn --no-transfer-progress package -Dspark.version=${spark_version} set +x set +e diff --git a/tests/travis/run_test.sh b/tests/travis/run_test.sh index ae73ece2b..8890c15bd 100755 --- a/tests/travis/run_test.sh +++ b/tests/travis/run_test.sh @@ -27,8 +27,8 @@ fi if [ ${TASK} == "java_test" ]; then set -e cd jvm-packages - mvn -q clean install -DskipTests -Dmaven.test.skip -Pdev - mvn -q test -Pdev + mvn -q clean install -DskipTests -Dmaven.test.skip + mvn -q test fi if [ ${TASK} == "cmake_test" ]; then