Document GPU objectives in NEWS. (#3866)

.clang-tidy

@@ -1,4 +1,4 @@
-Checks: 'modernize-*,-modernize-make-*,-modernize-use-auto,-modernize-raw-string-literal,google-*,-google-default-arguments,-clang-diagnostic-#pragma-messages,readability-identifier-naming'
+Checks: 'modernize-*,-modernize-make-*,-modernize-raw-string-literal,google-*,-google-default-arguments,-clang-diagnostic-#pragma-messages,readability-identifier-naming'
 CheckOptions:
   - { key: readability-identifier-naming.ClassCase,           value: CamelCase  }
   - { key: readability-identifier-naming.StructCase,          value: CamelCase  }

.gitignore

@@ -91,4 +91,3 @@ lib/
 metastore_db
 
 plugin/updater_gpu/test/cpp/data
-/include/xgboost/build_config.h

.travis.yml

@@ -6,7 +6,9 @@ os:
   - linux
   - osx
 
-osx_image: xcode9.3
+osx_image: xcode8
+
+group: deprecated-2017Q4
 
 # Use Build Matrix to do lint and build seperately
 env:
@@ -66,11 +68,6 @@ addons:
       - g++-4.8
       - gcc-7
       - g++-7
-  homebrew:
-    packages:
-      - gcc@7
-      - graphviz
-    update: true
 
 before_install:
   - source dmlc-core/scripts/travis/travis_setup_env.sh

CMakeLists.txt

@@ -8,23 +8,17 @@ set_default_configuration_release()
 msvc_use_static_runtime()
 
 # Options
-## GPUs
+option(USE_CUDA  "Build with GPU acceleration")
-option(USE_CUDA  "Build with GPU acceleration" OFF)
+option(JVM_BINDINGS "Build JVM bindings" OFF)
-option(USE_NCCL  "Build with multiple GPUs support" OFF)
+option(GOOGLE_TEST "Build google tests" OFF)
+option(R_LIB "Build shared library for R package" OFF)
 set(GPU_COMPUTE_VER "" CACHE STRING
   "Space separated list of compute versions to be built against, e.g. '35 61'")
-
-## Bindings
-option(JVM_BINDINGS "Build JVM bindings" OFF)
-option(R_LIB "Build shared library for R package" OFF)
-
-## Devs
 option(USE_SANITIZER "Use santizer flags" OFF)
 option(SANITIZER_PATH "Path to sanitizes.")
 set(ENABLED_SANITIZERS "address" "leak" CACHE STRING
   "Semicolon separated list of sanitizer names. E.g 'address;leak'. Supported sanitizers are
 address, leak and thread.")
-option(GOOGLE_TEST "Build google tests" OFF)
 
 # Plugins
 option(PLUGIN_LZ4 "Build lz4 plugin" OFF)
@@ -55,26 +49,6 @@ if(WIN32 AND MINGW)
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -static-libstdc++")
 endif()
 
-# Check existence of software pre-fetching
-include(CheckCXXSourceCompiles)
-check_cxx_source_compiles("
-#include <xmmintrin.h>
-int main() {
-  char data = 0;
-  const char* address = &data;
-  _mm_prefetch(address, _MM_HINT_NTA);
-  return 0;
-}
-" XGBOOST_MM_PREFETCH_PRESENT)
-check_cxx_source_compiles("
-int main() {
-  char data = 0;
-  const char* address = &data;
-  __builtin_prefetch(address, 0, 0);
-  return 0;
-}
-" XGBOOST_BUILTIN_PREFETCH_PRESENT)
-
 # Sanitizer
 if(USE_SANITIZER)
   include(cmake/Sanitizer.cmake)
@@ -108,12 +82,6 @@ include_directories (
     ${PROJECT_SOURCE_DIR}/rabit/include
 )
 
-# Generate configurable header
-set(CMAKE_LOCAL "${PROJECT_SOURCE_DIR}/cmake")
-set(INCLUDE_ROOT "${PROJECT_SOURCE_DIR}/include")
-message(STATUS "${CMAKE_LOCAL}/build_config.h.in -> ${INCLUDE_ROOT}/xgboost/build_config.h")
-configure_file("${CMAKE_LOCAL}/build_config.h.in" "${INCLUDE_ROOT}/xgboost/build_config.h")
-
 file(GLOB_RECURSE SOURCES
     src/*.cc
     src/*.h
@@ -123,6 +91,8 @@ file(GLOB_RECURSE SOURCES
 # Only add main function for executable target
 list(REMOVE_ITEM SOURCES ${PROJECT_SOURCE_DIR}/src/cli_main.cc)
 
+file(GLOB_RECURSE TEST_SOURCES "tests/cpp/*.cc")
+
 file(GLOB_RECURSE CUDA_SOURCES
     src/*.cu
     src/*.cuh
@@ -138,7 +108,7 @@ if(PLUGIN_DENSE_PARSER)
 endif()
 
 # rabit
-# TODO: Use CMakeLists.txt from rabit.
+# TODO: Create rabit cmakelists.txt
 set(RABIT_SOURCES
     rabit/src/allreduce_base.cc
     rabit/src/allreduce_robust.cc
@@ -149,7 +119,6 @@ set(RABIT_EMPTY_SOURCES
     rabit/src/engine_empty.cc
     rabit/src/c_api.cc
 )
-
 if(MINGW OR R_LIB)
   # build a dummy rabit library
   add_library(rabit STATIC ${RABIT_EMPTY_SOURCES})
@@ -157,11 +126,7 @@ else()
   add_library(rabit STATIC ${RABIT_SOURCES})
 endif()
 
-if (GENERATE_COMPILATION_DATABASE)
+if(USE_CUDA)
-  set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
-endif (GENERATE_COMPILATION_DATABASE)
-
-if(USE_CUDA AND (NOT GENERATE_COMPILATION_DATABASE))
   find_package(CUDA 8.0 REQUIRED)
   cmake_minimum_required(VERSION 3.5)
 
@@ -171,7 +136,7 @@ if(USE_CUDA AND (NOT GENERATE_COMPILATION_DATABASE))
 
   if(USE_NCCL)
     find_package(Nccl REQUIRED)
-    cuda_include_directories(${NCCL_INCLUDE_DIR})
+    include_directories(${NCCL_INCLUDE_DIR})
     add_definitions(-DXGBOOST_USE_NCCL)
   endif()
 
@@ -191,39 +156,6 @@ if(USE_CUDA AND (NOT GENERATE_COMPILATION_DATABASE))
     target_link_libraries(gpuxgboost ${NCCL_LIB_NAME})
   endif()
   list(APPEND LINK_LIBRARIES gpuxgboost)
-
-elseif (USE_CUDA AND GENERATE_COMPILATION_DATABASE)
-  # Enable CUDA language to generate a compilation database.
-  cmake_minimum_required(VERSION 3.8)
-
-  find_package(CUDA 8.0 REQUIRED)
-  enable_language(CUDA)
-  set(CMAKE_CUDA_COMPILER clang++)
-  set(CUDA_SEPARABLE_COMPILATION ON)
-  if (NOT CLANG_CUDA_GENCODE)
-    set(CLANG_CUDA_GENCODE "--cuda-gpu-arch=sm_35")
-  endif (NOT CLANG_CUDA_GENCODE)
-  set(CMAKE_CUDA_FLAGS " -Wno-deprecated  ${CLANG_CUDA_GENCODE} -fPIC ${GENCODE} -std=c++11 -x cuda")
-  message(STATUS "CMAKE_CUDA_FLAGS: ${CMAKE_CUDA_FLAGS}")
-
-  add_library(gpuxgboost STATIC ${CUDA_SOURCES})
-
-  if(USE_NCCL)
-    find_package(Nccl REQUIRED)
-    target_include_directories(gpuxgboost PUBLIC ${NCCL_INCLUDE_DIR})
-    target_compile_definitions(gpuxgboost PUBLIC -DXGBOOST_USE_NCCL)
-    target_link_libraries(gpuxgboost PUBLIC ${NCCL_LIB_NAME})
-  endif()
-
-  target_compile_definitions(gpuxgboost PUBLIC -DXGBOOST_USE_CUDA)
-  # A hack for CMake to make arguments valid for clang++
-  string(REPLACE "-x cu" "-x cuda" CMAKE_CUDA_COMPILE_PTX_COMPILATION
-    ${CMAKE_CUDA_COMPILE_PTX_COMPILATION})
-  string(REPLACE "-x cu" "-x cuda" CMAKE_CUDA_COMPILE_WHOLE_COMPILATION
-    ${CMAKE_CUDA_COMPILE_WHOLE_COMPILATION})
-  string(REPLACE "-x cu" "-x cuda" CMAKE_CUDA_COMPILE_SEPARABLE_COMPILATION
-    ${CMAKE_CUDA_COMPILE_SEPARABLE_COMPILATION})
-  target_include_directories(gpuxgboost PUBLIC cub)
 endif()
 
 
@@ -239,6 +171,7 @@ endif()
 
 add_library(objxgboost OBJECT ${SOURCES})
 
+
 # building shared library for R package
 if(R_LIB)
   find_package(LibR REQUIRED)
@@ -246,13 +179,13 @@ if(R_LIB)
   list(APPEND LINK_LIBRARIES "${LIBR_CORE_LIBRARY}")
   MESSAGE(STATUS "LIBR_CORE_LIBRARY " ${LIBR_CORE_LIBRARY})
 
-  # Shared library target for the R package
+  include_directories(
-  add_library(xgboost SHARED $<TARGET_OBJECTS:objxgboost>)
-  include_directories(xgboost
     "${LIBR_INCLUDE_DIRS}"
     "${PROJECT_SOURCE_DIR}"
   )
 
+  # Shared library target for the R package
+  add_library(xgboost SHARED $<TARGET_OBJECTS:objxgboost>)
   target_link_libraries(xgboost ${LINK_LIBRARIES})
   # R uses no lib prefix in shared library names of its packages
   set_target_properties(xgboost PROPERTIES PREFIX "")
@@ -264,7 +197,7 @@ if(R_LIB)
   # use a dummy location for any other remaining installs
   set(CMAKE_INSTALL_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/dummy_inst")
 
-  # main targets: shared library & exe
+# main targets: shared library & exe
 else()
   # Executable
   add_executable(runxgboost $<TARGET_OBJECTS:objxgboost> src/cli_main.cc)
@@ -287,20 +220,20 @@ else()
   add_dependencies(xgboost runxgboost)
 endif()
 
+
 # JVM
 if(JVM_BINDINGS)
     find_package(JNI QUIET REQUIRED)
 
+    include_directories(${JNI_INCLUDE_DIRS} jvm-packages/xgboost4j/src/native)
+
     add_library(xgboost4j SHARED
-      $<TARGET_OBJECTS:objxgboost>
+        $<TARGET_OBJECTS:objxgboost>
-      jvm-packages/xgboost4j/src/native/xgboost4j.cpp)
+        jvm-packages/xgboost4j/src/native/xgboost4j.cpp)
-    target_include_directories(xgboost4j
-      PRIVATE ${JNI_INCLUDE_DIRS}
-      PRIVATE jvm-packages/xgboost4j/src/native)
-    target_link_libraries(xgboost4j
-      ${LINK_LIBRARIES}
-      ${JAVA_JVM_LIBRARY})
     set_output_directory(xgboost4j ${PROJECT_SOURCE_DIR}/lib)
+    target_link_libraries(xgboost4j
+        ${LINK_LIBRARIES}
+        ${JAVA_JVM_LIBRARY})
 endif()
 
 
@@ -309,31 +242,18 @@ if(GOOGLE_TEST)
   enable_testing()
   find_package(GTest REQUIRED)
 
-  file(GLOB_RECURSE TEST_SOURCES "tests/cpp/*.cc")
   auto_source_group("${TEST_SOURCES}")
+  include_directories(${GTEST_INCLUDE_DIRS})
 
-  if(USE_CUDA AND (NOT GENERATE_COMPILATION_DATABASE))
+  if(USE_CUDA)
     file(GLOB_RECURSE CUDA_TEST_SOURCES "tests/cpp/*.cu")
-    cuda_include_directories(${GTEST_INCLUDE_DIRS})
     cuda_compile(CUDA_TEST_OBJS ${CUDA_TEST_SOURCES})
-  elseif (USE_CUDA AND GENERATE_COMPILATION_DATABASE)
-    file(GLOB_RECURSE CUDA_TEST_SOURCES "tests/cpp/*.cu")
   else()
     set(CUDA_TEST_OBJS "")
   endif()
 
-  if (USE_CUDA AND GENERATE_COMPILATION_DATABASE)
+  add_executable(testxgboost ${TEST_SOURCES} ${CUDA_TEST_OBJS} $<TARGET_OBJECTS:objxgboost>)
-    add_executable(testxgboost ${TEST_SOURCES} ${CUDA_TEST_SOURCES}
-      $<TARGET_OBJECTS:objxgboost>)
-    target_include_directories(testxgboost PRIVATE cub)
-  else ()
-    add_executable(testxgboost ${TEST_SOURCES} ${CUDA_TEST_OBJS}
-      $<TARGET_OBJECTS:objxgboost>)
-  endif ()
-
   set_output_directory(testxgboost ${PROJECT_SOURCE_DIR})
-  target_include_directories(testxgboost
-    PRIVATE ${GTEST_INCLUDE_DIRS})
   target_link_libraries(testxgboost ${GTEST_LIBRARIES} ${LINK_LIBRARIES})
 
   add_test(TestXGBoost testxgboost)

CONTRIBUTORS.md

@@ -85,6 +85,4 @@ List of Contributors
 * [Andrew Thia](https://github.com/BlueTea88)
   - Andrew Thia implemented feature interaction constraints
 * [Wei Tian](https://github.com/weitian)
-* [Chen Qin](https://github.com/chenqin)
+* [Chen Qin] (https://github.com/chenqin)
-* [Sam Wilkinson](https://samwilkinson.io)
-* [Matthew Jones](https://github.com/mt-jones)

Jenkinsfile

@@ -53,7 +53,7 @@ pipeline {
                     parallel (buildMatrix.findAll{it['enabled']}.collectEntries{ c ->
                         def buildName = utils.getBuildName(c)
                         utils.buildFactory(buildName, c, false, this.&buildPlatformCmake)
-                    } + [ "clang-tidy" : { buildClangTidyJob() } ])
+                    })
                 }
             }
         }
@@ -73,7 +73,7 @@ def buildPlatformCmake(buildName, conf, nodeReq, dockerTarget) {
     }
     def test_suite = conf["withGpu"] ? (conf["multiGpu"] ? "mgpu" : "gpu") : "cpu"
     // Build node - this is returned result
-    retry(1) {
+    retry(3) {
         node(nodeReq) {
             unstash name: 'srcs'
             echo """
@@ -96,32 +96,13 @@ def buildPlatformCmake(buildName, conf, nodeReq, dockerTarget) {
                 # Test the wheel for compatibility on a barebones CPU container
                 ${dockerRun} release ${dockerArgs} bash -c " \
                     pip install --user python-package/dist/xgboost-*-none-any.whl && \
-		    pytest -v --fulltrace -s tests/python"
+                    python -m nose -v tests/python"
                 # Test the wheel for compatibility on CUDA 10.0 container
                 ${dockerRun} gpu --build-arg CUDA_VERSION=10.0 bash -c " \
                     pip install --user python-package/dist/xgboost-*-none-any.whl && \
-		    pytest -v -s --fulltrace -m '(not mgpu) and (not slow)' tests/python-gpu"
+                    python -m nose -v --eval-attr='(not slow) and (not mgpu)' tests/python-gpu"
                 """
             }
         }
     }
 }
-
-/**
- * Run a clang-tidy job on a GPU machine
- */
-def buildClangTidyJob() {
-    def nodeReq = "linux && gpu && unrestricted"
-    node(nodeReq) {
-        unstash name: 'srcs'
-        echo "Running clang-tidy job..."
-        // Invoke command inside docker
-        // Install Google Test and Python yaml
-        dockerTarget = "clang_tidy"
-        dockerArgs = "--build-arg CUDA_VERSION=9.2"
-        sh """
-        ${dockerRun} ${dockerTarget} ${dockerArgs} tests/ci_build/clang_tidy.sh
-        """
-      }
-  }
-

Jenkinsfile-restricted

@@ -56,7 +56,7 @@ pipeline {
         stage('Jenkins: Build doc') {
             steps {
                 script {
-                    retry(1) {
+                    retry(3) {
                         node('linux && cpu && restricted') {
                             unstash name: 'srcs'
                             echo 'Building doc...'
@@ -99,7 +99,7 @@ def buildPlatformCmake(buildName, conf, nodeReq, dockerTarget) {
         dockerArgs = "--build-arg CUDA_VERSION=" + conf["cudaVersion"]
     }
     // Build node - this is returned result
-    retry(1) {
+    retry(3) {
         node(nodeReq) {
             unstash name: 'srcs'
             echo """

Makefile

@@ -260,8 +260,7 @@ Rpack: clean_all
 	cp ./LICENSE xgboost
 	cat R-package/src/Makevars.in|sed '2s/.*/PKGROOT=./' | sed '3s/.*/ENABLE_STD_THREAD=0/' > xgboost/src/Makevars.in
 	cp xgboost/src/Makevars.in xgboost/src/Makevars.win
-	sed -i -e 's/@OPENMP_CXXFLAGS@/$$\(SHLIB_OPENMP_CXXFLAGS\)/g' xgboost/src/Makevars.win
+	sed -i -e 's/@OPENMP_CXXFLAGS@/$$\(SHLIB_OPENMP_CFLAGS\)/g' xgboost/src/Makevars.win
-	sed -i -e 's/-pthread/$$\(SHLIB_PTHREAD_FLAGS\)/g' xgboost/src/Makevars.win
 	bash R-package/remove_warning_suppression_pragma.sh
 	rm xgboost/remove_warning_suppression_pragma.sh
 

NEWS.md

@@ -3,165 +3,6 @@ XGBoost Change Log
 
 This file records the changes in xgboost library in reverse chronological order.
 
-## v0.82 (2019.03.03)
-This release is packed with many new features and bug fixes.
-
-### Roadmap: better performance scaling for multi-core CPUs (#3957)
-* Poor performance scaling of the `hist` algorithm for multi-core CPUs has been under investigation (#3810). #3957 marks an important step toward better performance scaling, by using software pre-fetching and replacing STL vectors with C-style arrays. Special thanks to @Laurae2 and @SmirnovEgorRu.
-* See #3810 for latest progress on this roadmap.
-
-### New feature: Distributed Fast Histogram Algorithm (`hist`) (#4011, #4102, #4140, #4128)
-* It is now possible to run the `hist` algorithm in distributed setting. Special thanks to @CodingCat. The benefits include:
-  1. Faster local computation via feature binning
-  2. Support for monotonic constraints and feature interaction constraints
-  3. Simpler codebase than `approx`, allowing for future improvement
-* Depth-wise tree growing is now performed in a separate code path, so that cross-node syncronization is performed only once per level.
-
-### New feature: Multi-Node, Multi-GPU training (#4095)
-* Distributed training is now able to utilize clusters equipped with NVIDIA GPUs. In particular, the rabit AllReduce layer will communicate GPU device information. Special thanks to @mt-jones, @RAMitchell, @rongou, @trivialfis, @canonizer, and @jeffdk.
-* Resource management systems will be able to assign a rank for each GPU in the cluster.
-* In Dask, users will be able to construct a collection of XGBoost processes over an inhomogeneous device cluster (i.e. workers with different number and/or kinds of GPUs).
-
-### New feature: Multiple validation datasets in XGBoost4J-Spark (#3904, #3910)
-* You can now track the performance of the model during training with multiple evaluation datasets. By specifying `eval_sets` or call `setEvalSets` over a `XGBoostClassifier` or `XGBoostRegressor`, you can pass in multiple evaluation datasets typed as a `Map` from `String` to `DataFrame`. Special thanks to @CodingCat.
-* See the usage of multiple validation datasets [here](https://github.com/dmlc/xgboost/blob/0c1d5f1120c0a159f2567b267f0ec4ffadee00d0/jvm-packages/xgboost4j-example/src/main/scala/ml/dmlc/xgboost4j/scala/example/spark/SparkTraining.scala#L66-L78)
-
-### New feature: Additional metric functions for GPUs (#3952)
-* Element-wise metrics have been ported to GPU: `rmse`, `mae`, `logloss`, `poisson-nloglik`, `gamma-deviance`, `gamma-nloglik`, `error`, `tweedie-nloglik`. Special thanks to @trivialfis and @RAMitchell.
-* With supported metrics, XGBoost will select the correct devices based on your system and `n_gpus` parameter.
-
-### New feature: Column sampling at individual nodes (splits) (#3971)
-* Columns (features) can now be sampled at individual tree nodes, in addition to per-tree and per-level sampling. To enable per-node sampling, set `colsample_bynode` parameter, which represents the fraction of columns sampled at each node. This parameter is set to 1.0 by default (i.e. no sampling per node). Special thanks to @canonizer.
-* The `colsample_bynode` parameter works cumulatively with other `colsample_by*` parameters: for example, `{'colsample_bynode':0.5, 'colsample_bytree':0.5}` with 100 columns will give 25 features to choose from at each split.
-
-### Major API change: consistent logging level via `verbosity` (#3982, #4002, #4138)
-* XGBoost now allows fine-grained control over logging. You can set `verbosity` to 0 (silent), 1 (warning), 2 (info), and 3 (debug). This is useful for controlling the amount of logging outputs. Special thanks to @trivialfis.
-* Parameters `silent` and `debug_verbose` are now deprecated.
-* Note: Sometimes XGBoost tries to change configurations based on heuristics, which is displayed as warning message.  If there's unexpected behaviour, please try to increase value of verbosity.
-
-### Major bug fix: external memory (#4040, #4193)
-* Clarify object ownership in multi-threaded prefetcher, to avoid memory error.
-* Correctly merge two column batches (which uses [CSC layout](https://en.wikipedia.org/wiki/Sparse_matrix#Compressed_sparse_column_(CSC_or_CCS))).
-* Add unit tests for external memory.
-* Special thanks to @trivialfis and @hcho3.
-
-### Major bug fix: early stopping fixed in XGBoost4J and XGBoost4J-Spark (#3928, #4176)
-* Early stopping in XGBoost4J and XGBoost4J-Spark is now consistent with its counterpart in the Python package. Training stops if the current iteration is `earlyStoppingSteps` away from the best iteration. If there are multiple evaluation sets, only the last one is used to determinate early stop.
-* See the updated documentation [here](https://xgboost.readthedocs.io/en/release_0.82/jvm/xgboost4j_spark_tutorial.html#early-stopping)
-* Special thanks to @CodingCat, @yanboliang, and @mingyang.
-
-### Major bug fix: infrequent features should not crash distributed training (#4045)
-* For infrequently occuring features, some partitions may not get any instance. This scenario used to crash distributed training due to mal-formed ranges. The problem has now been fixed.
-* In practice, one-hot-encoded categorical variables tend to produce rare features, particularly when the cardinality is high.
-* Special thanks to @CodingCat.
-
-### Performance improvements
-* Faster, more space-efficient radix sorting in `gpu_hist` (#3895)
-* Subtraction trick in histogram calculation in `gpu_hist` (#3945)
-* More performant re-partition in XGBoost4J-Spark (#4049)
-
-### Bug-fixes
-* Fix semantics of `gpu_id` when running multiple XGBoost processes on a multi-GPU machine (#3851)
-* Fix page storage path for external memory on Windows (#3869)
-* Fix configuration setup so that DART utilizes GPU (#4024)
-* Eliminate NAN values from SHAP prediction (#3943)
-* Prevent empty quantile sketches in `hist` (#4155)
-* Enable running objectives with 0 GPU (#3878)
-* Parameters are no longer dependent on system locale (#3891, #3907)
-* Use consistent data type in the GPU coordinate descent code (#3917)
-* Remove undefined behavior in the CLI config parser on the ARM platform (#3976)
-* Initialize counters in GPU AllReduce (#3987)
-* Prevent deadlocks in GPU AllReduce (#4113)
-* Load correct values from sliced NumPy arrays (#4147, #4165)
-* Fix incorrect GPU device selection (#4161)
-* Make feature binning logic in `hist` aware of query groups when running a ranking task (#4115). For ranking task, query groups are weighted, not individual instances.
-* Generate correct C++ exception type for `LOG(FATAL)` macro (#4159)
-* Python package
-  - Python package should run on system without `PATH` environment variable (#3845)
-  - Fix `coef_` and `intercept_` signature to be compatible with `sklearn.RFECV` (#3873)
-  - Use UTF-8 encoding in Python package README, to support non-English locale (#3867)
-  - Add AUC-PR to list of metrics to maximize for early stopping (#3936)
-  - Allow loading pickles without `self.booster` attribute, for backward compatibility (#3938, #3944)
-  - White-list DART for feature importances (#4073)
-  - Update usage of [h2oai/datatable](https://github.com/h2oai/datatable) (#4123)
-* XGBoost4J-Spark
-  - Address scalability issue in prediction (#4033)
-  - Enforce the use of per-group weights for ranking task (#4118)
-  - Fix vector size of `rawPredictionCol` in `XGBoostClassificationModel` (#3932)
-  - More robust error handling in Spark tracker (#4046, #4108)
-  - Fix return type of `setEvalSets` (#4105)
-  - Return correct value of `getMaxLeaves` (#4114)
-
-### API changes
-* Add experimental parameter `single_precision_histogram` to use single-precision histograms for the `gpu_hist` algorithm (#3965)
-* Python package
-  - Add option to select type of feature importances in the scikit-learn inferface (#3876)
-  - Add `trees_to_df()` method to dump decision trees as Pandas data frame (#4153)
-  - Add options to control node shapes in the GraphViz plotting function (#3859)
-  - Add `xgb_model` option to `XGBClassifier`, to load previously saved model (#4092)
-  - Passing lists into `DMatrix` is now deprecated (#3970)
-* XGBoost4J
-  - Support multiple feature importance features (#3801)
-
-### Maintenance: Refactor C++ code for legibility and maintainability
-* Refactor `hist` algorithm code and add unit tests (#3836)
-* Minor refactoring of split evaluator in `gpu_hist` (#3889)
-* Removed unused leaf vector field in the tree model (#3989)
-* Simplify the tree representation by combining `TreeModel` and `RegTree` classes (#3995)
-* Simplify and harden tree expansion code (#4008, #4015)
-* De-duplicate parameter classes in the linear model algorithms (#4013)
-* Robust handling of ranges with C++20 span in `gpu_exact` and `gpu_coord_descent` (#4020, #4029)
-* Simplify tree training code (#3825). Also use Span class for robust handling of ranges.
-
-### Maintenance: testing, continuous integration, build system
-* Disallow `std::regex` since it's not supported by GCC 4.8.x (#3870)
-* Add multi-GPU tests for coordinate descent algorithm for linear models (#3893, #3974)
-* Enforce naming style in Python lint (#3896)
-* Refactor Python tests (#3897, #3901): Use pytest exclusively, display full trace upon failure
-* Address `DeprecationWarning` when using Python collections (#3909)
-* Use correct group for maven site plugin (#3937)
-* Jenkins CI is now using on-demand EC2 instances exclusively, due to unreliability of Spot instances (#3948)
-* Better GPU performance logging (#3945)
-* Fix GPU tests on machines with only 1 GPU (#4053)
-* Eliminate CRAN check warnings and notes (#3988)
-* Add unit tests for tree serialization (#3989)
-* Add unit tests for tree fitting functions in `hist` (#4155)
-* Add a unit test for `gpu_exact` algorithm (#4020)
-* Correct JVM CMake GPU flag (#4071)
-* Fix failing Travis CI on Mac (#4086)
-* Speed up Jenkins by not compiling CMake (#4099)
-* Analyze C++ and CUDA code using clang-tidy, as part of Jenkins CI pipeline (#4034)
-* Fix broken R test: Install Homebrew GCC (#4142)
-* Check for empty datasets in GPU unit tests (#4151)
-* Fix Windows compilation (#4139)
-* Comply with latest convention of cpplint (#4157)
-* Fix a unit test in `gpu_hist` (#4158)
-* Speed up data generation in Python tests (#4164)
-
-### Usability Improvements
-* Add link to [InfoWorld 2019 Technology of the Year Award](https://www.infoworld.com/article/3336072/application-development/infoworlds-2019-technology-of-the-year-award-winners.html) (#4116)
-* Remove outdated AWS YARN tutorial (#3885)
-* Document current limitation in number of features (#3886)
-* Remove unnecessary warning when `gblinear` is selected (#3888)
-* Document limitation of CSV parser: header not supported (#3934)
-* Log training parameters in XGBoost4J-Spark (#4091)
-* Clarify early stopping behavior in the scikit-learn interface (#3967)
-* Clarify behavior of `max_depth` parameter (#4078)
-* Revise Python docstrings for ranking task (#4121). In particular, weights must be per-group in learning-to-rank setting.
-* Document parameter `num_parallel_tree` (#4022)
-* Add Jenkins status badge (#4090)
-* Warn users against using internal functions of `Booster` object (#4066)
-* Reformat `benchmark_tree.py` to comply with Python style convention (#4126)
-* Clarify a comment in `objectiveTrait` (#4174)
-* Fix typos and broken links in documentation (#3890, #3872, #3902, #3919, #3975, #4027, #4156, #4167)
-
-### Acknowledgement
-**Contributors** (in no particular order): Jiaming Yuan (@trivialfis), Hyunsu Cho (@hcho3), Nan Zhu (@CodingCat), Rory Mitchell (@RAMitchell), Yanbo Liang (@yanboliang), Andy Adinets (@canonizer), Tong He (@hetong007), Yuan Tang (@terrytangyuan)
-
-**First-time Contributors** (in no particular order): Jelle Zijlstra (@JelleZijlstra), Jiacheng Xu (@jiachengxu), @ajing, Kashif Rasul (@kashif), @theycallhimavi, Joey Gao (@pjgao), Prabakaran Kumaresshan (@nixphix), Huafeng Wang (@huafengw), @lyxthe, Sam Wilkinson (@scwilkinson), Tatsuhito Kato (@stabacov), Shayak Banerjee (@shayakbanerjee), Kodi Arfer (@Kodiologist), @KyleLi1985, Egor Smirnov (@SmirnovEgorRu), @tmitanitky, Pasha Stetsenko (@st-pasha), Kenichi Nagahara (@keni-chi), Abhai Kollara Dilip (@abhaikollara), Patrick Ford (@pford221), @hshujuan, Matthew Jones (@mt-jones), Thejaswi Rao (@teju85), Adam November (@anovember)
-
-**First-time Reviewers** (in no particular order): Mingyang Hu (@mingyang), Theodore Vasiloudis (@thvasilo), Jakub Troszok (@troszok), Rong Ou (@rongou), @Denisevi4, Matthew Jones (@mt-jones), Jeff Kaplan (@jeffdk)
-
 ## v0.81 (2018.11.04)
 ### New feature: feature interaction constraints
 * Users are now able to control which features (independent variables) are allowed to interact by specifying feature interaction constraints (#3466).
@@ -338,7 +179,7 @@ This release is packed with many new features and bug fixes.
   - Latest master: https://xgboost.readthedocs.io/en/latest
   - 0.80 stable: https://xgboost.readthedocs.io/en/release_0.80
   - 0.72 stable: https://xgboost.readthedocs.io/en/release_0.72
-* Support for per-group weights in ranking objective (#3379)
+* Ranking task now uses instance weights (#3379)
 * Fix inaccurate decimal parsing (#3546)
 * New functionality
   - Query ID column support in LIBSVM data files (#2749). This is convenient for performing ranking task in distributed setting.

R-package/DESCRIPTION

@@ -1,8 +1,8 @@
 Package: xgboost
 Type: Package
 Title: Extreme Gradient Boosting
-Version: 0.82.0.1
+Version: 0.81.0.1
-Date: 2019-03-11
+Date: 2018-08-13
 Authors@R: c(
   person("Tianqi", "Chen", role = c("aut"),
          email = "tianqi.tchen@gmail.com"),

R-package/R/xgb.plot.deepness.R

@@ -27,7 +27,7 @@
 #' a tree's median absolute leaf weight changes through the iterations.
 #'
 #' This function was inspired by the blog post
-#' \url{https://github.com/aysent/random-forest-leaf-visualization}.
+#' \url{http://aysent.github.io/2015/11/08/random-forest-leaf-visualization.html}.
 #' 
 #' @return
 #' 

R-package/configure

@@ -1667,12 +1667,12 @@ OPENMP_CXXFLAGS=""
 
 if test `uname -s` = "Linux"
 then
-  OPENMP_CXXFLAGS="\$(SHLIB_OPENMP_CXXFLAGS)"
+  OPENMP_CXXFLAGS="\$(SHLIB_OPENMP_CFLAGS)"
 fi
 
 if test `uname -s` = "Darwin"
 then
-  OPENMP_CXXFLAGS="\$(SHLIB_OPENMP_CXXFLAGS)"
+  OPENMP_CXXFLAGS="\$(SHLIB_OPENMP_CFLAGS)"
   ac_pkg_openmp=no
   { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether OpenMP will work in a package" >&5
 $as_echo_n "checking whether OpenMP will work in a package... " >&6; }

R-package/configure.ac

@@ -8,12 +8,12 @@ OPENMP_CXXFLAGS=""
 
 if test `uname -s` = "Linux"
 then
-  OPENMP_CXXFLAGS="\$(SHLIB_OPENMP_CXXFLAGS)"
+  OPENMP_CXXFLAGS="\$(SHLIB_OPENMP_CFLAGS)"
 fi
 
 if test `uname -s` = "Darwin"
 then
-  OPENMP_CXXFLAGS="\$(SHLIB_OPENMP_CXXFLAGS)"
+  OPENMP_CXXFLAGS="\$(SHLIB_OPENMP_CFLAGS)"
   ac_pkg_openmp=no
   AC_MSG_CHECKING([whether OpenMP will work in a package])
   AC_LANG_CONFTEST(

R-package/demo/custom_objective.R

@@ -33,7 +33,7 @@ evalerror <- function(preds, dtrain) {
   return(list(metric = "error", value = err))
 }
 
-param <- list(max_depth=2, eta=1, nthread = 2, verbosity=0, 
+param <- list(max_depth=2, eta=1, nthread = 2, silent=1, 
               objective=logregobj, eval_metric=evalerror)
 print ('start training with user customized objective')
 # training with customized objective, we can also do step by step training
@@ -57,7 +57,7 @@ logregobjattr <- function(preds, dtrain) {
   hess <- preds * (1 - preds)
   return(list(grad = grad, hess = hess))
 }
-param <- list(max_depth=2, eta=1, nthread = 2, verbosity=0, 
+param <- list(max_depth=2, eta=1, nthread = 2, silent=1, 
               objective=logregobjattr, eval_metric=evalerror)
 print ('start training with user customized objective, with additional attributes in DMatrix')
 # training with customized objective, we can also do step by step training

R-package/demo/early_stopping.R

@@ -7,7 +7,7 @@ dtest <- xgb.DMatrix(agaricus.test$data, label = agaricus.test$label)
 # note: for customized objective function, we leave objective as default
 # note: what we are getting is margin value in prediction
 # you must know what you are doing
-param <- list(max_depth=2, eta=1, nthread=2, verbosity=0)
+param <- list(max_depth=2, eta=1, nthread = 2, silent=1)
 watchlist <- list(eval = dtest)
 num_round <- 20
 # user define objective function, given prediction, return gradient and second order gradient

R-package/man/xgb.plot.deepness.Rd

@@ -50,7 +50,7 @@ per tree with respect to tree number are created. And \code{which="med.weight"}
 a tree's median absolute leaf weight changes through the iterations.
 
 This function was inspired by the blog post
-\url{https://github.com/aysent/random-forest-leaf-visualization}.
+\url{http://aysent.github.io/2015/11/08/random-forest-leaf-visualization.html}.
 }
 \examples{
 

R-package/src/Makevars.in

@@ -17,8 +17,8 @@ endif
 $(foreach v, $(XGB_RFLAGS), $(warning $(v)))
 
 PKG_CPPFLAGS=  -I$(PKGROOT)/include -I$(PKGROOT)/dmlc-core/include -I$(PKGROOT)/rabit/include -I$(PKGROOT) $(XGB_RFLAGS)
-PKG_CXXFLAGS= @OPENMP_CXXFLAGS@ -pthread
+PKG_CXXFLAGS= @OPENMP_CXXFLAGS@ $(SHLIB_PTHREAD_FLAGS)
-PKG_LIBS = @OPENMP_CXXFLAGS@ -pthread
+PKG_LIBS = @OPENMP_CXXFLAGS@ $(SHLIB_PTHREAD_FLAGS)
 OBJECTS= ./xgboost_R.o ./xgboost_custom.o ./xgboost_assert.o ./init.o\
          $(PKGROOT)/amalgamation/xgboost-all0.o $(PKGROOT)/amalgamation/dmlc-minimum0.o\
          $(PKGROOT)/rabit/src/engine_empty.o $(PKGROOT)/rabit/src/c_api.o

R-package/src/Makevars.win

@@ -29,8 +29,8 @@ endif
 $(foreach v, $(XGB_RFLAGS), $(warning $(v)))
 
 PKG_CPPFLAGS=  -I$(PKGROOT)/include -I$(PKGROOT)/dmlc-core/include -I$(PKGROOT)/rabit/include -I$(PKGROOT) $(XGB_RFLAGS)
-PKG_CXXFLAGS= $(SHLIB_OPENMP_CXXFLAGS) $(SHLIB_PTHREAD_FLAGS)
+PKG_CXXFLAGS= $(SHLIB_OPENMP_CFLAGS) $(SHLIB_PTHREAD_FLAGS)
-PKG_LIBS = $(SHLIB_OPENMP_CXXFLAGS) $(SHLIB_PTHREAD_FLAGS)
+PKG_LIBS = $(SHLIB_OPENMP_CFLAGS) $(SHLIB_PTHREAD_FLAGS)
 OBJECTS= ./xgboost_R.o ./xgboost_custom.o ./xgboost_assert.o ./init.o\
          $(PKGROOT)/amalgamation/xgboost-all0.o $(PKGROOT)/amalgamation/dmlc-minimum0.o\
          $(PKGROOT)/rabit/src/engine_empty.o $(PKGROOT)/rabit/src/c_api.o

R-package/src/init.c

@@ -70,7 +70,7 @@ static const R_CallMethodDef CallEntries[] = {
 
 #if defined(_WIN32)
 __declspec(dllexport)
-#endif  // defined(_WIN32)
+#endif
 void R_init_xgboost(DllInfo *dll) {
   R_registerRoutines(dll, NULL, CallEntries, NULL, NULL);
   R_useDynamicSymbols(dll, FALSE);

R-package/src/xgboost_custom.cc

@@ -32,10 +32,7 @@ extern "C" {
 
 namespace xgboost {
 ConsoleLogger::~ConsoleLogger() {
-  if (cur_verbosity_ == LogVerbosity::kIgnore ||
+  dmlc::CustomLogMessage::Log(log_stream_.str());
-      cur_verbosity_ <= global_verbosity_) {
-    dmlc::CustomLogMessage::Log(log_stream_.str());
-  }
 }
 TrackerLogger::~TrackerLogger() {
   dmlc::CustomLogMessage::Log(log_stream_.str());
@@ -49,11 +46,10 @@ namespace common {
 bool CheckNAN(double v) {
   return ISNAN(v);
 }
-#if !defined(XGBOOST_USE_CUDA)
 double LogGamma(double v) {
   return lgammafn(v);
 }
-#endif  // !defined(XGBOOST_USE_CUDA)
+
 // customize random engine.
 void CustomGlobalRandomEngine::seed(CustomGlobalRandomEngine::result_type val) {
   // ignore the seed

R-package/tests/testthat/test_basic.R

@@ -182,7 +182,7 @@ test_that("xgb.cv works", {
   expect_is(cv, 'xgb.cv.synchronous')
   expect_false(is.null(cv$evaluation_log))
   expect_lt(cv$evaluation_log[, min(test_error_mean)], 0.03)
-  expect_lt(cv$evaluation_log[, min(test_error_std)], 0.008)
+  expect_lt(cv$evaluation_log[, min(test_error_std)], 0.004)
   expect_equal(cv$niter, 2)
   expect_false(is.null(cv$folds) && is.list(cv$folds))
   expect_length(cv$folds, 5)

R-package/tests/testthat/test_callbacks.R

@@ -282,7 +282,7 @@ test_that("prediction in xgb.cv works for gblinear too", {
 })
 
 test_that("prediction in early-stopping xgb.cv works", {
-  set.seed(11)
+  set.seed(1)
   expect_output(
     cv <- xgb.cv(param, dtrain, nfold = 5, eta = 0.1, nrounds = 20,
                  early_stopping_rounds = 5, maximize = FALSE, prediction = TRUE)

R-package/tests/testthat/test_interactions.R

@@ -81,39 +81,6 @@ test_that("predict feature interactions works", {
   expect_lt(max(abs(intr - gt_intr)), 0.1)
 })
 
-test_that("SHAP contribution values are not NAN", {
-  d <- data.frame(
-    x1 = c(-2.3, 1.4, 5.9, 2, 2.5, 0.3, -3.6, -0.2, 0.5, -2.8, -4.6, 3.3, -1.2,
-           -1.1, -2.3, 0.4, -1.5, -0.2, -1, 3.7),
-    x2 = c(291.179171, 269.198331, 289.942097, 283.191669, 269.673332,
-           294.158346, 287.255835, 291.530838, 285.899586, 269.290833,
-           268.649586, 291.530841, 280.074593, 269.484168, 293.94042,
-           294.327506, 296.20709, 295.441669, 283.16792, 270.227085),
-    y = c(9, 15, 5.7, 9.2, 22.4, 5, 9, 3.2, 7.2, 13.1, 7.8, 16.9, 6.5, 22.1,
-          5.3, 10.4, 11.1, 13.9, 11, 20.5),
-    fold = c(2, 2, 2, 1, 2, 2, 1, 2, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2))
-
-  ivs <- c("x1", "x2")
-
-  fit <- xgboost(
-    verbose = 0,
-    params = list(
-      objective = "reg:linear",
-      eval_metric = "rmse"),
-    data = as.matrix(subset(d, fold == 2)[, ivs]),
-    label = subset(d, fold == 2)$y,
-    nthread = 1,
-    nrounds = 3)
-
-  shaps <- as.data.frame(predict(fit,
-    newdata = as.matrix(subset(d, fold == 1)[, ivs]),
-    predcontrib = T))
-  result <- cbind(shaps, sum = rowSums(shaps), pred = predict(fit,
-      newdata = as.matrix(subset(d, fold == 1)[, ivs])))
-
-  expect_true(identical(TRUE, all.equal(result$sum, result$pred, tol = 1e-6)))
-})
-
 
 test_that("multiclass feature interactions work", {
   dm <- xgb.DMatrix(as.matrix(iris[,-5]), label=as.numeric(iris$Species)-1)

README.md

@@ -1,6 +1,5 @@
 <img src=https://raw.githubusercontent.com/dmlc/dmlc.github.io/master/img/logo-m/xgboost.png width=135/>  eXtreme Gradient Boosting
 ===========
-[![Build Status](https://xgboost-ci.net/job/xgboost/job/master/badge/icon?style=plastic)](https://xgboost-ci.net/blue/organizations/jenkins/xgboost/activity)
 [![Build Status](https://travis-ci.org/dmlc/xgboost.svg?branch=master)](https://travis-ci.org/dmlc/xgboost)
 [![Build Status](https://ci.appveyor.com/api/projects/status/5ypa8vaed6kpmli8?svg=true)](https://ci.appveyor.com/project/tqchen/xgboost)
 [![Documentation Status](https://readthedocs.org/projects/xgboost/badge/?version=latest)](https://xgboost.readthedocs.org)

amalgamation/xgboost-all0.cc

@@ -48,7 +48,7 @@
 #include "../src/tree/tree_model.cc"
 #include "../src/tree/tree_updater.cc"
 #include "../src/tree/updater_colmaker.cc"
-#include "../src/tree/updater_quantile_hist.cc"
+#include "../src/tree/updater_fast_hist.cc"
 #include "../src/tree/updater_prune.cc"
 #include "../src/tree/updater_refresh.cc"
 #include "../src/tree/updater_sync.cc"

appveyor.yml

@@ -44,12 +44,12 @@ install:
     - set DO_PYTHON=off
     - if /i "%target%" == "mingw" set DO_PYTHON=on
     - if /i "%target%_%ver%_%configuration%" == "msvc_2015_Release" set DO_PYTHON=on
-    - if /i "%DO_PYTHON%" == "on" conda install -y numpy scipy pandas matplotlib pytest scikit-learn graphviz python-graphviz
+    - if /i "%DO_PYTHON%" == "on" conda install -y numpy scipy pandas matplotlib nose scikit-learn graphviz python-graphviz
     # R: based on https://github.com/krlmlr/r-appveyor
     - ps: |
         if($env:target -eq 'rmingw' -or $env:target -eq 'rmsvc') {
           #$ErrorActionPreference = "Stop"
-          Invoke-WebRequest https://raw.githubusercontent.com/krlmlr/r-appveyor/master/scripts/appveyor-tool.ps1 -OutFile "$Env:TEMP\appveyor-tool.ps1"
+          Invoke-WebRequest http://raw.github.com/krlmlr/r-appveyor/master/scripts/appveyor-tool.ps1 -OutFile "$Env:TEMP\appveyor-tool.ps1"
           Import-Module "$Env:TEMP\appveyor-tool.ps1"
           Bootstrap
           $DEPS = "c('data.table','magrittr','stringi','ggplot2','DiagrammeR','Ckmeans.1d.dp','vcd','testthat','lintr','knitr','rmarkdown')"
@@ -96,7 +96,7 @@ build_script:
 
 test_script:
     - cd %APPVEYOR_BUILD_FOLDER%
-    - if /i "%DO_PYTHON%" == "on" python -m pytest tests/python
+    - if /i "%DO_PYTHON%" == "on" python -m nose tests/python
     # mingw R package: run the R check (which includes unit tests), and also keep the built binary package
     - if /i "%target%" == "rmingw" (
         set _R_CHECK_CRAN_INCOMING_=FALSE&&

cmake/build_config.h.in

@@ -1,11 +0,0 @@
-/*!
- * Copyright 2019 by Contributors
- * \file build_config.h
- */
-#ifndef XGBOOST_BUILD_CONFIG_H_
-#define XGBOOST_BUILD_CONFIG_H_
-
-#cmakedefine XGBOOST_MM_PREFETCH_PRESENT
-#cmakedefine XGBOOST_BUILTIN_PREFETCH_PRESENT
-
-#endif  // XGBOOST_BUILD_CONFIG_H_

demo/README.md

@@ -135,7 +135,6 @@ Send a PR to add a one sentence description:)
 
 ## Awards
 - [John Chambers Award](http://stat-computing.org/awards/jmc/winners.html) - 2016 Winner: XGBoost R Package, by Tong He (Simon Fraser University) and Tianqi Chen (University of Washington)
-- [InfoWorld’s 2019 Technology of the Year Award](https://www.infoworld.com/article/3336072/application-development/infoworlds-2019-technology-of-the-year-award-winners.html)
 
 ## Windows Binaries
 Unofficial windows binaries and instructions on how to use them are hosted on [Guido Tapia's blog](http://www.picnet.com.au/blogs/guido/post/2016/09/22/xgboost-windows-x64-binaries-for-download/)

demo/binary_classification/README.md

@@ -62,7 +62,7 @@ test:data = "agaricus.txt.test"
 We use the tree booster and logistic regression objective in our setting. This indicates that we accomplish our task using classic gradient boosting regression tree(GBRT), which is a promising method for binary classification.
 
 The parameters shown in the example gives the most common ones that are needed to use xgboost.
-If you are interested in more parameter settings, the complete parameter settings and detailed descriptions are [here](../../doc/parameter.rst). Besides putting the parameters in the configuration file, we can set them by passing them as arguments as below:
+If you are interested in more parameter settings, the complete parameter settings and detailed descriptions are [here](../../doc/parameter.md). Besides putting the parameters in the configuration file, we can set them by passing them as arguments as below:
 
 ```
 ../../xgboost mushroom.conf max_depth=6

demo/kaggle-otto/README.MD

@@ -1,4 +1,4 @@
-Benchmark for Otto Group Competition
+Benckmark for Otto Group Competition
 =========
 
 This is a folder containing the benchmark for the [Otto Group Competition on Kaggle](http://www.kaggle.com/c/otto-group-product-classification-challenge).
@@ -20,3 +20,5 @@ devtools::install_github('tqchen/xgboost',subdir='R-package')
 ```
 
 Windows users may need to install [RTools](http://cran.r-project.org/bin/windows/Rtools/) first.
+
+

doc/R-package/xgboostPresentation.md

@@ -176,7 +176,7 @@ In a *sparse* matrix, cells containing `0` are not stored in memory. Therefore,
 We will train decision tree model using the following parameters:
 
 * `objective = "binary:logistic"`: we will train a binary classification model ;
-* `max.depth = 2`: the trees won't be deep, because our case is very simple ;
+* `max.deph = 2`: the trees won't be deep, because our case is very simple ;
 * `nthread = 2`: the number of cpu threads we are going to use;
 * `nrounds = 2`: there will be two passes on the data, the second one will enhance the model by further reducing the difference between ground truth and prediction.
 
@@ -576,8 +576,8 @@ print(class(rawVec))
 bst3 <- xgb.load(rawVec)
 pred3 <- predict(bst3, test$data)
 
-# pred3 should be identical to pred
+# pred2 should be identical to pred
-print(paste("sum(abs(pred3-pred))=", sum(abs(pred3-pred))))
+print(paste("sum(abs(pred3-pred))=", sum(abs(pred2-pred))))
 ```
 
 ```

doc/build.rst

@@ -13,7 +13,7 @@ Installation Guide
     #   * xgboost-{version}-py2.py3-none-win_amd64.whl
     pip3 install xgboost
 
-  * The binary wheel will support GPU algorithms (`gpu_exact`, `gpu_hist`) on machines with NVIDIA GPUs. Please note that **training with multiple GPUs is only supported for Linux platform**. See :doc:`gpu/index`.
+  * The binary wheel will support GPU algorithms (`gpu_exact`, `gpu_hist`) on machines with NVIDIA GPUs. **However, it will not support multi-GPU training; only single GPU will be used.** To enable multi-GPU training, download and install the binary wheel from `this page <https://s3-us-west-2.amazonaws.com/xgboost-wheels/list.html>`_.
   * Currently, we provide binary wheels for 64-bit Linux and Windows.
 
 ****************************
@@ -70,21 +70,19 @@ Our goal is to build the shared library:
 The minimal building requirement is
 
 - A recent C++ compiler supporting C++11 (g++-4.8 or higher)
-- CMake 3.2 or higher
+
+We can edit ``make/config.mk`` to change the compile options, and then build by
+``make``. If everything goes well, we can go to the specific language installation section.
 
 Building on Ubuntu/Debian
 =========================
 
-On Ubuntu, one builds XGBoost by running CMake:
+On Ubuntu, one builds XGBoost by running
 
 .. code-block:: bash
 
   git clone --recursive https://github.com/dmlc/xgboost
-  cd xgboost
+  cd xgboost; make -j4
-  mkdir build
-  cd build
-  cmake ..
-  make -j4
 
 Building on OSX
 ===============
@@ -92,11 +90,11 @@ Building on OSX
 Install with pip: simple method
 --------------------------------
 
-First, obtain ``gcc-8`` with Homebrew (https://brew.sh/) to enable multi-threading (i.e. using multiple CPU threads for training). The default Apple Clang compiler does not support OpenMP, so using the default compiler would have disabled multi-threading.
+First, obtain ``gcc-7`` with Homebrew (https://brew.sh/) to enable multi-threading (i.e. using multiple CPU threads for training). The default Apple Clang compiler does not support OpenMP, so using the default compiler would have disabled multi-threading.
 
 .. code-block:: bash
 
-  brew install gcc@8
+  brew install gcc@7
 
 Then install XGBoost with ``pip``:
 
@@ -109,11 +107,11 @@ You might need to run the command with ``--user`` flag if you run into permissio
 Build from the source code - advanced method
 --------------------------------------------
 
-Obtain ``gcc-8`` from Homebrew:
+Obtain ``gcc-7`` from Homebrew:
 
 .. code-block:: bash
 
-  brew install gcc@8
+  brew install gcc@7
 
 Now clone the repository:
 
@@ -121,13 +119,13 @@ Now clone the repository:
 
   git clone --recursive https://github.com/dmlc/xgboost
 
-Create the ``build/`` directory and invoke CMake. Make sure to add ``CC=gcc-8 CXX=g++-8`` so that Homebrew GCC is selected. After invoking CMake, you can build XGBoost with ``make``:
+Create the ``build/`` directory and invoke CMake. Make sure to add ``CC=gcc-7 CXX=g++-7`` so that Homebrew GCC is selected. After invoking CMake, you can build XGBoost with ``make``:
 
 .. code-block:: bash
 
   mkdir build
   cd build
-  CC=gcc-8 CXX=g++-8 cmake ..
+  CC=gcc-7 CXX=g++-7 cmake ..
   make -j4
 
 You may now continue to `Python Package Installation`_.
@@ -144,20 +142,6 @@ We recommend you use `Git for Windows <https://git-for-windows.github.io/>`_, as
 
 XGBoost support compilation with Microsoft Visual Studio and MinGW.
 
-Compile XGBoost with Microsoft Visual Studio
---------------------------------------------
-To build with Visual Studio, we will need CMake. Make sure to install a recent version of CMake. Then run the following from the root of the XGBoost directory:
-
-.. code-block:: bash
-
-  mkdir build
-  cd build
-  cmake .. -G"Visual Studio 14 2015 Win64"
-
-This specifies an out of source build using the Visual Studio 64 bit generator. (Change the ``-G`` option appropriately if you have a different version of Visual Studio installed.) Open the ``.sln`` file in the build directory and build with Visual Studio.
-
-After the build process successfully ends, you will find a ``xgboost.dll`` library file inside ``./lib/`` folder.
-
 Compile XGBoost using MinGW
 ---------------------------
 After installing `Git for Windows <https://git-for-windows.github.io/>`_, you should have a shortcut named ``Git Bash``. You should run all subsequent steps in ``Git Bash``.
@@ -179,6 +163,22 @@ To build with MinGW, type:
 
 See :ref:`mingw_python` for buildilng XGBoost for Python.
 
+Compile XGBoost with Microsoft Visual Studio
+--------------------------------------------
+To build with Visual Studio, we will need CMake. Make sure to install a recent version of CMake. Then run the following from the root of the XGBoost directory:
+
+.. code-block:: bash
+
+  mkdir build
+  cd build
+  cmake .. -G"Visual Studio 12 2013 Win64"
+
+This specifies an out of source build using the MSVC 12 64 bit generator. Open the ``.sln`` file in the build directory and build with Visual Studio. To use the Python module you can copy ``xgboost.dll`` into ``python-package/xgboost``.
+
+After the build process successfully ends, you will find a ``xgboost.dll`` library file inside ``./lib/`` folder, copy this file to the the API package folder like ``python-package/xgboost`` if you are using Python API.
+
+Unofficial windows binaries and instructions on how to use them are hosted on `Guido Tapia's blog <http://www.picnet.com.au/blogs/guido/post/2016/09/22/xgboost-windows-x64-binaries-for-download/>`_.
+
 .. _build_gpu_support:
 
 Building with GPU support
@@ -207,7 +207,13 @@ From the command line on Linux starting from the XGBoost directory:
     cmake .. -DUSE_CUDA=ON -DUSE_NCCL=ON -DNCCL_ROOT=/path/to/nccl2
     make -j4
 
-On Windows, run CMake as follows:
+On Windows, see what options for generators you have for CMake, and choose one with ``[arch]`` replaced with Win64:
+
+.. code-block:: bash
+
+  cmake -help
+
+Then run CMake as follows:
 
 .. code-block:: bash
 
@@ -215,15 +221,13 @@ On Windows, run CMake as follows:
   cd build
   cmake .. -G"Visual Studio 14 2015 Win64" -DUSE_CUDA=ON
 
-(Change the ``-G`` option appropriately if you have a different version of Visual Studio installed.)
-
 .. note:: Visual Studio 2017 Win64 Generator may not work
 
   Choosing the Visual Studio 2017 generator may cause compilation failure. When it happens, specify the 2015 compiler by adding the ``-T`` option:
 
   .. code-block:: bash
 
-    make .. -G"Visual Studio 15 2017 Win64" -T v140,cuda=8.0 -DUSE_CUDA=ON
+    make .. -G"Visual Studio 15 2017 Win64" -T v140,cuda=8.0 -DR_LIB=ON -DUSE_CUDA=ON
 
 To speed up compilation, the compute version specific to your GPU could be passed to cmake as, e.g., ``-DGPU_COMPUTE_VER=50``.
 The above cmake configuration run will create an ``xgboost.sln`` solution file in the build directory. Build this solution in release mode as a x64 build, either from Visual studio or from command line:
@@ -237,15 +241,15 @@ To speed up compilation, run multiple jobs in parallel by appending option ``--
 Customized Building
 ===================
 
-We recommend the use of CMake for most use cases. See the full range of building options in CMakeLists.txt.
+The configuration file ``config.mk`` modifies several compilation flags:
-
-Alternatively, you may use Makefile. The Makefile uses a configuration file ``config.mk``, which lets you modify several compilation flags:
 - Whether to enable support for various distributed filesystems such as HDFS and Amazon S3
 - Which compiler to use
 - And some more
 
 To customize, first copy ``make/config.mk`` to the project root and then modify the copy.
 
+Alternatively, use CMake.
+
 Python Package Installation
 ===========================
 
@@ -271,9 +275,9 @@ package manager, e.g. in Debian use
   If you recompiled XGBoost, then you need to reinstall it again to make the new library take effect.
 
 2. Only set the environment variable ``PYTHONPATH`` to tell Python where to find
-   the library. For example, assume we cloned ``xgboost`` on the home directory
+   the library. For example, assume we cloned `xgboost` on the home directory
-   ``~``. then we can added the following line in ``~/.bashrc``.
+   `~`. then we can added the following line in `~/.bashrc`.
-   This option is **recommended for developers** who change the code frequently. The changes will be immediately reflected once you pulled the code and rebuild the project (no need to call ``setup`` again).
+   This option is **recommended for developers** who change the code frequently. The changes will be immediately reflected once you pulled the code and rebuild the project (no need to call ``setup`` again)
 
 .. code-block:: bash
 
@@ -285,22 +289,30 @@ package manager, e.g. in Debian use
 
   cd python-package; python setup.py develop --user
 
+4. If you are installing the latest XGBoost version which requires compilation, add MinGW to the system PATH:
+
+.. code-block:: bash
+
+    import os
+    os.environ['PATH'] = os.environ['PATH'] + ';C:\\Program Files\\mingw-w64\\x86_64-5.3.0-posix-seh-rt_v4-rev0\\mingw64\\bin'
+
 .. _mingw_python:
 
-Building XGBoost library for Python for Windows with MinGW-w64 (Advanced)
+Building XGBoost library for Python for Windows with MinGW-w64
--------------------------------------------------------------------------
+--------------------------------------------------------------
 
-Windows versions of Python are built with Microsoft Visual Studio. Usually Python binary modules are built with the same compiler the interpreter is built with. However, you may not be able to use Visual Studio, for following reasons:
+Windows versions of Python are built with Microsoft Visual Studio. Usually Python binary modules are built with the same compiler the interpreter is built with, raising several potential concerns.
 
-1. VS is proprietary and commercial software. Microsoft provides a freeware "Community" edition, but its licensing terms impose restrictions as to where and how it can be used.
+1. VS is proprietary and commercial software. Microsoft provides a freeware "Community" edition, but its licensing terms are unsuitable for many organizations.
-2. Visual Studio contains telemetry, as documented in `Microsoft Visual Studio Licensing Terms <https://visualstudio.microsoft.com/license-terms/mt736442/>`_. Running software with telemetry may be against the policy of your organization.
+2. Visual Studio contains telemetry, as documented in `Microsoft Visual Studio Licensing Terms <https://visualstudio.microsoft.com/license-terms/mt736442/>`_. It `has been inserting telemetry <https://old.reddit.com/r/cpp/comments/4ibauu/visual_studio_adding_telemetry_function_calls_to/>`_ into apps for some time. In order to download VS distribution from MS servers one has to run the application containing telemetry. These facts have raised privacy and security concerns among some users and system administrators. Running software with telemetry may be against the policy of your organization.
+3. g++ usually generates faster code on ``-O3``.
 
-So you may want to build XGBoost with GCC own your own risk. This presents some difficulties because MSVC uses Microsoft runtime and MinGW-w64 uses own runtime, and the runtimes have different incompatible memory allocators. But in fact this setup is usable if you know how to deal with it. Here is some experience.
+So you may want to build XGBoost with g++ own your own risk. This opens a can of worms, because MSVC uses Microsoft runtime and MinGW-w64 uses own runtime, and the runtimes have different incompatible memory allocators. But in fact this setup is usable if you know how to deal with it. Here is some experience.
 
 1. The Python interpreter will crash on exit if XGBoost was used. This is usually not a big issue.
 2. ``-O3`` is OK.
 3. ``-mtune=native`` is also OK.
-4. Don't use ``-march=native`` gcc flag. Using it causes the Python interpreter to crash if the DLL was actually used.
+4. Don't use ``-march=native`` gcc flag. Using it causes the Python interpreter to crash if the dll was actually used.
 5. You may need to provide the lib with the runtime libs. If ``mingw32/bin`` is not in ``PATH``, build a wheel (``python setup.py bdist_wheel``), open it with an archiver and put the needed dlls to the directory where ``xgboost.dll`` is situated. Then you can install the wheel with ``pip``.
 
 R Package Installation
@@ -343,7 +355,7 @@ In this case, just start R as you would normally do and run the following:
   setwd('wherever/you/cloned/it/xgboost/R-package/')
   install.packages('.', repos = NULL, type="source")
 
-The package could also be built and installed with CMake (and Visual C++ 2015 on Windows) using instructions from :ref:`r_gpu_support`, but without GPU support (omit the ``-DUSE_CUDA=ON`` cmake parameter).
+The package could also be built and installed with cmake (and Visual C++ 2015 on Windows) using instructions from the next section, but without GPU support (omit the ``-DUSE_CUDA=ON`` cmake parameter).
 
 If all fails, try `Building the shared library`_ to see whether a problem is specific to R package or not.
 
@@ -352,11 +364,11 @@ If all fails, try `Building the shared library`_ to see whether a problem is spe
 Installing R package on Mac OSX with multi-threading
 ----------------------------------------------------
 
-First, obtain ``gcc-8`` with Homebrew (https://brew.sh/) to enable multi-threading (i.e. using multiple CPU threads for training). The default Apple Clang compiler does not support OpenMP, so using the default compiler would have disabled multi-threading.
+First, obtain ``gcc-7`` with Homebrew (https://brew.sh/) to enable multi-threading (i.e. using multiple CPU threads for training). The default Apple Clang compiler does not support OpenMP, so using the default compiler would have disabled multi-threading.
 
 .. code-block:: bash
 
-  brew install gcc@8
+  brew install gcc@7
 
 Now, clone the repository:
 
@@ -364,7 +376,7 @@ Now, clone the repository:
 
   git clone --recursive https://github.com/dmlc/xgboost
 
-Create the ``build/`` directory and invoke CMake with option ``R_LIB=ON``. Make sure to add ``CC=gcc-8 CXX=g++-8`` so that Homebrew GCC is selected. After invoking CMake, you can install the R package by running ``make`` and ``make install``:
+Create the ``build/`` directory and invoke CMake with option ``R_LIB=ON``. Make sure to add ``CC=gcc-7 CXX=g++-7`` so that Homebrew GCC is selected. After invoking CMake, you can install the R package by running ``make`` and ``make install``:
 
 .. code-block:: bash
 
@@ -374,8 +386,6 @@ Create the ``build/`` directory and invoke CMake with option ``R_LIB=ON``. Make
   make -j4
   make install
 
-.. _r_gpu_support:
-
 Installing R package with GPU support
 -------------------------------------
 
@@ -391,7 +401,7 @@ On Linux, starting from the XGBoost directory type:
   make install -j
 
 When default target is used, an R package shared library would be built in the ``build`` area.
-The ``install`` target, in addition, assembles the package files with this shared library under ``build/R-package`` and runs ``R CMD INSTALL``.
+The ``install`` target, in addition, assembles the package files with this shared library under ``build/R-package``, and runs ``R CMD INSTALL``.
 
 On Windows, CMake with Visual C++ Build Tools (or Visual Studio) has to be used to build an R package with GPU support. Rtools must also be installed (perhaps, some other MinGW distributions with ``gendef.exe`` and ``dlltool.exe`` would work, but that was not tested).
 
@@ -402,8 +412,8 @@ On Windows, CMake with Visual C++ Build Tools (or Visual Studio) has to be used
   cmake .. -G"Visual Studio 14 2015 Win64" -DUSE_CUDA=ON -DR_LIB=ON
   cmake --build . --target install --config Release
 
-When ``--target xgboost`` is used, an R package DLL would be built under ``build/Release``.
+When ``--target xgboost`` is used, an R package dll would be built under ``build/Release``.
-The ``--target install``, in addition, assembles the package files with this dll under ``build/R-package`` and runs ``R CMD INSTALL``.
+The ``--target install``, in addition, assembles the package files with this dll under ``build/R-package``, and runs ``R CMD INSTALL``.
 
 If cmake can't find your R during the configuration step, you might provide the location of its executable to cmake like this: ``-DLIBR_EXECUTABLE="C:/Program Files/R/R-3.4.1/bin/x64/R.exe"``.
 

doc/contribute.rst

@@ -19,7 +19,6 @@ Everyone is more than welcome to contribute. It is a way to make the project bet
 * `Documents`_
 * `Testcases`_
 * `Sanitizers`_
-* `clang-tidy`_
 * `Examples`_
 * `Core Library`_
 * `Python Package`_
@@ -170,31 +169,6 @@ environment variable:
 
 For details, please consult `official documentation <https://github.com/google/sanitizers/wiki>`_ for sanitizers.
 
-**********
-clang-tidy
-**********
-To run clang-tidy on both C++ and CUDA source code,  run the following command
-from the top level source tree:
-
-  .. code-black:: bash
-    cd /path/to/xgboost/
-    python3 tests/ci_build/tidy.py --gtest-path=/path/to/google-test
-
-The script requires the full path of Google Test library via the ``--gtest-path`` argument.
-
-Also, the script accepts two optional integer arguments, namely ``--cpp`` and ``--cuda``.
-By default they are both set to 1.  If you want to exclude CUDA source from
-linting, use:
-
-  .. code-black:: bash
-    cd /path/to/xgboost/
-    python3 tests/ci_build/tidy.py --cuda=0
-
-Similarly, if you want to exclude C++ source from linting:
-
-  .. code-black:: bash
-    cd /path/to/xgboost/
-    python3 tests/ci_build/tidy.py --cpp=0
 
 ********
 Examples

doc/gpu/index.rst

@@ -37,37 +37,33 @@ Supported parameters
 .. |tick| unicode:: U+2714 
 .. |cross| unicode:: U+2718 
 
-+--------------------------------+---------------+--------------+
++--------------------------+---------------+--------------+
-| parameter                      | ``gpu_exact`` | ``gpu_hist`` |
+| parameter                | ``gpu_exact`` | ``gpu_hist`` |
-+================================+===============+==============+
++==========================+===============+==============+
-| ``subsample``                  | |cross|       | |tick|       |
+| ``subsample``            | |cross|       | |tick|       |
-+--------------------------------+---------------+--------------+
++--------------------------+---------------+--------------+
-| ``colsample_bytree``           | |cross|       | |tick|       |
+| ``colsample_bytree``     | |cross|       | |tick|       |
-+--------------------------------+---------------+--------------+
++--------------------------+---------------+--------------+
-| ``colsample_bylevel``          | |cross|       | |tick|       |
+| ``colsample_bylevel``    | |cross|       | |tick|       |
-+--------------------------------+---------------+--------------+
++--------------------------+---------------+--------------+
-| ``max_bin``                    | |cross|       | |tick|       |
+| ``max_bin``              | |cross|       | |tick|       |
-+--------------------------------+---------------+--------------+
++--------------------------+---------------+--------------+
-| ``gpu_id``                     | |tick|        | |tick|       |
+| ``gpu_id``               | |tick|        | |tick|       |
-+--------------------------------+---------------+--------------+
++--------------------------+---------------+--------------+
-| ``n_gpus``                     | |cross|       | |tick|       |
+| ``n_gpus``               | |cross|       | |tick|       |
-+--------------------------------+---------------+--------------+
++--------------------------+---------------+--------------+
-| ``predictor``                  | |tick|        | |tick|       |
+| ``predictor``            | |tick|        | |tick|       |
-+--------------------------------+---------------+--------------+
++--------------------------+---------------+--------------+
-| ``grow_policy``                | |cross|       | |tick|       |
+| ``grow_policy``          | |cross|       | |tick|       |
-+--------------------------------+---------------+--------------+
++--------------------------+---------------+--------------+
-| ``monotone_constraints``       | |cross|       | |tick|       |
+| ``monotone_constraints`` | |cross|       | |tick|       |
-+--------------------------------+---------------+--------------+
++--------------------------+---------------+--------------+
-| ``single_precision_histogram`` | |cross|       | |tick|       |
-+--------------------------------+---------------+--------------+
 
 GPU accelerated prediction is enabled by default for the above mentioned ``tree_method`` parameters but can be switched to CPU prediction by setting ``predictor`` to ``cpu_predictor``. This could be useful if you want to conserve GPU memory. Likewise when using CPU algorithms, GPU accelerated prediction can be enabled by setting ``predictor`` to ``gpu_predictor``.
 
-The experimental parameter ``single_precision_histogram`` can be set to True to enable building histograms using single precision. This may improve speed, in particular on older architectures.
-
 The device ordinal can be selected using the ``gpu_id`` parameter, which defaults to 0.
 
-Multiple GPUs can be used with the ``gpu_hist`` tree method using the ``n_gpus`` parameter. which defaults to 1. If this is set to -1 all available GPUs will be used.  If ``gpu_id`` is specified as non-zero, the selected gpu devices will be from ``gpu_id`` to ``gpu_id+n_gpus``, please note that ``gpu_id+n_gpus`` must be less than or equal to the number of available GPUs on your system.  As with GPU vs. CPU, multi-GPU will not always be faster than a single GPU due to PCI bus bandwidth that can limit performance.
+Multiple GPUs can be used with the ``gpu_hist`` tree method using the ``n_gpus`` parameter. which defaults to 1. If this is set to -1 all available GPUs will be used.  If ``gpu_id`` is specified as non-zero, the gpu device order is ``mod(gpu_id + i) % n_visible_devices`` for ``i=0`` to ``n_gpus-1``.  As with GPU vs. CPU, multi-GPU will not always be faster than a single GPU due to PCI bus bandwidth that can limit performance.
 
 .. note:: Enabling multi-GPU training
 
@@ -82,95 +78,6 @@ The GPU algorithms currently work with CLI, Python and R packages. See :doc:`/bu
   param['max_bin'] = 16
   param['tree_method'] = 'gpu_hist'
 
-Objective functions
-===================
-Most of the objective functions implemented in XGBoost can be run on GPU.  Following table shows current support status.
-
-.. |tick| unicode:: U+2714
-.. |cross| unicode:: U+2718
-
-+-----------------+-------------+
-| Objectives      | GPU support |
-+-----------------+-------------+
-| reg:linear      | |tick|      |
-+-----------------+-------------+
-| reg:logistic    | |tick|      |
-+-----------------+-------------+
-| binary:logistic | |tick|      |
-+-----------------+-------------+
-| binary:logitraw | |tick|      |
-+-----------------+-------------+
-| binary:hinge    | |tick|      |
-+-----------------+-------------+
-| count:poisson   | |tick|      |
-+-----------------+-------------+
-| reg:gamma       | |tick|      |
-+-----------------+-------------+
-| reg:tweedie     | |tick|      |
-+-----------------+-------------+
-| multi:softmax   | |tick|      |
-+-----------------+-------------+
-| multi:softprob  | |tick|      |
-+-----------------+-------------+
-| survival:cox    | |cross|     |
-+-----------------+-------------+
-| rank:pairwise   | |cross|     |
-+-----------------+-------------+
-| rank:ndcg       | |cross|     |
-+-----------------+-------------+
-| rank:map        | |cross|     |
-+-----------------+-------------+
-
-For multi-gpu support, objective functions also honor the ``n_gpus`` parameter,
-which, by default is set to 1.  To disable running objectives on GPU, just set
-``n_gpus`` to 0.
-
-Metric functions
-===================
-Following table shows current support status for evaluation metrics on the GPU.
-
-.. |tick| unicode:: U+2714
-.. |cross| unicode:: U+2718
-
-+-----------------+-------------+
-| Metric          | GPU Support |
-+=================+=============+
-| rmse            | |tick|      |
-+-----------------+-------------+
-| mae             | |tick|      |
-+-----------------+-------------+
-| logloss         | |tick|      |
-+-----------------+-------------+
-| error           | |tick|      |
-+-----------------+-------------+
-| merror          | |cross|     |
-+-----------------+-------------+
-| mlogloss        | |cross|     |
-+-----------------+-------------+
-| auc             | |cross|     |
-+-----------------+-------------+
-| aucpr           | |cross|     |
-+-----------------+-------------+
-| ndcg            | |cross|     |
-+-----------------+-------------+
-| map             | |cross|     |
-+-----------------+-------------+
-| poisson-nloglik | |tick|      |
-+-----------------+-------------+
-| gamma-nloglik   | |tick|      |
-+-----------------+-------------+
-| cox-nloglik     | |cross|     |
-+-----------------+-------------+
-| gamma-deviance  | |tick|      |
-+-----------------+-------------+
-| tweedie-nloglik | |tick|      |
-+-----------------+-------------+
-
-As for objective functions, metrics honor the ``n_gpus`` parameter,
-which, by default is set to 1.  To disable running metrics on GPU, just set
-``n_gpus`` to 0.
-
-
 Benchmarks
 ==========
 You can run benchmarks on synthetic data for binary classification:
@@ -202,16 +109,13 @@ References
 
 `Nvidia Parallel Forall: Gradient Boosting, Decision Trees and XGBoost with CUDA <https://devblogs.nvidia.com/parallelforall/gradient-boosting-decision-trees-xgboost-cuda/>`_
 
-Contributors
+Authors
 =======
-Many thanks to the following contributors (alphabetical order):
-* Andrey Adinets
-* Jiaming Yuan
-* Jonathan C. McKinney
-* Matthew Jones
-* Philip Cho
 * Rory Mitchell
+* Jonathan C. McKinney
 * Shankara Rao Thejaswi Nanditale
 * Vinay Deshpande
+* ... and the rest of the H2O.ai and NVIDIA team.
 
 Please report bugs to the user forum https://discuss.xgboost.ai/.
+

doc/jvm/xgboost4j_spark_tutorial.rst

@@ -57,7 +57,7 @@ and then refer to the snapshot dependency by adding:
 
   <dependency>
       <groupId>ml.dmlc</groupId>
-      <artifactId>xgboost4j-spark</artifactId>
+      <artifactId>xgboost4j</artifactId>
       <version>next_version_num-SNAPSHOT</version>
   </dependency>
 
@@ -194,16 +194,11 @@ After we set XGBoostClassifier parameters and feature/label column, we can build
 Early Stopping
 ----------------
 
-Early stopping is a feature to prevent the unnecessary training iterations. By specifying ``num_early_stopping_rounds`` or directly call ``setNumEarlyStoppingRounds`` over a XGBoostClassifier or XGBoostRegressor, we can define number of rounds if the evaluation metric going away from the best iteration and early stop training iterations.
+Early stopping is a feature to prevent the unnecessary training iterations. By specifying ``num_early_stopping_rounds`` or directly call ``setNumEarlyStoppingRounds`` over a XGBoostClassifier or XGBoostRegressor, we can define number of rounds for the evaluation metric going to the unexpected direction to tolerate before stopping the training.
 
 In additional to ``num_early_stopping_rounds``, you also need to define ``maximize_evaluation_metrics`` or call ``setMaximizeEvaluationMetrics`` to specify whether you want to maximize or minimize the metrics in training.
 
-For example, we need to maximize the evaluation metrics (set ``maximize_evaluation_metrics`` with true), and set ``num_early_stopping_rounds`` with 5. The evaluation metric of 10th iteration is the maximum one until now. In the following iterations, if there is no evaluation metric greater than the 10th iteration's (best one), the traning would be early stopped at 15th iteration.  
+After specifying these two parameters, the training would stop when the metrics goes to the other direction against the one specified by ``maximize_evaluation_metrics`` for ``num_early_stopping_rounds`` iterations.
-
-Training with Evaluation Sets
-----------------
-
-You can also monitor the performance of the model during training with multiple evaluation datasets. By specifying ``eval_sets`` or call ``setEvalSets`` over a XGBoostClassifier or XGBoostRegressor, you can pass in multiple evaluation datasets typed as a Map from String to DataFrame.
 
 Prediction
 ==========

doc/parameter.rst

@@ -23,16 +23,9 @@ General Parameters
 
   - Which booster to use. Can be ``gbtree``, ``gblinear`` or ``dart``; ``gbtree`` and ``dart`` use tree based models while ``gblinear`` uses linear functions.
 
-* ``silent`` [default=0] [Deprecated]
+* ``silent`` [default=0]
 
-  - Deprecated.  Please use ``verbosity`` instead.
+  - 0 means printing running messages, 1 means silent mode
-
-* ``verbosity`` [default=1]
-
-  - Verbosity of printing messages.  Valid values are 0 (silent),
-    1 (warning), 2 (info), 3 (debug).  Sometimes XGBoost tries to change
-    configurations based on heuristics, which is displayed as warning message.
-    If there's unexpected behaviour, please try to increase value of verbosity.
 
 * ``nthread`` [default to maximum number of threads available if not set]
 
@@ -64,8 +57,8 @@ Parameters for Tree Booster
 
 * ``max_depth`` [default=6]
 
-  - Maximum depth of a tree. Increasing this value will make the model more complex and more likely to overfit. 0 is only accepted in ``lossguided`` growing policy when tree_method is set as ``hist`` and it indicates no limit on depth. Beware that XGBoost aggressively consumes memory when training a deep tree.
+  - Maximum depth of a tree. Increasing this value will make the model more complex and more likely to overfit. 0 indicates no limit. Note that limit is required when ``grow_policy`` is set of ``depthwise``.
-  - range: [0,∞] (0 is only accepted in ``lossguided`` growing policy when tree_method is set as ``hist``)
+  - range: [0,∞]
 
 * ``min_child_weight`` [default=1]
 
@@ -82,22 +75,15 @@ Parameters for Tree Booster
   - Subsample ratio of the training instances. Setting it to 0.5 means that XGBoost would randomly sample half of the training data prior to growing trees. and this will prevent overfitting. Subsampling will occur once in every boosting iteration.
   - range: (0,1]
 
-* ``colsample_bytree``, ``colsample_bylevel``, ``colsample_bynode`` [default=1]
+* ``colsample_bytree`` [default=1]
-  - This is a family of parameters for subsampling of columns.
+
-  - All ``colsample_by*`` parameters have a range of (0, 1], the default value of 1, and
+  - Subsample ratio of columns when constructing each tree. Subsampling will occur once in every boosting iteration.
-    specify the fraction of columns to be subsampled.
+  - range: (0,1]
-  - ``colsample_bytree`` is the subsample ratio of columns when constructing each
+
-    tree. Subsampling occurs once for every tree constructed.
+* ``colsample_bylevel`` [default=1]
-  - ``colsample_bylevel`` is the subsample ratio of columns for each level. Subsampling
+
-    occurs once for every new depth level reached in a tree. Columns are subsampled from
+  - Subsample ratio of columns for each split, in each level. Subsampling will occur each time a new split is made.
-    the set of columns chosen for the current tree.
+  - range: (0,1]
-  - ``colsample_bynode`` is the subsample ratio of columns for each node
-    (split). Subsampling occurs once every time a new split is evaluated. Columns are
-    subsampled from the set of columns chosen for the current level.
-  - ``colsample_by*`` parameters work cumulatively. For instance,
-    the combination ``{'colsample_bytree':0.5, 'colsample_bylevel':0.5,
-    'colsample_bynode':0.5}`` with 64 features will leave 4 features to choose from at
-    each split.
 
 * ``lambda`` [default=1, alias: ``reg_lambda``]
 
@@ -110,7 +96,7 @@ Parameters for Tree Booster
 * ``tree_method`` string [default= ``auto``]
 
   - The tree construction algorithm used in XGBoost. See description in the `reference paper <http://arxiv.org/abs/1603.02754>`_.
-  - XGBoost supports ``hist`` and ``approx`` for distributed training and only support ``approx`` for external memory version.
+  - Distributed and external memory version only support ``tree_method=approx``.
   - Choices: ``auto``, ``exact``, ``approx``, ``hist``, ``gpu_exact``, ``gpu_hist``
 
     - ``auto``: Use heuristic to choose the fastest method.
@@ -152,7 +138,7 @@ Parameters for Tree Booster
     - ``refresh``: refreshes tree's statistics and/or leaf values based on the current data. Note that no random subsampling of data rows is performed.
     - ``prune``: prunes the splits where loss < min_split_loss (or gamma).
 
-  - In a distributed setting, the implicit updater sequence value would be adjusted to ``grow_histmaker,prune`` by default, and you can set ``tree_method`` as ``hist`` to use ``grow_histmaker``. 
+  - In a distributed setting, the implicit updater sequence value would be adjusted to ``grow_histmaker,prune``.
 
 * ``refresh_leaf`` [default=1]
 
@@ -192,10 +178,6 @@ Parameters for Tree Booster
     - ``cpu_predictor``: Multicore CPU prediction algorithm.
     - ``gpu_predictor``: Prediction using GPU. Default when ``tree_method`` is ``gpu_exact`` or ``gpu_hist``.
 
-* ``num_parallel_tree``, [default=1]
-  - Number of parallel trees constructed during each iteration. This
-     option is used to support boosted random forest
-
 Additional parameters for Dart Booster (``booster=dart``)
 =========================================================
 
@@ -301,6 +283,9 @@ Specify the learning task and the corresponding learning objective. The objectiv
   - ``binary:logistic``: logistic regression for binary classification, output probability
   - ``binary:logitraw``: logistic regression for binary classification, output score before logistic transformation
   - ``binary:hinge``: hinge loss for binary classification. This makes predictions of 0 or 1, rather than producing probabilities.
+  - ``gpu:reg:linear``, ``gpu:reg:logistic``, ``gpu:binary:logistic``, ``gpu:binary:logitraw``: versions
+    of the corresponding objective functions evaluated on the GPU; note that like the GPU histogram algorithm,
+    they can only be used when the entire training session uses the same dataset
   - ``count:poisson`` --poisson regression for count data, output mean of poisson distribution
 
     - ``max_delta_step`` is set to 0.7 by default in poisson regression (used to safeguard optimization)

doc/python/python_intro.rst

@@ -48,15 +48,9 @@ The data is stored in a :py:class:`DMatrix <xgboost.DMatrix>` object.
     dtrain = xgb.DMatrix('train.csv?format=csv&label_column=0')
     dtest = xgb.DMatrix('test.csv?format=csv&label_column=0')
 
-  .. note:: Categorical features not supported
+  (Note that XGBoost does not support categorical features; if your data contains
-
+  categorical features, load it as a NumPy array first and then perform
-    Note that XGBoost does not support categorical features; if your data contains
+  `one-hot encoding <http://scikit-learn.org/stable/modules/generated/sklearn.preprocessing.OneHotEncoder.html>`_.)
-    categorical features, load it as a NumPy array first and then perform
-    `one-hot encoding <http://scikit-learn.org/stable/modules/generated/sklearn.preprocessing.OneHotEncoder.html>`_.
-
-  .. note:: Use Pandas to load CSV files with headers
-
-    Currently, the DMLC data parser cannot parse CSV files with headers. Use Pandas (see below) to read CSV files with headers.
 
 * To load a NumPy array into :py:class:`DMatrix <xgboost.DMatrix>`:
 
@@ -101,10 +95,6 @@ The data is stored in a :py:class:`DMatrix <xgboost.DMatrix>` object.
     w = np.random.rand(5, 1)
     dtrain = xgb.DMatrix(data, label=label, missing=-999.0, weight=w)
 
-When performing ranking tasks, the number of weights should be equal
-to number of groups.
-
-
 Setting Parameters
 ------------------
 XGBoost can use either a list of pairs or a dictionary to set :doc:`parameters </parameter>`. For instance:
@@ -165,10 +155,6 @@ A saved model can be loaded as follows:
   bst = xgb.Booster({'nthread': 4})  # init model
   bst.load_model('model.bin')  # load data
 
-Methods including `update` and `boost` from `xgboost.Booster` are designed for
-internal usage only.  The wrapper function `xgboost.train` does some
-pre-configuration including setting up caches and some other parameters.
-
 Early Stopping
 --------------
 If you have a validation set, you can use early stopping to find the optimal number of boosting rounds.
@@ -223,3 +209,4 @@ When you use ``IPython``, you can use the :py:meth:`xgboost.to_graphviz` functio
 .. code-block:: python
 
   xgb.to_graphviz(bst, num_trees=2)
+

doc/tutorials/aws_yarn.rst

@@ -1,8 +1,216 @@
 ###############################
 Distributed XGBoost YARN on AWS
 ###############################
-[This page is under construction.]
+This is a step-by-step tutorial on how to setup and run distributed `XGBoost <https://github.com/dmlc/xgboost>`_
+on an AWS EC2 cluster. Distributed XGBoost runs on various platforms such as MPI, SGE and Hadoop YARN.
+In this tutorial, we use YARN as an example since this is a widely used solution for distributed computing.
 
 .. note:: XGBoost with Spark
 
   If you are preprocessing training data with Spark, consider using :doc:`XGBoost4J-Spark </jvm/xgboost4j_spark_tutorial>`.
+
+************
+Prerequisite
+************
+We need to get a `AWS key-pair <http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-key-pairs.html>`_
+to access the AWS services. Let us assume that we are using a key ``mykey`` and  the corresponding permission file ``mypem.pem``.
+
+We also need `AWS credentials <https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-getting-started.html>`_,
+which includes an ``ACCESS_KEY_ID`` and a ``SECRET_ACCESS_KEY``.
+
+Finally, we will need a S3 bucket to host the data and the model, ``s3://mybucket/``
+
+***************************
+Setup a Hadoop YARN Cluster
+***************************
+This sections shows how to start a Hadoop YARN cluster from scratch.
+You can skip this step if you have already have one.
+We will be using `yarn-ec2 <https://github.com/tqchen/yarn-ec2>`_ to start the cluster.
+
+We can first clone the yarn-ec2 script by the following command.
+
+.. code-block:: bash
+
+  git clone https://github.com/tqchen/yarn-ec2
+
+To use the script, we must set the environment variables ``AWS_ACCESS_KEY_ID`` and
+``AWS_SECRET_ACCESS_KEY`` properly. This can be done by adding the following two lines in
+``~/.bashrc`` (replacing the strings with the correct ones)
+
+.. code-block:: bash
+
+  export AWS_ACCESS_KEY_ID=[your access ID]
+  export AWS_SECRET_ACCESS_KEY=[your secret access key]
+
+Now we can launch a master machine of the cluster from EC2:
+
+.. code-block:: bash
+
+  ./yarn-ec2 -k mykey -i mypem.pem launch xgboost
+
+Wait a few mininutes till the master machine gets up.
+
+After the master machine gets up, we can query the public DNS of the master machine using the following command.
+
+.. code-block:: bash
+
+  ./yarn-ec2 -k mykey -i mypem.pem get-master xgboost
+
+It will show the public DNS of the master machine like ``ec2-xx-xx-xx.us-west-2.compute.amazonaws.com``
+Now we can open the browser, and type (replace the DNS with the master DNS)
+
+.. code-block:: none
+
+  ec2-xx-xx-xx.us-west-2.compute.amazonaws.com:8088
+
+This will show the job tracker of the YARN cluster. Note that we may have to wait a few minutes before the master finishes bootstrapping and starts the
+job tracker.
+
+After the master machine gets up, we can freely add more slave machines to the cluster.
+The following command add m3.xlarge instances to the cluster.
+
+.. code-block:: bash
+
+  ./yarn-ec2 -k mykey -i mypem.pem -t m3.xlarge -s 2 addslave xgboost
+
+We can also choose to add two spot instances
+
+.. code-block:: bash
+
+  ./yarn-ec2 -k mykey -i mypem.pem -t m3.xlarge -s 2 addspot xgboost
+
+The slave machines will start up, bootstrap  and report to the master.
+You can check if the slave machines are connected by clicking on the Nodes link on the job tracker.
+Or simply type the following URL (replace DNS ith the master DNS)
+
+.. code-block:: none
+
+  ec2-xx-xx-xx.us-west-2.compute.amazonaws.com:8088/cluster/nodes
+
+One thing we should note is that not all the links in the job tracker work.
+This is due to that many of them use the private IP of AWS, which can only be accessed by EC2.
+We can use ssh proxy to access these packages.
+Now that we have set up a cluster with one master and two slaves, we are ready to run the experiment.
+
+*********************
+Build XGBoost with S3
+*********************
+We can log into the master machine by the following command.
+
+.. code-block:: bash
+
+  ./yarn-ec2 -k mykey -i mypem.pem login xgboost
+
+We will be using S3 to host the data and the result model, so the data won't get lost after the cluster shutdown.
+To do so, we will need to build XGBoost with S3 support. The only thing we need to do is to set ``USE_S3``
+variable to be true. This can be achieved by the following command.
+
+.. code-block:: bash
+
+  git clone --recursive https://github.com/dmlc/xgboost
+  cd xgboost
+  cp make/config.mk config.mk
+  echo "USE_S3=1" >> config.mk
+  make -j4
+
+Now we have built the XGBoost with S3 support. You can also enable HDFS support if you plan to store data on HDFS by turning on ``USE_HDFS`` option.
+XGBoost also relies on the environment variable to access S3, so you will need to add the following two lines to ``~/.bashrc`` (replacing the strings with the correct ones)
+on the master machine as well.
+
+.. code-block:: bash
+
+  export AWS_ACCESS_KEY_ID=AKIAIOSFODNN7EXAMPLE
+  export AWS_SECRET_ACCESS_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
+  export BUCKET=mybucket
+
+*******************
+Host the Data on S3
+*******************
+In this example, we will copy the example dataset in XGBoost to the S3 bucket as input.
+In normal usecases, the dataset is usually created from existing distributed processing pipeline.
+We can use `s3cmd <http://s3tools.org/s3cmd>`_ to copy the data into mybucket (replace ``${BUCKET}`` with the real bucket name).
+
+.. code-block:: bash
+
+  cd xgboost
+  s3cmd put demo/data/agaricus.txt.train s3://${BUCKET}/xgb-demo/train/
+  s3cmd put demo/data/agaricus.txt.test s3://${BUCKET}/xgb-demo/test/
+
+***************
+Submit the Jobs
+***************
+Now everything is ready, we can submit the XGBoost distributed job to the YARN cluster.
+We will use the `dmlc-submit <https://github.com/dmlc/dmlc-core/tree/master/tracker>`_ script to submit the job.
+
+Now we can run the following script in the distributed training folder (replace ``${BUCKET}`` with the real bucket name)
+
+.. code-block:: bash
+
+  cd xgboost/demo/distributed-training
+  # Use dmlc-submit to submit the job.
+  ../../dmlc-core/tracker/dmlc-submit --cluster=yarn --num-workers=2 --worker-cores=2\
+      ../../xgboost mushroom.aws.conf nthread=2\
+      data=s3://${BUCKET}/xgb-demo/train\
+      eval[test]=s3://${BUCKET}/xgb-demo/test\
+      model_dir=s3://${BUCKET}/xgb-demo/model
+
+All the configurations such as ``data`` and ``model_dir`` can also be directly written into the configuration file.
+Note that we only specified the folder path to the file, instead of the file name.
+XGBoost will read in all the files in that folder as the training and evaluation data.
+
+In this command, we are using two workers, and each worker uses two running threads.
+XGBoost can benefit from using multiple cores in each worker.
+A common choice of working cores can range from 4 to 8.
+The trained model will be saved into the specified model folder. You can browse the model folder.
+
+.. code-block:: bash
+
+  s3cmd ls s3://${BUCKET}/xgb-demo/model/
+
+The following is an example output from distributed training.
+
+.. code-block:: none
+
+  16/02/26 05:41:59 INFO dmlc.Client: jobname=DMLC[nworker=2]:xgboost,username=ubuntu
+  16/02/26 05:41:59 INFO dmlc.Client: Submitting application application_1456461717456_0015
+  16/02/26 05:41:59 INFO impl.YarnClientImpl: Submitted application application_1456461717456_0015
+  2016-02-26 05:42:05,230 INFO @tracker All of 2 nodes getting started
+  2016-02-26 05:42:14,027 INFO [05:42:14] [0]  test-error:0.016139        train-error:0.014433
+  2016-02-26 05:42:14,186 INFO [05:42:14] [1]  test-error:0.000000        train-error:0.001228
+  2016-02-26 05:42:14,947 INFO @tracker All nodes finishes job
+  2016-02-26 05:42:14,948 INFO @tracker 9.71754479408 secs between node start and job finish
+  Application application_1456461717456_0015 finished with state FINISHED at 1456465335961
+
+*****************
+Analyze the Model
+*****************
+After the model is trained, we can analyse the learnt model and use it for future prediction tasks.
+XGBoost is a portable framework, meaning the models in all platforms are *exchangeable*.
+This means we can load the trained model in python/R/Julia and take benefit of data science pipelines
+in these languages to do model analysis and prediction.
+
+For example, you can use `this IPython notebook <https://github.com/dmlc/xgboost/tree/master/demo/distributed-training/plot_model.ipynb>`_
+to plot feature importance and visualize the learnt model.
+
+***************
+Troubleshooting
+***************
+
+If you encounter a problem, the best way might be to use the following command
+to get logs of stdout and stderr of the containers and check what causes the problem.
+
+.. code-block:: bash
+
+  yarn logs -applicationId yourAppId
+
+*****************
+Future Directions
+*****************
+You have learned to use distributed XGBoost on YARN in this tutorial.
+XGBoost is a portable and scalable framework for gradient boosting.
+You can check out more examples and resources in the `resources page <https://github.com/dmlc/xgboost/blob/master/demo/README.md>`_.
+
+The project goal is to make the best scalable machine learning solution available to all platforms.
+The API is designed to be able to portable, and the same code can also run on other platforms such as MPI and SGE.
+XGBoost is actively evolving and we are working on even more exciting features
+such as distributed XGBoost python/R package.

doc/tutorials/feature_interaction_constraint.rst

@@ -143,11 +143,11 @@ first and second constraints (``[0, 1]``, ``[2, 3, 4]``).
 Enforcing Feature Interaction Constraints in XGBoost
 ****************************************************
 
-It is very simple to enforce feature interaction constraints in XGBoost.  Here we will
+It is very simple to enforce monotonicity constraints in XGBoost.  Here we will
 give an example using Python, but the same general idea generalizes to other
 platforms.
 
-Suppose the following code fits your model without feature interaction constraints:
+Suppose the following code fits your model without monotonicity constraints:
 
 .. code-block:: python
 
@@ -155,7 +155,7 @@ Suppose the following code fits your model without feature interaction constrain
                                    num_boost_round = 1000, evals = evallist,
                                    early_stopping_rounds = 10)
 
-Then fitting with feature interaction constraints only requires adding a single
+Then fitting with monotonicity constraints only requires adding a single
 parameter:
 
 .. code-block:: python

include/xgboost/base.h

@@ -16,7 +16,7 @@
  */
 #ifndef XGBOOST_STRICT_R_MODE
 #define XGBOOST_STRICT_R_MODE 0
-#endif  // XGBOOST_STRICT_R_MODE
+#endif
 
 /*!
  * \brief Whether always log console message with time.
@@ -26,21 +26,21 @@
  */
 #ifndef XGBOOST_LOG_WITH_TIME
 #define XGBOOST_LOG_WITH_TIME 1
-#endif  // XGBOOST_LOG_WITH_TIME
+#endif
 
 /*!
  * \brief Whether customize the logger outputs.
  */
 #ifndef XGBOOST_CUSTOMIZE_LOGGER
 #define XGBOOST_CUSTOMIZE_LOGGER XGBOOST_STRICT_R_MODE
-#endif  // XGBOOST_CUSTOMIZE_LOGGER
+#endif
 
 /*!
  * \brief Whether to customize global PRNG.
  */
 #ifndef XGBOOST_CUSTOMIZE_GLOBAL_PRNG
 #define XGBOOST_CUSTOMIZE_GLOBAL_PRNG XGBOOST_STRICT_R_MODE
-#endif  // XGBOOST_CUSTOMIZE_GLOBAL_PRNG
+#endif
 
 /*!
  * \brief Check if alignas(*) keyword is supported. (g++ 4.8 or higher)
@@ -49,7 +49,7 @@
 #define XGBOOST_ALIGNAS(X) alignas(X)
 #else
 #define XGBOOST_ALIGNAS(X)
-#endif  // defined(__GNUC__) && ((__GNUC__ == 4 && __GNUC_MINOR__ >= 8) || __GNUC__ > 4)
+#endif
 
 #if defined(__GNUC__) && ((__GNUC__ == 4 && __GNUC_MINOR__ >= 8) || __GNUC__ > 4) && \
     !defined(__CUDACC__)
@@ -64,7 +64,7 @@
 #else
 #define XGBOOST_PARALLEL_SORT(X, Y, Z) std::sort((X), (Y), (Z))
 #define XGBOOST_PARALLEL_STABLE_SORT(X, Y, Z) std::stable_sort((X), (Y), (Z))
-#endif  // GLIBC VERSION
+#endif
 
 /*!
  * \brief Tag function as usable by device
@@ -73,7 +73,7 @@
 #define XGBOOST_DEVICE __host__ __device__
 #else
 #define XGBOOST_DEVICE
-#endif  // defined (__CUDA__) || defined(__NVCC__)
+#endif
 
 /*! \brief namespace of xgboost*/
 namespace xgboost {
@@ -215,11 +215,7 @@ using bst_omp_uint = dmlc::omp_uint;  // NOLINT
 #if __GNUC__ == 4 && __GNUC_MINOR__ < 8
 #define override
 #define final
-#endif  // __GNUC__ == 4 && __GNUC_MINOR__ < 8
+#endif
-#endif  // DMLC_USE_CXX11 && defined(__GNUC__) && !defined(__clang_version__)
+#endif
 }  // namespace xgboost
-
-/* Always keep this #include at the bottom of xgboost/base.h */
-#include <xgboost/build_config.h>
-
 #endif  // XGBOOST_BASE_H_

include/xgboost/build_config.h

@@ -1,18 +0,0 @@
-/*!
- * Copyright 2019 by Contributors
- * \file build_config.h
- */
-#ifndef XGBOOST_BUILD_CONFIG_H_
-#define XGBOOST_BUILD_CONFIG_H_
-
-/* default logic for software pre-fetching */
-#if (defined(_MSC_VER) && (defined(_M_IX86) || defined(_M_AMD64))) || defined(__INTEL_COMPILER)
-// Enable _mm_prefetch for Intel compiler and MSVC+x86
-  #define XGBOOST_MM_PREFETCH_PRESENT
-  #define XGBOOST_BUILTIN_PREFETCH_PRESENT
-#elif defined(__GNUC__)
-// Enable __builtin_prefetch for GCC
-#define XGBOOST_BUILTIN_PREFETCH_PRESENT
-#endif  // GUARDS
-
-#endif  // XGBOOST_BUILD_CONFIG_H_

include/xgboost/c_api.h

@@ -10,12 +10,11 @@
 #ifdef __cplusplus
 #define XGB_EXTERN_C extern "C"
 #include <cstdio>
-#include <cstdint>
 #else
 #define XGB_EXTERN_C
 #include <stdio.h>
 #include <stdint.h>
-#endif  // __cplusplus
+#endif
 
 // XGBoost C API will include APIs in Rabit C API
 #include <rabit/c_api.h>
@@ -24,7 +23,7 @@
 #define XGB_DLL XGB_EXTERN_C __declspec(dllexport)
 #else
 #define XGB_DLL XGB_EXTERN_C
-#endif  // defined(_MSC_VER) || defined(_WIN32)
+#endif
 
 // manually define unsigned long
 typedef uint64_t bst_ulong;  // NOLINT(*)
@@ -50,7 +49,7 @@ typedef struct {  // NOLINT(*)
   long* offset; // NOLINT(*)
 #else
   int64_t* offset;  // NOLINT(*)
-#endif  // __APPLE__
+#endif
   /*! \brief labels of each instance */
   float* label;
   /*! \brief weight of each instance, can be NULL */

include/xgboost/data.h

@@ -9,7 +9,6 @@
 
 #include <dmlc/base.h>
 #include <dmlc/data.h>
-#include <rabit/rabit.h>
 #include <cstring>
 #include <memory>
 #include <numeric>
@@ -170,16 +169,8 @@ class SparsePage {
   inline Inst operator[](size_t i) const {
     const auto& data_vec = data.HostVector();
     const auto& offset_vec = offset.HostVector();
-    size_t size;
-    // in distributed mode, some partitions may not get any instance for a feature. Therefore
-    // we should set the size as zero
-    if (rabit::IsDistributed() && i + 1 >= offset_vec.size()) {
-      size = 0;
-    } else {
-      size = offset_vec[i + 1] - offset_vec[i];
-    }
     return {data_vec.data() + offset_vec[i],
-            static_cast<Inst::index_type>(size)};
+            static_cast<Inst::index_type>(offset_vec[i + 1] - offset_vec[i])};
   }
 
   /*! \brief constructor */
@@ -250,17 +241,42 @@ class SparsePage {
    * \brief Push row block into the page.
    * \param batch the row batch.
    */
-  void Push(const dmlc::RowBlock<uint32_t>& batch);
+  inline void Push(const dmlc::RowBlock<uint32_t>& batch) {
+    auto& data_vec = data.HostVector();
+    auto& offset_vec = offset.HostVector();
+    data_vec.reserve(data.Size() + batch.offset[batch.size] - batch.offset[0]);
+    offset_vec.reserve(offset.Size() + batch.size);
+    CHECK(batch.index != nullptr);
+    for (size_t i = 0; i < batch.size; ++i) {
+      offset_vec.push_back(offset_vec.back() + batch.offset[i + 1] - batch.offset[i]);
+    }
+    for (size_t i = batch.offset[0]; i < batch.offset[batch.size]; ++i) {
+      uint32_t index = batch.index[i];
+      bst_float fvalue = batch.value == nullptr ? 1.0f : batch.value[i];
+      data_vec.emplace_back(index, fvalue);
+    }
+    CHECK_EQ(offset_vec.back(), data.Size());
+  }
   /*!
    * \brief Push a sparse page
    * \param batch the row page
    */
-  void Push(const SparsePage &batch);
+  inline void Push(const SparsePage &batch) {
-  /*!
+    auto& data_vec = data.HostVector();
-   * \brief Push a SparsePage stored in CSC format
+    auto& offset_vec = offset.HostVector();
-   * \param batch The row batch to be pushed
+    const auto& batch_offset_vec = batch.offset.HostVector();
-   */
+    const auto& batch_data_vec = batch.data.HostVector();
-  void PushCSC(const SparsePage& batch);
+    size_t top = offset_vec.back();
+    data_vec.resize(top + batch.data.Size());
+    std::memcpy(dmlc::BeginPtr(data_vec) + top,
+                dmlc::BeginPtr(batch_data_vec),
+                sizeof(Entry) * batch.data.Size());
+    size_t begin = offset.Size();
+    offset_vec.resize(begin + batch.Size());
+    for (size_t i = 0; i < batch.Size(); ++i) {
+      offset_vec[i + begin] = top + batch_offset_vec[i + 1];
+    }
+  }
   /*!
    * \brief Push one instance into page
    *  \param inst an instance row
@@ -269,6 +285,7 @@ class SparsePage {
     auto& data_vec = data.HostVector();
     auto& offset_vec = offset.HostVector();
     offset_vec.push_back(offset_vec.back() + inst.size());
+
     size_t begin = data_vec.size();
     data_vec.resize(begin + inst.size());
     if (inst.size() != 0) {

include/xgboost/learner.h

@@ -11,7 +11,6 @@
 #include <rabit/rabit.h>
 #include <utility>
 #include <map>
-#include <memory>
 #include <string>
 #include <vector>
 #include "./base.h"

include/xgboost/logging.h

@@ -9,13 +9,8 @@
 #define XGBOOST_LOGGING_H_
 
 #include <dmlc/logging.h>
-#include <dmlc/parameter.h>
 #include <dmlc/thread_local.h>
 #include <sstream>
-#include <map>
-#include <string>
-#include <utility>
-#include <vector>
 #include "./base.h"
 
 namespace xgboost {
@@ -25,7 +20,7 @@ class BaseLogger {
   BaseLogger() {
 #if XGBOOST_LOG_WITH_TIME
     log_stream_ << "[" << dmlc::DateLogger().HumanDate() << "] ";
-#endif  // XGBOOST_LOG_WITH_TIME
+#endif
   }
   std::ostream& stream() { return log_stream_; }  // NOLINT
 
@@ -33,55 +28,8 @@ class BaseLogger {
   std::ostringstream log_stream_;
 };
 
-// Parsing both silent and debug_verbose is to provide backward compatibility.
-struct ConsoleLoggerParam : public dmlc::Parameter<ConsoleLoggerParam> {
-  bool silent;  // deprecated.
-  int verbosity;
-
-  DMLC_DECLARE_PARAMETER(ConsoleLoggerParam) {
-    DMLC_DECLARE_FIELD(silent)
-        .set_default(false)
-        .describe("Do not print information during training.");
-    DMLC_DECLARE_FIELD(verbosity)
-        .set_range(0, 3)
-        .set_default(1)  // shows only warning
-        .describe("Flag to print out detailed breakdown of runtime.");
-    DMLC_DECLARE_ALIAS(verbosity, debug_verbose);
-  }
-};
-
 class ConsoleLogger : public BaseLogger {
  public:
-  enum class LogVerbosity {
-    kSilent = 0,
-    kWarning = 1,
-    kInfo = 2,   // information may interests users.
-    kDebug = 3,  // information only interesting to developers.
-    kIgnore = 4  // ignore global setting
-  };
-  using LV = LogVerbosity;
-
- private:
-  static LogVerbosity global_verbosity_;
-  static ConsoleLoggerParam param_;
-
-  LogVerbosity cur_verbosity_;
-  static void Configure(const std::map<std::string, std::string>& args);
-
- public:
-  template <typename ArgIter>
-  static void Configure(ArgIter begin, ArgIter end) {
-    std::map<std::string, std::string> args(begin, end);
-    Configure(args);
-  }
-
-  static LogVerbosity GlobalVerbosity();
-  static LogVerbosity DefaultVerbosity();
-  static bool ShouldLog(LogVerbosity verbosity);
-
-  ConsoleLogger() = delete;
-  explicit ConsoleLogger(LogVerbosity cur_verb);
-  ConsoleLogger(const std::string& file, int line, LogVerbosity cur_verb);
   ~ConsoleLogger();
 };
 
@@ -116,47 +64,17 @@ class LogCallbackRegistry {
     return nullptr;
   }
 };
-#endif  // !defined(XGBOOST_STRICT_R_MODE) || XGBOOST_STRICT_R_MODE == 0
+#endif
 
 using LogCallbackRegistryStore = dmlc::ThreadLocalStore<LogCallbackRegistry>;
 
-// Redefines LOG_WARNING for controling verbosity
-#if defined(LOG_WARNING)
-#undef  LOG_WARNING
-#endif  // defined(LOG_WARNING)
-#define LOG_WARNING                                                            \
-  if (::xgboost::ConsoleLogger::ShouldLog(                                     \
-          ::xgboost::ConsoleLogger::LV::kWarning))                             \
-  ::xgboost::ConsoleLogger(__FILE__, __LINE__,                                 \
-                           ::xgboost::ConsoleLogger::LogVerbosity::kWarning)
-
-// Redefines LOG_INFO for controling verbosity
-#if defined(LOG_INFO)
-#undef  LOG_INFO
-#endif  // defined(LOG_INFO)
-#define LOG_INFO                                                               \
-  if (::xgboost::ConsoleLogger::ShouldLog(                                     \
-          ::xgboost::ConsoleLogger::LV::kInfo))                                \
-  ::xgboost::ConsoleLogger(__FILE__, __LINE__,                                 \
-                           ::xgboost::ConsoleLogger::LogVerbosity::kInfo)
-
-#if defined(LOG_DEBUG)
-#undef LOG_DEBUG
-#endif  // defined(LOG_DEBUG)
-#define LOG_DEBUG                                                              \
-  if (::xgboost::ConsoleLogger::ShouldLog(                                     \
-          ::xgboost::ConsoleLogger::LV::kDebug))                               \
-  ::xgboost::ConsoleLogger(__FILE__, __LINE__,                                 \
-                           ::xgboost::ConsoleLogger::LogVerbosity::kDebug)
-
 // redefines the logging macro if not existed
 #ifndef LOG
 #define LOG(severity) LOG_##severity.stream()
-#endif  // LOG
+#endif
 
 // Enable LOG(CONSOLE) for print messages to console.
-#define LOG_CONSOLE ::xgboost::ConsoleLogger(           \
+#define LOG_CONSOLE ::xgboost::ConsoleLogger()
-    ::xgboost::ConsoleLogger::LogVerbosity::kIgnore)
 // Enable LOG(TRACKER) for print messages to tracker
 #define LOG_TRACKER ::xgboost::TrackerLogger()
 }  // namespace xgboost.

include/xgboost/metric.h

@@ -11,11 +11,8 @@
 #include <vector>
 #include <string>
 #include <functional>
-#include <utility>
-
 #include "./data.h"
 #include "./base.h"
-#include "../../src/common/host_device_vector.h"
 
 namespace xgboost {
 /*!
@@ -24,23 +21,6 @@ namespace xgboost {
  */
 class Metric {
  public:
-  /*!
-   * \brief Configure the Metric with the specified parameters.
-   * \param args arguments to the objective function.
-   */
-  virtual void Configure(
-      const std::vector<std::pair<std::string, std::string> >& args) {}
-  /*!
-   * \brief set configuration from pair iterators.
-   * \param begin The beginning iterator.
-   * \param end The end iterator.
-   * \tparam PairIter iterator<std::pair<std::string, std::string> >
-   */
-  template<typename PairIter>
-  inline void Configure(PairIter begin, PairIter end) {
-    std::vector<std::pair<std::string, std::string> > vec(begin, end);
-    this->Configure(vec);
-  }
   /*!
    * \brief evaluate a specific metric
    * \param preds prediction
@@ -49,9 +29,9 @@ class Metric {
    *        the average statistics across all the node,
    *        this is only supported by some metrics
    */
-  virtual bst_float Eval(const HostDeviceVector<bst_float>& preds,
+  virtual bst_float Eval(const std::vector<bst_float>& preds,
                          const MetaInfo& info,
-                         bool distributed) = 0;
+                         bool distributed) const = 0;
   /*! \return name of metric */
   virtual const char* Name() const = 0;
   /*! \brief virtual destructor */

include/xgboost/predictor.h

@@ -7,14 +7,11 @@
 #pragma once
 #include <xgboost/base.h>
 #include <xgboost/data.h>
-
 #include <functional>
 #include <memory>
 #include <string>
-#include <unordered_map>
 #include <utility>
 #include <vector>
-
 #include "../../src/gbm/gbtree_model.h"
 #include "../../src/common/host_device_vector.h"
 

jvm-packages/create_jni.py

@@ -19,7 +19,7 @@ CONFIG = {
     "USE_AZURE": "OFF",
     "USE_S3": "OFF",
 
-    "USE_CUDA": "OFF",
+    "PLUGIN_UPDATER_GPU": "OFF",
     "JVM_BINDINGS": "ON"
 }
 

jvm-packages/dev/build.sh

@@ -17,5 +17,5 @@ rm /usr/bin/python
 ln -s /opt/rh/python27/root/usr/bin/python /usr/bin/python
 
 # build xgboost
-cd /xgboost/jvm-packages;ulimit -c unlimited;mvn package
+cd /xgboost/jvm-packages;mvn package
 

jvm-packages/pom.xml

@@ -6,7 +6,7 @@
 
     <groupId>ml.dmlc</groupId>
     <artifactId>xgboost-jvm</artifactId>
-    <version>0.82</version>
+    <version>0.81</version>
     <packaging>pom</packaging>
     <name>XGBoost JVM Package</name>
     <description>JVM Package for XGBoost</description>
@@ -34,7 +34,7 @@
         <maven.compiler.source>1.7</maven.compiler.source>
         <maven.compiler.target>1.7</maven.compiler.target>
         <flink.version>1.5.0</flink.version>
-        <spark.version>2.3.3</spark.version>
+        <spark.version>2.3.1</spark.version>
         <scala.version>2.11.12</scala.version>
         <scala.binary.version>2.11</scala.binary.version>
     </properties>
@@ -237,7 +237,7 @@
                 </executions>
             </plugin>
             <plugin>
-                <groupId>org.apache.maven.plugins</groupId>
+                <groupId>net.alchim31.maven</groupId>
                 <artifactId>maven-site-plugin</artifactId>
                 <version>3.0</version>
                 <configuration>
@@ -335,6 +335,25 @@
                     </execution>
                 </executions>
             </plugin>
+            <plugin>
+                <groupId>org.jacoco</groupId>
+                <artifactId>jacoco-maven-plugin</artifactId>
+                <version>0.7.9</version>
+                <executions>
+                    <execution>
+                        <goals>
+                            <goal>prepare-agent</goal>
+                        </goals>
+                    </execution>
+                    <execution>
+                        <id>report</id>
+                        <phase>test</phase>
+                        <goals>
+                            <goal>report</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
         </plugins>
     </build>
     <dependencies>

jvm-packages/xgboost4j-example/README.md

@@ -23,7 +23,7 @@ XGBoost4J Code Examples
 * [External Memory](src/main/scala/ml/dmlc/xgboost4j/scala/example/ExternalMemory.scala)
 
 ## Spark API
-* [Distributed Training with Spark](src/main/scala/ml/dmlc/xgboost4j/scala/example/spark/SparkMLlibPipeline.scala)
+* [Distributed Training with Spark](src/main/scala/ml/dmlc/xgboost4j/scala/example/spark/SparkWithDataFrame.scala)
 
 ## Flink API
 * [Distributed Training with Flink](src/main/scala/ml/dmlc/xgboost4j/scala/example/flink/DistTrainWithFlink.scala)

jvm-packages/xgboost4j-example/pom.xml

@@ -6,10 +6,10 @@
     <parent>
         <groupId>ml.dmlc</groupId>
         <artifactId>xgboost-jvm</artifactId>
-        <version>0.82</version>
+        <version>0.81</version>
     </parent>
     <artifactId>xgboost4j-example</artifactId>
-    <version>0.82</version>
+    <version>0.81</version>
     <packaging>jar</packaging>
     <build>
         <plugins>
@@ -26,7 +26,7 @@
         <dependency>
             <groupId>ml.dmlc</groupId>
             <artifactId>xgboost4j-spark</artifactId>
-            <version>0.82</version>
+            <version>0.81</version>
         </dependency>
         <dependency>
             <groupId>org.apache.spark</groupId>
@@ -37,7 +37,7 @@
         <dependency>
             <groupId>ml.dmlc</groupId>
             <artifactId>xgboost4j-flink</artifactId>
-            <version>0.82</version>
+            <version>0.81</version>
         </dependency>
         <dependency>
             <groupId>org.apache.commons</groupId>

jvm-packages/xgboost4j-example/src/main/scala/ml/dmlc/xgboost4j/scala/example/spark/SparkTraining.scala

@@ -31,6 +31,7 @@ object SparkTraining {
       println("Usage: program input_path")
       sys.exit(1)
     }
+
     val spark = SparkSession.builder().getOrCreate()
     val inputPath = args(0)
     val schema = new StructType(Array(
@@ -39,7 +40,7 @@ object SparkTraining {
       StructField("petal length", DoubleType, true),
       StructField("petal width", DoubleType, true),
       StructField("class", StringType, true)))
-    val rawInput = spark.read.schema(schema).csv(inputPath)
+    val rawInput = spark.read.schema(schema).csv(args(0))
 
     // transform class to index to make xgboost happy
     val stringIndexer = new StringIndexer()
@@ -54,8 +55,6 @@ object SparkTraining {
     val xgbInput = vectorAssembler.transform(labelTransformed).select("features",
       "classIndex")
 
-    val Array(train, eval1, eval2, test) = xgbInput.randomSplit(Array(0.6, 0.2, 0.1, 0.1))
-
     /**
      * setup  "timeout_request_workers" -> 60000L to make this application if it cannot get enough resources
      * to get 2 workers within 60000 ms
@@ -68,13 +67,12 @@ object SparkTraining {
       "objective" -> "multi:softprob",
       "num_class" -> 3,
       "num_round" -> 100,
-      "num_workers" -> 2,
+      "num_workers" -> 2)
-      "eval_sets" -> Map("eval1" -> eval1, "eval2" -> eval2))
     val xgbClassifier = new XGBoostClassifier(xgbParam).
       setFeaturesCol("features").
       setLabelCol("classIndex")
-    val xgbClassificationModel = xgbClassifier.fit(train)
+    val xgbClassificationModel = xgbClassifier.fit(xgbInput)
-    val results = xgbClassificationModel.transform(test)
+    val results = xgbClassificationModel.transform(xgbInput)
     results.show()
   }
 }

jvm-packages/xgboost4j-flink/pom.xml

@@ -6,10 +6,10 @@
     <parent>
         <groupId>ml.dmlc</groupId>
         <artifactId>xgboost-jvm</artifactId>
-        <version>0.82</version>
+        <version>0.81</version>
     </parent>
     <artifactId>xgboost4j-flink</artifactId>
-    <version>0.82</version>
+    <version>0.81</version>
     <build>
         <plugins>
             <plugin>
@@ -26,7 +26,7 @@
         <dependency>
             <groupId>ml.dmlc</groupId>
             <artifactId>xgboost4j</artifactId>
-            <version>0.82</version>
+            <version>0.81</version>
         </dependency>
         <dependency>
             <groupId>org.apache.commons</groupId>

jvm-packages/xgboost4j-spark/pom.xml

@@ -6,7 +6,7 @@
     <parent>
         <groupId>ml.dmlc</groupId>
         <artifactId>xgboost-jvm</artifactId>
-        <version>0.82</version>
+        <version>0.81</version>
     </parent>
     <artifactId>xgboost4j-spark</artifactId>
     <build>
@@ -24,7 +24,7 @@
         <dependency>
             <groupId>ml.dmlc</groupId>
             <artifactId>xgboost4j</artifactId>
-            <version>0.82</version>
+            <version>0.81</version>
         </dependency>
         <dependency>
             <groupId>org.apache.spark</groupId>

jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/DataUtils.scala

@@ -20,11 +20,6 @@ import ml.dmlc.xgboost4j.{LabeledPoint => XGBLabeledPoint}
 
 import org.apache.spark.ml.feature.{LabeledPoint => MLLabeledPoint}
 import org.apache.spark.ml.linalg.{DenseVector, SparseVector, Vector, Vectors}
-import org.apache.spark.ml.param.Param
-import org.apache.spark.rdd.RDD
-import org.apache.spark.sql.{Column, DataFrame, Row}
-import org.apache.spark.sql.functions.col
-import org.apache.spark.sql.types.{FloatType, IntegerType}
 
 object DataUtils extends Serializable {
   private[spark] implicit class XGBLabeledPointFeatures(
@@ -72,38 +67,4 @@ object DataUtils extends Serializable {
         XGBLabeledPoint(0.0f, v.indices, v.values.map(_.toFloat))
     }
   }
-
-  private[spark] def convertDataFrameToXGBLabeledPointRDDs(
-      labelCol: Column,
-      featuresCol: Column,
-      weight: Column,
-      baseMargin: Column,
-      group: Option[Column],
-      dataFrames: DataFrame*): Array[RDD[XGBLabeledPoint]] = {
-    val selectedColumns = group.map(groupCol => Seq(labelCol.cast(FloatType),
-      featuresCol,
-      weight.cast(FloatType),
-      groupCol.cast(IntegerType),
-      baseMargin.cast(FloatType))).getOrElse(Seq(labelCol.cast(FloatType),
-      featuresCol,
-      weight.cast(FloatType),
-      baseMargin.cast(FloatType)))
-    dataFrames.toArray.map {
-      df => df.select(selectedColumns: _*).rdd.map {
-        case Row(label: Float, features: Vector, weight: Float, group: Int, baseMargin: Float) =>
-          val (indices, values) = features match {
-            case v: SparseVector => (v.indices, v.values.map(_.toFloat))
-            case v: DenseVector => (null, v.values.map(_.toFloat))
-          }
-          XGBLabeledPoint(label, indices, values, weight, group, baseMargin)
-        case Row(label: Float, features: Vector, weight: Float, baseMargin: Float) =>
-          val (indices, values) = features match {
-            case v: SparseVector => (v.indices, v.values.map(_.toFloat))
-            case v: DenseVector => (null, v.values.map(_.toFloat))
-          }
-          XGBLabeledPoint(label, indices, values, weight, baseMargin = baseMargin)
-      }
-    }
-  }
-
 }

jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/XGBoost.scala

@@ -19,7 +19,6 @@ package ml.dmlc.xgboost4j.scala.spark
 import java.io.File
 import java.nio.file.Files
 
-import scala.collection.mutable.ListBuffer
 import scala.collection.{AbstractIterator, mutable}
 import scala.util.Random
 
@@ -27,12 +26,12 @@ import ml.dmlc.xgboost4j.java.{IRabitTracker, Rabit, XGBoostError, RabitTracker
 import ml.dmlc.xgboost4j.scala.rabit.RabitTracker
 import ml.dmlc.xgboost4j.scala.{XGBoost => SXGBoost, _}
 import ml.dmlc.xgboost4j.{LabeledPoint => XGBLabeledPoint}
+
 import org.apache.commons.io.FileUtils
 import org.apache.commons.logging.LogFactory
-
 import org.apache.spark.rdd.RDD
 import org.apache.spark.{SparkContext, SparkParallelismTracker, TaskContext}
-import org.apache.spark.sql.{DataFrame, SparkSession}
+import org.apache.spark.sql.SparkSession
 
 
 /**
@@ -115,12 +114,13 @@ object XGBoost extends Serializable {
       round: Int,
       obj: ObjectiveTrait,
       eval: EvalTrait,
-      prevBooster: Booster): Iterator[(Booster, Map[String, Array[Float]])] = {
+      prevBooster: Booster)
+    : Iterator[(Booster, Map[String, Array[Float]])] = {
 
     // to workaround the empty partitions in training dataset,
     // this might not be the best efficient implementation, see
     // (https://github.com/dmlc/xgboost/issues/1277)
-    if (watches.toMap("train").rowNum == 0) {
+    if (watches.train.rowNum == 0) {
       throw new XGBoostError(
         s"detected an empty partition in the training data, partition ID:" +
           s" ${TaskContext.getPartitionId()}")
@@ -138,7 +138,7 @@ object XGBoost extends Serializable {
         }
       }
       val metrics = Array.tabulate(watches.size)(_ => Array.ofDim[Float](round))
-      val booster = SXGBoost.train(watches.toMap("train"), params, round,
+      val booster = SXGBoost.train(watches.train, params, round,
         watches.toMap, metrics, obj, eval,
         earlyStoppingRound = numEarlyStoppingRounds, prevBooster)
       Iterator(booster -> watches.toMap.keys.zip(metrics).toMap)
@@ -175,52 +175,6 @@ object XGBoost extends Serializable {
     tracker
   }
 
-  class IteratorWrapper[T](arrayOfXGBLabeledPoints: Array[(String, Iterator[T])])
-    extends Iterator[(String, Iterator[T])] {
-
-    private var currentIndex = 0
-
-    override def hasNext: Boolean = currentIndex <= arrayOfXGBLabeledPoints.length - 1
-
-    override def next(): (String, Iterator[T]) = {
-      currentIndex += 1
-      arrayOfXGBLabeledPoints(currentIndex - 1)
-    }
-  }
-
-  private def coPartitionNoGroupSets(
-      trainingData: RDD[XGBLabeledPoint],
-      evalSets: Map[String, RDD[XGBLabeledPoint]],
-      nWorkers: Int) = {
-    // eval_sets is supposed to be set by the caller of [[trainDistributed]]
-    val allDatasets = Map("train" -> trainingData) ++ evalSets
-    val repartitionedDatasets = allDatasets.map{case (name, rdd) =>
-      if (rdd.getNumPartitions != nWorkers) {
-        (name, rdd.repartition(nWorkers))
-      } else {
-        (name, rdd)
-      }
-    }
-    repartitionedDatasets.foldLeft(trainingData.sparkContext.parallelize(
-      Array.fill[(String, Iterator[XGBLabeledPoint])](nWorkers)(null), nWorkers)){
-      case (rddOfIterWrapper, (name, rddOfIter)) =>
-        rddOfIterWrapper.zipPartitions(rddOfIter){
-          (itrWrapper, itr) =>
-            if (!itr.hasNext) {
-              logger.error("when specifying eval sets as dataframes, you have to ensure that " +
-                "the number of elements in each dataframe is larger than the number of workers")
-              throw new Exception("too few elements in evaluation sets")
-            }
-            val itrArray = itrWrapper.toArray
-            if (itrArray.head != null) {
-              new IteratorWrapper(itrArray :+ (name -> itr))
-            } else {
-              new IteratorWrapper(Array(name -> itr))
-            }
-        }
-    }
-  }
-
   /**
    * Check to see if Spark expects SSL encryption (`spark.ssl.enabled` set to true).
    * If so, throw an exception unless this safety measure has been explicitly overridden
@@ -253,25 +207,24 @@ object XGBoost extends Serializable {
     }
   }
 
-  private def parameterFetchAndValidation(params: Map[String, Any], sparkContext: SparkContext) = {
+  /**
-    val nWorkers = params("num_workers").asInstanceOf[Int]
+   * @return A tuple of the booster and the metrics used to build training summary
-    val round = params("num_round").asInstanceOf[Int]
+   */
-    val useExternalMemory = params("use_external_memory").asInstanceOf[Boolean]
+  @throws(classOf[XGBoostError])
-    val obj = params.getOrElse("custom_obj", null).asInstanceOf[ObjectiveTrait]
+  private[spark] def trainDistributed(
-    val eval = params.getOrElse("custom_eval", null).asInstanceOf[EvalTrait]
+      trainingData: RDD[XGBLabeledPoint],
-    val missing = params.getOrElse("missing", Float.NaN).asInstanceOf[Float]
+      params: Map[String, Any],
-    validateSparkSslConf(sparkContext)
+      round: Int,
-
+      nWorkers: Int,
+      obj: ObjectiveTrait = null,
+      eval: EvalTrait = null,
+      useExternalMemory: Boolean = false,
+      missing: Float = Float.NaN,
+      hasGroup: Boolean = false): (Booster, Map[String, Array[Float]]) = {
+    validateSparkSslConf(trainingData.context)
     if (params.contains("tree_method")) {
-      require(params("tree_method") == "hist" ||
+      require(params("tree_method") != "hist", "xgboost4j-spark does not support fast histogram" +
-        params("tree_method") == "approx" ||
+        " for now")
-        params("tree_method") == "auto", "xgboost4j-spark only supports tree_method as 'hist'," +
-        " 'approx' and 'auto'")
-    }
-    if (params.contains("train_test_ratio")) {
-      logger.warn("train_test_ratio is deprecated since XGBoost 0.82, we recommend to explicitly" +
-        " pass a training and multiple evaluation datasets by passing 'eval_sets' and " +
-        "'eval_set_names'")
     }
     require(nWorkers > 0, "you must specify more than 0 workers")
     if (obj != null) {
@@ -292,89 +245,11 @@ object XGBoost extends Serializable {
         " an instance of Long.")
     }
     val (checkpointPath, checkpointInterval) = CheckpointManager.extractParams(params)
-    (nWorkers, round, useExternalMemory, obj, eval, missing, trackerConf, timeoutRequestWorkers,
-      checkpointPath, checkpointInterval)
-  }
 
-  private def trainForNonRanking(
-      trainingData: RDD[XGBLabeledPoint],
-      params: Map[String, Any],
-      rabitEnv: java.util.Map[String, String],
-      checkpointRound: Int,
-      prevBooster: Booster,
-      evalSetsMap: Map[String, RDD[XGBLabeledPoint]]): RDD[(Booster, Map[String, Array[Float]])] = {
-    val (nWorkers, _, useExternalMemory, obj, eval, missing, _, _, _, _) =
-      parameterFetchAndValidation(params, trainingData.sparkContext)
-    val partitionedData = repartitionForTraining(trainingData, nWorkers)
-    if (evalSetsMap.isEmpty) {
-      partitionedData.mapPartitions(labeledPoints => {
-        val watches = Watches.buildWatches(params,
-          removeMissingValues(labeledPoints, missing),
-          getCacheDirName(useExternalMemory))
-        buildDistributedBooster(watches, params, rabitEnv, checkpointRound,
-          obj, eval, prevBooster)
-      }).cache()
-    } else {
-      coPartitionNoGroupSets(partitionedData, evalSetsMap, nWorkers).mapPartitions {
-        nameAndLabeledPointSets =>
-          val watches = Watches.buildWatches(
-            nameAndLabeledPointSets.map {
-              case (name, iter) => (name, removeMissingValues(iter, missing))},
-            getCacheDirName(useExternalMemory))
-          buildDistributedBooster(watches, params, rabitEnv, checkpointRound,
-            obj, eval, prevBooster)
-      }.cache()
-    }
-  }
-
-  private def trainForRanking(
-      trainingData: RDD[XGBLabeledPoint],
-      params: Map[String, Any],
-      rabitEnv: java.util.Map[String, String],
-      checkpointRound: Int,
-      prevBooster: Booster,
-      evalSetsMap: Map[String, RDD[XGBLabeledPoint]]): RDD[(Booster, Map[String, Array[Float]])] = {
-    val (nWorkers, _, useExternalMemory, obj, eval, missing, _, _, _, _) =
-      parameterFetchAndValidation(params, trainingData.sparkContext)
-    val partitionedTrainingSet = repartitionForTrainingGroup(trainingData, nWorkers)
-    if (evalSetsMap.isEmpty) {
-      partitionedTrainingSet.mapPartitions(labeledPointGroups => {
-        val watches = Watches.buildWatchesWithGroup(params,
-          removeMissingValuesWithGroup(labeledPointGroups, missing),
-          getCacheDirName(useExternalMemory))
-        buildDistributedBooster(watches, params, rabitEnv, checkpointRound, obj, eval, prevBooster)
-      }).cache()
-    } else {
-      coPartitionGroupSets(partitionedTrainingSet, evalSetsMap, nWorkers).mapPartitions(
-        labeledPointGroupSets => {
-          val watches = Watches.buildWatchesWithGroup(
-            labeledPointGroupSets.map {
-              case (name, iter) => (name, removeMissingValuesWithGroup(iter, missing))
-            },
-            getCacheDirName(useExternalMemory))
-          buildDistributedBooster(watches, params, rabitEnv, checkpointRound, obj, eval,
-            prevBooster)
-        }).cache()
-    }
-  }
-
-  /**
-   * @return A tuple of the booster and the metrics used to build training summary
-   */
-  @throws(classOf[XGBoostError])
-  private[spark] def trainDistributed(
-      trainingData: RDD[XGBLabeledPoint],
-      params: Map[String, Any],
-      hasGroup: Boolean = false,
-      evalSetsMap: Map[String, RDD[XGBLabeledPoint]] = Map()):
-    (Booster, Map[String, Array[Float]]) = {
-    logger.info(s"XGBoost training with parameters:\n${params.mkString("\n")}")
-    val (nWorkers, round, _, _, _, _, trackerConf, timeoutRequestWorkers,
-      checkpointPath, checkpointInterval) = parameterFetchAndValidation(params,
-      trainingData.sparkContext)
     val sc = trainingData.sparkContext
     val checkpointManager = new CheckpointManager(sc, checkpointPath)
-    checkpointManager.cleanUpHigherVersions(round.asInstanceOf[Int])
+    checkpointManager.cleanUpHigherVersions(round)
+
     var prevBooster = checkpointManager.loadCheckpointAsBooster
     // Train for every ${savingRound} rounds and save the partially completed booster
     checkpointManager.getCheckpointRounds(checkpointInterval, round).map {
@@ -384,12 +259,27 @@ object XGBoost extends Serializable {
           val overriddenParams = overrideParamsAccordingToTaskCPUs(params, sc)
           val parallelismTracker = new SparkParallelismTracker(sc, timeoutRequestWorkers, nWorkers)
           val rabitEnv = tracker.getWorkerEnvs
-          val boostersAndMetrics = if (hasGroup) {
+          val boostersAndMetrics = hasGroup match {
-            trainForRanking(trainingData, overriddenParams, rabitEnv, checkpointRound,
+            case true => {
-              prevBooster, evalSetsMap)
+              val partitionedData = repartitionForTrainingGroup(trainingData, nWorkers)
-          } else {
+              partitionedData.mapPartitions(labeledPointGroups => {
-            trainForNonRanking(trainingData, overriddenParams, rabitEnv, checkpointRound,
+                val watches = Watches.buildWatchesWithGroup(overriddenParams,
-              prevBooster, evalSetsMap)
+                  removeMissingValuesWithGroup(labeledPointGroups, missing),
+                  getCacheDirName(useExternalMemory))
+                buildDistributedBooster(watches, overriddenParams, rabitEnv, checkpointRound,
+                  obj, eval, prevBooster)
+              }).cache()
+            }
+            case false => {
+              val partitionedData = repartitionForTraining(trainingData, nWorkers)
+              partitionedData.mapPartitions(labeledPoints => {
+                val watches = Watches.buildWatches(overriddenParams,
+                  removeMissingValues(labeledPoints, missing),
+                  getCacheDirName(useExternalMemory))
+                 buildDistributedBooster(watches, overriddenParams, rabitEnv, checkpointRound,
+                   obj, eval, prevBooster)
+              }).cache()
+            }
           }
           val sparkJobThread = new Thread() {
             override def run() {
@@ -423,7 +313,8 @@ object XGBoost extends Serializable {
     }
   }
 
-  private def aggByGroupInfo(trainingData: RDD[XGBLabeledPoint]) = {
+  private[spark] def repartitionForTrainingGroup(
+      trainingData: RDD[XGBLabeledPoint], nWorkers: Int): RDD[Array[XGBLabeledPoint]] = {
     val normalGroups: RDD[Array[XGBLabeledPoint]] = trainingData.mapPartitions(
       // LabeledPointGroupIterator returns (Boolean, Array[XGBLabeledPoint])
       new LabeledPointGroupIterator(_)).filter(!_.isEdgeGroup).map(_.points)
@@ -431,60 +322,22 @@ object XGBoost extends Serializable {
     // edge groups with partition id.
     val edgeGroups: RDD[(Int, XGBLabeledPointGroup)] = trainingData.mapPartitions(
       new LabeledPointGroupIterator(_)).filter(_.isEdgeGroup).map(
-      group => (TaskContext.getPartitionId(), group))
+        group => (TaskContext.getPartitionId(), group))
 
     // group chunks from different partitions together by group id in XGBLabeledPoint.
-    // use groupBy instead of aggregateBy since all groups within a partition have unique group ids.
+    // use groupBy instead of aggregateBy since all groups within a partition have unique groud ids.
     val stitchedGroups: RDD[Array[XGBLabeledPoint]] = edgeGroups.groupBy(_._2.groupId).map(
       groups => {
         val it: Iterable[(Int, XGBLabeledPointGroup)] = groups._2
         // sorted by partition id and merge list of Array[XGBLabeledPoint] into one array
-        it.toArray.sortBy(_._1).flatMap(_._2.points)
+        it.toArray.sortBy(_._1).map(_._2.points).flatten
       })
-    normalGroups.union(stitchedGroups)
-  }
 
-  private[spark] def repartitionForTrainingGroup(
+    var allGroups = normalGroups.union(stitchedGroups)
-      trainingData: RDD[XGBLabeledPoint], nWorkers: Int): RDD[Array[XGBLabeledPoint]] = {
-    val allGroups = aggByGroupInfo(trainingData)
     logger.info(s"repartitioning training group set to $nWorkers partitions")
     allGroups.repartition(nWorkers)
   }
 
-  private def coPartitionGroupSets(
-      aggedTrainingSet: RDD[Array[XGBLabeledPoint]],
-      evalSets: Map[String, RDD[XGBLabeledPoint]],
-      nWorkers: Int): RDD[(String, Iterator[Array[XGBLabeledPoint]])] = {
-    val repartitionedDatasets = Map("train" -> aggedTrainingSet) ++ evalSets.map {
-      case (name, rdd) => {
-        val aggedRdd = aggByGroupInfo(rdd)
-        if (aggedRdd.getNumPartitions != nWorkers) {
-          name -> aggedRdd.repartition(nWorkers)
-        } else {
-          name -> aggedRdd
-        }
-      }
-    }
-    repartitionedDatasets.foldLeft(aggedTrainingSet.sparkContext.parallelize(
-      Array.fill[(String, Iterator[Array[XGBLabeledPoint]])](nWorkers)(null), nWorkers)){
-      case (rddOfIterWrapper, (name, rddOfIter)) =>
-        rddOfIterWrapper.zipPartitions(rddOfIter){
-          (itrWrapper, itr) =>
-            if (!itr.hasNext) {
-              logger.error("when specifying eval sets as dataframes, you have to ensure that " +
-                "the number of elements in each dataframe is larger than the number of workers")
-              throw new Exception("too few elements in evaluation sets")
-            }
-            val itrArray = itrWrapper.toArray
-            if (itrArray.head != null) {
-              new IteratorWrapper(itrArray :+ (name -> itr))
-            } else {
-              new IteratorWrapper(Array(name -> itr))
-            }
-        }
-    }
-  }
-
   private def postTrackerReturnProcessing(
       trackerReturnVal: Int,
       distributedBoostersAndMetrics: RDD[(Booster, Map[String, Array[Float]])],
@@ -515,13 +368,12 @@ object XGBoost extends Serializable {
 }
 
 private class Watches private(
-    val datasets: Array[DMatrix],
+    val train: DMatrix,
-    val names: Array[String],
+    val test: DMatrix,
-    val cacheDirName: Option[String]) {
+    private val cacheDirName: Option[String]) {
 
-  def toMap: Map[String, DMatrix] = {
+  def toMap: Map[String, DMatrix] = Map("train" -> train, "test" -> test)
-    names.zip(datasets).toMap.filter { case (_, matrix) => matrix.rowNum > 0 }
+    .filter { case (_, matrix) => matrix.rowNum > 0 }
-  }
 
   def size: Int = toMap.size
 
@@ -561,26 +413,6 @@ private object Watches {
     }
   }
 
-  def buildWatches(
-      nameAndLabeledPointSets: Iterator[(String, Iterator[XGBLabeledPoint])],
-      cachedDirName: Option[String]): Watches = {
-    val dms = nameAndLabeledPointSets.map {
-      case (name, labeledPoints) =>
-        val baseMargins = new mutable.ArrayBuilder.ofFloat
-        val duplicatedItr = labeledPoints.map(labeledPoint => {
-          baseMargins += labeledPoint.baseMargin
-          labeledPoint
-        })
-        val dMatrix = new DMatrix(duplicatedItr, cachedDirName.map(_ + s"/$name").orNull)
-        val baseMargin = fromBaseMarginsToArray(baseMargins.result().iterator)
-        if (baseMargin.isDefined) {
-          dMatrix.setBaseMargin(baseMargin.get)
-        }
-        (name, dMatrix)
-    }.toArray
-    new Watches(dms.map(_._2), dms.map(_._1), cachedDirName)
-  }
-
   def buildWatches(
       params: Map[String, Any],
       labeledPoints: Iterator[XGBLabeledPoint],
@@ -609,46 +441,7 @@ private object Watches {
     if (trainMargin.isDefined) trainMatrix.setBaseMargin(trainMargin.get)
     if (testMargin.isDefined) testMatrix.setBaseMargin(testMargin.get)
 
-    new Watches(Array(trainMatrix, testMatrix), Array("train", "test"), cacheDirName)
+    new Watches(trainMatrix, testMatrix, cacheDirName)
-  }
-
-  def buildWatchesWithGroup(
-      nameAndlabeledPointGroupSets: Iterator[(String, Iterator[Array[XGBLabeledPoint]])],
-      cachedDirName: Option[String]): Watches = {
-    val dms = nameAndlabeledPointGroupSets.map {
-      case (name, labeledPointsGroups) =>
-        val baseMargins = new mutable.ArrayBuilder.ofFloat
-        val groupsInfo = new mutable.ArrayBuilder.ofInt
-        val weights = new mutable.ArrayBuilder.ofFloat
-        val iter = labeledPointsGroups.filter(labeledPointGroup => {
-          var groupWeight = -1.0f
-          var groupSize = 0
-          labeledPointGroup.map { labeledPoint => {
-            if (groupWeight < 0) {
-              groupWeight = labeledPoint.weight
-            } else if (groupWeight != labeledPoint.weight) {
-              throw new IllegalArgumentException("the instances in the same group have to be" +
-                s" assigned with the same weight (unexpected weight ${labeledPoint.weight}")
-            }
-            baseMargins += labeledPoint.baseMargin
-            groupSize += 1
-            labeledPoint
-          }
-          }
-          weights += groupWeight
-          groupsInfo += groupSize
-          true
-        })
-        val dMatrix = new DMatrix(iter.flatMap(_.iterator), cachedDirName.map(_ + s"/$name").orNull)
-        val baseMargin = fromBaseMarginsToArray(baseMargins.result().iterator)
-        if (baseMargin.isDefined) {
-          dMatrix.setBaseMargin(baseMargin.get)
-        }
-        dMatrix.setGroup(groupsInfo.result())
-        dMatrix.setWeight(weights.result())
-        (name, dMatrix)
-    }.toArray
-    new Watches(dms.map(_._2), dms.map(_._1), cachedDirName)
   }
 
   def buildWatchesWithGroup(
@@ -661,46 +454,20 @@ private object Watches {
     val testPoints = mutable.ArrayBuilder.make[XGBLabeledPoint]
     val trainBaseMargins = new mutable.ArrayBuilder.ofFloat
     val testBaseMargins = new mutable.ArrayBuilder.ofFloat
-
     val trainGroups = new mutable.ArrayBuilder.ofInt
     val testGroups = new mutable.ArrayBuilder.ofInt
 
-    val trainWeights = new mutable.ArrayBuilder.ofFloat
-    val testWeights = new mutable.ArrayBuilder.ofFloat
-
     val trainLabelPointGroups = labeledPointGroups.filter { labeledPointGroup =>
       val accepted = r.nextDouble() <= trainTestRatio
       if (!accepted) {
-        var groupWeight = -1.0f
-        var groupSize = 0
         labeledPointGroup.foreach(labeledPoint => {
           testPoints += labeledPoint
           testBaseMargins += labeledPoint.baseMargin
-          if (groupWeight < 0) {
-            groupWeight = labeledPoint.weight
-          } else if (labeledPoint.weight != groupWeight) {
-            throw new IllegalArgumentException("the instances in the same group have to be" +
-              s" assigned with the same weight (unexpected weight ${labeledPoint.weight}")
-          }
-          groupSize += 1
         })
-        testWeights += groupWeight
+        testGroups += labeledPointGroup.length
-        testGroups += groupSize
       } else {
-        var groupWeight = -1.0f
+        labeledPointGroup.foreach(trainBaseMargins += _.baseMargin)
-        var groupSize = 0
+        trainGroups += labeledPointGroup.length
-        labeledPointGroup.foreach { labeledPoint => {
-          if (groupWeight < 0) {
-            groupWeight = labeledPoint.weight
-          } else if (labeledPoint.weight != groupWeight) {
-            throw new IllegalArgumentException("the instances in the same group have to be" +
-              s" assigned with the same weight (unexpected weight ${labeledPoint.weight}")
-          }
-          trainBaseMargins += labeledPoint.baseMargin
-          groupSize += 1
-        }}
-        trainWeights += groupWeight
-        trainGroups += groupSize
       }
       accepted
     }
@@ -708,12 +475,10 @@ private object Watches {
     val trainPoints = trainLabelPointGroups.flatMap(_.iterator)
     val trainMatrix = new DMatrix(trainPoints, cacheDirName.map(_ + "/train").orNull)
     trainMatrix.setGroup(trainGroups.result())
-    trainMatrix.setWeight(trainWeights.result())
 
     val testMatrix = new DMatrix(testPoints.result().iterator, cacheDirName.map(_ + "/test").orNull)
     if (trainTestRatio < 1.0) {
       testMatrix.setGroup(testGroups.result())
-      testMatrix.setWeight(testWeights.result())
     }
 
     val trainMargin = fromBaseMarginsToArray(trainBaseMargins.result().iterator)
@@ -721,7 +486,7 @@ private object Watches {
     if (trainMargin.isDefined) trainMatrix.setBaseMargin(trainMargin.get)
     if (testMargin.isDefined) testMatrix.setBaseMargin(testMargin.get)
 
-    new Watches(Array(trainMatrix, testMatrix), Array("train", "test"), cacheDirName)
+    new Watches(trainMatrix, testMatrix, cacheDirName)
   }
 }
 
@@ -740,7 +505,7 @@ private[spark] class LabeledPointGroupIterator(base: Iterator[XGBLabeledPoint])
   private var isNewGroup = false
 
   override def hasNext: Boolean = {
-    base.hasNext || isNewGroup
+    return base.hasNext || isNewGroup
   }
 
   override def next(): XGBLabeledPointGroup = {

jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/XGBoostClassifier.scala

@@ -43,7 +43,7 @@ import org.apache.spark.broadcast.Broadcast
 
 private[spark] trait XGBoostClassifierParams extends GeneralParams with LearningTaskParams
   with BoosterParams with HasWeightCol with HasBaseMarginCol with HasNumClass with ParamMapFuncs
-  with HasLeafPredictionCol with HasContribPredictionCol with NonParamVariables
+  with HasLeafPredictionCol with HasContribPredictionCol
 
 class XGBoostClassifier (
     override val uid: String,
@@ -182,19 +182,24 @@ class XGBoostClassifier (
       col($(baseMarginCol))
     }
 
-    val trainingSet: RDD[XGBLabeledPoint] = DataUtils.convertDataFrameToXGBLabeledPointRDDs(
+    val instances: RDD[XGBLabeledPoint] = dataset.select(
-      col($(labelCol)), col($(featuresCol)), weight, baseMargin,
+      col($(featuresCol)),
-      None, dataset.asInstanceOf[DataFrame]).head
+      col($(labelCol)).cast(FloatType),
-    val evalRDDMap = getEvalSets(xgboostParams).map {
+      baseMargin.cast(FloatType),
-      case (name, dataFrame) => (name,
+      weight.cast(FloatType)
-        DataUtils.convertDataFrameToXGBLabeledPointRDDs(col($(labelCol)), col($(featuresCol)),
+    ).rdd.map { case Row(features: Vector, label: Float, baseMargin: Float, weight: Float) =>
-          weight, baseMargin, None, dataFrame).head)
+      val (indices, values) = features match {
+        case v: SparseVector => (v.indices, v.values.map(_.toFloat))
+        case v: DenseVector => (null, v.values.map(_.toFloat))
+      }
+      XGBLabeledPoint(label, indices, values, baseMargin = baseMargin, weight = weight)
     }
     transformSchema(dataset.schema, logging = true)
     val derivedXGBParamMap = MLlib2XGBoostParams
     // All non-null param maps in XGBoostClassifier are in derivedXGBParamMap.
-    val (_booster, _metrics) = XGBoost.trainDistributed(trainingSet, derivedXGBParamMap,
+    val (_booster, _metrics) = XGBoost.trainDistributed(instances, derivedXGBParamMap,
-      hasGroup = false, evalRDDMap)
+      $(numRound), $(numWorkers), $(customObj), $(customEval), $(useExternalMemory),
+      $(missing), hasGroup = false)
     val model = new XGBoostClassificationModel(uid, _numClasses, _booster)
     val summary = XGBoostTrainingSummary(_metrics)
     model.setSummary(summary)
@@ -285,12 +290,12 @@ class XGBoostClassificationModel private[ml](
     val bBooster = dataset.sparkSession.sparkContext.broadcast(_booster)
     val appName = dataset.sparkSession.sparkContext.appName
 
-    val inputRDD = dataset.asInstanceOf[Dataset[Row]].rdd
+    val rdd = dataset.asInstanceOf[Dataset[Row]].rdd.mapPartitions { rowIterator =>
-    val predictionRDD = dataset.asInstanceOf[Dataset[Row]].rdd.mapPartitions { rowIterator =>
       if (rowIterator.hasNext) {
         val rabitEnv = Array("DMLC_TASK_ID" -> TaskContext.getPartitionId().toString).toMap
         Rabit.init(rabitEnv.asJava)
-        val featuresIterator = rowIterator.map(row => row.getAs[Vector](
+        val (rowItr1, rowItr2) = rowIterator.duplicate
+        val featuresIterator = rowItr2.map(row => row.getAs[Vector](
           $(featuresCol))).toList.iterator
         import DataUtils._
         val cacheInfo = {
@@ -307,27 +312,19 @@ class XGBoostClassificationModel private[ml](
           val Array(rawPredictionItr, probabilityItr, predLeafItr, predContribItr) =
             producePredictionItrs(bBooster, dm)
           Rabit.shutdown()
-          Iterator(rawPredictionItr, probabilityItr, predLeafItr,
+          produceResultIterator(rowItr1, rawPredictionItr, probabilityItr, predLeafItr,
             predContribItr)
         } finally {
           dm.delete()
         }
       } else {
-        Iterator()
+        Iterator[Row]()
       }
     }
-    val resultRDD = inputRDD.zipPartitions(predictionRDD, preservesPartitioning = true) {
-      case (inputIterator, predictionItr) =>
-        if (inputIterator.hasNext) {
-          produceResultIterator(inputIterator, predictionItr.next(), predictionItr.next(),
-            predictionItr.next(), predictionItr.next())
-        } else {
-          Iterator()
-        }
-    }
 
     bBooster.unpersist(blocking = false)
-    dataset.sparkSession.createDataFrame(resultRDD, generateResultSchema(schema))
+
+    dataset.sparkSession.createDataFrame(rdd, generateResultSchema(schema))
   }
 
   private def produceResultIterator(
@@ -419,9 +416,7 @@ class XGBoostClassificationModel private[ml](
     var numColsOutput = 0
 
     val rawPredictionUDF = udf { rawPrediction: mutable.WrappedArray[Float] =>
-      val raw = rawPrediction.map(_.toDouble).toArray
+      Vectors.dense(rawPrediction.map(_.toDouble).toArray)
-      val rawPredictions = if (numClasses == 2) Array(-raw(0), raw(0)) else raw
-      Vectors.dense(rawPredictions)
     }
 
     val probabilityUDF = udf { probability: mutable.WrappedArray[Float] =>

jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/XGBoostRegressor.scala

@@ -43,7 +43,7 @@ import org.apache.spark.broadcast.Broadcast
 
 private[spark] trait XGBoostRegressorParams extends GeneralParams with BoosterParams
   with LearningTaskParams with HasBaseMarginCol with HasWeightCol with HasGroupCol
-  with ParamMapFuncs with HasLeafPredictionCol with HasContribPredictionCol with NonParamVariables
+  with ParamMapFuncs with HasLeafPredictionCol with HasContribPredictionCol
 
 class XGBoostRegressor (
     override val uid: String,
@@ -174,19 +174,27 @@ class XGBoostRegressor (
       col($(baseMarginCol))
     }
     val group = if (!isDefined(groupCol) || $(groupCol).isEmpty) lit(-1) else col($(groupCol))
-    val trainingSet: RDD[XGBLabeledPoint] = DataUtils.convertDataFrameToXGBLabeledPointRDDs(
+
-      col($(labelCol)), col($(featuresCol)), weight, baseMargin, Some(group),
+    val instances: RDD[XGBLabeledPoint] = dataset.select(
-      dataset.asInstanceOf[DataFrame]).head
+      col($(labelCol)).cast(FloatType),
-    val evalRDDMap = getEvalSets(xgboostParams).map {
+      col($(featuresCol)),
-      case (name, dataFrame) => (name,
+      weight.cast(FloatType),
-        DataUtils.convertDataFrameToXGBLabeledPointRDDs(col($(labelCol)), col($(featuresCol)),
+      group.cast(IntegerType),
-          weight, baseMargin, Some(group), dataFrame).head)
+      baseMargin.cast(FloatType)
+    ).rdd.map {
+      case Row(label: Float, features: Vector, weight: Float, group: Int, baseMargin: Float) =>
+        val (indices, values) = features match {
+          case v: SparseVector => (v.indices, v.values.map(_.toFloat))
+          case v: DenseVector => (null, v.values.map(_.toFloat))
+        }
+        XGBLabeledPoint(label, indices, values, weight, group, baseMargin)
     }
     transformSchema(dataset.schema, logging = true)
     val derivedXGBParamMap = MLlib2XGBoostParams
     // All non-null param maps in XGBoostRegressor are in derivedXGBParamMap.
-    val (_booster, _metrics) = XGBoost.trainDistributed(trainingSet, derivedXGBParamMap,
+    val (_booster, _metrics) = XGBoost.trainDistributed(instances, derivedXGBParamMap,
-      hasGroup = group != lit(-1), evalRDDMap)
+      $(numRound), $(numWorkers), $(customObj), $(customEval), $(useExternalMemory),
+      $(missing), hasGroup = group != lit(-1))
     val model = new XGBoostRegressionModel(uid, _booster)
     val summary = XGBoostTrainingSummary(_metrics)
     model.setSummary(summary)
@@ -257,12 +265,13 @@ class XGBoostRegressionModel private[ml] (
 
     val bBooster = dataset.sparkSession.sparkContext.broadcast(_booster)
     val appName = dataset.sparkSession.sparkContext.appName
-    val inputRDD = dataset.asInstanceOf[Dataset[Row]].rdd
+
-    val predictionRDD = dataset.asInstanceOf[Dataset[Row]].rdd.mapPartitions { rowIterator =>
+    val rdd = dataset.asInstanceOf[Dataset[Row]].rdd.mapPartitions { rowIterator =>
       if (rowIterator.hasNext) {
         val rabitEnv = Array("DMLC_TASK_ID" -> TaskContext.getPartitionId().toString).toMap
         Rabit.init(rabitEnv.asJava)
-        val featuresIterator = rowIterator.map(row => row.getAs[Vector](
+        val (rowItr1, rowItr2) = rowIterator.duplicate
+        val featuresIterator = rowItr2.map(row => row.getAs[Vector](
           $(featuresCol))).toList.iterator
         import DataUtils._
         val cacheInfo = {
@@ -272,6 +281,7 @@ class XGBoostRegressionModel private[ml] (
             null
           }
         }
+
         val dm = new DMatrix(
           XGBoost.removeMissingValues(featuresIterator.map(_.asXGB), $(missing)),
           cacheInfo)
@@ -279,25 +289,16 @@ class XGBoostRegressionModel private[ml] (
           val Array(originalPredictionItr, predLeafItr, predContribItr) =
             producePredictionItrs(bBooster, dm)
           Rabit.shutdown()
-          Iterator(originalPredictionItr, predLeafItr, predContribItr)
+          produceResultIterator(rowItr1, originalPredictionItr, predLeafItr, predContribItr)
         } finally {
           dm.delete()
         }
       } else {
-        Iterator()
+        Iterator[Row]()
       }
     }
-    val resultRDD = inputRDD.zipPartitions(predictionRDD, preservesPartitioning = true) {
-      case (inputIterator, predictionItr) =>
-        if (inputIterator.hasNext) {
-          produceResultIterator(inputIterator, predictionItr.next(), predictionItr.next(),
-            predictionItr.next())
-        } else {
-          Iterator()
-        }
-    }
     bBooster.unpersist(blocking = false)
-    dataset.sparkSession.createDataFrame(resultRDD, generateResultSchema(schema))
+    dataset.sparkSession.createDataFrame(rdd, generateResultSchema(schema))
   }
 
   private def produceResultIterator(

jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/XGBoostTrainingSummary.scala

@@ -18,17 +18,12 @@ package ml.dmlc.xgboost4j.scala.spark
 
 class XGBoostTrainingSummary private(
     val trainObjectiveHistory: Array[Float],
-    val validationObjectiveHistory: (String, Array[Float])*) extends Serializable {
+    val testObjectiveHistory: Option[Array[Float]]
-
+) extends Serializable {
   override def toString: String = {
-    val train = trainObjectiveHistory.mkString(",")
+    val train = trainObjectiveHistory.toList
-    val vaidationObjectiveHistoryString = {
+    val test = testObjectiveHistory.map(_.toList)
-      validationObjectiveHistory.map {
+    s"XGBoostTrainingSummary(trainObjectiveHistory=$train, testObjectiveHistory=$test)"
-        case (name, metrics) =>
-          s"${name}ObjectiveHistory=${metrics.mkString(",")}"
-      }.mkString(";")
-    }
-    s"XGBoostTrainingSummary(trainObjectiveHistory=$train; $vaidationObjectiveHistoryString)"
   }
 }
 
@@ -36,6 +31,6 @@ private[xgboost4j] object XGBoostTrainingSummary {
   def apply(metrics: Map[String, Array[Float]]): XGBoostTrainingSummary = {
     new XGBoostTrainingSummary(
       trainObjectiveHistory = metrics("train"),
-      metrics.filter(_._1 != "train").toSeq: _*)
+      testObjectiveHistory = metrics.get("test"))
   }
 }

jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/params/BoosterParams.scala

@@ -50,21 +50,10 @@ private[spark] trait BoosterParams extends Params {
    * overfitting. [default=6] range: [1, Int.MaxValue]
    */
   final val maxDepth = new IntParam(this, "maxDepth", "maximum depth of a tree, increase this " +
-    "value will make model more complex/likely to be overfitting.", (value: Int) => value >= 0)
+    "value will make model more complex/likely to be overfitting.", (value: Int) => value >= 1)
 
   final def getMaxDepth: Int = $(maxDepth)
 
-
-  /**
-   * Maximum number of nodes to be added. Only relevant when grow_policy=lossguide is set.
-   */
-  final val maxLeaves = new IntParam(this, "maxLeaves",
-    "Maximum number of nodes to be added. Only relevant when grow_policy=lossguide is set.",
-    (value: Int) => value >= 0)
-
-  final def getMaxLeaves: Int = $(maxLeaves)
-
-
   /**
    * minimum sum of instance weight(hessian) needed in a child. If the tree partition step results
    * in a leaf node with the sum of instance weight less than min_child_weight, then the building
@@ -158,9 +147,7 @@ private[spark] trait BoosterParams extends Params {
    * growth policy for fast histogram algorithm
    */
   final val growPolicy = new Param[String](this, "growPolicy",
-    "Controls a way new nodes are added to the tree. Currently supported only if" +
+    "growth policy for fast histogram algorithm",
-      " tree_method is set to hist. Choices: depthwise, lossguide. depthwise: split at nodes" +
-      " closest to the root. lossguide: split at nodes with highest loss change.",
     (value: String) => BoosterParams.supportedGrowthPolicies.contains(value))
 
   final def getGrowPolicy: String = $(growPolicy)
@@ -255,22 +242,6 @@ private[spark] trait BoosterParams extends Params {
 
   final def getTreeLimit: Int = $(treeLimit)
 
-  final val monotoneConstraints = new Param[String](this, name = "monotoneConstraints",
-    doc = "a list in length of number of features, 1 indicate monotonic increasing, - 1 means " +
-      "decreasing, 0 means no constraint. If it is shorter than number of features, 0 will be " +
-      "padded ")
-
-  final def getMonotoneConstraints: String = $(monotoneConstraints)
-
-  final val interactionConstraints = new Param[String](this,
-    name = "interactionConstraints",
-    doc = "Constraints for interaction representing permitted interactions. The constraints" +
-      " must be specified in the form of a nest list, e.g. [[0, 1], [2, 3, 4]]," +
-      " where each inner list is a group of indices of features that are allowed to interact" +
-      " with each other. See tutorial for more information")
-
-  final def getInteractionConstraints: String = $(interactionConstraints)
-
   setDefault(eta -> 0.3, gamma -> 0, maxDepth -> 6,
     minChildWeight -> 1, maxDeltaStep -> 0,
     growPolicy -> "depthwise", maxBins -> 16,

jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/params/CustomParams.scala

@@ -22,7 +22,26 @@ import org.json4s.{DefaultFormats, Extraction, NoTypeHints}
 import org.json4s.jackson.JsonMethods.{compact, parse, render}
 
 import org.apache.spark.ml.param.{Param, ParamPair, Params}
-import org.apache.spark.sql.DataFrame
+
+class GroupDataParam(
+    parent: Params,
+    name: String,
+    doc: String) extends Param[Seq[Seq[Int]]](parent, name, doc) {
+
+  /** Creates a param pair with the given value (for Java). */
+  override def w(value: Seq[Seq[Int]]): ParamPair[Seq[Seq[Int]]] = super.w(value)
+
+  override def jsonEncode(value: Seq[Seq[Int]]): String = {
+    import org.json4s.jackson.Serialization
+    implicit val formats = Serialization.formats(NoTypeHints)
+    compact(render(Extraction.decompose(value)))
+  }
+
+  override def jsonDecode(json: String): Seq[Seq[Int]] = {
+    implicit val formats = DefaultFormats
+    parse(json).extract[Seq[Seq[Int]]]
+  }
+}
 
 class CustomEvalParam(
     parent: Params,

jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/params/GeneralParams.scala

@@ -22,14 +22,6 @@ import ml.dmlc.xgboost4j.scala.spark.TrackerConf
 import org.apache.spark.ml.param._
 import scala.collection.mutable
 
-import ml.dmlc.xgboost4j.{LabeledPoint => XGBLabeledPoint}
-
-import org.apache.spark.ml.linalg.{DenseVector, SparseVector, Vector}
-import org.apache.spark.rdd.RDD
-import org.apache.spark.sql.{Column, DataFrame, Row}
-import org.apache.spark.sql.functions.col
-import org.apache.spark.sql.types.{FloatType, IntegerType}
-
 private[spark] trait GeneralParams extends Params {
 
   /**
@@ -65,27 +57,14 @@ private[spark] trait GeneralParams extends Params {
   final def getUseExternalMemory: Boolean = $(useExternalMemory)
 
   /**
-   * Deprecated. Please use verbosity instead.
    * 0 means printing running messages, 1 means silent mode. default: 0
    */
   final val silent = new IntParam(this, "silent",
-    "Deprecated. Please use verbosity instead. " +
     "0 means printing running messages, 1 means silent mode.",
     (value: Int) => value >= 0 && value <= 1)
 
   final def getSilent: Int = $(silent)
 
-  /**
-   * Verbosity of printing messages. Valid values are 0 (silent), 1 (warning), 2 (info), 3 (debug).
-   * default: 1
-   */
-  final val verbosity = new IntParam(this, "verbosity",
-    "Verbosity of printing messages. Valid values are 0 (silent), 1 (warning), 2 (info), " +
-    "3 (debug).",
-    (value: Int) => value >= 0 && value <= 3)
-
-  final def getVerbosity: Int = $(verbosity)
-
   /**
    * customized objective function provided by user. default: null
    */
@@ -172,10 +151,11 @@ private[spark] trait GeneralParams extends Params {
   final def getSeed: Long = $(seed)
 
   setDefault(numRound -> 1, numWorkers -> 1, nthread -> 1,
-    useExternalMemory -> false, silent -> 0, verbosity -> 1,
+    useExternalMemory -> false, silent -> 0,
     customObj -> null, customEval -> null, missing -> Float.NaN,
     trackerConf -> TrackerConf(), seed -> 0, timeoutRequestWorkers -> 30 * 60 * 1000L,
-    checkpointPath -> "", checkpointInterval -> -1)
+    checkpointPath -> "", checkpointInterval -> -1
+  )
 }
 
 trait HasLeafPredictionCol extends Params {
@@ -244,11 +224,10 @@ private[spark] trait ParamMapFuncs extends Params {
   def XGBoostToMLlibParams(xgboostParams: Map[String, Any]): Unit = {
     for ((paramName, paramValue) <- xgboostParams) {
       if ((paramName == "booster" && paramValue != "gbtree") ||
-        (paramName == "updater" && paramValue != "grow_histmaker,prune" &&
+        (paramName == "updater" && paramValue != "grow_histmaker,prune")) {
-          paramValue != "hist")) {
         throw new IllegalArgumentException(s"you specified $paramName as $paramValue," +
           s" XGBoost-Spark only supports gbtree as booster type" +
-          " and grow_histmaker,prune or hist as the updater type")
+          " and grow_histmaker,prune as the updater type")
       }
       val name = CaseFormat.LOWER_UNDERSCORE.to(CaseFormat.LOWER_CAMEL, paramName)
       params.find(_.name == name) match {

jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/params/NonParamVariables.scala

@@ -1,36 +0,0 @@
-/*
- Copyright (c) 2014 by Contributors
-
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
- */
-
-package ml.dmlc.xgboost4j.scala.spark.params
-
-import org.apache.spark.sql.DataFrame
-
-trait NonParamVariables {
-  protected var evalSetsMap: Map[String, DataFrame] = Map.empty
-
-  def setEvalSets(evalSets: Map[String, DataFrame]): this.type = {
-    evalSetsMap = evalSets
-    this
-  }
-
-  def getEvalSets(params: Map[String, Any]): Map[String, DataFrame] = {
-    if (params.contains("eval_sets")) {
-      params("eval_sets").asInstanceOf[Map[String, DataFrame]]
-    } else {
-      evalSetsMap
-    }
-  }
-}

jvm-packages/xgboost4j-spark/src/main/scala/org/apache/spark/SparkParallelismTracker.scala

@@ -20,7 +20,7 @@ import java.net.URL
 
 import org.apache.commons.logging.LogFactory
 
-import org.apache.spark.scheduler._
+import org.apache.spark.scheduler.{SparkListener, SparkListenerTaskEnd}
 import org.codehaus.jackson.map.ObjectMapper
 import scala.collection.JavaConverters._
 import scala.concurrent.ExecutionContext.Implicits.global
@@ -98,11 +98,9 @@ class SparkParallelismTracker(
    */
   def execute[T](body: => T): T = {
     if (timeout <= 0) {
-      logger.info("starting training without setting timeout for waiting for resources")
       body
     } else {
       try {
-        logger.info(s"starting training with timeout set as $timeout ms for waiting for resources")
         waitForCondition(numAliveCores >= requestedCores, timeout)
       } catch {
         case _: TimeoutException =>
@@ -114,26 +112,16 @@ class SparkParallelismTracker(
   }
 }
 
+private class ErrorInXGBoostTraining(msg: String) extends ControlThrowable {
+  override def toString: String = s"ErrorInXGBoostTraining: $msg"
+}
+
 private[spark] class TaskFailedListener extends SparkListener {
-
-  private[this] val logger = LogFactory.getLog("XGBoostTaskFailedListener")
-
   override def onTaskEnd(taskEnd: SparkListenerTaskEnd): Unit = {
     taskEnd.reason match {
-      case taskEndReason: TaskFailedReason =>
+      case reason: TaskFailedReason =>
-        logger.error(s"Training Task Failed during XGBoost Training: " +
+        throw new ErrorInXGBoostTraining(s"ExecutorLost during XGBoost Training: " +
-            s"$taskEndReason, stopping SparkContext")
+          s"${reason.toErrorString}")
-        // Spark does not allow ListenerThread to shutdown SparkContext so that we have to do it
-        // in a separate thread
-        val sparkContextKiller = new Thread() {
-          override def run(): Unit = {
-            LiveListenerBus.withinListenerThread.withValue(false) {
-              SparkContext.getOrCreate().stop()
-            }
-          }
-        }
-        sparkContextKiller.setDaemon(true)
-        sparkContextKiller.start()
       case _ =>
     }
   }

jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/TrainTestData.scala

@@ -98,12 +98,3 @@ object Ranking extends TrainTestData {
     getResourceLines(resource).map(_.toInt).toList
   }
 }
-
-object Synthetic extends {
-  val train: Seq[XGBLabeledPoint] = Seq(
-    XGBLabeledPoint(1.0f, Array(0, 1), Array(1.0f, 2.0f)),
-    XGBLabeledPoint(0.0f, Array(0, 1, 2), Array(1.0f, 2.0f, 3.0f)),
-    XGBLabeledPoint(0.0f, Array(0, 1, 2), Array(1.0f, 2.0f, 3.0f)),
-    XGBLabeledPoint(1.0f, Array(0, 1), Array(1.0f, 2.0f))
-  )
-}

jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/XGBoostClassifierSuite.scala

@@ -17,14 +17,11 @@
 package ml.dmlc.xgboost4j.scala.spark
 
 import ml.dmlc.xgboost4j.scala.{DMatrix, XGBoost => ScalaXGBoost}
-
 import org.apache.spark.ml.linalg._
 import org.apache.spark.ml.param.ParamMap
 import org.apache.spark.sql._
 import org.scalatest.FunSuite
 
-import org.apache.spark.Partitioner
-
 class XGBoostClassifierSuite extends FunSuite with PerTest {
 
   test("XGBoost-Spark XGBoostClassifier ouput should match XGBoost4j") {
@@ -63,11 +60,10 @@ class XGBoostClassifierSuite extends FunSuite with PerTest {
       collect().map(row => (row.getAs[Int]("id"), row.getAs[DenseVector]("rawPrediction"))).toMap
 
     assert(testDF.count() === prediction4.size)
-    // the vector length in rawPrediction column is 2 since we have to fit to the evaluator in Spark
     for (i <- prediction3.indices) {
-      assert(prediction3(i).length === prediction4(i).values.length - 1)
+      assert(prediction3(i).length === prediction4(i).values.length)
       for (j <- prediction3(i).indices) {
-        assert(prediction3(i)(j) === prediction4(i)(j + 1))
+        assert(prediction3(i)(j) === prediction4(i)(j))
       }
     }
 
@@ -140,7 +136,7 @@ class XGBoostClassifierSuite extends FunSuite with PerTest {
     assert(predictionDF.columns.contains("final_prediction") === false)
 
     assert(model.summary.trainObjectiveHistory.length === 5)
-    assert(model.summary.validationObjectiveHistory.isEmpty)
+    assert(model.summary.testObjectiveHistory.isEmpty)
   }
 
   test("XGBoost and Spark parameters synchronize correctly") {
@@ -194,6 +190,31 @@ class XGBoostClassifierSuite extends FunSuite with PerTest {
     assert(count != 0)
   }
 
+  test("training summary") {
+    val paramMap = Map("eta" -> "1", "max_depth" -> "6", "silent" -> "1",
+      "objective" -> "binary:logistic", "num_round" -> 5, "nWorkers" -> numWorkers)
+
+    val trainingDF = buildDataFrame(Classification.train)
+    val xgb = new XGBoostClassifier(paramMap)
+    val model = xgb.fit(trainingDF)
+
+    assert(model.summary.trainObjectiveHistory.length === 5)
+    assert(model.summary.testObjectiveHistory.isEmpty)
+  }
+
+  test("train/test split") {
+    val paramMap = Map("eta" -> "1", "max_depth" -> "6", "silent" -> "1",
+      "objective" -> "binary:logistic", "train_test_ratio" -> "0.5",
+      "num_round" -> 5, "num_workers" -> numWorkers)
+    val training = buildDataFrame(Classification.train)
+
+    val xgb = new XGBoostClassifier(paramMap)
+    val model = xgb.fit(training)
+    val Some(testObjectiveHistory) = model.summary.testObjectiveHistory
+    assert(testObjectiveHistory.length === 5)
+    assert(model.summary.trainObjectiveHistory !== testObjectiveHistory)
+  }
+
   test("test predictionLeaf") {
     val paramMap = Map("eta" -> "1", "max_depth" -> "6", "silent" -> "1",
       "objective" -> "binary:logistic", "train_test_ratio" -> "0.5",
@@ -266,46 +287,4 @@ class XGBoostClassifierSuite extends FunSuite with PerTest {
     assert(resultDF.columns.contains("predictLeaf"))
     assert(resultDF.columns.contains("predictContrib"))
   }
-
-  test("infrequent features") {
-    val paramMap = Map("eta" -> "1", "max_depth" -> "6", "silent" -> "1",
-      "objective" -> "binary:logistic",
-      "num_round" -> 5, "num_workers" -> 2)
-    import DataUtils._
-    val sparkSession = SparkSession.builder().getOrCreate()
-    import sparkSession.implicits._
-    val repartitioned = sc.parallelize(Synthetic.train, 3).map(lp => (lp.label, lp)).partitionBy(
-      new Partitioner {
-        override def numPartitions: Int = 2
-
-        override def getPartition(key: Any): Int = key.asInstanceOf[Float].toInt
-      }
-    ).map(_._2).zipWithIndex().map {
-      case (lp, id) =>
-        (id, lp.label, lp.features)
-    }.toDF("id", "label", "features")
-    val xgb = new XGBoostClassifier(paramMap)
-    xgb.fit(repartitioned)
-  }
-
-  test("infrequent features (use_external_memory)") {
-    val paramMap = Map("eta" -> "1", "max_depth" -> "6", "silent" -> "1",
-      "objective" -> "binary:logistic",
-      "num_round" -> 5, "num_workers" -> 2, "use_external_memory" -> true)
-    import DataUtils._
-    val sparkSession = SparkSession.builder().getOrCreate()
-    import sparkSession.implicits._
-    val repartitioned = sc.parallelize(Synthetic.train, 3).map(lp => (lp.label, lp)).partitionBy(
-      new Partitioner {
-        override def numPartitions: Int = 2
-
-        override def getPartition(key: Any): Int = key.asInstanceOf[Float].toInt
-      }
-    ).map(_._2).zipWithIndex().map {
-      case (lp, id) =>
-        (id, lp.label, lp.features)
-    }.toDF("id", "label", "features")
-    val xgb = new XGBoostClassifier(paramMap)
-    xgb.fit(repartitioned)
-  }
 }

jvm-packages/xgboost4j-spark/src/test/scala/ml/dmlc/xgboost4j/scala/spark/XGBoostGeneralSuite.scala

@@ -18,21 +18,18 @@ package ml.dmlc.xgboost4j.scala.spark
 
 import java.nio.file.Files
 import java.util.concurrent.LinkedBlockingDeque
-
+import ml.dmlc.xgboost4j.java.Rabit
 import ml.dmlc.xgboost4j.{LabeledPoint => XGBLabeledPoint}
 import ml.dmlc.xgboost4j.scala.DMatrix
 import ml.dmlc.xgboost4j.scala.rabit.RabitTracker
 import ml.dmlc.xgboost4j.scala.{XGBoost => SXGBoost, _}
 import org.apache.hadoop.fs.{FileSystem, Path}
-
 import org.apache.spark.TaskContext
 import org.apache.spark.ml.linalg.Vectors
 import org.apache.spark.sql._
 import org.scalatest.FunSuite
 import scala.util.Random
 
-import ml.dmlc.xgboost4j.java.Rabit
-
 class XGBoostGeneralSuite extends FunSuite with PerTest {
 
   test("test Rabit allreduce to validate Scala-implemented Rabit tracker") {
@@ -80,11 +77,11 @@ class XGBoostGeneralSuite extends FunSuite with PerTest {
     val trainingRDD = sc.parallelize(Classification.train)
     val (booster, metrics) = XGBoost.trainDistributed(
       trainingRDD,
-      List("eta" -> "1", "max_depth" -> "6",
+      List("eta" -> "1", "max_depth" -> "6", "silent" -> "1",
-        "objective" -> "binary:logistic", "num_round" -> 5, "num_workers" -> numWorkers,
+        "objective" -> "binary:logistic").toMap,
-        "custom_eval" -> null, "custom_obj" -> null, "use_external_memory" -> false,
+      round = 5, nWorkers = numWorkers, eval = null, obj = null, useExternalMemory = false,
-        "missing" -> Float.NaN).toMap,
+      hasGroup = false, missing = Float.NaN)
-      hasGroup = false)
+
     assert(booster != null)
   }
 
@@ -92,7 +89,7 @@ class XGBoostGeneralSuite extends FunSuite with PerTest {
     val eval = new EvalError()
     val training = buildDataFrame(Classification.train)
     val testDM = new DMatrix(Classification.test.iterator)
-    val paramMap = Map("eta" -> "1", "max_depth" -> "6",
+    val paramMap = Map("eta" -> "1", "max_depth" -> "6", "silent" -> "1",
       "objective" -> "binary:logistic", "num_round" -> 5, "num_workers" -> numWorkers,
       "use_external_memory" -> true)
     val model = new XGBoostClassifier(paramMap).fit(training)
@@ -104,120 +101,73 @@ class XGBoostGeneralSuite extends FunSuite with PerTest {
     val eval = new EvalError()
     val training = buildDataFrame(Classification.train)
     val testDM = new DMatrix(Classification.test.iterator)
-    val paramMap = Map("eta" -> "1", "max_depth" -> "6",
+    val paramMap = Map("eta" -> "1", "max_depth" -> "6", "silent" -> "1",
       "objective" -> "binary:logistic", "num_round" -> 5, "num_workers" -> numWorkers,
       "tracker_conf" -> TrackerConf(60 * 60 * 1000, "scala"))
     val model = new XGBoostClassifier(paramMap).fit(training)
     assert(eval.eval(model._booster.predict(testDM, outPutMargin = true), testDM) < 0.1)
   }
 
-  test("test with quantile hist with monotone_constraints (lossguide)") {
-    val eval = new EvalError()
-    val training = buildDataFrame(Classification.train)
-    val testDM = new DMatrix(Classification.test.iterator)
-    val paramMap = Map("eta" -> "1",
-      "max_depth" -> "6",
-      "objective" -> "binary:logistic", "tree_method" -> "hist", "grow_policy" -> "lossguide",
-      "num_round" -> 5, "num_workers" -> numWorkers, "monotone_constraints" -> "(1, 0)")
-    val model = new XGBoostClassifier(paramMap).fit(training)
-    assert(eval.eval(model._booster.predict(testDM, outPutMargin = true), testDM) < 0.1)
-  }
 
-  test("test with quantile hist with interaction_constraints (lossguide)") {
+  ignore("test with fast histo depthwise") {
     val eval = new EvalError()
     val training = buildDataFrame(Classification.train)
     val testDM = new DMatrix(Classification.test.iterator)
-    val paramMap = Map("eta" -> "1",
+    val paramMap = Map("eta" -> "1", "gamma" -> "0.5", "max_depth" -> "6", "silent" -> "1",
-      "max_depth" -> "6",
-      "objective" -> "binary:logistic", "tree_method" -> "hist", "grow_policy" -> "lossguide",
-      "num_round" -> 5, "num_workers" -> numWorkers, "interaction_constraints" -> "[[1,2],[2,3,4]]")
-    val model = new XGBoostClassifier(paramMap).fit(training)
-    assert(eval.eval(model._booster.predict(testDM, outPutMargin = true), testDM) < 0.1)
-  }
-
-  test("test with quantile hist with monotone_constraints (depthwise)") {
-    val eval = new EvalError()
-    val training = buildDataFrame(Classification.train)
-    val testDM = new DMatrix(Classification.test.iterator)
-    val paramMap = Map("eta" -> "1",
-      "max_depth" -> "6",
       "objective" -> "binary:logistic", "tree_method" -> "hist", "grow_policy" -> "depthwise",
-      "num_round" -> 5, "num_workers" -> numWorkers, "monotone_constraints" -> "(1, 0)")
+      "eval_metric" -> "error", "num_round" -> 5, "num_workers" -> math.min(numWorkers, 2))
+    // TODO: histogram algorithm seems to be very very sensitive to worker number
     val model = new XGBoostClassifier(paramMap).fit(training)
     assert(eval.eval(model._booster.predict(testDM, outPutMargin = true), testDM) < 0.1)
   }
 
-  test("test with quantile hist with interaction_constraints (depthwise)") {
+  ignore("test with fast histo lossguide") {
     val eval = new EvalError()
     val training = buildDataFrame(Classification.train)
     val testDM = new DMatrix(Classification.test.iterator)
-    val paramMap = Map("eta" -> "1",
+    val paramMap = Map("eta" -> "1", "gamma" -> "0.5", "max_depth" -> "0", "silent" -> "1",
-      "max_depth" -> "6",
-      "objective" -> "binary:logistic", "tree_method" -> "hist", "grow_policy" -> "depthwise",
-      "num_round" -> 5, "num_workers" -> numWorkers, "interaction_constraints" -> "[[1,2],[2,3,4]]")
-    val model = new XGBoostClassifier(paramMap).fit(training)
-    assert(eval.eval(model._booster.predict(testDM, outPutMargin = true), testDM) < 0.1)
-  }
-
-  test("test with quantile hist depthwise") {
-    val eval = new EvalError()
-    val training = buildDataFrame(Classification.train)
-    val testDM = new DMatrix(Classification.test.iterator)
-    val paramMap = Map("eta" -> "1",
-      "max_depth" -> "6",
-      "objective" -> "binary:logistic", "tree_method" -> "hist", "grow_policy" -> "depthwise",
-      "num_round" -> 5, "num_workers" -> numWorkers)
-    val model = new XGBoostClassifier(paramMap).fit(training)
-    assert(eval.eval(model._booster.predict(testDM, outPutMargin = true), testDM) < 0.1)
-  }
-
-  test("test with quantile hist lossguide") {
-    val eval = new EvalError()
-    val training = buildDataFrame(Classification.train)
-    val testDM = new DMatrix(Classification.test.iterator)
-    val paramMap = Map("eta" -> "1", "gamma" -> "0.5", "max_depth" -> "0",
       "objective" -> "binary:logistic", "tree_method" -> "hist", "grow_policy" -> "lossguide",
-      "max_leaves" -> "8", "num_round" -> 5,
+      "max_leaves" -> "8", "eval_metric" -> "error", "num_round" -> 5,
-      "num_workers" -> numWorkers)
+      "num_workers" -> math.min(numWorkers, 2))
     val model = new XGBoostClassifier(paramMap).fit(training)
     val x = eval.eval(model._booster.predict(testDM, outPutMargin = true), testDM)
     assert(x < 0.1)
   }
 
-  test("test with quantile hist lossguide with max bin") {
+  ignore("test with fast histo lossguide with max bin") {
     val eval = new EvalError()
     val training = buildDataFrame(Classification.train)
     val testDM = new DMatrix(Classification.test.iterator)
-    val paramMap = Map("eta" -> "1", "gamma" -> "0.5", "max_depth" -> "0",
+    val paramMap = Map("eta" -> "1", "gamma" -> "0.5", "max_depth" -> "0", "silent" -> "0",
       "objective" -> "binary:logistic", "tree_method" -> "hist",
       "grow_policy" -> "lossguide", "max_leaves" -> "8", "max_bin" -> "16",
-      "eval_metric" -> "error", "num_round" -> 5, "num_workers" -> numWorkers)
+      "eval_metric" -> "error", "num_round" -> 5, "num_workers" -> math.min(numWorkers, 2))
     val model = new XGBoostClassifier(paramMap).fit(training)
     val x = eval.eval(model._booster.predict(testDM, outPutMargin = true), testDM)
     assert(x < 0.1)
   }
 
-  test("test with quantile hist depthwidth with max depth") {
+  ignore("test with fast histo depthwidth with max depth") {
     val eval = new EvalError()
     val training = buildDataFrame(Classification.train)
     val testDM = new DMatrix(Classification.test.iterator)
-    val paramMap = Map("eta" -> "1", "gamma" -> "0.5", "max_depth" -> "6",
+    val paramMap = Map("eta" -> "1", "gamma" -> "0.5", "max_depth" -> "0", "silent" -> "0",
       "objective" -> "binary:logistic", "tree_method" -> "hist",
-      "grow_policy" -> "depthwise", "max_depth" -> "2",
+      "grow_policy" -> "depthwise", "max_leaves" -> "8", "max_depth" -> "2",
-      "eval_metric" -> "error", "num_round" -> 10, "num_workers" -> numWorkers)
+      "eval_metric" -> "error", "num_round" -> 10, "num_workers" -> math.min(numWorkers, 2))
     val model = new XGBoostClassifier(paramMap).fit(training)
     val x = eval.eval(model._booster.predict(testDM, outPutMargin = true), testDM)
     assert(x < 0.1)
   }
 
-  test("test with quantile hist depthwidth with max depth and max bin") {
+  ignore("test with fast histo depthwidth with max depth and max bin") {
     val eval = new EvalError()
     val training = buildDataFrame(Classification.train)
     val testDM = new DMatrix(Classification.test.iterator)
-    val paramMap = Map("eta" -> "1", "gamma" -> "0.5", "max_depth" -> "6",
+    val paramMap = Map("eta" -> "1", "gamma" -> "0.5", "max_depth" -> "0", "silent" -> "0",
       "objective" -> "binary:logistic", "tree_method" -> "hist",
       "grow_policy" -> "depthwise", "max_depth" -> "2", "max_bin" -> "2",
-      "eval_metric" -> "error", "num_round" -> 10, "num_workers" -> numWorkers)
+      "eval_metric" -> "error", "num_round" -> 10, "num_workers" -> math.min(numWorkers, 2))
     val model = new XGBoostClassifier(paramMap).fit(training)
     val x = eval.eval(model._booster.predict(testDM, outPutMargin = true), testDM)
     assert(x < 0.1)
@@ -241,7 +191,7 @@ class XGBoostGeneralSuite extends FunSuite with PerTest {
     }
 
     val denseDF = buildDenseDataFrame().repartition(4)
-    val paramMap = List("eta" -> "1", "max_depth" -> "2",
+    val paramMap = List("eta" -> "1", "max_depth" -> "2", "silent" -> "1",
       "objective" -> "binary:logistic", "missing" -> -0.1f, "num_workers" -> numWorkers).toMap
     val model = new XGBoostClassifier(paramMap).fit(denseDF)
     model.transform(denseDF).collect()
@@ -251,7 +201,7 @@ class XGBoostGeneralSuite extends FunSuite with PerTest {
     val eval = new EvalError()
     val training = buildDataFrame(Classification.train)
     val testDM = new DMatrix(Classification.test.iterator)
-    val paramMap = Map("eta" -> "1", "max_depth" -> "6",
+    val paramMap = Map("eta" -> "1", "max_depth" -> "6", "silent" -> "1",
       "objective" -> "binary:logistic", "timeout_request_workers" -> 0L,
       "num_round" -> 5, "num_workers" -> numWorkers)
     val model = new XGBoostClassifier(paramMap).fit(training)
@@ -265,7 +215,7 @@ class XGBoostGeneralSuite extends FunSuite with PerTest {
     val testDM = new DMatrix(Classification.test.iterator)
 
     val tmpPath = Files.createTempDirectory("model1").toAbsolutePath.toString
-    val paramMap = Map("eta" -> "1", "max_depth" -> 2,
+    val paramMap = Map("eta" -> "1", "max_depth" -> 2, "silent" -> "1",
       "objective" -> "binary:logistic", "checkpoint_path" -> tmpPath,
       "checkpoint_interval" -> 2, "num_workers" -> numWorkers)
 
@@ -317,101 +267,13 @@ class XGBoostGeneralSuite extends FunSuite with PerTest {
 
   test("distributed training with group data") {
     val trainingRDD = sc.parallelize(Ranking.train, 5)
-    val (booster, _) = XGBoost.trainDistributed(
+    val (booster, metrics) = XGBoost.trainDistributed(
       trainingRDD,
-      List("eta" -> "1", "max_depth" -> "6",
+      List("eta" -> "1", "max_depth" -> "6", "silent" -> "1",
-        "objective" -> "rank:pairwise", "num_round" -> 5, "num_workers" -> numWorkers,
+        "objective" -> "binary:logistic").toMap,
-        "custom_eval" -> null, "custom_obj" -> null, "use_external_memory" -> false,
+      round = 5, nWorkers = numWorkers, eval = null, obj = null, useExternalMemory = false,
-        "missing" -> Float.NaN).toMap,
+      hasGroup = true, missing = Float.NaN)
-      hasGroup = true)
 
     assert(booster != null)
   }
-
-  test("training summary") {
-    val paramMap = Map("eta" -> "1", "max_depth" -> "6",
-      "objective" -> "binary:logistic", "num_round" -> 5, "nWorkers" -> numWorkers)
-
-    val trainingDF = buildDataFrame(Classification.train)
-    val xgb = new XGBoostClassifier(paramMap)
-    val model = xgb.fit(trainingDF)
-
-    assert(model.summary.trainObjectiveHistory.length === 5)
-    assert(model.summary.validationObjectiveHistory.isEmpty)
-  }
-
-  test("train/test split") {
-    val paramMap = Map("eta" -> "1", "max_depth" -> "6",
-      "objective" -> "binary:logistic", "train_test_ratio" -> "0.5",
-      "num_round" -> 5, "num_workers" -> numWorkers)
-    val training = buildDataFrame(Classification.train)
-
-    val xgb = new XGBoostClassifier(paramMap)
-    val model = xgb.fit(training)
-    assert(model.summary.validationObjectiveHistory.length === 1)
-    assert(model.summary.validationObjectiveHistory(0)._1 === "test")
-    assert(model.summary.validationObjectiveHistory(0)._2.length === 5)
-    assert(model.summary.trainObjectiveHistory !== model.summary.validationObjectiveHistory(0))
-  }
-
-  test("train with multiple validation datasets (non-ranking)") {
-    val training = buildDataFrame(Classification.train)
-    val Array(train, eval1, eval2) = training.randomSplit(Array(0.6, 0.2, 0.2))
-    val paramMap1 = Map("eta" -> "1", "max_depth" -> "6",
-      "objective" -> "binary:logistic",
-      "num_round" -> 5, "num_workers" -> numWorkers)
-
-    val xgb1 = new XGBoostClassifier(paramMap1).setEvalSets(Map("eval1" -> eval1, "eval2" -> eval2))
-    val model1 = xgb1.fit(train)
-    assert(model1.summary.validationObjectiveHistory.length === 2)
-    assert(model1.summary.validationObjectiveHistory.map(_._1).toSet === Set("eval1", "eval2"))
-    assert(model1.summary.validationObjectiveHistory(0)._2.length === 5)
-    assert(model1.summary.validationObjectiveHistory(1)._2.length === 5)
-    assert(model1.summary.trainObjectiveHistory !== model1.summary.validationObjectiveHistory(0))
-    assert(model1.summary.trainObjectiveHistory !== model1.summary.validationObjectiveHistory(1))
-
-    val paramMap2 = Map("eta" -> "1", "max_depth" -> "6",
-      "objective" -> "binary:logistic",
-      "num_round" -> 5, "num_workers" -> numWorkers,
-      "eval_sets" -> Map("eval1" -> eval1, "eval2" -> eval2))
-    val xgb2 = new XGBoostClassifier(paramMap2)
-    val model2 = xgb2.fit(train)
-    assert(model2.summary.validationObjectiveHistory.length === 2)
-    assert(model2.summary.validationObjectiveHistory.map(_._1).toSet === Set("eval1", "eval2"))
-    assert(model2.summary.validationObjectiveHistory(0)._2.length === 5)
-    assert(model2.summary.validationObjectiveHistory(1)._2.length === 5)
-    assert(model2.summary.trainObjectiveHistory !== model2.summary.validationObjectiveHistory(0))
-    assert(model2.summary.trainObjectiveHistory !== model2.summary.validationObjectiveHistory(1))
-  }
-
-  test("train with multiple validation datasets (ranking)") {
-    val training = buildDataFrameWithGroup(Ranking.train, 5)
-    val Array(train, eval1, eval2) = training.randomSplit(Array(0.6, 0.2, 0.2))
-    val paramMap1 = Map("eta" -> "1", "max_depth" -> "6",
-      "objective" -> "rank:pairwise",
-      "num_round" -> 5, "num_workers" -> numWorkers, "group_col" -> "group")
-    val xgb1 = new XGBoostRegressor(paramMap1).setEvalSets(Map("eval1" -> eval1, "eval2" -> eval2))
-    val model1 = xgb1.fit(train)
-    assert(model1 != null)
-    assert(model1.summary.validationObjectiveHistory.length === 2)
-    assert(model1.summary.validationObjectiveHistory.map(_._1).toSet === Set("eval1", "eval2"))
-    assert(model1.summary.validationObjectiveHistory(0)._2.length === 5)
-    assert(model1.summary.validationObjectiveHistory(1)._2.length === 5)
-    assert(model1.summary.trainObjectiveHistory !== model1.summary.validationObjectiveHistory(0))
-    assert(model1.summary.trainObjectiveHistory !== model1.summary.validationObjectiveHistory(1))
-
-    val paramMap2 = Map("eta" -> "1", "max_depth" -> "6",
-      "objective" -> "rank:pairwise",
-      "num_round" -> 5, "num_workers" -> numWorkers, "group_col" -> "group",
-      "eval_sets" -> Map("eval1" -> eval1, "eval2" -> eval2))
-    val xgb2 = new XGBoostRegressor(paramMap2)
-    val model2 = xgb2.fit(train)
-    assert(model2 != null)
-    assert(model2.summary.validationObjectiveHistory.length === 2)
-    assert(model2.summary.validationObjectiveHistory.map(_._1).toSet === Set("eval1", "eval2"))
-    assert(model2.summary.validationObjectiveHistory(0)._2.length === 5)
-    assert(model2.summary.validationObjectiveHistory(1)._2.length === 5)
-    assert(model2.summary.trainObjectiveHistory !== model2.summary.validationObjectiveHistory(0))
-    assert(model2.summary.trainObjectiveHistory !== model2.summary.validationObjectiveHistory(1))
-  }
 }

jvm-packages/xgboost4j/pom.xml

@@ -6,10 +6,10 @@
     <parent>
         <groupId>ml.dmlc</groupId>
         <artifactId>xgboost-jvm</artifactId>
-        <version>0.82</version>
+        <version>0.81</version>
     </parent>
     <artifactId>xgboost4j</artifactId>
-    <version>0.82</version>
+    <version>0.81</version>
     <packaging>jar</packaging>
 
     <dependencies>

jvm-packages/xgboost4j/src/main/java/ml/dmlc/xgboost4j/java/Booster.java

@@ -16,12 +16,9 @@
 package ml.dmlc.xgboost4j.java;
 
 import java.io.*;
-import java.util.Arrays;
 import java.util.HashMap;
-import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
-import java.util.Set;
 
 import com.esotericsoftware.kryo.Kryo;
 import com.esotericsoftware.kryo.KryoSerializable;
@@ -398,25 +395,6 @@ public class Booster implements Serializable, KryoSerializable {
     return modelInfos[0];
   }
 
-  /**
-   * Supported feature importance types
-   *
-   * WEIGHT = Number of nodes that a feature was used to determine a split
-   * GAIN = Average information gain per split for a feature
-   * COVER = Average cover per split for a feature
-   * TOTAL_GAIN = Total information gain over all splits of a feature
-   * TOTAL_COVER = Total cover over all splits of a feature
-   */
-  public static class FeatureImportanceType {
-    public static final String WEIGHT = "weight";
-    public static final String GAIN = "gain";
-    public static final String COVER = "cover";
-    public static final String TOTAL_GAIN = "total_gain";
-    public static final String TOTAL_COVER = "total_cover";
-    public static final Set<String> ACCEPTED_TYPES = new HashSet<>(
-            Arrays.asList(WEIGHT, GAIN, COVER, TOTAL_GAIN, TOTAL_COVER));
-  }
-
   /**
    * Get importance of each feature with specified feature names.
    *
@@ -425,28 +403,6 @@ public class Booster implements Serializable, KryoSerializable {
    */
   public Map<String, Integer> getFeatureScore(String[] featureNames) throws XGBoostError {
     String[] modelInfos = getModelDump(featureNames, false);
-    return getFeatureWeightsFromModel(modelInfos);
-  }
-
-  /**
-   * Get importance of each feature
-   *
-   * @return featureScoreMap  key: feature index, value: feature importance score, can be nill
-   * @throws XGBoostError native error
-   */
-  public Map<String, Integer> getFeatureScore(String featureMap) throws XGBoostError {
-    String[] modelInfos = getModelDump(featureMap, false);
-    return getFeatureWeightsFromModel(modelInfos);
-  }
-
-  /**
-   * Get the importance of each feature based purely on weights (number of splits)
-   *
-   * @return featureScoreMap key: feature index,
-   * value: feature importance score based on weight
-   * @throws XGBoostError native error
-   */
-  private Map<String, Integer> getFeatureWeightsFromModel(String[] modelInfos) throws XGBoostError {
     Map<String, Integer> featureScore = new HashMap<>();
     for (String tree : modelInfos) {
       for (String node : tree.split("\n")) {
@@ -467,91 +423,30 @@ public class Booster implements Serializable, KryoSerializable {
   }
 
   /**
-   * Get the feature importances for gain or cover (average or total)
+   * Get importance of each feature
    *
-   * @return featureImportanceMap key: feature index,
+   * @return featureScoreMap  key: feature index, value: feature importance score, can be nill
-   * values: feature importance score based on gain or cover
    * @throws XGBoostError native error
    */
-  public Map<String, Double> getScore(
+  public Map<String, Integer> getFeatureScore(String featureMap) throws XGBoostError {
-          String[] featureNames, String importanceType) throws XGBoostError {
+    String[] modelInfos = getModelDump(featureMap, false);
-    String[] modelInfos = getModelDump(featureNames, true);
+    Map<String, Integer> featureScore = new HashMap<>();
-    return getFeatureImportanceFromModel(modelInfos, importanceType);
+    for (String tree : modelInfos) {
-  }
+      for (String node : tree.split("\n")) {
-
-  /**
-   * Get the feature importances for gain or cover (average or total), with feature names
-   *
-   * @return featureImportanceMap key: feature name,
-   * values: feature importance score based on gain or cover
-   * @throws XGBoostError native error
-   */
-  public Map<String, Double> getScore(
-          String featureMap, String importanceType) throws XGBoostError {
-    String[] modelInfos = getModelDump(featureMap, true);
-    return getFeatureImportanceFromModel(modelInfos, importanceType);
-  }
-
-  /**
-   * Get the importance of each feature based on information gain or cover
-   *
-   * @return featureImportanceMap key: feature index, value: feature importance score
-   * based on information gain or cover
-   * @throws XGBoostError native error
-   */
-  private Map<String, Double> getFeatureImportanceFromModel(
-          String[] modelInfos, String importanceType) throws XGBoostError {
-    if (!FeatureImportanceType.ACCEPTED_TYPES.contains(importanceType)) {
-      throw new AssertionError(String.format("Importance type %s is not supported",
-              importanceType));
-    }
-    Map<String, Double> importanceMap = new HashMap<>();
-    Map<String, Double> weightMap = new HashMap<>();
-    if (importanceType == FeatureImportanceType.WEIGHT) {
-      Map<String, Integer> importanceWeights = getFeatureWeightsFromModel(modelInfos);
-      for (String feature: importanceWeights.keySet()) {
-        importanceMap.put(feature, new Double(importanceWeights.get(feature)));
-      }
-      return importanceMap;
-    }
-    /* Each split in the tree has this text form:
-    "0:[f28<-9.53674316e-07] yes=1,no=2,missing=1,gain=4000.53101,cover=1628.25"
-    So the line has to be split according to whether cover or gain is desired */
-    String splitter = "gain=";
-    if (importanceType == FeatureImportanceType.COVER
-        || importanceType == FeatureImportanceType.TOTAL_COVER) {
-      splitter = "cover=";
-    }
-    for (String tree: modelInfos) {
-      for (String node: tree.split("\n")) {
         String[] array = node.split("\\[");
         if (array.length == 1) {
           continue;
         }
-        String[] fidWithImportance = array[1].split("\\]");
+        String fid = array[1].split("\\]")[0];
-        // Extract gain or cover from string after closing bracket
+        fid = fid.split("<")[0];
-        Double importance = Double.parseDouble(
+        if (featureScore.containsKey(fid)) {
-            fidWithImportance[1].split(splitter)[1].split(",")[0]
+          featureScore.put(fid, 1 + featureScore.get(fid));
-        );
-        String fid = fidWithImportance[0].split("<")[0];
-        if (importanceMap.containsKey(fid)) {
-          importanceMap.put(fid, importance + importanceMap.get(fid));
-          weightMap.put(fid, 1d + weightMap.get(fid));
         } else {
-          importanceMap.put(fid, importance);
+          featureScore.put(fid, 1);
-          weightMap.put(fid, 1d);
         }
       }
     }
-    /* By default we calculate total gain and total cover.
+    return featureScore;
-    Divide by the number of nodes per feature to get gain / cover */
-    if (importanceType == FeatureImportanceType.COVER
-        || importanceType == FeatureImportanceType.GAIN) {
-      for (String fid: importanceMap.keySet()) {
-        importanceMap.put(fid, importanceMap.get(fid)/weightMap.get(fid));
-      }
-    }
-    return importanceMap;
   }
 
   /**

jvm-packages/xgboost4j/src/main/java/ml/dmlc/xgboost4j/java/XGBoost.java

@@ -140,8 +140,6 @@ public class XGBoost {
     //collect eval matrixs
     String[] evalNames;
     DMatrix[] evalMats;
-    float bestScore;
-    int bestIteration;
     List<String> names = new ArrayList<String>();
     List<DMatrix> mats = new ArrayList<DMatrix>();
 
@@ -152,12 +150,6 @@ public class XGBoost {
 
     evalNames = names.toArray(new String[names.size()]);
     evalMats = mats.toArray(new DMatrix[mats.size()]);
-    if (isMaximizeEvaluation(params)) {
-      bestScore = -Float.MAX_VALUE;
-    } else {
-      bestScore = Float.MAX_VALUE;
-    }
-    bestIteration = 0;
     metrics = metrics == null ? new float[evalNames.length][round] : metrics;
 
     //collect all data matrixs
@@ -204,27 +196,12 @@ public class XGBoost {
         for (int i = 0; i < metricsOut.length; i++) {
           metrics[i][iter] = metricsOut[i];
         }
-
-        // If there is more than one evaluation datasets, the last one would be used
-        // to determinate early stop.
-        float score = metricsOut[metricsOut.length - 1];
-        if (isMaximizeEvaluation(params)) {
-          // Update best score if the current score is better (no update when equal)
-          if (score > bestScore) {
-            bestScore = score;
-            bestIteration = iter;
-          }
-        } else {
-          if (score < bestScore) {
-            bestScore = score;
-            bestIteration = iter;
-          }
-        }
         if (earlyStoppingRounds > 0) {
-          if (shouldEarlyStop(earlyStoppingRounds, iter, bestIteration)) {
+          boolean onTrack = judgeIfTrainingOnTrack(params, earlyStoppingRounds, metrics, iter);
+          if (!onTrack) {
+            String reversedDirection = getReversedDirection(params);
             Rabit.trackerPrint(String.format(
-                    "early stopping after %d rounds away from the best iteration",
+                    "early stopping after %d %s rounds", earlyStoppingRounds, reversedDirection));
-                        earlyStoppingRounds));
             break;
           }
         }
@@ -237,11 +214,30 @@ public class XGBoost {
     return booster;
   }
 
-  static boolean shouldEarlyStop(int earlyStoppingRounds, int iter, int bestIteration) {
+  static boolean judgeIfTrainingOnTrack(
-    return iter - bestIteration >= earlyStoppingRounds;
+          Map<String, Object> params, int earlyStoppingRounds, float[][] metrics, int iter) {
+    boolean maximizeEvaluationMetrics = getMetricsExpectedDirection(params);
+    boolean onTrack = false;
+    float[] criterion = metrics[metrics.length - 1];
+    for (int shift = 0; shift < Math.min(iter, earlyStoppingRounds) - 1; shift++) {
+      onTrack |= maximizeEvaluationMetrics ?
+              criterion[iter - shift] >= criterion[iter - shift - 1] :
+              criterion[iter - shift] <= criterion[iter - shift - 1];
+    }
+    return onTrack;
   }
 
-  private static boolean isMaximizeEvaluation(Map<String, Object> params) {
+  private static String getReversedDirection(Map<String, Object> params) {
+    String reversedDirection = null;
+    if (Boolean.valueOf(String.valueOf(params.get("maximize_evaluation_metrics")))) {
+      reversedDirection = "descending";
+    } else if (!Boolean.valueOf(String.valueOf(params.get("maximize_evaluation_metrics")))) {
+      reversedDirection = "ascending";
+    }
+    return reversedDirection;
+  }
+
+  private static boolean getMetricsExpectedDirection(Map<String, Object> params) {
     try {
       String maximize = String.valueOf(params.get("maximize_evaluation_metrics"));
       assert(maximize != null);

jvm-packages/xgboost4j/src/main/scala/ml/dmlc/xgboost4j/scala/Booster.scala

@@ -204,7 +204,7 @@ class Booster private[xgboost4j](private[xgboost4j] var booster: JBooster)
 
 
   /**
-   * Get importance of each feature based on weight only (number of splits)
+   * Get importance of each feature
    *
    * @return featureScoreMap  key: feature index, value: feature importance score
    */
@@ -214,8 +214,7 @@ class Booster private[xgboost4j](private[xgboost4j] var booster: JBooster)
   }
 
   /**
-    * Get importance of each feature based on weight only
+    * Get importance of each feature with specified feature names.
-    * (number of splits), with specified feature names.
     *
     * @return featureScoreMap  key: feature name, value: feature importance score
     */
@@ -224,31 +223,6 @@ class Booster private[xgboost4j](private[xgboost4j] var booster: JBooster)
     booster.getFeatureScore(featureNames).asScala
   }
 
-  /**
-    * Get importance of each feature based on information gain or cover
-    * Supported: ["gain, "cover", "total_gain", "total_cover"]
-    *
-    * @return featureScoreMap  key: feature index, value: feature importance score
-    */
-  @throws(classOf[XGBoostError])
-  def getScore(featureMap: String, importanceType: String): Map[String, Double] = {
-    Map(booster.getScore(featureMap, importanceType)
-        .asScala.mapValues(_.doubleValue).toSeq: _*)
-  }
-
-  /**
-    * Get importance of each feature based on information gain or cover
-    * , with specified feature names.
-    * Supported: ["gain, "cover", "total_gain", "total_cover"]
-    *
-    * @return featureScoreMap  key: feature name, value: feature importance score
-    */
-  @throws(classOf[XGBoostError])
-  def getScore(featureNames: Array[String], importanceType: String): Map[String, Double] = {
-    Map(booster.getScore(featureNames, importanceType)
-        .asScala.mapValues(_.doubleValue).toSeq: _*)
-  }
-
   def getVersion: Int = booster.getVersion
 
   def toByteArray: Array[Byte] = {

jvm-packages/xgboost4j/src/main/scala/ml/dmlc/xgboost4j/scala/ObjectiveTrait.scala

@@ -27,7 +27,7 @@ trait ObjectiveTrait extends IObjective {
    *
    * @param predicts untransformed margin predicts
    * @param dtrain   training data
-   * @return List with two float array, correspond to grad and hess
+   * @return List with two float array, correspond to first order grad and second order grad
    */
   def getGradient(predicts: Array[Array[Float]], dtrain: DMatrix): List[Array[Float]]
 

jvm-packages/xgboost4j/src/test/java/ml/dmlc/xgboost4j/java/BoosterImplTest.java

@@ -139,7 +139,7 @@ public class BoosterImplTest {
   }
 
   private static class IncreasingEval implements IEvaluation {
-    private int value = 1;
+    private int value = 0;
 
     @Override
     public String getMetric() {
@@ -152,167 +152,71 @@ public class BoosterImplTest {
     }
   }
 
-  @Test
-  public void testDescendMetricsWithBoundaryCondition() {
-    // maximize_evaluation_metrics = false
-    int totalIterations = 11;
-    int earlyStoppingRound = 10;
-    float[][] metrics = new float[1][totalIterations];
-    for (int i = 0; i < totalIterations; i++) {
-      metrics[0][i] = i;
-    }
-    int bestIteration = 0;
-
-    for (int itr = 0; itr < totalIterations; itr++) {
-      boolean es = XGBoost.shouldEarlyStop(earlyStoppingRound, itr, bestIteration);
-      if (itr == totalIterations - 1) {
-        TestCase.assertTrue(es);
-      } else {
-        TestCase.assertFalse(es);
-      }
-    }
-  }
-
-  @Test
-  public void testEarlyStoppingForMultipleMetrics() {
-    // maximize_evaluation_metrics = true
-    int earlyStoppingRound = 3;
-    int totalIterations = 5;
-    int numOfMetrics = 3;
-    float[][] metrics = new float[numOfMetrics][totalIterations];
-    // Only assign metric values to the first dataset, zeros for other datasets
-    for (int i = 0; i < numOfMetrics; i++) {
-      for (int j = 0; j < totalIterations; j++) {
-        metrics[0][j] = j;
-      }
-    }
-    int bestIteration;
-
-    for (int i = 0; i < totalIterations; i++) {
-      bestIteration = i;
-      boolean es = XGBoost.shouldEarlyStop(earlyStoppingRound, i, bestIteration);
-      TestCase.assertFalse(es);
-    }
-
-    // when we have multiple datasets, only the last one was used to determinate early stop
-    // Here we changed the metric of the first dataset, it doesn't have any effect to the final result
-    for (int i = 0; i < totalIterations; i++) {
-      metrics[0][i] = totalIterations - i;
-    }
-    for (int i = 0; i < totalIterations; i++) {
-      bestIteration = i;
-      boolean es = XGBoost.shouldEarlyStop(earlyStoppingRound, i, bestIteration);
-      TestCase.assertFalse(es);
-    }
-
-    // Now assign metric values to the last dataset.
-    for (int i = 0; i < totalIterations; i++) {
-      metrics[2][i] = totalIterations - i;
-    }
-    bestIteration = 0;
-
-    for (int i = 0; i < totalIterations; i++) {
-      // if any metrics off, we need to stop
-      boolean es = XGBoost.shouldEarlyStop(earlyStoppingRound, i, bestIteration);
-      if (i >= earlyStoppingRound) {
-        TestCase.assertTrue(es);
-      } else {
-        TestCase.assertFalse(es);
-      }
-    }
-  }
-
   @Test
   public void testDescendMetrics() {
-    // maximize_evaluation_metrics = false
+    Map<String, Object> paramMap = new HashMap<String, Object>() {
-    int totalIterations = 10;
+      {
-    int earlyStoppingRounds = 5;
+        put("max_depth", 3);
-    float[][] metrics = new float[1][totalIterations];
+        put("silent", 1);
-    for (int i = 0; i < totalIterations; i++) {
+        put("objective", "binary:logistic");
+        put("maximize_evaluation_metrics", "false");
+      }
+    };
+    float[][] metrics = new float[1][5];
+    for (int i = 0; i < 5; i++) {
       metrics[0][i] = i;
     }
-    int bestIteration = 0;
+    boolean onTrack = XGBoost.judgeIfTrainingOnTrack(paramMap, 5, metrics, 4);
-
+    TestCase.assertFalse(onTrack);
-    boolean es = XGBoost.shouldEarlyStop(earlyStoppingRounds, totalIterations - 1, bestIteration);
+    for (int i = 0; i < 5; i++) {
-    TestCase.assertTrue(es);
+      metrics[0][i] = 5 - i;
-    for (int i = 0; i < totalIterations; i++) {
-      metrics[0][i] = totalIterations - i;
     }
-    bestIteration = totalIterations - 1;
+    onTrack = XGBoost.judgeIfTrainingOnTrack(paramMap, 5, metrics, 4);
-
+    TestCase.assertTrue(onTrack);
-    es = XGBoost.shouldEarlyStop(earlyStoppingRounds, totalIterations - 1, bestIteration);
+    for (int i = 0; i < 5; i++) {
-    TestCase.assertFalse(es);
+      metrics[0][i] = 5 - i;
-
-    for (int i = 0; i < totalIterations; i++) {
-      metrics[0][i] = totalIterations - i;
-    }
-    metrics[0][4] = 1;
-    metrics[0][9] = 5;
-
-    bestIteration = 4;
-
-    es = XGBoost.shouldEarlyStop(earlyStoppingRounds, totalIterations - 1, bestIteration);
-    TestCase.assertTrue(es);
-  }
-
-  @Test
-  public void testAscendMetricsWithBoundaryCondition() {
-    // maximize_evaluation_metrics = true
-    int totalIterations = 11;
-    int earlyStoppingRounds = 10;
-    float[][] metrics = new float[1][totalIterations];
-    for (int i = 0; i < totalIterations; i++) {
-      metrics[0][i] = totalIterations - i;
-    }
-    int bestIteration = 0;
-
-    for (int itr = 0; itr < totalIterations; itr++) {
-      boolean es = XGBoost.shouldEarlyStop(earlyStoppingRounds, itr, bestIteration);
-      if (itr == totalIterations - 1) {
-        TestCase.assertTrue(es);
-      } else {
-        TestCase.assertFalse(es);
-      }
     }
+    metrics[0][0] = 1;
+    metrics[0][2] = 5;
+    onTrack = XGBoost.judgeIfTrainingOnTrack(paramMap, 5, metrics, 4);
+    TestCase.assertTrue(onTrack);
   }
 
   @Test
   public void testAscendMetrics() {
-    // maximize_evaluation_metrics = true
+    Map<String, Object> paramMap = new HashMap<String, Object>() {
-    int totalIterations = 10;
+      {
-    int earlyStoppingRounds = 5;
+        put("max_depth", 3);
-    float[][] metrics = new float[1][totalIterations];
+        put("silent", 1);
-    for (int i = 0; i < totalIterations; i++) {
+        put("objective", "binary:logistic");
-      metrics[0][i] = totalIterations - i;
+        put("maximize_evaluation_metrics", "true");
-    }
+      }
-    int bestIteration = 0;
+    };
-
+    float[][] metrics = new float[1][5];
-    boolean es = XGBoost.shouldEarlyStop(earlyStoppingRounds, totalIterations - 1, bestIteration);
+    for (int i = 0; i < 5; i++) {
-    TestCase.assertTrue(es);
-    for (int i = 0; i < totalIterations; i++) {
       metrics[0][i] = i;
     }
-    bestIteration = totalIterations - 1;
+    boolean onTrack = XGBoost.judgeIfTrainingOnTrack(paramMap, 5, metrics, 4);
-
+    TestCase.assertTrue(onTrack);
-    es = XGBoost.shouldEarlyStop(earlyStoppingRounds, totalIterations - 1, bestIteration);
+    for (int i = 0; i < 5; i++) {
-    TestCase.assertFalse(es);
+      metrics[0][i] = 5 - i;
-
+    }
-    for (int i = 0; i < totalIterations; i++) {
+    onTrack = XGBoost.judgeIfTrainingOnTrack(paramMap, 5, metrics, 4);
+    TestCase.assertFalse(onTrack);
+    for (int i = 0; i < 5; i++) {
       metrics[0][i] = i;
     }
-    metrics[0][4] = 9;
+    metrics[0][0] = 6;
-    metrics[0][9] = 4;
+    metrics[0][2] = 1;
-
+    onTrack = XGBoost.judgeIfTrainingOnTrack(paramMap, 5, metrics, 4);
-    bestIteration = 4;
+    TestCase.assertTrue(onTrack);
-
-    es = XGBoost.shouldEarlyStop(earlyStoppingRounds, totalIterations - 1, bestIteration);
-    TestCase.assertTrue(es);
   }
 
   @Test
   public void testBoosterEarlyStop() throws XGBoostError, IOException {
     DMatrix trainMat = new DMatrix("../../demo/data/agaricus.txt.train");
     DMatrix testMat = new DMatrix("../../demo/data/agaricus.txt.test");
+    // testBoosterWithFastHistogram(trainMat, testMat);
     Map<String, Object> paramMap = new HashMap<String, Object>() {
       {
         put("max_depth", 3);
@@ -332,12 +236,6 @@ public class BoosterImplTest {
             earlyStoppingRound);
 
     // Make sure we've stopped early.
-    for (int w = 0; w < watches.size(); w++) {
-      for (int r = 0; r <= earlyStoppingRound; r++) {
-        TestCase.assertFalse(0.0f == metrics[w][r]);
-      }
-    }
-
     for (int w = 0; w < watches.size(); w++) {
       for (int r = earlyStoppingRound + 1; r < round; r++) {
         TestCase.assertEquals(0.0f, metrics[w][r]);
@@ -345,27 +243,27 @@ public class BoosterImplTest {
     }
   }
 
-  private void testWithQuantileHisto(DMatrix trainingSet, Map<String, DMatrix> watches, int round,
+  private void testWithFastHisto(DMatrix trainingSet, Map<String, DMatrix> watches, int round,
                                       Map<String, Object> paramMap, float threshold) throws XGBoostError {
     float[][] metrics = new float[watches.size()][round];
     Booster booster = XGBoost.train(trainingSet, paramMap, round, watches,
             metrics, null, null, 0);
     for (int i = 0; i < metrics.length; i++)
       for (int j = 1; j < metrics[i].length; j++) {
-        TestCase.assertTrue(metrics[i][j] >= metrics[i][j - 1] ||
+        TestCase.assertTrue(metrics[i][j] >= metrics[i][j - 1]);
-                Math.abs(metrics[i][j] - metrics[i][j - 1]) < 0.1);
       }
     for (int i = 0; i < metrics.length; i++)
       for (int j = 0; j < metrics[i].length; j++) {
-        TestCase.assertTrue(metrics[i][j] >= threshold);
+      TestCase.assertTrue(metrics[i][j] >= threshold);
       }
     booster.dispose();
   }
 
   @Test
-  public void testQuantileHistoDepthWise() throws XGBoostError {
+  public void testFastHistoDepthWise() throws XGBoostError {
     DMatrix trainMat = new DMatrix("../../demo/data/agaricus.txt.train");
     DMatrix testMat = new DMatrix("../../demo/data/agaricus.txt.test");
+    // testBoosterWithFastHistogram(trainMat, testMat);
     Map<String, Object> paramMap = new HashMap<String, Object>() {
       {
         put("max_depth", 3);
@@ -379,13 +277,14 @@ public class BoosterImplTest {
     Map<String, DMatrix> watches = new HashMap<>();
     watches.put("training", trainMat);
     watches.put("test", testMat);
-    testWithQuantileHisto(trainMat, watches, 10, paramMap, 0.95f);
+    testWithFastHisto(trainMat, watches, 10, paramMap, 0.0f);
   }
 
   @Test
-  public void testQuantileHistoLossGuide() throws XGBoostError {
+  public void testFastHistoLossGuide() throws XGBoostError {
     DMatrix trainMat = new DMatrix("../../demo/data/agaricus.txt.train");
     DMatrix testMat = new DMatrix("../../demo/data/agaricus.txt.test");
+    // testBoosterWithFastHistogram(trainMat, testMat);
     Map<String, Object> paramMap = new HashMap<String, Object>() {
       {
         put("max_depth", 0);
@@ -400,13 +299,14 @@ public class BoosterImplTest {
     Map<String, DMatrix> watches = new HashMap<>();
     watches.put("training", trainMat);
     watches.put("test", testMat);
-    testWithQuantileHisto(trainMat, watches, 10, paramMap, 0.95f);
+    testWithFastHisto(trainMat, watches, 10, paramMap, 0.0f);
   }
 
   @Test
-  public void testQuantileHistoLossGuideMaxBin() throws XGBoostError {
+  public void testFastHistoLossGuideMaxBin() throws XGBoostError {
     DMatrix trainMat = new DMatrix("../../demo/data/agaricus.txt.train");
     DMatrix testMat = new DMatrix("../../demo/data/agaricus.txt.test");
+    // testBoosterWithFastHistogram(trainMat, testMat);
     Map<String, Object> paramMap = new HashMap<String, Object>() {
       {
         put("max_depth", 0);
@@ -421,7 +321,7 @@ public class BoosterImplTest {
     };
     Map<String, DMatrix> watches = new HashMap<>();
     watches.put("training", trainMat);
-    testWithQuantileHisto(trainMat, watches, 10, paramMap, 0.95f);
+    testWithFastHisto(trainMat, watches, 10, paramMap, 0.0f);
   }
 
   @Test
@@ -441,7 +341,7 @@ public class BoosterImplTest {
   }
 
   @Test
-  public void testGetFeatureScore() throws XGBoostError {
+  public void testGetFeatureImportance() throws XGBoostError {
     DMatrix trainMat = new DMatrix("../../demo/data/agaricus.txt.train");
     DMatrix testMat = new DMatrix("../../demo/data/agaricus.txt.test");
 
@@ -453,81 +353,38 @@ public class BoosterImplTest {
   }
 
   @Test
-  public void testGetFeatureImportanceGain() throws XGBoostError {
+  public void testFastHistoDepthwiseMaxDepth() throws XGBoostError {
     DMatrix trainMat = new DMatrix("../../demo/data/agaricus.txt.train");
     DMatrix testMat = new DMatrix("../../demo/data/agaricus.txt.test");
-
+    // testBoosterWithFastHistogram(trainMat, testMat);
-    Booster booster = trainBooster(trainMat, testMat);
-    String[] featureNames = new String[126];
-    for(int i = 0; i < 126; i++) featureNames[i] = "test_feature_name_" + i;
-    Map<String, Double> scoreMap = booster.getScore(featureNames, "gain");
-    for (String fName: scoreMap.keySet()) TestCase.assertTrue(fName.startsWith("test_feature_name_"));
-  }
-
-  @Test
-  public void testGetFeatureImportanceTotalGain() throws XGBoostError {
-    DMatrix trainMat = new DMatrix("../../demo/data/agaricus.txt.train");
-    DMatrix testMat = new DMatrix("../../demo/data/agaricus.txt.test");
-
-    Booster booster = trainBooster(trainMat, testMat);
-    String[] featureNames = new String[126];
-    for(int i = 0; i < 126; i++) featureNames[i] = "test_feature_name_" + i;
-    Map<String, Double> scoreMap = booster.getScore(featureNames, "total_gain");
-    for (String fName: scoreMap.keySet()) TestCase.assertTrue(fName.startsWith("test_feature_name_"));
-  }
-
-  @Test
-  public void testGetFeatureImportanceCover() throws XGBoostError {
-    DMatrix trainMat = new DMatrix("../../demo/data/agaricus.txt.train");
-    DMatrix testMat = new DMatrix("../../demo/data/agaricus.txt.test");
-
-    Booster booster = trainBooster(trainMat, testMat);
-    String[] featureNames = new String[126];
-    for(int i = 0; i < 126; i++) featureNames[i] = "test_feature_name_" + i;
-    Map<String, Double> scoreMap = booster.getScore(featureNames, "cover");
-    for (String fName: scoreMap.keySet()) TestCase.assertTrue(fName.startsWith("test_feature_name_"));
-  }
-
-  @Test
-  public void testGetFeatureImportanceTotalCover() throws XGBoostError {
-    DMatrix trainMat = new DMatrix("../../demo/data/agaricus.txt.train");
-    DMatrix testMat = new DMatrix("../../demo/data/agaricus.txt.test");
-
-    Booster booster = trainBooster(trainMat, testMat);
-    String[] featureNames = new String[126];
-    for(int i = 0; i < 126; i++) featureNames[i] = "test_feature_name_" + i;
-    Map<String, Double> scoreMap = booster.getScore(featureNames, "total_cover");
-    for (String fName: scoreMap.keySet()) TestCase.assertTrue(fName.startsWith("test_feature_name_"));
-  }
-
-  @Test
-  public void testQuantileHistoDepthwiseMaxDepth() throws XGBoostError {
-    DMatrix trainMat = new DMatrix("../../demo/data/agaricus.txt.train");
     Map<String, Object> paramMap = new HashMap<String, Object>() {
       {
         put("max_depth", 3);
         put("silent", 1);
         put("objective", "binary:logistic");
         put("tree_method", "hist");
+        put("max_depth", 2);
         put("grow_policy", "depthwise");
         put("eval_metric", "auc");
       }
     };
     Map<String, DMatrix> watches = new HashMap<>();
     watches.put("training", trainMat);
-    testWithQuantileHisto(trainMat, watches, 10, paramMap, 0.95f);
+    testWithFastHisto(trainMat, watches, 10, paramMap, 0.85f);
   }
 
   @Test
-  public void testQuantileHistoDepthwiseMaxDepthMaxBin() throws XGBoostError {
+  public void testFastHistoDepthwiseMaxDepthMaxBin() throws XGBoostError {
     DMatrix trainMat = new DMatrix("../../demo/data/agaricus.txt.train");
     DMatrix testMat = new DMatrix("../../demo/data/agaricus.txt.test");
+    // testBoosterWithFastHistogram(trainMat, testMat);
     Map<String, Object> paramMap = new HashMap<String, Object>() {
       {
         put("max_depth", 3);
         put("silent", 1);
         put("objective", "binary:logistic");
         put("tree_method", "hist");
+        put("max_depth", 2);
         put("max_bin", 2);
         put("grow_policy", "depthwise");
         put("eval_metric", "auc");
@@ -535,7 +392,7 @@ public class BoosterImplTest {
     };
     Map<String, DMatrix> watches = new HashMap<>();
     watches.put("training", trainMat);
-    testWithQuantileHisto(trainMat, watches, 10, paramMap, 0.95f);
+    testWithFastHisto(trainMat, watches, 10, paramMap, 0.85f);
   }
 
   /**

jvm-packages/xgboost4j/src/test/scala/ml/dmlc/xgboost4j/scala/ScalaBoosterImplSuite.scala

@@ -77,7 +77,7 @@ class ScalaBoosterImplSuite extends FunSuite {
     XGBoost.train(trainMat, paramMap, round, watches)
   }
 
-  private def trainBoosterWithQuantileHisto(
+  private def trainBoosterWithFastHisto(
       trainMat: DMatrix,
       watches: Map[String, DMatrix],
       round: Int,
@@ -146,57 +146,57 @@ class ScalaBoosterImplSuite extends FunSuite {
     XGBoost.crossValidation(trainMat, params, round, nfold)
   }
 
-  test("test with quantile histo depthwise") {
+  test("test with fast histo depthwise") {
     val trainMat = new DMatrix("../../demo/data/agaricus.txt.train")
     val testMat = new DMatrix("../../demo/data/agaricus.txt.test")
     val paramMap = List("max_depth" -> "3", "silent" -> "0",
       "objective" -> "binary:logistic", "tree_method" -> "hist",
       "grow_policy" -> "depthwise", "eval_metric" -> "auc").toMap
-    trainBoosterWithQuantileHisto(trainMat, Map("training" -> trainMat, "test" -> testMat),
+    trainBoosterWithFastHisto(trainMat, Map("training" -> trainMat, "test" -> testMat),
-      round = 10, paramMap, 0.95f)
+      round = 10, paramMap, 0.0f)
   }
 
-  test("test with quantile histo lossguide") {
+  test("test with fast histo lossguide") {
     val trainMat = new DMatrix("../../demo/data/agaricus.txt.train")
     val testMat = new DMatrix("../../demo/data/agaricus.txt.test")
     val paramMap = List("max_depth" -> "0", "silent" -> "0",
       "objective" -> "binary:logistic", "tree_method" -> "hist",
       "grow_policy" -> "lossguide", "max_leaves" -> "8", "eval_metric" -> "auc").toMap
-    trainBoosterWithQuantileHisto(trainMat, Map("training" -> trainMat, "test" -> testMat),
+    trainBoosterWithFastHisto(trainMat, Map("training" -> trainMat, "test" -> testMat),
-      round = 10, paramMap, 0.95f)
+      round = 10, paramMap, 0.0f)
   }
 
-  test("test with quantile histo lossguide with max bin") {
+  test("test with fast histo lossguide with max bin") {
     val trainMat = new DMatrix("../../demo/data/agaricus.txt.train")
     val testMat = new DMatrix("../../demo/data/agaricus.txt.test")
     val paramMap = List("max_depth" -> "0", "silent" -> "0",
       "objective" -> "binary:logistic", "tree_method" -> "hist",
       "grow_policy" -> "lossguide", "max_leaves" -> "8", "max_bin" -> "16",
       "eval_metric" -> "auc").toMap
-    trainBoosterWithQuantileHisto(trainMat, Map("training" -> trainMat),
+    trainBoosterWithFastHisto(trainMat, Map("training" -> trainMat),
-      round = 10, paramMap, 0.95f)
+      round = 10, paramMap, 0.0f)
   }
 
-  test("test with quantile histo depthwidth with max depth") {
+  test("test with fast histo depthwidth with max depth") {
     val trainMat = new DMatrix("../../demo/data/agaricus.txt.train")
     val testMat = new DMatrix("../../demo/data/agaricus.txt.test")
     val paramMap = List("max_depth" -> "0", "silent" -> "0",
       "objective" -> "binary:logistic", "tree_method" -> "hist",
       "grow_policy" -> "depthwise", "max_leaves" -> "8", "max_depth" -> "2",
       "eval_metric" -> "auc").toMap
-    trainBoosterWithQuantileHisto(trainMat, Map("training" -> trainMat),
+    trainBoosterWithFastHisto(trainMat, Map("training" -> trainMat),
-      round = 10, paramMap, 0.95f)
+      round = 10, paramMap, 0.85f)
   }
 
-  test("test with quantile histo depthwidth with max depth and max bin") {
+  test("test with fast histo depthwidth with max depth and max bin") {
     val trainMat = new DMatrix("../../demo/data/agaricus.txt.train")
     val testMat = new DMatrix("../../demo/data/agaricus.txt.test")
     val paramMap = List("max_depth" -> "0", "silent" -> "0",
       "objective" -> "binary:logistic", "tree_method" -> "hist",
       "grow_policy" -> "depthwise", "max_depth" -> "2", "max_bin" -> "2",
       "eval_metric" -> "auc").toMap
-    trainBoosterWithQuantileHisto(trainMat, Map("training" -> trainMat),
+    trainBoosterWithFastHisto(trainMat, Map("training" -> trainMat),
-      round = 10, paramMap, 0.95f)
+      round = 10, paramMap, 0.85f)
   }
 
   test("test training from existing model in scala") {

python-package/.pylintrc

@@ -2,25 +2,8 @@
 
 ignore=tests
 
-extension-pkg-whitelist=numpy
-
 disiable=unexpected-special-method-signature,too-many-nested-blocks
 
 dummy-variables-rgx=(unused|)_.*
 
 reports=no
-
-[BASIC]
-
-# Enforce naming convention
-const-naming-style=UPPER_CASE
-class-naming-style=PascalCase
-function-naming-style=snake_case
-method-naming-style=snake_case
-attr-naming-style=snake_case
-argument-naming-style=snake_case
-variable-naming-style=snake_case
-class-attribute-naming-style=snake_case
-
-# Allow single-letter variables
-variable-rgx=[a-zA-Z_][a-z0-9_]{0,30}$

python-package/README.rst

@@ -28,8 +28,8 @@ Please install ``gcc@5`` from `Homebrew <https://brew.sh/>`_::
 
 After installing ``gcc@5``, set it as your compiler::
 
-    export CC=gcc-5
+    export CC = gcc-5
-    export CXX=g++-5
+    export CXX = g++-5
 
 Linux
 -----

python-package/setup.py

@@ -1,7 +1,6 @@
 # pylint: disable=invalid-name, exec-used
 """Setup xgboost package."""
 from __future__ import absolute_import
-import io
 import sys
 import os
 from setuptools import setup, find_packages
@@ -32,7 +31,7 @@ print("Install libxgboost from: %s" % LIB_PATH)
 setup(name='xgboost',
       version=open(os.path.join(CURRENT_DIR, 'xgboost/VERSION')).read().strip(),
       description="XGBoost Python Package",
-      long_description=io.open(os.path.join(CURRENT_DIR, 'README.rst'), encoding='utf-8').read(),
+      long_description=open(os.path.join(CURRENT_DIR, 'README.rst')).read(),
       install_requires=[
           'numpy',
           'scipy',

python-package/xgboost/VERSION

@@ -1 +1 @@
-0.82
+0.81

python-package/xgboost/build-python.sh

@@ -25,9 +25,6 @@ if echo "${OSTYPE}" | grep -q "darwin"; then
   elif which g++-7; then
     export CC=gcc-7
     export CXX=g++-7
-  elif which g++-8; then
-    export CC=gcc-8
-    export CXX=g++-8
   elif which clang++; then
     export CC=clang
     export CXX=clang++

python-package/xgboost/callback.py

@@ -188,8 +188,8 @@ def early_stop(stopping_rounds, maximize=False, verbose=True):
             msg = ("Multiple eval metrics have been passed: "
                    "'{0}' will be used for early stopping.\n\n")
             rabit.tracker_print(msg.format(env.evaluation_result_list[-1][0]))
-        maximize_metrics = ('auc', 'aucpr', 'map', 'ndcg')
+        maximize_metrics = ('auc', 'map', 'ndcg')
-        maximize_at_n_metrics = ('auc@', 'aucpr@' 'map@', 'ndcg@')
+        maximize_at_n_metrics = ('auc@', 'map@', 'ndcg@')
         maximize_score = maximize
         metric_label = env.evaluation_result_list[-1][0]
         metric = metric_label.split('-', 1)[-1]

python-package/xgboost/compat.py

@@ -49,11 +49,7 @@ except ImportError:
 
 # dt
 try:
-    import datatable
+    from datatable import DataTable
-    if hasattr(datatable, "Frame"):
-        DataTable = datatable.Frame
-    else:
-        DataTable = datatable.DataTable
     DT_INSTALLED = True
 except ImportError:
 

python-package/xgboost/core.py

@@ -3,27 +3,19 @@
 # pylint: disable=too-many-branches, too-many-lines, W0141
 """Core XGBoost Library."""
 from __future__ import absolute_import
+
 import collections
-# pylint: disable=no-name-in-module,import-error
-try:
-    from collections.abc import Mapping  # Python 3
-except ImportError:
-    from collections import Mapping  # Python 2
-# pylint: enable=no-name-in-module,import-error
 import ctypes
 import os
 import re
 import sys
-import warnings
 
 import numpy as np
 import scipy.sparse
 
-from .compat import (STRING_TYPES, PY3, DataFrame, MultiIndex, py_str,
+from .compat import STRING_TYPES, PY3, DataFrame, MultiIndex, py_str, PANDAS_INSTALLED, DataTable
-                     PANDAS_INSTALLED, DataTable)
 from .libpath import find_lib_path
 
-
 # c_bst_ulong corresponds to bst_ulong defined in xgboost/c_api.h
 c_bst_ulong = ctypes.c_uint64
 
@@ -125,23 +117,18 @@ def _load_lib():
     lib_paths = find_lib_path()
     if len(lib_paths) == 0:
         return None
-    try:
+    pathBackup = os.environ['PATH']
-        pathBackup = os.environ['PATH'].split(os.pathsep)
-    except KeyError:
-        pathBackup = []
     lib_success = False
     os_error_list = []
     for lib_path in lib_paths:
         try:
             # needed when the lib is linked with non-system-available dependencies
-            os.environ['PATH'] = os.pathsep.join(pathBackup + [os.path.dirname(lib_path)])
+            os.environ['PATH'] = pathBackup + os.pathsep + os.path.dirname(lib_path)
             lib = ctypes.cdll.LoadLibrary(lib_path)
             lib_success = True
         except OSError as e:
             os_error_list.append(str(e))
             continue
-        finally:
-            os.environ['PATH'] = os.pathsep.join(pathBackup)
     if not lib_success:
         libname = os.path.basename(lib_paths[0])
         raise XGBoostError(
@@ -287,10 +274,10 @@ def _maybe_dt_data(data, feature_names, feature_types):
         return data, feature_names, feature_types
 
     data_types_names = tuple(lt.name for lt in data.ltypes)
-    bad_fields = [data.names[i]
+    if not all(type_name in DT_TYPE_MAPPER for type_name in data_types_names):
-                  for i, type_name in enumerate(data_types_names)
+        bad_fields = [data.names[i] for i, type_name in
-                  if type_name not in DT_TYPE_MAPPER]
+                      enumerate(data_types_names) if type_name not in DT_TYPE_MAPPER]
-    if bad_fields:
+
         msg = """DataFrame.types for data must be int, float or bool.
                 Did not expect the data types in fields """
         raise ValueError(msg + ', '.join(bad_fields))
@@ -317,7 +304,7 @@ def _maybe_dt_array(array):
 
     # below requires new dt version
     # extract first column
-    array = array.to_numpy()[:, 0].astype('float')
+    array = array.tonumpy()[:, 0].astype('float')
 
     return array
 
@@ -340,7 +327,7 @@ class DMatrix(object):
         """
         Parameters
         ----------
-        data : string/numpy.array/scipy.sparse/pd.DataFrame/dt.Frame
+        data : string/numpy array/scipy.sparse/pd.DataFrame/DataTable
             Data source of DMatrix.
             When data is string type, it represents the path libsvm format txt file,
             or binary file that xgboost can read from.
@@ -351,14 +338,6 @@ class DMatrix(object):
             None, defaults to np.nan.
         weight : list or numpy 1-D array , optional
             Weight for each instance.
-
-            .. note:: For ranking task, weights are per-group.
-
-                In ranking task, one weight is assigned to each group (not each data
-                point). This is because we only care about the relative ordering of
-                data points within each group, so it doesn't make sense to assign
-                weights to individual data points.
-
         silent : boolean, optional
             Whether print messages during construction
         feature_names : list, optional
@@ -390,10 +369,6 @@ class DMatrix(object):
         label = _maybe_dt_array(label)
         weight = _maybe_dt_array(weight)
 
-        if isinstance(data, list):
-            warnings.warn('Initializing DMatrix from List is deprecated.',
-                          DeprecationWarning)
-
         if isinstance(data, STRING_TYPES):
             self.handle = ctypes.c_void_p()
             _check_call(_LIB.XGDMatrixCreateFromFile(c_str(data),
@@ -497,20 +472,16 @@ class DMatrix(object):
 
     def _init_from_dt(self, data, nthread):
         """
-        Initialize data from a datatable Frame.
+        Initialize data from a DataTable
         """
+        cols = []
         ptrs = (ctypes.c_void_p * data.ncols)()
-        if hasattr(data, "internal") and hasattr(data.internal, "column"):
+        for icol in range(data.ncols):
-            # datatable>0.8.0
+            col = data.internal.column(icol)
-            for icol in range(data.ncols):
+            cols.append(col)
-                col = data.internal.column(icol)
+            # int64_t (void*)
-                ptr = col.data_pointer
+            ptr = col.data_pointer
-                ptrs[icol] = ctypes.c_void_p(ptr)
+            ptrs[icol] = ctypes.c_void_p(ptr)
-        else:
-            # datatable<=0.8.0
-            from datatable.internal import frame_column_data_r
-            for icol in range(data.ncols):
-                ptrs[icol] = frame_column_data_r(data, icol)
 
         # always return stypes for dt ingestion
         feature_type_strings = (ctypes.c_char_p * data.ncols)()
@@ -584,11 +555,6 @@ class DMatrix(object):
         data: numpy array
             The array of data to be set
         """
-        if getattr(data, 'base', None) is not None and \
-           data.base is not None and isinstance(data, np.ndarray) \
-           and isinstance(data.base, np.ndarray) and (not data.flags.c_contiguous):
-            self.set_float_info_npy2d(field, data)
-            return
         c_data = c_array(ctypes.c_float, data)
         _check_call(_LIB.XGDMatrixSetFloatInfo(self.handle,
                                                c_str(field),
@@ -607,14 +573,7 @@ class DMatrix(object):
         data: numpy array
             The array of data to be set
         """
-        if getattr(data, 'base', None) is not None and \
+        data = np.array(data, copy=False, dtype=np.float32)
-           data.base is not None and isinstance(data, np.ndarray) \
-           and isinstance(data.base, np.ndarray) and (not data.flags.c_contiguous):
-            warnings.warn("Use subset (sliced data) of np.ndarray is not recommended " +
-                          "because it will generate extra copies and increase memory consumption")
-            data = np.array(data, copy=True, dtype=np.float32)
-        else:
-            data = np.array(data, copy=False, dtype=np.float32)
         c_data = data.ctypes.data_as(ctypes.POINTER(ctypes.c_float))
         _check_call(_LIB.XGDMatrixSetFloatInfo(self.handle,
                                                c_str(field),
@@ -632,14 +591,6 @@ class DMatrix(object):
         data: numpy array
             The array of data to be set
         """
-        if getattr(data, 'base', None) is not None and \
-           data.base is not None and isinstance(data, np.ndarray) \
-           and isinstance(data.base, np.ndarray) and (not data.flags.c_contiguous):
-            warnings.warn("Use subset (sliced data) of np.ndarray is not recommended " +
-                          "because it will generate extra copies and increase memory consumption")
-            data = np.array(data, copy=True, dtype=ctypes.c_uint)
-        else:
-            data = np.array(data, copy=False, dtype=ctypes.c_uint)
         _check_call(_LIB.XGDMatrixSetUIntInfo(self.handle,
                                               c_str(field),
                                               c_array(ctypes.c_uint, data),
@@ -687,13 +638,6 @@ class DMatrix(object):
         ----------
         weight : array like
             Weight for each data point
-
-            .. note:: For ranking task, weights are per-group.
-
-                In ranking task, one weight is assigned to each group (not each data
-                point). This is because we only care about the relative ordering of
-                data points within each group, so it doesn't make sense to assign
-                weights to individual data points.
         """
         self.set_float_info('weight', weight)
 
@@ -705,13 +649,6 @@ class DMatrix(object):
         ----------
         weight : array like
             Weight for each data point in numpy 2D array
-
-            .. note:: For ranking task, weights are per-group.
-
-                In ranking task, one weight is assigned to each group (not each data
-                point). This is because we only care about the relative ordering of
-                data points within each group, so it doesn't make sense to assign
-                weights to individual data points.
         """
         self.set_float_info_npy2d('weight', weight)
 
@@ -913,7 +850,6 @@ class DMatrix(object):
 
 
 class Booster(object):
-    # pylint: disable=too-many-public-methods
     """A Booster of XGBoost.
 
     Booster is the model of xgboost, that contains low level routines for
@@ -1080,7 +1016,7 @@ class Booster(object):
         value: optional
            value of the specified parameter, when params is str key
         """
-        if isinstance(params, Mapping):
+        if isinstance(params, collections.Mapping):
             params = params.items()
         elif isinstance(params, STRING_TYPES) and value is not None:
             params = [(params, value)]
@@ -1088,8 +1024,8 @@ class Booster(object):
             _check_call(_LIB.XGBoosterSetParam(self.handle, c_str(key), c_str(str(val))))
 
     def update(self, dtrain, iteration, fobj=None):
-        """Update for one iteration, with objective function calculated
+        """
-        internally.  This function should not be called directly by users.
+        Update for one iteration, with objective function calculated internally.
 
         Parameters
         ----------
@@ -1099,7 +1035,6 @@ class Booster(object):
             Current iteration number.
         fobj : function
             Customized objective function.
-
         """
         if not isinstance(dtrain, DMatrix):
             raise TypeError('invalid training matrix: {}'.format(type(dtrain).__name__))
@@ -1114,9 +1049,8 @@ class Booster(object):
             self.boost(dtrain, grad, hess)
 
     def boost(self, dtrain, grad, hess):
-        """Boost the booster for one iteration, with customized gradient
+        """
-        statistics.  Like :func:`xgboost.core.Booster.update`, this
+        Boost the booster for one iteration, with customized gradient statistics.
-        function should not be called directly by users.
 
         Parameters
         ----------
@@ -1126,7 +1060,6 @@ class Booster(object):
             The first order of gradient.
         hess : list
             The second order of gradient.
-
         """
         if len(grad) != len(hess):
             raise ValueError('grad / hess length mismatch: {} / {}'.format(len(grad), len(hess)))
@@ -1498,7 +1431,8 @@ class Booster(object):
         importance_type: str, default 'weight'
             One of the importance types defined above.
         """
-        if getattr(self, 'booster', None) is not None and self.booster not in {'gbtree', 'dart'}:
+
+        if self.booster != 'gbtree':
             raise ValueError('Feature importance is not defined for Booster type {}'
                              .format(self.booster))
 
@@ -1579,91 +1513,6 @@ class Booster(object):
 
             return gmap
 
-    def trees_to_dataframe(self, fmap=''):
-        """Parse a boosted tree model text dump into a pandas DataFrame structure.
-
-        This feature is only defined when the decision tree model is chosen as base
-        learner (`booster in {gbtree, dart}`). It is not defined for other base learner
-        types, such as linear learners (`booster=gblinear`).
-
-        Parameters
-        ----------
-        fmap: str (optional)
-           The name of feature map file.
-        """
-        # pylint: disable=too-many-locals
-        if not PANDAS_INSTALLED:
-            raise Exception(('pandas must be available to use this method.'
-                             'Install pandas before calling again.'))
-
-        if getattr(self, 'booster', None) is not None and self.booster not in {'gbtree', 'dart'}:
-            raise ValueError('This method is not defined for Booster type {}'
-                             .format(self.booster))
-
-        tree_ids = []
-        node_ids = []
-        fids = []
-        splits = []
-        y_directs = []
-        n_directs = []
-        missings = []
-        gains = []
-        covers = []
-
-        trees = self.get_dump(fmap, with_stats=True)
-        for i, tree in enumerate(trees):
-            for line in tree.split('\n'):
-                arr = line.split('[')
-                # Leaf node
-                if len(arr) == 1:
-                    # Last element of line.split is an empy string
-                    if arr == ['']:
-                        continue
-                    # parse string
-                    parse = arr[0].split(':')
-                    stats = re.split('=|,', parse[1])
-
-                    # append to lists
-                    tree_ids.append(i)
-                    node_ids.append(int(re.findall(r'\b\d+\b', parse[0])[0]))
-                    fids.append('Leaf')
-                    splits.append(float('NAN'))
-                    y_directs.append(float('NAN'))
-                    n_directs.append(float('NAN'))
-                    missings.append(float('NAN'))
-                    gains.append(float(stats[1]))
-                    covers.append(float(stats[3]))
-                # Not a Leaf Node
-                else:
-                    # parse string
-                    fid = arr[1].split(']')
-                    parse = fid[0].split('<')
-                    stats = re.split('=|,', fid[1])
-
-                    # append to lists
-                    tree_ids.append(i)
-                    node_ids.append(int(re.findall(r'\b\d+\b', arr[0])[0]))
-                    fids.append(parse[0])
-                    splits.append(float(parse[1]))
-                    str_i = str(i)
-                    y_directs.append(str_i + '-' + stats[1])
-                    n_directs.append(str_i + '-' + stats[3])
-                    missings.append(str_i + '-' + stats[5])
-                    gains.append(float(stats[7]))
-                    covers.append(float(stats[9]))
-
-        ids = [str(t_id) + '-' + str(n_id) for t_id, n_id in zip(tree_ids, node_ids)]
-        df = DataFrame({'Tree': tree_ids, 'Node': node_ids, 'ID': ids,
-                        'Feature': fids, 'Split': splits, 'Yes': y_directs,
-                        'No': n_directs, 'Missing': missings, 'Gain': gains,
-                        'Cover': covers})
-
-        if callable(getattr(df, 'sort_values', None)):
-            # pylint: disable=no-member
-            return df.sort_values(['Tree', 'Node']).reset_index(drop=True)
-        # pylint: disable=no-member
-        return df.sort(['Tree', 'Node']).reset_index(drop=True)
-
     def _validate_features(self, data):
         """
         Validate Booster and data's feature_names are identical.

python-package/xgboost/plotting.py

@@ -16,6 +16,7 @@ def plot_importance(booster, ax=None, height=0.2,
                     xlabel='F score', ylabel='Features',
                     importance_type='weight', max_num_features=None,
                     grid=True, show_values=True, **kwargs):
+
     """Plot importance based on fitted trees.
 
     Parameters
@@ -123,17 +124,17 @@ _EDGEPAT = re.compile(r'yes=(\d+),no=(\d+),missing=(\d+)')
 _EDGEPAT2 = re.compile(r'yes=(\d+),no=(\d+)')
 
 
-def _parse_node(graph, text, condition_node_params, leaf_node_params):
+def _parse_node(graph, text):
     """parse dumped node"""
     match = _NODEPAT.match(text)
     if match is not None:
         node = match.group(1)
-        graph.node(node, label=match.group(2), **condition_node_params)
+        graph.node(node, label=match.group(2), shape='circle')
         return node
     match = _LEAFPAT.match(text)
     if match is not None:
         node = match.group(1)
-        graph.node(node, label=match.group(2), **leaf_node_params)
+        graph.node(node, label=match.group(2), shape='box')
         return node
     raise ValueError('Unable to parse node: {0}'.format(text))
 
@@ -163,8 +164,8 @@ def _parse_edge(graph, node, text, yes_color='#0000FF', no_color='#FF0000'):
 
 
 def to_graphviz(booster, fmap='', num_trees=0, rankdir='UT',
-                yes_color='#0000FF', no_color='#FF0000',
+                yes_color='#0000FF', no_color='#FF0000', **kwargs):
-                condition_node_params=None, leaf_node_params=None, **kwargs):
+
     """Convert specified tree to graphviz instance. IPython can automatically plot the
     returned graphiz instance. Otherwise, you should call .render() method
     of the returned graphiz instance.
@@ -183,18 +184,6 @@ def to_graphviz(booster, fmap='', num_trees=0, rankdir='UT',
         Edge color when meets the node condition.
     no_color : str, default '#FF0000'
         Edge color when doesn't meet the node condition.
-    condition_node_params : dict (optional)
-        condition node configuration,
-        {'shape':'box',
-               'style':'filled,rounded',
-               'fillcolor':'#78bceb'
-        }
-    leaf_node_params : dict (optional)
-        leaf node configuration
-        {'shape':'box',
-               'style':'filled',
-               'fillcolor':'#e48038'
-        }
     kwargs :
         Other keywords passed to graphviz graph_attr
 
@@ -203,11 +192,6 @@ def to_graphviz(booster, fmap='', num_trees=0, rankdir='UT',
     ax : matplotlib Axes
     """
 
-    if condition_node_params is None:
-        condition_node_params = {}
-    if leaf_node_params is None:
-        leaf_node_params = {}
-
     try:
         from graphviz import Digraph
     except ImportError:
@@ -228,9 +212,7 @@ def to_graphviz(booster, fmap='', num_trees=0, rankdir='UT',
 
     for i, text in enumerate(tree):
         if text[0].isdigit():
-            node = _parse_node(
+            node = _parse_node(graph, text)
-                graph, text, condition_node_params=condition_node_params,
-                leaf_node_params=leaf_node_params)
         else:
             if i == 0:
                 # 1st string must be node
@@ -274,8 +256,7 @@ def plot_tree(booster, fmap='', num_trees=0, rankdir='UT', ax=None, **kwargs):
     if ax is None:
         _, ax = plt.subplots(1, 1)
 
-    g = to_graphviz(booster, fmap=fmap, num_trees=num_trees,
+    g = to_graphviz(booster, fmap=fmap, num_trees=num_trees, rankdir=rankdir, **kwargs)
-                    rankdir=rankdir, **kwargs)
 
     s = BytesIO()
     s.write(g.pipe(format='png'))

python-package/xgboost/sklearn.py

@@ -100,9 +100,6 @@ class XGBModel(XGBModelBase):
     missing : float, optional
         Value in the data which needs to be present as a missing value. If
         None, defaults to np.nan.
-    importance_type: string, default "gain"
-        The feature importance type for the feature_importances_ property: either "gain",
-        "weight", "cover", "total_gain" or "total_cover".
     \*\*kwargs : dict, optional
         Keyword arguments for XGBoost Booster object.  Full documentation of parameters can
         be found here: https://github.com/dmlc/xgboost/blob/master/doc/parameter.rst.
@@ -136,8 +133,7 @@ class XGBModel(XGBModelBase):
                  n_jobs=1, nthread=None, gamma=0, min_child_weight=1, max_delta_step=0,
                  subsample=1, colsample_bytree=1, colsample_bylevel=1,
                  reg_alpha=0, reg_lambda=1, scale_pos_weight=1,
-                 base_score=0.5, random_state=0, seed=None, missing=None,
+                 base_score=0.5, random_state=0, seed=None, missing=None, **kwargs):
-                 importance_type="gain", **kwargs):
         if not SKLEARN_INSTALLED:
             raise XGBoostError('sklearn needs to be installed in order to use this module')
         self.max_depth = max_depth
@@ -163,7 +159,6 @@ class XGBModel(XGBModelBase):
         self.random_state = random_state
         self.nthread = nthread
         self.n_jobs = n_jobs
-        self.importance_type = importance_type
 
     def __setstate__(self, state):
         # backward compatibility code
@@ -237,7 +232,7 @@ class XGBModel(XGBModelBase):
         else:
             xgb_params['nthread'] = n_jobs
 
-        xgb_params['verbosity'] = 0 if self.silent else 0
+        xgb_params['silent'] = 1 if self.silent else 0
 
         if xgb_params['nthread'] <= 0:
             xgb_params.pop('nthread', None)
@@ -518,12 +513,12 @@ class XGBModel(XGBModelBase):
         feature_importances_ : array of shape ``[n_features]``
 
         """
-        if getattr(self, 'booster', None) is not None and self.booster != 'gbtree':
+        if self.booster != 'gbtree':
             raise AttributeError('Feature importance is not defined for Booster type {}'
                                  .format(self.booster))
         b = self.get_booster()
-        score = b.get_score(importance_type=self.importance_type)
+        fs = b.get_fscore()
-        all_features = [score.get(f, 0.) for f in b.feature_names]
+        all_features = [fs.get(f, 0.) for f in b.feature_names]
         all_features = np.array(all_features, dtype=np.float32)
         return all_features / all_features.sum()
 
@@ -540,21 +535,13 @@ class XGBModel(XGBModelBase):
 
         Returns
         -------
-        coef_ : array of shape ``[n_features]`` or ``[n_classes, n_features]``
+        coef_ : array of shape ``[n_features]``
         """
-        if getattr(self, 'booster', None) is not None and self.booster != 'gblinear':
+        if self.booster != 'gblinear':
             raise AttributeError('Coefficients are not defined for Booster type {}'
                                  .format(self.booster))
         b = self.get_booster()
-        coef = np.array(json.loads(b.get_dump(dump_format='json')[0])['weight'])
+        return json.loads(b.get_dump(dump_format='json')[0])['weight']
-        # Logic for multiclass classification
-        n_classes = getattr(self, 'n_classes_', None)
-        if n_classes is not None:
-            if n_classes > 2:
-                assert len(coef.shape) == 1
-                assert coef.shape[0] % n_classes == 0
-                coef = coef.reshape((n_classes, -1))
-        return coef
 
     @property
     def intercept_(self):
@@ -569,13 +556,13 @@ class XGBModel(XGBModelBase):
 
         Returns
         -------
-        intercept_ : array of shape ``(1,)`` or ``[n_classes]``
+        intercept_ : array of shape ``[n_features]``
         """
-        if getattr(self, 'booster', None) is not None and self.booster != 'gblinear':
+        if self.booster != 'gblinear':
             raise AttributeError('Intercept (bias) is not defined for Booster type {}'
                                  .format(self.booster))
         b = self.get_booster()
-        return np.array(json.loads(b.get_dump(dump_format='json')[0])['bias'])
+        return json.loads(b.get_dump(dump_format='json')[0])['bias']
 
 
 class XGBClassifier(XGBModel, XGBClassifierBase):
@@ -631,11 +618,11 @@ class XGBClassifier(XGBModel, XGBClassifierBase):
         early_stopping_rounds : int, optional
             Activates early stopping. Validation error needs to decrease at
             least every <early_stopping_rounds> round(s) to continue training.
-            Requires at least one item in evals. If there's more than one,
+            Requires at least one item in evals.  If there's more than one,
-            will use the last. If early stopping occurs, the model will have
+            will use the last. Returns the model from the last iteration
-            three additional fields: bst.best_score, bst.best_iteration and
+            (not the best one). If early stopping occurs, the model will
-            bst.best_ntree_limit (bst.best_ntree_limit is the ntree_limit parameter
+            have three additional fields: bst.best_score, bst.best_iteration
-            default value in predict method if not any other value is specified).
+            and bst.best_ntree_limit.
             (Use bst.best_ntree_limit to get the correct value if num_parallel_tree
             and/or num_class appears in the parameters)
         verbose : bool
@@ -709,7 +696,7 @@ class XGBClassifier(XGBModel, XGBClassifierBase):
                               evals=evals,
                               early_stopping_rounds=early_stopping_rounds,
                               evals_result=evals_result, obj=obj, feval=feval,
-                              verbose_eval=verbose, xgb_model=xgb_model,
+                              verbose_eval=verbose, xgb_model=None,
                               callbacks=callbacks)
 
         self.objective = xgb_options["objective"]
@@ -885,7 +872,7 @@ class XGBRanker(XGBModel):
             Whether to print messages while running boosting.
         objective : string
             Specify the learning task and the corresponding learning objective.
-            The objective name must start with "rank:".
+            Only "rank:pairwise" is supported currently.
         booster: string
             Specify which booster to use: gbtree, gblinear or dart.
         nthread : int
@@ -999,29 +986,13 @@ class XGBRanker(XGBModel):
         group : array_like
             group size of training data
         sample_weight : array_like
-            group weights
+            instance weights
-
-            .. note:: Weights are per-group for ranking tasks
-
-                In ranking task, one weight is assigned to each group (not each data
-                point). This is because we only care about the relative ordering of
-                data points within each group, so it doesn't make sense to assign
-                weights to individual data points.
-
         eval_set : list, optional
             A list of (X, y) tuple pairs to use as a validation set for
             early-stopping
         sample_weight_eval_set : list, optional
             A list of the form [L_1, L_2, ..., L_n], where each L_i is a list of
-            group weights on the i-th validation set.
+            instance weights on the i-th validation set.
-
-            .. note:: Weights are per-group for ranking tasks
-
-                In ranking task, one weight is assigned to each group (not each data
-                point). This is because we only care about the relative ordering of
-                data points within each group, so it doesn't make sense to assign
-                weights to individual data points.
-
         eval_group : list of arrays, optional
             A list that contains the group size corresponds to each
             (X, y) pair in eval_set

src/cli_main.cc

@@ -19,7 +19,7 @@
 #include <cstdio>
 #include <cstring>
 #include <vector>
-#include "./common/common.h"
+#include "./common/sync.h"
 #include "./common/config.h"
 
 
@@ -34,6 +34,8 @@ enum CLITask {
 struct CLIParam : public dmlc::Parameter<CLIParam> {
   /*! \brief the task name */
   int task;
+  /*! \brief whether silent */
+  int silent;
   /*! \brief whether evaluate training statistics */
   bool eval_train;
   /*! \brief number of boosting iterations */
@@ -81,6 +83,8 @@ struct CLIParam : public dmlc::Parameter<CLIParam> {
         .add_enum("dump", kDumpModel)
         .add_enum("pred", kPredict)
         .describe("Task to be performed by the CLI program.");
+    DMLC_DECLARE_FIELD(silent).set_default(0).set_range(0, 2)
+        .describe("Silent level during the task.");
     DMLC_DECLARE_FIELD(eval_train).set_default(false)
         .describe("Whether evaluate on training data during training.");
     DMLC_DECLARE_FIELD(num_round).set_default(10).set_lower_bound(1)
@@ -122,10 +126,10 @@ struct CLIParam : public dmlc::Parameter<CLIParam> {
     DMLC_DECLARE_ALIAS(name_fmap, fmap);
   }
   // customized configure function of CLIParam
-  inline void Configure(const std::vector<std::pair<std::string, std::string> >& _cfg) {
+  inline void Configure(const std::vector<std::pair<std::string, std::string> >& cfg) {
-    this->cfg = _cfg;
+    this->cfg = cfg;
-    this->InitAllowUnknown(_cfg);
+    this->InitAllowUnknown(cfg);
-    for (const auto& kv : _cfg) {
+    for (const auto& kv : cfg) {
       if (!strncmp("eval[", kv.first.c_str(), 5)) {
         char evname[256];
         CHECK_EQ(sscanf(kv.first.c_str(), "eval[%[^]]", evname), 1)
@@ -137,13 +141,13 @@ struct CLIParam : public dmlc::Parameter<CLIParam> {
     // constraint.
     if (name_pred == "stdout") {
       save_period = 0;
-      this->cfg.emplace_back(std::make_pair("silent", "0"));
+      silent = 1;
     }
     if (dsplit == 0 && rabit::IsDistributed()) {
       dsplit = 2;
     }
     if (rabit::GetRank() != 0) {
-      this->cfg.emplace_back(std::make_pair("silent", "1"));
+      silent = 2;
     }
   }
 };
@@ -158,20 +162,15 @@ void CLITrain(const CLIParam& param) {
   }
   // load in data.
   std::shared_ptr<DMatrix> dtrain(
-      DMatrix::Load(
+      DMatrix::Load(param.train_path, param.silent != 0, param.dsplit == 2));
-          param.train_path,
-          ConsoleLogger::GlobalVerbosity() > ConsoleLogger::DefaultVerbosity(),
-          param.dsplit == 2));
   std::vector<std::shared_ptr<DMatrix> > deval;
   std::vector<std::shared_ptr<DMatrix> > cache_mats;
   std::vector<DMatrix*> eval_datasets;
   cache_mats.push_back(dtrain);
   for (size_t i = 0; i < param.eval_data_names.size(); ++i) {
     deval.emplace_back(
-        std::shared_ptr<DMatrix>(DMatrix::Load(
+        std::shared_ptr<DMatrix>(DMatrix::Load(param.eval_data_paths[i],
-            param.eval_data_paths[i],
+                                               param.silent != 0, param.dsplit == 2)));
-            ConsoleLogger::GlobalVerbosity() > ConsoleLogger::DefaultVerbosity(),
-            param.dsplit == 2)));
     eval_datasets.push_back(deval.back().get());
     cache_mats.push_back(deval.back());
   }
@@ -195,14 +194,17 @@ void CLITrain(const CLIParam& param) {
       learner->InitModel();
     }
   }
-  LOG(INFO) << "Loading data: " << dmlc::GetTime() - tstart_data_load << " sec";
+  if (param.silent == 0) {
-
+    LOG(INFO) << "Loading data: " << dmlc::GetTime() - tstart_data_load << " sec";
+  }
   // start training.
   const double start = dmlc::GetTime();
   for (int i = version / 2; i < param.num_round; ++i) {
     double elapsed = dmlc::GetTime() - start;
     if (version % 2 == 0) {
-      LOG(INFO) << "boosting round " << i << ", " << elapsed << " sec elapsed";
+      if (param.silent == 0) {
+        LOG(CONSOLE) << "boosting round " << i << ", " << elapsed << " sec elapsed";
+      }
       learner->UpdateOneIter(i, dtrain.get());
       if (learner->AllowLazyCheckPoint()) {
         rabit::LazyCheckPoint(learner.get());
@@ -218,7 +220,9 @@ void CLITrain(const CLIParam& param) {
         LOG(TRACKER) << res;
       }
     } else {
-      LOG(CONSOLE) << res;
+      if (param.silent < 2) {
+        LOG(CONSOLE) << res;
+      }
     }
     if (param.save_period != 0 &&
         (i + 1) % param.save_period == 0 &&
@@ -257,8 +261,10 @@ void CLITrain(const CLIParam& param) {
     learner->Save(fo.get());
   }
 
-  double elapsed = dmlc::GetTime() - start;
+  if (param.silent == 0) {
-  LOG(INFO) << "update end, " << elapsed << " sec in all";
+    double elapsed = dmlc::GetTime() - start;
+    LOG(CONSOLE) << "update end, " << elapsed << " sec in all";
+  }
 }
 
 void CLIDumpModel(const CLIParam& param) {
@@ -305,10 +311,7 @@ void CLIPredict(const CLIParam& param) {
       << "Test dataset parameter test:data must be specified.";
   // load data
   std::unique_ptr<DMatrix> dtest(
-      DMatrix::Load(
+      DMatrix::Load(param.test_path, param.silent != 0, param.dsplit == 2));
-          param.test_path,
-          ConsoleLogger::GlobalVerbosity() > ConsoleLogger::DefaultVerbosity(),
-          param.dsplit == 2));
   // load model
   CHECK_NE(param.model_in, "NULL")
       << "Must specify model_in for predict";
@@ -318,11 +321,14 @@ void CLIPredict(const CLIParam& param) {
   learner->Load(fi.get());
   learner->Configure(param.cfg);
 
-  LOG(INFO) << "start prediction...";
+  if (param.silent == 0) {
+    LOG(CONSOLE) << "start prediction...";
+  }
   HostDeviceVector<bst_float> preds;
   learner->Predict(dtest.get(), param.pred_margin, &preds, param.ntree_limit);
-  LOG(CONSOLE) << "writing prediction to " << param.name_pred;
+  if (param.silent == 0) {
-
+    LOG(CONSOLE) << "writing prediction to " << param.name_pred;
+  }
   std::unique_ptr<dmlc::Stream> fo(
       dmlc::Stream::Create(param.name_pred.c_str(), "w"));
   dmlc::ostream os(fo.get());

src/common/common.cc

@@ -27,6 +27,6 @@ GlobalRandomEngine& GlobalRandom() {
 int AllVisibleImpl::AllVisible() {
   return 0;
 }
-#endif  // !defined(XGBOOST_USE_CUDA)
+#endif
 
 }  // namespace xgboost

src/common/common.h

@@ -26,7 +26,7 @@
 
 #define WITH_CUDA() false
 
-#endif  // defined(__CUDACC__)
+#endif
 
 namespace dh {
 #if defined(__CUDACC__)
@@ -44,7 +44,7 @@ inline cudaError_t ThrowOnCudaError(cudaError_t code, const char *file,
   }
   return code;
 }
-#endif  // defined(__CUDACC__)
+#endif
 }  // namespace dh
 
 namespace xgboost {
@@ -147,86 +147,61 @@ struct AllVisibleImpl {
  */
 class GPUSet {
  public:
-  using GpuIdType = int;
-  static constexpr GpuIdType kAll = -1;
-
   explicit GPUSet(int start = 0, int ndevices = 0)
       : devices_(start, start + ndevices) {}
 
   static GPUSet Empty() { return GPUSet(); }
 
-  static GPUSet Range(GpuIdType start, GpuIdType n_gpus) {
+  static GPUSet Range(int start, int ndevices) {
-    return n_gpus <= 0 ? Empty() : GPUSet{start, n_gpus};
+    return ndevices <= 0 ? Empty() : GPUSet{start, ndevices};
   }
-  /*! \brief n_gpus and num_rows both are upper bounds. */
+  /*! \brief ndevices and num_rows both are upper bounds. */
-  static GPUSet All(GpuIdType gpu_id, GpuIdType n_gpus,
+  static GPUSet All(int ndevices, int num_rows = std::numeric_limits<int>::max()) {
-                    GpuIdType num_rows = std::numeric_limits<GpuIdType>::max()) {
+    int n_devices_visible = AllVisible().Size();
-    CHECK_GE(gpu_id, 0) << "gpu_id must be >= 0.";
+    if (ndevices < 0 || ndevices >  n_devices_visible) {
-    CHECK_GE(n_gpus, -1) << "n_gpus must be >= -1.";
+      ndevices = n_devices_visible;
-
-    GpuIdType const n_devices_visible = AllVisible().Size();
-    if (n_devices_visible == 0 || n_gpus == 0) { return Empty(); }
-
-    GpuIdType const n_available_devices = n_devices_visible - gpu_id;
-
-    if (n_gpus == kAll) {  // Use all devices starting from `gpu_id'.
-      CHECK(gpu_id < n_devices_visible)
-          << "\ngpu_id should be less than number of visible devices.\ngpu_id: "
-          << gpu_id
-          << ", number of visible devices: "
-          << n_devices_visible;
-      GpuIdType n_devices =
-          n_available_devices < num_rows ? n_available_devices : num_rows;
-      return Range(gpu_id, n_devices);
-    } else {  // Use devices in ( gpu_id, gpu_id + n_gpus ).
-      CHECK_LE(n_gpus, n_available_devices)
-          << "Starting from gpu id: " << gpu_id << ", there are only "
-          << n_available_devices << " available devices, while n_gpus is set to: "
-          << n_gpus;
-      GpuIdType n_devices = n_gpus < num_rows ? n_gpus : num_rows;
-      return Range(gpu_id, n_devices);
     }
+    // fix-up device number to be limited by number of rows
+    ndevices = ndevices > num_rows ? num_rows : ndevices;
+    return Range(0, ndevices);
   }
-
   static GPUSet AllVisible() {
-    GpuIdType n =  AllVisibleImpl::AllVisible();
+    int n =  AllVisibleImpl::AllVisible();
     return Range(0, n);
   }
-
+  /*! \brief Ensure gpu_id is correct, so not dependent upon user knowing details */
-  size_t Size() const {
+  static int GetDeviceIdx(int gpu_id) {
-    GpuIdType size = *devices_.end() - *devices_.begin();
+    auto devices = AllVisible();
-    GpuIdType res = size < 0 ? 0 : size;
+    CHECK(!devices.IsEmpty()) << "Empty device.";
-    return static_cast<size_t>(res);
+    return (std::abs(gpu_id) + 0) % devices.Size();
+  }
+  /*! \brief Counting from gpu_id */
+  GPUSet Normalised(int gpu_id) const {
+    return Range(gpu_id, Size());
+  }
+  /*! \brief Counting from 0 */
+  GPUSet Unnormalised() const {
+    return Range(0, Size());
   }
 
-  /*
+  int Size() const {
-   * By default, we have two configurations of identifying device, one
+    int res = *devices_.end() - *devices_.begin();
-   * is the device id obtained from `cudaGetDevice'.  But we sometimes
+    return res < 0 ? 0 : res;
-   * store objects that allocated one for each device in a list, which
-   * requires a zero-based index.
-   *
-   * Hence, `DeviceId' converts a zero-based index to actual device id,
-   * `Index' converts a device id to a zero-based index.
-   */
-  GpuIdType DeviceId(size_t index) const {
-    GpuIdType result = *devices_.begin() + static_cast<GpuIdType>(index);
-    CHECK(Contains(result)) << "\nDevice " << result << " is not in GPUSet."
-                            << "\nIndex: " << index
-                            << "\nGPUSet: (" << *begin() << ", " << *end() << ")"
-                            << std::endl;
-    return result;
   }
-  size_t Index(GpuIdType device) const {
+  /*! \brief Get normalised device id. */
-    CHECK(Contains(device)) << "\nDevice " << device << " is not in GPUSet."
+  int operator[](int index) const {
-                            << "\nGPUSet: (" << *begin() << ", " << *end() << ")"
+    CHECK(index >= 0 && index < Size());
-                            << std::endl;
+    return *devices_.begin() + index;
-    size_t result = static_cast<size_t>(device - *devices_.begin());
-    return result;
   }
 
   bool IsEmpty() const { return Size() == 0; }
+  /*! \brief Get un-normalised index. */
+  int Index(int device) const {
+    CHECK(Contains(device));
+    return device - *devices_.begin();
+  }
 
-  bool Contains(GpuIdType device) const {
+  bool Contains(int device) const {
     return *devices_.begin() <= device && device < *devices_.end();
   }
 

src/common/compressed_iterator.h

@@ -10,7 +10,7 @@
 
 #ifdef __CUDACC__
 #include "device_helpers.cuh"
-#endif  // __CUDACC__
+#endif
 
 namespace xgboost {
 namespace common {
@@ -115,7 +115,7 @@ class CompressedBufferWriter {
       symbol >>= 8;
     }
   }
-#endif  // __CUDACC__
+#endif
 
   template <typename IterT>
   void Write(CompressedByteT *buffer, IterT input_begin, IterT input_end) {

src/common/config.h

@@ -58,12 +58,12 @@ class ConfigReaderBase {
    * \brief to be implemented by subclass,
    * get next token, return EOF if end of file
    */
-  virtual int GetChar() = 0;
+  virtual char GetChar() = 0;
   /*! \brief to be implemented by child, check if end of stream */
   virtual bool IsEnd() = 0;
 
  private:
-  int ch_buf_;
+  char ch_buf_;
   std::string s_name_, s_val_, s_buf_;
 
   inline void SkipLine() {
@@ -79,7 +79,7 @@ class ConfigReaderBase {
         case '\"': return;
         case '\r':
         case '\n': LOG(FATAL)<< "ConfigReader: unterminated string";
-        default: *tok += static_cast<char>(ch_buf_);
+        default: *tok += ch_buf_;
       }
     }
     LOG(FATAL) << "ConfigReader: unterminated string";
@@ -89,7 +89,7 @@ class ConfigReaderBase {
       switch (ch_buf_) {
         case '\\': *tok += this->GetChar(); break;
         case '\'': return;
-        default: *tok += static_cast<char>(ch_buf_);
+        default: *tok += ch_buf_;
       }
     }
     LOG(FATAL) << "unterminated string";
@@ -128,7 +128,7 @@ class ConfigReaderBase {
           if (tok->length() != 0) return new_line;
           break;
         default:
-          *tok += static_cast<char>(ch_buf_);
+          *tok += ch_buf_;
           ch_buf_ = this->GetChar();
           break;
       }
@@ -152,7 +152,7 @@ class ConfigStreamReader: public ConfigReaderBase {
   explicit ConfigStreamReader(std::istream &fin) : fin_(fin) {}
 
  protected:
-  int GetChar() override {
+  char GetChar() override {
     return fin_.get();
   }
   /*! \brief to be implemented by child, check if end of stream */

src/common/device_helpers.cuh

@@ -19,11 +19,9 @@
 #include <sstream>
 #include <string>
 #include <vector>
-#include "timer.h"
 
 #ifdef XGBOOST_USE_NCCL
 #include "nccl.h"
-#include "../common/io.h"
 #endif
 
 // Uncomment to enable
@@ -55,16 +53,6 @@ T *Raw(thrust::device_vector<T> &v) {  //  NOLINT
   return raw_pointer_cast(v.data());
 }
 
-inline void CudaCheckPointerDevice(void* ptr) {
-  cudaPointerAttributes attr;
-  dh::safe_cuda(cudaPointerGetAttributes(&attr, ptr));
-  int ptr_device = attr.device;
-  int cur_device = -1;
-  cudaGetDevice(&cur_device);
-  CHECK_EQ(ptr_device, cur_device) << "pointer device: " << ptr_device
-                                   << "current device: " << cur_device;
-}
-
 template <typename T>
 const T *Raw(const thrust::device_vector<T> &v) {  //  NOLINT
   return raw_pointer_cast(v.data());
@@ -73,7 +61,7 @@ const T *Raw(const thrust::device_vector<T> &v) {  //  NOLINT
 // if n_devices=-1, then use all visible devices
 inline void SynchronizeNDevices(xgboost::GPUSet devices) {
   devices = devices.IsEmpty() ? xgboost::GPUSet::AllVisible() : devices;
-  for (auto const d : devices) {
+  for (auto const d : devices.Unnormalised()) {
     safe_cuda(cudaSetDevice(d));
     safe_cuda(cudaDeviceSynchronize());
   }
@@ -259,14 +247,6 @@ class DVec {
 
   const T *Data() const { return ptr_; }
 
-  xgboost::common::Span<const T> GetSpan() const {
-    return xgboost::common::Span<const T>(ptr_, this->Size());
-  }
-
-  xgboost::common::Span<T> GetSpan() {
-    return xgboost::common::Span<T>(ptr_, this->Size());
-  }
-
   std::vector<T> AsVector() const {
     std::vector<T> h_vector(Size());
     safe_cuda(cudaSetDevice(device_idx_));
@@ -379,11 +359,6 @@ class DVec2 {
   DVec<T> &D2() { return d2_; }
 
   T *Current() { return buff_.Current(); }
-  xgboost::common::Span<T> CurrentSpan() {
-    return xgboost::common::Span<T>{
-      buff_.Current(),
-      static_cast<typename xgboost::common::Span<T>::index_type>(Size())};
-  }
 
   DVec<T> &CurrentDVec() { return buff_.selector == 0 ? D1() : D2(); }
 
@@ -486,7 +461,7 @@ class BulkAllocator {
   }
 
   template <typename... Args>
-  void Allocate(int device_idx, Args... args) {
+  void Allocate(int device_idx, bool silent, Args... args) {
     size_t size = GetSizeBytes(args...);
 
     char *ptr = AllocateDevice(device_idx, size, MemoryT);
@@ -512,9 +487,8 @@ struct CubMemory {
   ~CubMemory() { Free(); }
 
   template <typename T>
-  xgboost::common::Span<T> GetSpan(size_t size) {
+  T *Pointer() {
-    this->LazyAllocate(size * sizeof(T));
+    return static_cast<T *>(d_temp_storage);
-    return xgboost::common::Span<T>(static_cast<T*>(d_temp_storage), size);
   }
 
   void Free() {
@@ -769,12 +743,10 @@ void SumReduction(dh::CubMemory &tmp_mem, dh::DVec<T> &in, dh::DVec<T> &out,
 * @param nVals number of elements in the input array
 */
 template <typename T>
-typename std::iterator_traits<T>::value_type SumReduction(
+typename std::iterator_traits<T>::value_type SumReduction(dh::CubMemory &tmp_mem, T in, int nVals) {
-    dh::CubMemory &tmp_mem, T in, int nVals) {
   using ValueT = typename std::iterator_traits<T>::value_type;
   size_t tmpSize;
-  ValueT *dummy_out = nullptr;
+  dh::safe_cuda(cub::DeviceReduce::Sum(nullptr, tmpSize, in, in, nVals));
-  dh::safe_cuda(cub::DeviceReduce::Sum(nullptr, tmpSize, in, dummy_out, nVals));
   // Allocate small extra memory for the return value
   tmp_mem.LazyAllocate(tmpSize + sizeof(ValueT));
   auto ptr = reinterpret_cast<ValueT *>(tmp_mem.d_temp_storage) + 1;
@@ -797,7 +769,7 @@ typename std::iterator_traits<T>::value_type SumReduction(
 template <typename T, int BlkDim = 256, int ItemsPerThread = 4>
 void FillConst(int device_idx, T *out, int len, T def) {
   dh::LaunchN<ItemsPerThread, BlkDim>(device_idx, len,
-                                      [=] __device__(int i) { out[i] = def; });
+                                       [=] __device__(int i) { out[i] = def; });
 }
 
 /**
@@ -847,20 +819,14 @@ void Gather(int device_idx, T *out, const T *in, const int *instId, int nVals) {
  */
 
 class AllReducer {
-  bool initialised_;
+  bool initialised;
-  size_t allreduce_bytes_;  // Keep statistics of the number of bytes communicated
-  size_t allreduce_calls_;  // Keep statistics of the number of reduce calls
 #ifdef XGBOOST_USE_NCCL
   std::vector<ncclComm_t> comms;
   std::vector<cudaStream_t> streams;
-  std::vector<int> device_ordinals;  // device id from CUDA
+  std::vector<int> device_ordinals;
-  std::vector<int> device_counts;  // device count from CUDA
-  ncclUniqueId id;
 #endif
-
  public:
-  AllReducer() : initialised_(false), allreduce_bytes_(0),
+  AllReducer() : initialised(false) {}
-                 allreduce_calls_(0) {}
 
   /**
    * \fn  void Init(const std::vector<int> &device_ordinals)
@@ -873,45 +839,17 @@ class AllReducer {
 
   void Init(const std::vector<int> &device_ordinals) {
 #ifdef XGBOOST_USE_NCCL
-    /** \brief this >monitor . init. */
     this->device_ordinals = device_ordinals;
-    this->device_counts.resize(rabit::GetWorldSize());
+    comms.resize(device_ordinals.size());
-    this->comms.resize(device_ordinals.size());
+    dh::safe_nccl(ncclCommInitAll(comms.data(),
-    this->streams.resize(device_ordinals.size());
+                                  static_cast<int>(device_ordinals.size()),
-    this->id = GetUniqueId();
+                                  device_ordinals.data()));
-
+    streams.resize(device_ordinals.size());
-    device_counts.at(rabit::GetRank()) = device_ordinals.size();
-    for (size_t i = 0; i < device_counts.size(); i++) {
-      int dev_count = device_counts.at(i);
-      rabit::Allreduce<rabit::op::Sum, int>(&dev_count, 1);
-      device_counts.at(i) = dev_count;
-    }
-
-    int nccl_rank = 0;
-    int nccl_rank_offset = std::accumulate(device_counts.begin(),
-                             device_counts.begin() + rabit::GetRank(), 0);
-    int nccl_nranks = std::accumulate(device_counts.begin(),
-                        device_counts.end(), 0);
-    nccl_rank += nccl_rank_offset;
-    
-    GroupStart();
     for (size_t i = 0; i < device_ordinals.size(); i++) {
-      int dev = device_ordinals.at(i);
+      safe_cuda(cudaSetDevice(device_ordinals[i]));
-      dh::safe_cuda(cudaSetDevice(dev));
+      safe_cuda(cudaStreamCreate(&streams[i]));
-      dh::safe_nccl(ncclCommInitRank(
-        &comms.at(i),
-        nccl_nranks, id, 
-        nccl_rank));
-
-      nccl_rank++;
     }
-    GroupEnd();
+    initialised = true;
-
-    for (size_t i = 0; i < device_ordinals.size(); i++) {
-      safe_cuda(cudaSetDevice(device_ordinals.at(i)));
-      safe_cuda(cudaStreamCreate(&streams.at(i)));
-    }
-    initialised_ = true;
 #else
     CHECK_EQ(device_ordinals.size(), 1)
         << "XGBoost must be compiled with NCCL to use more than one GPU.";
@@ -919,7 +857,7 @@ class AllReducer {
   }
   ~AllReducer() {
 #ifdef XGBOOST_USE_NCCL
-    if (initialised_) {
+    if (initialised) {
       for (auto &stream : streams) {
         dh::safe_cuda(cudaStreamDestroy(stream));
       }
@@ -927,11 +865,6 @@ class AllReducer {
         ncclCommDestroy(comm);
       }
     }
-    if (xgboost::ConsoleLogger::ShouldLog(xgboost::ConsoleLogger::LV::kDebug)) {
-      LOG(CONSOLE) << "======== NCCL Statistics========";
-      LOG(CONSOLE) << "AllReduce calls: " << allreduce_calls_;
-      LOG(CONSOLE) << "AllReduce total MB communicated: " << allreduce_bytes_/1000000;
-    }
 #endif
   }
 
@@ -966,42 +899,12 @@ class AllReducer {
   void AllReduceSum(int communication_group_idx, const double *sendbuff,
                     double *recvbuff, int count) {
 #ifdef XGBOOST_USE_NCCL
-    CHECK(initialised_);
+    CHECK(initialised);
-    dh::safe_cuda(cudaSetDevice(device_ordinals.at(communication_group_idx)));
+
+    dh::safe_cuda(cudaSetDevice(device_ordinals[communication_group_idx]));
     dh::safe_nccl(ncclAllReduce(sendbuff, recvbuff, count, ncclDouble, ncclSum,
-                                comms.at(communication_group_idx),
+                                comms[communication_group_idx],
-                                streams.at(communication_group_idx)));
+                                streams[communication_group_idx]));
-    if(communication_group_idx == 0)
-    {
-      allreduce_bytes_ += count * sizeof(double);
-      allreduce_calls_ += 1;
-    }
-#endif
-  }
-
-  /**
-   * \brief Allreduce. Use in exactly the same way as NCCL but without needing
-   * streams or comms.
-   *
-   * \param communication_group_idx Zero-based index of the communication group.
-   * \param sendbuff                The sendbuff.
-   * \param recvbuff                The recvbuff.
-   * \param count                   Number of elements.
-   */
-
-  void AllReduceSum(int communication_group_idx, const float *sendbuff,
-                    float *recvbuff, int count) {
-#ifdef XGBOOST_USE_NCCL
-    CHECK(initialised_);
-    dh::safe_cuda(cudaSetDevice(device_ordinals.at(communication_group_idx)));
-    dh::safe_nccl(ncclAllReduce(sendbuff, recvbuff, count, ncclFloat, ncclSum,
-                                comms.at(communication_group_idx),
-                                streams.at(communication_group_idx)));
-    if(communication_group_idx == 0)
-    {
-      allreduce_bytes_ += count * sizeof(float);
-      allreduce_calls_ += 1;
-    }
 #endif
   }
 
@@ -1019,7 +922,7 @@ class AllReducer {
   void AllReduceSum(int communication_group_idx, const int64_t *sendbuff,
                     int64_t *recvbuff, int count) {
 #ifdef XGBOOST_USE_NCCL
-    CHECK(initialised_);
+    CHECK(initialised);
 
     dh::safe_cuda(cudaSetDevice(device_ordinals[communication_group_idx]));
     dh::safe_nccl(ncclAllReduce(sendbuff, recvbuff, count, ncclInt64, ncclSum,
@@ -1035,35 +938,12 @@ class AllReducer {
    */
   void Synchronize() {
 #ifdef XGBOOST_USE_NCCL
-    for (size_t i = 0; i < device_ordinals.size(); i++) {
+    for (int i = 0; i < device_ordinals.size(); i++) {
       dh::safe_cuda(cudaSetDevice(device_ordinals[i]));
       dh::safe_cuda(cudaStreamSynchronize(streams[i]));
     }
 #endif
-  };
-
-#ifdef XGBOOST_USE_NCCL
-  /**
-   * \fn  ncclUniqueId GetUniqueId()
-   *
-   * \brief Gets the Unique ID from NCCL to be used in setting up interprocess
-   * communication
-   *
-   * \return the Unique ID
-   */
-  ncclUniqueId GetUniqueId() {
-    static const int RootRank = 0;
-    ncclUniqueId id;
-    if (rabit::GetRank() == RootRank) {
-      dh::safe_nccl(ncclGetUniqueId(&id));
-    }
-    rabit::Broadcast(
-      (void*)&id,
-      (size_t)sizeof(ncclUniqueId),
-      (int)RootRank);
-    return id;
   }
-#endif
 };
 
 class SaveCudaContext {
@@ -1089,6 +969,27 @@ class SaveCudaContext {
   }
 };
 
+/**
+ * \brief Executes some operation on each element of the input vector, using a
+ * single controlling thread for each element.
+ *
+ * \tparam  T       Generic type parameter.
+ * \tparam  FunctionT  Type of the function t.
+ * \param shards  The shards.
+ * \param f       The func_t to process.
+ */
+
+template <typename T, typename FunctionT>
+void ExecuteShards(std::vector<T> *shards, FunctionT f) {
+  SaveCudaContext {
+    [&](){
+#pragma omp parallel for schedule(static, 1) if (shards->size() > 1)
+      for (int shard = 0; shard < shards->size(); ++shard) {
+        f(shards->at(shard));
+      }
+    }};
+}
+
 /**
  * \brief Executes some operation on each element of the input vector, using a
  * single controlling thread for each element. In addition, passes the shard index
@@ -1102,13 +1003,13 @@ class SaveCudaContext {
 
 template <typename T, typename FunctionT>
 void ExecuteIndexShards(std::vector<T> *shards, FunctionT f) {
-  SaveCudaContext{[&]() {
+  SaveCudaContext {
-    const long shards_size = static_cast<long>(shards->size());
+    [&](){
-#pragma omp parallel for schedule(static, 1) if (shards_size > 1)
+#pragma omp parallel for schedule(static, 1) if (shards->size() > 1)
-    for (long shard = 0; shard < shards_size; ++shard) {
+      for (int shard = 0; shard < shards->size(); ++shard) {
-      f(shard, shards->at(shard));
+        f(shard, shards->at(shard));
-    }
+      }
-  }};
+    }};
 }
 
 /**
@@ -1154,71 +1055,4 @@ xgboost::common::Span<T> ToSpan(thrust::device_vector<T>& vec,
   using IndexT = typename xgboost::common::Span<T>::index_type;
   return ToSpan(vec, static_cast<IndexT>(offset), static_cast<IndexT>(size));
 }
-
-template <typename FunctionT>
-class LauncherItr {
-public:
-  int idx;
-  FunctionT f;
-  XGBOOST_DEVICE LauncherItr() : idx(0) {}
-  XGBOOST_DEVICE LauncherItr(int idx, FunctionT f) : idx(idx), f(f) {}
-  XGBOOST_DEVICE LauncherItr &operator=(int output) {
-    f(idx, output);
-    return *this;
-  }
-};
-
-/**
- * \brief Thrust compatible iterator type - discards algorithm output and launches device lambda
- *        with the index of the output and the algorithm output as arguments.
- *
- * \author  Rory
- * \date  7/9/2017
- *
- * \tparam  FunctionT Type of the function t.
- */
-template <typename FunctionT>
-class DiscardLambdaItr {
-public:
- // Required iterator traits
- using self_type = DiscardLambdaItr;  // NOLINT
- using difference_type = ptrdiff_t;   // NOLINT
- using value_type = void;       // NOLINT
- using pointer = value_type *;  // NOLINT
- using reference = LauncherItr<FunctionT>;  // NOLINT
- using iterator_category = typename thrust::detail::iterator_facade_category<
-     thrust::any_system_tag, thrust::random_access_traversal_tag, value_type,
-     reference>::type;  // NOLINT
-private:
-  difference_type offset_;
-  FunctionT f_;
-public:
- XGBOOST_DEVICE explicit DiscardLambdaItr(FunctionT f) : offset_(0), f_(f) {}
- XGBOOST_DEVICE DiscardLambdaItr(difference_type offset, FunctionT f)
-     : offset_(offset), f_(f) {}
- XGBOOST_DEVICE self_type operator+(const int &b) const {
-   return DiscardLambdaItr(offset_ + b, f_);
-  }
-  XGBOOST_DEVICE self_type operator++() {
-    offset_++;
-    return *this;
-  }
-  XGBOOST_DEVICE self_type operator++(int) {
-    self_type retval = *this;
-    offset_++;
-    return retval;
-  }
-  XGBOOST_DEVICE self_type &operator+=(const int &b) {
-    offset_ += b;
-    return *this;
-  }
-  XGBOOST_DEVICE reference operator*() const {
-    return LauncherItr<FunctionT>(offset_, f_);
-  }
-  XGBOOST_DEVICE reference operator[](int idx) {
-    self_type offset = (*this) + idx;
-    return *offset;
-  }
-};
-
 }  // namespace dh

src/common/group_data.h

@@ -23,7 +23,7 @@ namespace common {
  * \tparam ValueType type of entries in the sparse matrix
  * \tparam SizeType type of the index range holder
  */
-template<typename ValueType, typename SizeType = std::size_t>
+template<typename ValueType, typename SizeType = size_t>
 struct ParallelGroupBuilder {
  public:
   // parallel group builder of data
@@ -44,9 +44,9 @@ struct ParallelGroupBuilder {
    * \param nkeys number of keys in the matrix, can be smaller than expected
    * \param nthread number of thread that will be used in construction
    */
-  inline void InitBudget(std::size_t nkeys, int nthread) {
+  inline void InitBudget(size_t nkeys, int nthread) {
     thread_rptr_.resize(nthread);
-    for (std::size_t i = 0;  i < thread_rptr_.size(); ++i) {
+    for (size_t i = 0;  i < thread_rptr_.size(); ++i) {
       thread_rptr_[i].resize(nkeys);
       std::fill(thread_rptr_[i].begin(), thread_rptr_[i].end(), 0);
     }
@@ -57,7 +57,7 @@ struct ParallelGroupBuilder {
    * \param threadid the id of thread that calls this function
    * \param nelem number of element budget add to this row
    */
-  inline void AddBudget(std::size_t key, int threadid, SizeType nelem = 1) {
+  inline void AddBudget(size_t key, int threadid, SizeType nelem = 1) {
     std::vector<SizeType> &trptr = thread_rptr_[threadid];
     if (trptr.size() < key + 1) {
       trptr.resize(key + 1, 0);
@@ -67,23 +67,23 @@ struct ParallelGroupBuilder {
   /*! \brief step 3: initialize the necessary storage */
   inline void InitStorage() {
     // set rptr to correct size
-    for (std::size_t tid = 0; tid < thread_rptr_.size(); ++tid) {
+    for (size_t tid = 0; tid < thread_rptr_.size(); ++tid) {
       if (rptr_.size() <= thread_rptr_[tid].size()) {
-        rptr_.resize(thread_rptr_[tid].size() + 1);  // key + 1
+        rptr_.resize(thread_rptr_[tid].size() + 1);
       }
     }
     // initialize rptr to be beginning of each segment
-    std::size_t start = 0;
+    size_t start = 0;
-    for (std::size_t i = 0; i + 1 < rptr_.size(); ++i) {
+    for (size_t i = 0; i + 1 < rptr_.size(); ++i) {
-      for (std::size_t tid = 0; tid < thread_rptr_.size(); ++tid) {
+      for (size_t tid = 0; tid < thread_rptr_.size(); ++tid) {
         std::vector<SizeType> &trptr = thread_rptr_[tid];
-        if (i < trptr.size()) {         // i^th row is assigned for this thread
+        if (i < trptr.size()) {
-          std::size_t ncnt = trptr[i];  // how many entries in this row
+          size_t ncnt = trptr[i];
           trptr[i] = start;
           start += ncnt;
         }
       }
-      rptr_[i + 1] = start;  // pointer accumulated from all thread
+      rptr_[i + 1] = start;
     }
     data_.resize(start);
   }
@@ -95,7 +95,7 @@ struct ParallelGroupBuilder {
    * \param value The value to be pushed to the group.
    * \param threadid the id of thread that calls this function
    */
-  void Push(std::size_t key, ValueType value, int threadid) {
+  inline void Push(size_t key, ValueType value, int threadid) {
     SizeType &rp = thread_rptr_[threadid][key];
     data_[rp++] = value;
   }

src/common/hist_util.cc

@@ -1,48 +1,22 @@
 /*!
- * Copyright 2017-2019 by Contributors
+ * Copyright 2017 by Contributors
  * \file hist_util.h
+ * \brief Utilities to store histograms
+ * \author Philip Cho, Tianqi Chen
  */
-#include <rabit/rabit.h>
 #include <dmlc/omp.h>
 #include <numeric>
 #include <vector>
-
+#include "./sync.h"
 #include "./random.h"
 #include "./column_matrix.h"
 #include "./hist_util.h"
 #include "./quantile.h"
 
-#if defined(XGBOOST_MM_PREFETCH_PRESENT)
-  #include <xmmintrin.h>
-  #define PREFETCH_READ_T0(addr) _mm_prefetch(reinterpret_cast<const char*>(addr), _MM_HINT_T0)
-#elif defined(XGBOOST_BUILTIN_PREFETCH_PRESENT)
-  #define PREFETCH_READ_T0(addr) __builtin_prefetch(reinterpret_cast<const char*>(addr), 0, 3)
-#else  // no SW pre-fetching available; PREFETCH_READ_T0 is no-op
-  #define PREFETCH_READ_T0(addr) do {} while (0)
-#endif  // defined(XGBOOST_MM_PREFETCH_PRESENT)
-
 namespace xgboost {
 namespace common {
 
-HistCutMatrix::HistCutMatrix() {
-  monitor_.Init("HistCutMatrix");
-}
-
-size_t HistCutMatrix::SearchGroupIndFromBaseRow(
-    std::vector<bst_uint> const& group_ptr, size_t const base_rowid) const {
-  using KIt = std::vector<bst_uint>::const_iterator;
-  KIt res = std::lower_bound(group_ptr.cbegin(), group_ptr.cend() - 1, base_rowid);
-  // Cannot use CHECK_NE because it will try to print the iterator.
-  bool const found = res != group_ptr.cend() - 1;
-  if (!found) {
-    LOG(FATAL) << "Row " << base_rowid << " does not lie in any group!\n";
-  }
-  size_t group_ind = std::distance(group_ptr.cbegin(), res);
-  return group_ind;
-}
-
 void HistCutMatrix::Init(DMatrix* p_fmat, uint32_t max_num_bins) {
-  monitor_.Start("Init");
   const MetaInfo& info = p_fmat->Info();
 
   // safe factor for better accuracy
@@ -51,50 +25,30 @@ void HistCutMatrix::Init(DMatrix* p_fmat, uint32_t max_num_bins) {
 
   const int nthread = omp_get_max_threads();
 
-  unsigned const nstep =
+  auto nstep = static_cast<unsigned>((info.num_col_ + nthread - 1) / nthread);
-      static_cast<unsigned>((info.num_col_ + nthread - 1) / nthread);
+  auto ncol = static_cast<unsigned>(info.num_col_);
-  unsigned const ncol = static_cast<unsigned>(info.num_col_);
   sketchs.resize(info.num_col_);
   for (auto& s : sketchs) {
     s.Init(info.num_row_, 1.0 / (max_num_bins * kFactor));
   }
 
   const auto& weights = info.weights_.HostVector();
-
-  // Data groups, used in ranking.
-  std::vector<bst_uint> const& group_ptr = info.group_ptr_;
-  size_t const num_groups = group_ptr.size() == 0 ? 0 : group_ptr.size() - 1;
-  // Use group index for weights?
-  bool const use_group_ind = num_groups != 0 && weights.size() != info.num_row_;
-
   for (const auto &batch : p_fmat->GetRowBatches()) {
-    size_t group_ind = 0;
+    #pragma omp parallel num_threads(nthread)
-    if (use_group_ind) {
-      group_ind = this->SearchGroupIndFromBaseRow(group_ptr, batch.base_rowid);
-    }
-#pragma omp parallel num_threads(nthread) firstprivate(group_ind, use_group_ind)
     {
       CHECK_EQ(nthread, omp_get_num_threads());
       auto tid = static_cast<unsigned>(omp_get_thread_num());
       unsigned begin = std::min(nstep * tid, ncol);
       unsigned end = std::min(nstep * (tid + 1), ncol);
-
       // do not iterate if no columns are assigned to the thread
       if (begin < end && end <= ncol) {
         for (size_t i = 0; i < batch.Size(); ++i) { // NOLINT(*)
-          size_t const ridx = batch.base_rowid + i;
+          size_t ridx = batch.base_rowid + i;
-          SparsePage::Inst const inst = batch[i];
+          SparsePage::Inst inst = batch[i];
-          if (use_group_ind &&
+          for (auto& ins : inst) {
-              group_ptr[group_ind] == ridx &&
+            if (ins.index >= begin && ins.index < end) {
-              // maximum equals to weights.size() - 1
+              sketchs[ins.index].Push(ins.fvalue,
-              group_ind < num_groups - 1) {
+                                      weights.size() > 0 ? weights[ridx] : 1.0f);
-            // move to next group
-            group_ind++;
-          }
-          for (auto const& entry : inst) {
-            if (entry.index >= begin && entry.index < end) {
-              size_t w_idx = use_group_ind ? group_ind : ridx;
-              sketchs[entry.index].Push(entry.fvalue, info.GetWeight(w_idx));
             }
           }
         }
@@ -103,7 +57,6 @@ void HistCutMatrix::Init(DMatrix* p_fmat, uint32_t max_num_bins) {
   }
 
   Init(&sketchs, max_num_bins);
-  monitor_.Stop("Init");
 }
 
 void HistCutMatrix::Init
@@ -120,9 +73,9 @@ void HistCutMatrix::Init
     summary_array[i].Reserve(max_num_bins * kFactor);
     summary_array[i].SetPrune(out, max_num_bins * kFactor);
   }
-  CHECK_EQ(summary_array.size(), in_sketchs->size());
   size_t nbytes = WXQSketch::SummaryContainer::CalcMemCost(max_num_bins * kFactor);
   sreducer.Allreduce(dmlc::BeginPtr(summary_array), nbytes, summary_array.size());
+
   this->min_val.resize(sketchs.size());
   row_ptr.push_back(0);
   for (size_t fid = 0; fid < summary_array.size(); ++fid) {
@@ -148,17 +101,14 @@ void HistCutMatrix::Init
       }
     }
     // push a value that is greater than anything
-    const bst_float cpt
+    if (a.size != 0) {
-      = (a.size > 0) ? a.data[a.size - 1].value : this->min_val[fid];
+      bst_float cpt = a.data[a.size - 1].value;
-    // this must be bigger than last value in a scale
+      // this must be bigger than last value in a scale
-    const bst_float last = cpt + (fabs(cpt) + 1e-5);
+      bst_float last = cpt + (fabs(cpt) + 1e-5);
-    cut.push_back(last);
+      cut.push_back(last);
+    }
 
-    // Ensure that every feature gets at least one quantile point
+    row_ptr.push_back(static_cast<bst_uint>(cut.size()));
-    CHECK_LE(cut.size(), std::numeric_limits<uint32_t>::max());
-    auto cut_size = static_cast<uint32_t>(cut.size());
-    CHECK_GT(cut_size, row_ptr.back());
-    row_ptr.push_back(cut_size);
   }
 }
 
@@ -168,9 +118,7 @@ uint32_t HistCutMatrix::GetBinIdx(const Entry& e) {
   auto cend = cut.begin() + row_ptr[fid + 1];
   CHECK(cbegin != cend);
   auto it = std::upper_bound(cbegin, cend, e.fvalue);
-  if (it == cend) {
+  if (it == cend) it = cend - 1;
-    it = cend - 1;
-  }
   uint32_t idx = static_cast<uint32_t>(it - cut.begin());
   return idx;
 }
@@ -203,7 +151,6 @@ void GHistIndexMatrix::Init(DMatrix* p_fmat, int max_num_bins) {
       SparsePage::Inst inst = batch[i];
 
       CHECK_EQ(ibegin + inst.size(), iend);
-
       for (bst_uint j = 0; j < inst.size(); ++j) {
         uint32_t idx = cut.GetBinIdx(inst[j]);
 
@@ -269,7 +216,7 @@ FindGroups(const std::vector<unsigned>& feature_list,
            const std::vector<size_t>& feature_nnz,
            const ColumnMatrix& colmat,
            size_t nrow,
-           const tree::TrainParam& param) {
+           const FastHistParam& param) {
   /* Goal: Bundle features together that has little or no "overlap", i.e.
            only a few data points should have nonzero values for
            member features.
@@ -331,7 +278,7 @@ FindGroups(const std::vector<unsigned>& feature_list,
 inline std::vector<std::vector<unsigned>>
 FastFeatureGrouping(const GHistIndexMatrix& gmat,
                     const ColumnMatrix& colmat,
-                    const tree::TrainParam& param) {
+                    const FastHistParam& param) {
   const size_t nrow = gmat.row_ptr.size() - 1;
   const size_t nfeature = gmat.cut.row_ptr.size() - 1;
 
@@ -385,7 +332,7 @@ FastFeatureGrouping(const GHistIndexMatrix& gmat,
 
 void GHistIndexBlockMatrix::Init(const GHistIndexMatrix& gmat,
                                  const ColumnMatrix& colmat,
-                                 const tree::TrainParam& param) {
+                                 const FastHistParam& param) {
   cut_ = &gmat.cut;
 
   const size_t nrow = gmat.row_ptr.size() - 1;
@@ -451,89 +398,56 @@ void GHistBuilder::BuildHist(const std::vector<GradientPair>& gpair,
                              const RowSetCollection::Elem row_indices,
                              const GHistIndexMatrix& gmat,
                              GHistRow hist) {
-  const size_t nthread = static_cast<size_t>(this->nthread_);
+  data_.resize(nbins_ * nthread_, GHistEntry());
-  data_.resize(nbins_ * nthread_);
+  std::fill(data_.begin(), data_.end(), GHistEntry());
 
-  const size_t* rid =  row_indices.begin;
+  constexpr int kUnroll = 8;  // loop unrolling factor
-  const size_t nrows = row_indices.Size();
+  const auto nthread = static_cast<bst_omp_uint>(this->nthread_);
-  const uint32_t* index = gmat.index.data();
+  const size_t nrows = row_indices.end - row_indices.begin;
-  const size_t* row_ptr =  gmat.row_ptr.data();
+  const size_t rest = nrows % kUnroll;
-  const float* pgh = reinterpret_cast<const float*>(gpair.data());
 
-  double* hist_data = reinterpret_cast<double*>(hist.data());
+  #pragma omp parallel for num_threads(nthread) schedule(guided)
-  double* data = reinterpret_cast<double*>(data_.data());
+  for (bst_omp_uint i = 0; i < nrows - rest; i += kUnroll) {
-
+    const bst_omp_uint tid = omp_get_thread_num();
-  const size_t block_size = 512;
+    const size_t off = tid * nbins_;
-  size_t n_blocks = nrows/block_size;
+    size_t rid[kUnroll];
-  n_blocks += !!(nrows - n_blocks*block_size);
+    size_t ibegin[kUnroll];
-
+    size_t iend[kUnroll];
-  const size_t nthread_to_process = std::min(nthread,  n_blocks);
+    GradientPair stat[kUnroll];
-  memset(thread_init_.data(), '\0', nthread_to_process*sizeof(size_t));
+    for (int k = 0; k < kUnroll; ++k) {
-
+      rid[k] = row_indices.begin[i + k];
-  const size_t cache_line_size = 64;
-  const size_t prefetch_offset = 10;
-  size_t no_prefetch_size = prefetch_offset + cache_line_size/sizeof(*rid);
-  no_prefetch_size = no_prefetch_size > nrows ? nrows : no_prefetch_size;
-
-#pragma omp parallel for num_threads(nthread_to_process) schedule(guided)
-  for (bst_omp_uint iblock = 0; iblock < n_blocks; iblock++) {
-    dmlc::omp_uint tid = omp_get_thread_num();
-    double* data_local_hist = ((nthread_to_process == 1) ? hist_data :
-                               reinterpret_cast<double*>(data_.data() + tid * nbins_));
-
-    if (!thread_init_[tid]) {
-      memset(data_local_hist, '\0', 2*nbins_*sizeof(double));
-      thread_init_[tid] = true;
     }
-
+    for (int k = 0; k < kUnroll; ++k) {
-    const size_t istart = iblock*block_size;
+      ibegin[k] = gmat.row_ptr[rid[k]];
-    const size_t iend = (((iblock+1)*block_size > nrows) ? nrows : istart + block_size);
+      iend[k] = gmat.row_ptr[rid[k] + 1];
-    for (size_t i = istart; i < iend; ++i) {
+    }
-      const size_t icol_start = row_ptr[rid[i]];
+    for (int k = 0; k < kUnroll; ++k) {
-      const size_t icol_end = row_ptr[rid[i]+1];
+      stat[k] = gpair[rid[k]];
-
+    }
-      if (i < nrows - no_prefetch_size) {
+    for (int k = 0; k < kUnroll; ++k) {
-        PREFETCH_READ_T0(row_ptr + rid[i + prefetch_offset]);
+      for (size_t j = ibegin[k]; j < iend[k]; ++j) {
-        PREFETCH_READ_T0(pgh + 2*rid[i + prefetch_offset]);
+        const uint32_t bin = gmat.index[j];
-      }
+        data_[off + bin].Add(stat[k]);
-
-      for (size_t j = icol_start; j < icol_end; ++j) {
-        const uint32_t idx_bin = 2*index[j];
-        const size_t idx_gh = 2*rid[i];
-
-        data_local_hist[idx_bin] += pgh[idx_gh];
-        data_local_hist[idx_bin+1] += pgh[idx_gh+1];
       }
     }
   }
-
+  for (size_t i = nrows - rest; i < nrows; ++i) {
-  if (nthread_to_process > 1) {
+    const size_t rid = row_indices.begin[i];
-    const size_t size = (2*nbins_);
+    const size_t ibegin = gmat.row_ptr[rid];
-    const size_t block_size = 1024;
+    const size_t iend = gmat.row_ptr[rid + 1];
-    size_t n_blocks = size/block_size;
+    const GradientPair stat = gpair[rid];
-    n_blocks += !!(size - n_blocks*block_size);
+    for (size_t j = ibegin; j < iend; ++j) {
-
+      const uint32_t bin = gmat.index[j];
-    size_t n_worked_bins = 0;
+      data_[bin].Add(stat);
-    for (size_t i = 0; i < nthread_to_process; ++i) {
-      if (thread_init_[i]) {
-        thread_init_[n_worked_bins++] = i;
-      }
     }
+  }
 
-#pragma omp parallel for num_threads(std::min(nthread, n_blocks)) schedule(guided)
+  /* reduction */
-    for (bst_omp_uint iblock = 0; iblock < n_blocks; iblock++) {
+  const uint32_t nbins = nbins_;
-      const size_t istart = iblock * block_size;
+  #pragma omp parallel for num_threads(nthread) schedule(static)
-      const size_t iend = (((iblock + 1) * block_size > size) ? size : istart + block_size);
+  for (bst_omp_uint bin_id = 0; bin_id < bst_omp_uint(nbins); ++bin_id) {
-
+    for (bst_omp_uint tid = 0; tid < nthread; ++tid) {
-      const size_t bin = 2 * thread_init_[0] * nbins_;
+      hist.begin[bin_id].Add(data_[tid * nbins_ + bin_id]);
-      memcpy(hist_data + istart, (data + bin + istart), sizeof(double) * (iend - istart));
-
-      for (size_t i_bin_part = 1; i_bin_part < n_worked_bins; ++i_bin_part) {
-        const size_t bin = 2 * thread_init_[i_bin_part] * nbins_;
-        for (size_t i = istart; i < iend; i++) {
-          hist_data[i] += data[bin + i];
-        }
-      }
     }
   }
 }
@@ -549,10 +463,9 @@ void GHistBuilder::BuildBlockHist(const std::vector<GradientPair>& gpair,
 
 #if defined(_OPENMP)
   const auto nthread = static_cast<bst_omp_uint>(this->nthread_);
-#endif  // defined(_OPENMP)
+#endif
-  tree::GradStats* p_hist = hist.data();
 
-#pragma omp parallel for num_threads(nthread) schedule(guided)
+  #pragma omp parallel for num_threads(nthread) schedule(guided)
   for (bst_omp_uint bid = 0; bid < nblock; ++bid) {
     auto gmat = gmatb[bid];
 
@@ -561,17 +474,20 @@ void GHistBuilder::BuildBlockHist(const std::vector<GradientPair>& gpair,
       size_t ibegin[kUnroll];
       size_t iend[kUnroll];
       GradientPair stat[kUnroll];
-
       for (int k = 0; k < kUnroll; ++k) {
         rid[k] = row_indices.begin[i + k];
+      }
+      for (int k = 0; k < kUnroll; ++k) {
         ibegin[k] = gmat.row_ptr[rid[k]];
         iend[k] = gmat.row_ptr[rid[k] + 1];
+      }
+      for (int k = 0; k < kUnroll; ++k) {
         stat[k] = gpair[rid[k]];
       }
       for (int k = 0; k < kUnroll; ++k) {
         for (size_t j = ibegin[k]; j < iend[k]; ++j) {
           const uint32_t bin = gmat.index[j];
-          p_hist[bin].Add(stat[k]);
+          hist.begin[bin].Add(stat[k]);
         }
       }
     }
@@ -582,7 +498,7 @@ void GHistBuilder::BuildBlockHist(const std::vector<GradientPair>& gpair,
       const GradientPair stat = gpair[rid];
       for (size_t j = ibegin; j < iend; ++j) {
         const uint32_t bin = gmat.index[j];
-        p_hist[bin].Add(stat);
+        hist.begin[bin].Add(stat);
       }
     }
   }
@@ -595,28 +511,25 @@ void GHistBuilder::SubtractionTrick(GHistRow self, GHistRow sibling, GHistRow pa
 
 #if defined(_OPENMP)
   const auto nthread = static_cast<bst_omp_uint>(this->nthread_);
-#endif  // defined(_OPENMP)
+#endif
-  tree::GradStats* p_self = self.data();
-  tree::GradStats* p_sibling = sibling.data();
-  tree::GradStats* p_parent = parent.data();
 
-#pragma omp parallel for num_threads(nthread) schedule(static)
+  #pragma omp parallel for num_threads(nthread) schedule(static)
   for (bst_omp_uint bin_id = 0;
        bin_id < static_cast<bst_omp_uint>(nbins - rest); bin_id += kUnroll) {
-    tree::GradStats pb[kUnroll];
+    GHistEntry pb[kUnroll];
-    tree::GradStats sb[kUnroll];
+    GHistEntry sb[kUnroll];
     for (int k = 0; k < kUnroll; ++k) {
-      pb[k] = p_parent[bin_id + k];
+      pb[k] = parent.begin[bin_id + k];
     }
     for (int k = 0; k < kUnroll; ++k) {
-      sb[k] = p_sibling[bin_id + k];
+      sb[k] = sibling.begin[bin_id + k];
     }
     for (int k = 0; k < kUnroll; ++k) {
-      p_self[bin_id + k].SetSubstract(pb[k], sb[k]);
+      self.begin[bin_id + k].SetSubtract(pb[k], sb[k]);
     }
   }
   for (uint32_t bin_id = nbins - rest; bin_id < nbins; ++bin_id) {
-    p_self[bin_id].SetSubstract(p_parent[bin_id], p_sibling[bin_id]);
+    self.begin[bin_id].SetSubtract(parent.begin[bin_id], sibling.begin[bin_id]);
   }
 }
 

src/common/hist_util.cu

@@ -116,19 +116,19 @@ struct GPUSketcher {
       n_rows_(row_end - row_begin), param_(std::move(param)) {
     }
 
-    void Init(const SparsePage& row_batch, const MetaInfo& info, int gpu_batch_nrows) {
+    void Init(const SparsePage& row_batch, const MetaInfo& info) {
       num_cols_ = info.num_col_;
       has_weights_ = info.weights_.Size() > 0;
 
       // find the batch size
-      if (gpu_batch_nrows == 0) {
+      if (param_.gpu_batch_nrows == 0) {
         // By default, use no more than 1/16th of GPU memory
         gpu_batch_nrows_ = dh::TotalMemory(device_) /
           (16 * num_cols_ * sizeof(Entry));
-      } else if (gpu_batch_nrows == -1) {
+      } else if (param_.gpu_batch_nrows == -1) {
         gpu_batch_nrows_ = n_rows_;
       } else {
-        gpu_batch_nrows_ = gpu_batch_nrows;
+        gpu_batch_nrows_ = param_.gpu_batch_nrows;
       }
       if (gpu_batch_nrows_ > n_rows_) {
         gpu_batch_nrows_ = n_rows_;
@@ -346,24 +346,21 @@ struct GPUSketcher {
     }
   };
 
-  void Sketch(const SparsePage& batch, const MetaInfo& info,
+  void Sketch(const SparsePage& batch, const MetaInfo& info, HistCutMatrix* hmat) {
-              HistCutMatrix* hmat, int gpu_batch_nrows) {
     // create device shards
     shards_.resize(dist_.Devices().Size());
     dh::ExecuteIndexShards(&shards_, [&](int i, std::unique_ptr<DeviceShard>& shard) {
         size_t start = dist_.ShardStart(info.num_row_, i);
         size_t size = dist_.ShardSize(info.num_row_, i);
-        shard = std::unique_ptr<DeviceShard>(
+        shard = std::unique_ptr<DeviceShard>
-            new DeviceShard(dist_.Devices().DeviceId(i),
+          (new DeviceShard(dist_.Devices()[i], start, start + size, param_));
-                            start, start + size, param_));
       });
 
     // compute sketches for each shard
-    dh::ExecuteIndexShards(&shards_,
+    dh::ExecuteShards(&shards_, [&](std::unique_ptr<DeviceShard>& shard) {
-                           [&](int idx, std::unique_ptr<DeviceShard>& shard) {
+        shard->Init(batch, info);
-                             shard->Init(batch, info, gpu_batch_nrows);
+        shard->Sketch(batch, info);
-                             shard->Sketch(batch, info);
+      });
-                           });
 
     // merge the sketches from all shards
     // TODO(canonizer): do it in a tree-like reduction
@@ -382,7 +379,8 @@ struct GPUSketcher {
   }
 
   GPUSketcher(tree::TrainParam param, size_t n_rows) : param_(std::move(param)) {
-    dist_ = GPUDistribution::Block(GPUSet::All(param_.gpu_id, param_.n_gpus, n_rows));
+    dist_ = GPUDistribution::Block(GPUSet::All(param_.n_gpus, n_rows).
+                                   Normalised(param_.gpu_id));
   }
 
   std::vector<std::unique_ptr<DeviceShard>> shards_;
@@ -392,9 +390,9 @@ struct GPUSketcher {
 
 void DeviceSketch
   (const SparsePage& batch, const MetaInfo& info,
-   const tree::TrainParam& param, HistCutMatrix* hmat, int gpu_batch_nrows) {
+   const tree::TrainParam& param, HistCutMatrix* hmat) {
   GPUSketcher sketcher(param, info.num_row_);
-  sketcher.Sketch(batch, info, hmat, gpu_batch_nrows);
+  sketcher.Sketch(batch, info, hmat);
 }
 
 }  // namespace common

src/common/hist_util.h

@@ -11,14 +11,48 @@
 #include <limits>
 #include <vector>
 #include "row_set.h"
+#include "../tree/fast_hist_param.h"
 #include "../tree/param.h"
 #include "./quantile.h"
-#include "./timer.h"
-#include "../include/rabit/rabit.h"
 
 namespace xgboost {
+
 namespace common {
 
+using tree::FastHistParam;
+
+/*! \brief sums of gradient statistics corresponding to a histogram bin */
+struct GHistEntry {
+  /*! \brief sum of first-order gradient statistics */
+  double sum_grad{0};
+  /*! \brief sum of second-order gradient statistics */
+  double sum_hess{0};
+
+  GHistEntry()  = default;
+
+  inline void Clear() {
+    sum_grad = sum_hess = 0;
+  }
+
+  /*! \brief add a GradientPair to the sum */
+  inline void Add(const GradientPair& e) {
+    sum_grad += e.GetGrad();
+    sum_hess += e.GetHess();
+  }
+
+  /*! \brief add a GHistEntry to the sum */
+  inline void Add(const GHistEntry& e) {
+    sum_grad += e.sum_grad;
+    sum_hess += e.sum_hess;
+  }
+
+  /*! \brief set sum to be difference of two GHistEntry's */
+  inline void SetSubtract(const GHistEntry& a, const GHistEntry& b) {
+    sum_grad = a.sum_grad - b.sum_grad;
+    sum_hess = a.sum_hess - b.sum_hess;
+  }
+};
+
 /*! \brief Cut configuration for all the features. */
 struct HistCutMatrix {
   /*! \brief Unit pointer to rows by element position */
@@ -36,26 +70,26 @@ struct HistCutMatrix {
   void Init(DMatrix* p_fmat, uint32_t max_num_bins);
 
   void Init(std::vector<WXQSketch>* sketchs, uint32_t max_num_bins);
-
-  HistCutMatrix();
-
- protected:
-  virtual size_t SearchGroupIndFromBaseRow(
-      std::vector<bst_uint> const& group_ptr, size_t const base_rowid) const;
-
-  Monitor monitor_;
 };
 
 /*! \brief Builds the cut matrix on the GPU */
 void DeviceSketch
   (const SparsePage& batch, const MetaInfo& info,
-   const tree::TrainParam& param, HistCutMatrix* hmat, int gpu_batch_nrows);
+   const tree::TrainParam& param, HistCutMatrix* hmat);
 
 /*!
  * \brief A single row in global histogram index.
  *  Directly represent the global index in the histogram entry.
  */
-using GHistIndexRow = Span<uint32_t const>;
+struct GHistIndexRow {
+  /*! \brief The index of the histogram */
+  const uint32_t* index;
+  /*! \brief The size of the histogram */
+  size_t size;
+  GHistIndexRow() = default;
+  GHistIndexRow(const uint32_t* index, size_t size)
+      : index(index), size(size) {}
+};
 
 /*!
  * \brief preprocessed global index matrix, in CSR format
@@ -75,9 +109,7 @@ struct GHistIndexMatrix {
   void Init(DMatrix* p_fmat, int max_num_bins);
   // get i-th row
   inline GHistIndexRow operator[](size_t i) const {
-    return {&index[0] + row_ptr[i],
+    return {&index[0] + row_ptr[i], row_ptr[i + 1] - row_ptr[i]};
-            static_cast<GHistIndexRow::index_type>(
-                row_ptr[i + 1] - row_ptr[i])};
   }
   inline void GetFeatureCounts(size_t* counts) const {
     auto nfeature = cut.row_ptr.size() - 1;
@@ -100,6 +132,11 @@ struct GHistIndexBlock {
 
   inline GHistIndexBlock(const size_t* row_ptr, const uint32_t* index)
     : row_ptr(row_ptr), index(index) {}
+
+  // get i-th row
+  inline GHistIndexRow operator[](size_t i) const {
+    return {&index[0] + row_ptr[i], row_ptr[i + 1] - row_ptr[i]};
+  }
 };
 
 class ColumnMatrix;
@@ -108,7 +145,7 @@ class GHistIndexBlockMatrix {
  public:
   void Init(const GHistIndexMatrix& gmat,
             const ColumnMatrix& colmat,
-            const tree::TrainParam& param);
+            const FastHistParam& param);
 
   inline GHistIndexBlock operator[](size_t i) const {
     return {blocks_[i].row_ptr_begin, blocks_[i].index_begin};
@@ -133,11 +170,20 @@ class GHistIndexBlockMatrix {
 
 /*!
  * \brief histogram of graident statistics for a single node.
- *  Consists of multiple GradStats, each entry showing total graident statistics
+ *  Consists of multiple GHistEntry's, each entry showing total graident statistics
  *     for that particular bin
  *  Uses global bin id so as to represent all features simultaneously
  */
-using GHistRow = Span<tree::GradStats>;
+struct GHistRow {
+  /*! \brief base pointer to first entry */
+  GHistEntry* begin;
+  /*! \brief number of entries */
+  uint32_t size;
+
+  GHistRow() = default;
+  GHistRow(GHistEntry* begin, uint32_t size)
+    : begin(begin), size(size) {}
+};
 
 /*!
  * \brief histogram of gradient statistics for multiple nodes
@@ -145,29 +191,27 @@ using GHistRow = Span<tree::GradStats>;
 class HistCollection {
  public:
   // access histogram for i-th node
-  GHistRow operator[](bst_uint nid) const {
+  inline GHistRow operator[](bst_uint nid) const {
     constexpr uint32_t kMax = std::numeric_limits<uint32_t>::max();
     CHECK_NE(row_ptr_[nid], kMax);
-    tree::GradStats* ptr =
+    return {const_cast<GHistEntry*>(dmlc::BeginPtr(data_) + row_ptr_[nid]), nbins_};
-        const_cast<tree::GradStats*>(dmlc::BeginPtr(data_) + row_ptr_[nid]);
-    return {ptr, nbins_};
   }
 
   // have we computed a histogram for i-th node?
-  bool RowExists(bst_uint nid) const {
+  inline bool RowExists(bst_uint nid) const {
     const uint32_t k_max = std::numeric_limits<uint32_t>::max();
     return (nid < row_ptr_.size() && row_ptr_[nid] != k_max);
   }
 
   // initialize histogram collection
-  void Init(uint32_t nbins) {
+  inline void Init(uint32_t nbins) {
     nbins_ = nbins;
     row_ptr_.clear();
     data_.clear();
   }
 
   // create an empty histogram for i-th node
-  void AddHistRow(bst_uint nid) {
+  inline void AddHistRow(bst_uint nid) {
     constexpr uint32_t kMax = std::numeric_limits<uint32_t>::max();
     if (nid >= row_ptr_.size()) {
       row_ptr_.resize(nid + 1, kMax);
@@ -182,7 +226,7 @@ class HistCollection {
   /*! \brief number of all bins over all features */
   uint32_t nbins_;
 
-  std::vector<tree::GradStats> data_;
+  std::vector<GHistEntry> data_;
 
   /*! \brief row_ptr_[nid] locates bin for historgram of node nid */
   std::vector<size_t> row_ptr_;
@@ -197,7 +241,6 @@ class GHistBuilder {
   inline void Init(size_t nthread, uint32_t nbins) {
     nthread_ = nthread;
     nbins_ = nbins;
-    thread_init_.resize(nthread_);
   }
 
   // construct a histogram via histogram aggregation
@@ -213,17 +256,12 @@ class GHistBuilder {
   // construct a histogram via subtraction trick
   void SubtractionTrick(GHistRow self, GHistRow sibling, GHistRow parent);
 
-  uint32_t GetNumBins() {
-      return nbins_;
-  }
-
  private:
   /*! \brief number of threads for parallel computation */
   size_t nthread_;
   /*! \brief number of all bins over all features */
   uint32_t nbins_;
-  std::vector<size_t> thread_init_;
+  std::vector<GHistEntry> data_;
-  std::vector<tree::GradStats> data_;
 };
 
 

src/common/host_device_vector.cc

@@ -159,4 +159,4 @@ template class HostDeviceVector<size_t>;
 
 }  // namespace xgboost
 
-#endif  // XGBOOST_USE_CUDA
+#endif

src/common/host_device_vector.cu

@@ -46,13 +46,14 @@ template <typename T>
 struct HostDeviceVectorImpl {
   struct DeviceShard {
     DeviceShard()
-        : proper_size_(0), device_(-1), start_(0), perm_d_(false),
+      : index_(-1), proper_size_(0), device_(-1), start_(0), perm_d_(false),
         cached_size_(~0), vec_(nullptr) {}
 
     void Init(HostDeviceVectorImpl<T>* vec, int device) {
       if (vec_ == nullptr) { vec_ = vec; }
       CHECK_EQ(vec, vec_);
       device_ = device;
+      index_ = vec_->distribution_.devices_.Index(device);
       LazyResize(vec_->Size());
       perm_d_ = vec_->perm_h_.Complementary();
     }
@@ -61,6 +62,7 @@ struct HostDeviceVectorImpl {
       if (vec_ == nullptr) { vec_ = vec; }
       CHECK_EQ(vec, vec_);
       device_ = other.device_;
+      index_ = other.index_;
       cached_size_ = other.cached_size_;
       start_ = other.start_;
       proper_size_ = other.proper_size_;
@@ -112,11 +114,10 @@ struct HostDeviceVectorImpl {
       if (new_size == cached_size_) { return; }
       // resize is required
       int ndevices = vec_->distribution_.devices_.Size();
-      int device_index = vec_->distribution_.devices_.Index(device_);
+      start_ = vec_->distribution_.ShardStart(new_size, index_);
-      start_ = vec_->distribution_.ShardStart(new_size, device_index);
+      proper_size_ = vec_->distribution_.ShardProperSize(new_size, index_);
-      proper_size_ = vec_->distribution_.ShardProperSize(new_size, device_index);
       // The size on this device.
-      size_t size_d = vec_->distribution_.ShardSize(new_size, device_index);
+      size_t size_d = vec_->distribution_.ShardSize(new_size, index_);
       SetDevice();
       data_.resize(size_d);
       cached_size_ = new_size;
@@ -153,6 +154,7 @@ struct HostDeviceVectorImpl {
       }
     }
 
+    int index_;
     int device_;
     thrust::device_vector<T> data_;
     // cached vector size
@@ -181,13 +183,13 @@ struct HostDeviceVectorImpl {
       distribution_(other.distribution_), mutex_() {
     shards_.resize(other.shards_.size());
     dh::ExecuteIndexShards(&shards_, [&](int i, DeviceShard& shard) {
-        shard.Init(this, other.shards_.at(i));
+        shard.Init(this, other.shards_[i]);
       });
   }
 
-  // Initializer can be std::vector<T> or std::initializer_list<T>
+  // Init can be std::vector<T> or std::initializer_list<T>
-  template <class Initializer>
+  template <class Init>
-  HostDeviceVectorImpl(const Initializer& init, GPUDistribution distribution)
+  HostDeviceVectorImpl(const Init& init, GPUDistribution distribution)
     : distribution_(distribution), perm_h_(distribution.IsEmpty()), size_d_(0) {
     if (!distribution_.IsEmpty()) {
       size_d_ = init.size();
@@ -202,7 +204,7 @@ struct HostDeviceVectorImpl {
     int ndevices = distribution_.devices_.Size();
     shards_.resize(ndevices);
     dh::ExecuteIndexShards(&shards_, [&](int i, DeviceShard& shard) {
-        shard.Init(this, distribution_.devices_.DeviceId(i));
+        shard.Init(this, distribution_.devices_[i]);
       });
   }
 
@@ -215,20 +217,20 @@ struct HostDeviceVectorImpl {
   T* DevicePointer(int device) {
     CHECK(distribution_.devices_.Contains(device));
     LazySyncDevice(device, GPUAccess::kWrite);
-    return shards_.at(distribution_.devices_.Index(device)).data_.data().get();
+    return shards_[distribution_.devices_.Index(device)].data_.data().get();
   }
 
   const T* ConstDevicePointer(int device) {
     CHECK(distribution_.devices_.Contains(device));
     LazySyncDevice(device, GPUAccess::kRead);
-    return shards_.at(distribution_.devices_.Index(device)).data_.data().get();
+    return shards_[distribution_.devices_.Index(device)].data_.data().get();
   }
 
   common::Span<T> DeviceSpan(int device) {
     GPUSet devices = distribution_.devices_;
     CHECK(devices.Contains(device));
     LazySyncDevice(device, GPUAccess::kWrite);
-    return {shards_.at(devices.Index(device)).data_.data().get(),
+    return {shards_[devices.Index(device)].data_.data().get(),
         static_cast<typename common::Span<T>::index_type>(DeviceSize(device))};
   }
 
@@ -236,20 +238,20 @@ struct HostDeviceVectorImpl {
     GPUSet devices = distribution_.devices_;
     CHECK(devices.Contains(device));
     LazySyncDevice(device, GPUAccess::kRead);
-    return {shards_.at(devices.Index(device)).data_.data().get(),
+    return {shards_[devices.Index(device)].data_.data().get(),
         static_cast<typename common::Span<const T>::index_type>(DeviceSize(device))};
   }
 
   size_t DeviceSize(int device) {
     CHECK(distribution_.devices_.Contains(device));
     LazySyncDevice(device, GPUAccess::kRead);
-    return shards_.at(distribution_.devices_.Index(device)).data_.size();
+    return shards_[distribution_.devices_.Index(device)].data_.size();
   }
 
   size_t DeviceStart(int device) {
     CHECK(distribution_.devices_.Contains(device));
     LazySyncDevice(device, GPUAccess::kRead);
-    return shards_.at(distribution_.devices_.Index(device)).start_;
+    return shards_[distribution_.devices_.Index(device)].start_;
   }
 
   thrust::device_ptr<T> tbegin(int device) {  // NOLINT
@@ -275,7 +277,7 @@ struct HostDeviceVectorImpl {
                                (end - begin) * sizeof(T),
                                cudaMemcpyDeviceToHost));
     } else {
-    dh::ExecuteIndexShards(&shards_, [&](int idx, DeviceShard& shard) {
+      dh::ExecuteShards(&shards_, [&](DeviceShard& shard) {
         shard.ScatterFrom(begin.get());
       });
     }
@@ -288,7 +290,7 @@ struct HostDeviceVectorImpl {
                                data_h_.size() * sizeof(T),
                                cudaMemcpyHostToDevice));
     } else {
-      dh::ExecuteIndexShards(&shards_, [&](int idx, DeviceShard& shard) { shard.GatherTo(begin); });
+      dh::ExecuteShards(&shards_, [&](DeviceShard& shard) { shard.GatherTo(begin); });
     }
   }
 
@@ -296,7 +298,7 @@ struct HostDeviceVectorImpl {
     if (perm_h_.CanWrite()) {
       std::fill(data_h_.begin(), data_h_.end(), v);
     } else {
-      dh::ExecuteIndexShards(&shards_, [&](int idx, DeviceShard& shard) { shard.Fill(v); });
+      dh::ExecuteShards(&shards_, [&](DeviceShard& shard) { shard.Fill(v); });
     }
   }
 
@@ -314,7 +316,7 @@ struct HostDeviceVectorImpl {
       size_d_ = other->size_d_;
     }
     dh::ExecuteIndexShards(&shards_, [&](int i, DeviceShard& shard) {
-        shard.Copy(&other->shards_.at(i));
+        shard.Copy(&other->shards_[i]);
       });
   }
 
@@ -323,7 +325,7 @@ struct HostDeviceVectorImpl {
     if (perm_h_.CanWrite()) {
       std::copy(other.begin(), other.end(), data_h_.begin());
     } else {
-      dh::ExecuteIndexShards(&shards_, [&](int idx, DeviceShard& shard) {
+      dh::ExecuteShards(&shards_, [&](DeviceShard& shard) {
           shard.ScatterFrom(other.data());
         });
     }
@@ -334,7 +336,7 @@ struct HostDeviceVectorImpl {
     if (perm_h_.CanWrite()) {
       std::copy(other.begin(), other.end(), data_h_.begin());
     } else {
-      dh::ExecuteIndexShards(&shards_, [&](int idx, DeviceShard& shard) {
+      dh::ExecuteShards(&shards_, [&](DeviceShard& shard) {
           shard.ScatterFrom(other.begin());
         });
     }
@@ -387,14 +389,14 @@ struct HostDeviceVectorImpl {
     if (perm_h_.CanAccess(access)) { return; }
     if (perm_h_.CanRead()) {
       // data is present, just need to deny access to the device
-      dh::ExecuteIndexShards(&shards_, [&](int idx, DeviceShard& shard) {
+      dh::ExecuteShards(&shards_, [&](DeviceShard& shard) {
           shard.perm_d_.DenyComplementary(access);
         });
       perm_h_.Grant(access);
       return;
     }
     if (data_h_.size() != size_d_) { data_h_.resize(size_d_); }
-    dh::ExecuteIndexShards(&shards_, [&](int idx, DeviceShard& shard) {
+    dh::ExecuteShards(&shards_, [&](DeviceShard& shard) {
         shard.LazySyncHost(access);
       });
     perm_h_.Grant(access);
@@ -403,7 +405,7 @@ struct HostDeviceVectorImpl {
   void LazySyncDevice(int device, GPUAccess access) {
     GPUSet devices = distribution_.Devices();
     CHECK(devices.Contains(device));
-    shards_.at(devices.Index(device)).LazySyncDevice(access);
+    shards_[devices.Index(device)].LazySyncDevice(access);
   }
 
   bool HostCanAccess(GPUAccess access) { return perm_h_.CanAccess(access); }
@@ -411,7 +413,7 @@ struct HostDeviceVectorImpl {
   bool DeviceCanAccess(int device, GPUAccess access) {
     GPUSet devices = distribution_.Devices();
     if (!devices.Contains(device)) { return false; }
-    return shards_.at(devices.Index(device)).perm_d_.CanAccess(access);
+    return shards_[devices.Index(device)].perm_d_.CanAccess(access);
   }
 
   std::vector<T> data_h_;
@@ -459,8 +461,9 @@ HostDeviceVector<T>& HostDeviceVector<T>::operator=
 
 template <typename T>
 HostDeviceVector<T>::~HostDeviceVector() {
-  delete impl_;
+  HostDeviceVectorImpl<T>* tmp = impl_;
   impl_ = nullptr;
+  delete tmp;
 }
 
 template <typename T>